program(1.3)
[buildInfo = dict<string, string>({{"coremlc-component-MIL", "3600.16.1"}, {"coremlc-version", "3600.22.1"}})]
{
    func main<ios18>(tensor<fp16, [1, 6144, 3]> conv0, tensor<fp16, [1, 6144, 3]> conv1, tensor<fp16, [1, 6144, 3]> conv10, tensor<fp16, [1, 6144, 3]> conv12, tensor<fp16, [1, 6144, 3]> conv13, tensor<fp16, [1, 6144, 3]> conv14, tensor<fp16, [1, 6144, 3]> conv16, tensor<fp16, [1, 6144, 3]> conv17, tensor<fp16, [1, 6144, 3]> conv18, tensor<fp16, [1, 6144, 3]> conv2, tensor<fp16, [1, 6144, 3]> conv4, tensor<fp16, [1, 6144, 3]> conv5, tensor<fp16, [1, 6144, 3]> conv6, tensor<fp16, [1, 6144, 3]> conv8, tensor<fp16, [1, 6144, 3]> conv9, tensor<fp16, [1, 1, 1, 64]> cos, tensor<int32, [1]> current_pos, tensor<fp16, [1, 1024]> hidden, state<tensor<fp16, [4, 2, 2, 2048, 256]>> kv_cache, tensor<fp16, [1, 1, 2048]> mask3, tensor<fp16, [1, 16, 128, 128]> rec0, tensor<fp16, [1, 16, 128, 128]> rec1, tensor<fp16, [1, 16, 128, 128]> rec10, tensor<fp16, [1, 16, 128, 128]> rec12, tensor<fp16, [1, 16, 128, 128]> rec13, tensor<fp16, [1, 16, 128, 128]> rec14, tensor<fp16, [1, 16, 128, 128]> rec16, tensor<fp16, [1, 16, 128, 128]> rec17, tensor<fp16, [1, 16, 128, 128]> rec18, tensor<fp16, [1, 16, 128, 128]> rec2, tensor<fp16, [1, 16, 128, 128]> rec4, tensor<fp16, [1, 16, 128, 128]> rec5, tensor<fp16, [1, 16, 128, 128]> rec6, tensor<fp16, [1, 16, 128, 128]> rec8, tensor<fp16, [1, 16, 128, 128]> rec9, tensor<fp16, [1, 1, 1, 64]> sin) {
            int32 var_118 = const()[name = string("op_118"), val = int32(-1)];
            tensor<fp16, [1, 1024]> var_133_cast_fp16 = mul(x = hidden, y = hidden)[name = string("op_133_cast_fp16")];
            tensor<int32, [1]> variance_1_axes_0 = const()[name = string("variance_1_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_1_keep_dims_0 = const()[name = string("variance_1_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 1]> variance_1_cast_fp16 = reduce_mean(axes = variance_1_axes_0, keep_dims = variance_1_keep_dims_0, x = var_133_cast_fp16)[name = string("variance_1_cast_fp16")];
            fp16 var_136_to_fp16 = const()[name = string("op_136_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 1]> var_137_cast_fp16 = add(x = variance_1_cast_fp16, y = var_136_to_fp16)[name = string("op_137_cast_fp16")];
            fp32 var_138_epsilon_0 = const()[name = string("op_138_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 1]> var_138_cast_fp16 = rsqrt(epsilon = var_138_epsilon_0, x = var_137_cast_fp16)[name = string("op_138_cast_fp16")];
            tensor<fp16, [1, 1024]> var_139_cast_fp16 = mul(x = hidden, y = var_138_cast_fp16)[name = string("op_139_cast_fp16")];
            tensor<fp16, [1024]> var_141_to_fp16 = const()[name = string("op_141_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(64)))];
            tensor<fp16, [1, 1024]> input_1_cast_fp16 = mul(x = var_139_cast_fp16, y = var_141_to_fp16)[name = string("input_1_cast_fp16")];
            tensor<fp16, [6144, 1024]> groups_0_0_in_proj_qkv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(2176))), lut = tensor<fp16, [384, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(4720832))))[name = string("groups_0_0_in_proj_qkv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [6144]> linear_0_bias_0_to_fp16 = const()[name = string("linear_0_bias_0_to_fp16"), val = tensor<fp16, [6144]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(4770048)))];
            tensor<fp16, [1, 6144]> linear_0_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = groups_0_0_in_proj_qkv_weight_promoted_to_fp16_palettized, x = input_1_cast_fp16)[name = string("linear_0_cast_fp16")];
            tensor<int32, [2]> var_145_perm_0 = const()[name = string("op_145_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> mixed_1_axes_0 = const()[name = string("mixed_1_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [6144, 1]> var_145_cast_fp16 = transpose(perm = var_145_perm_0, x = linear_0_cast_fp16)[name = string("transpose_141")];
            tensor<fp16, [1, 6144, 1]> mixed_1_cast_fp16 = expand_dims(axes = mixed_1_axes_0, x = var_145_cast_fp16)[name = string("mixed_1_cast_fp16")];
            bool convolution_input_1_interleave_0 = const()[name = string("convolution_input_1_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 6144, 4]> convolution_input_1_cast_fp16 = concat(axis = var_118, interleave = convolution_input_1_interleave_0, values = (conv0, mixed_1_cast_fp16))[name = string("convolution_input_1_cast_fp16")];
            string input_3_pad_type_0 = const()[name = string("input_3_pad_type_0"), val = string("valid")];
            int32 input_3_groups_0 = const()[name = string("input_3_groups_0"), val = int32(6144)];
            tensor<int32, [1]> input_3_strides_0 = const()[name = string("input_3_strides_0"), val = tensor<int32, [1]>([1])];
            tensor<int32, [2]> input_3_pad_0 = const()[name = string("input_3_pad_0"), val = tensor<int32, [2]>([0, 0])];
            tensor<int32, [1]> input_3_dilations_0 = const()[name = string("input_3_dilations_0"), val = tensor<int32, [1]>([1])];
            tensor<fp16, [6144, 1, 4]> groups_0_0_conv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1, 4]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(4782400))), lut = tensor<fp16, [384, 1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(4800896))))[name = string("groups_0_0_conv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 6144, 1]> input_3_cast_fp16 = conv(dilations = input_3_dilations_0, groups = input_3_groups_0, pad = input_3_pad_0, pad_type = input_3_pad_type_0, strides = input_3_strides_0, weight = groups_0_0_conv_weight_promoted_to_fp16_palettized, x = convolution_input_1_cast_fp16)[name = string("input_3_cast_fp16")];
            tensor<fp16, [1, 6144, 1]> var_154_cast_fp16 = silu(x = input_3_cast_fp16)[name = string("op_154_cast_fp16")];
            tensor<int32, [3]> var_155_perm_0 = const()[name = string("op_155_perm_0"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [3]> var_158_begin_0 = const()[name = string("op_158_begin_0"), val = tensor<int32, [3]>([0, 0, 1])];
            tensor<int32, [3]> var_158_end_0 = const()[name = string("op_158_end_0"), val = tensor<int32, [3]>([1, 6144, 4])];
            tensor<bool, [3]> var_158_end_mask_0 = const()[name = string("op_158_end_mask_0"), val = tensor<bool, [3]>([true, true, true])];
            tensor<fp16, [1, 6144, 3]> conv0_out = slice_by_index(begin = var_158_begin_0, end = var_158_end_0, end_mask = var_158_end_mask_0, x = convolution_input_1_cast_fp16)[name = string("op_158_cast_fp16")];
            tensor<int32, [3]> var_159 = const()[name = string("op_159"), val = tensor<int32, [3]>([2048, 2048, 2048])];
            int32 var_160_axis_0 = const()[name = string("op_160_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 1, 6144]> var_155_cast_fp16 = transpose(perm = var_155_perm_0, x = var_154_cast_fp16)[name = string("transpose_140")];
            tensor<fp16, [1, 1, 2048]> var_160_cast_fp16_0, tensor<fp16, [1, 1, 2048]> var_160_cast_fp16_1, tensor<fp16, [1, 1, 2048]> var_160_cast_fp16_2 = split(axis = var_160_axis_0, split_sizes = var_159, x = var_155_cast_fp16)[name = string("op_160_cast_fp16")];
            tensor<int32, [4]> var_164 = const()[name = string("op_164"), val = tensor<int32, [4]>([1, 1, 16, 128])];
            tensor<fp16, [1, 1, 16, 128]> value_5_cast_fp16 = reshape(shape = var_164, x = var_160_cast_fp16_0)[name = string("value_5_cast_fp16")];
            tensor<int32, [4]> var_166 = const()[name = string("op_166"), val = tensor<int32, [4]>([1, 1, 16, 128])];
            tensor<fp16, [1, 1, 16, 128]> value_7_cast_fp16 = reshape(shape = var_166, x = var_160_cast_fp16_1)[name = string("value_7_cast_fp16")];
            tensor<int32, [4]> var_168 = const()[name = string("op_168"), val = tensor<int32, [4]>([1, 1, 16, 128])];
            tensor<fp16, [1, 1, 16, 128]> value_9_cast_fp16 = reshape(shape = var_168, x = var_160_cast_fp16_2)[name = string("value_9_cast_fp16")];
            tensor<fp16, [1, 1, 16, 128]> var_170_cast_fp16 = mul(x = value_5_cast_fp16, y = value_5_cast_fp16)[name = string("op_170_cast_fp16")];
            tensor<int32, [1]> var_172_axes_0 = const()[name = string("op_172_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_172_keep_dims_0 = const()[name = string("op_172_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 1, 16, 1]> var_172_cast_fp16 = reduce_sum(axes = var_172_axes_0, keep_dims = var_172_keep_dims_0, x = var_170_cast_fp16)[name = string("op_172_cast_fp16")];
            fp16 var_173_to_fp16 = const()[name = string("op_173_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 1, 16, 1]> var_174_cast_fp16 = add(x = var_172_cast_fp16, y = var_173_to_fp16)[name = string("op_174_cast_fp16")];
            fp32 var_175_epsilon_0 = const()[name = string("op_175_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 1, 16, 1]> var_175_cast_fp16 = rsqrt(epsilon = var_175_epsilon_0, x = var_174_cast_fp16)[name = string("op_175_cast_fp16")];
            tensor<fp16, [1, 1, 16, 128]> var_176_cast_fp16 = mul(x = value_5_cast_fp16, y = var_175_cast_fp16)[name = string("op_176_cast_fp16")];
            tensor<int32, [4]> var_177_perm_0 = const()[name = string("op_177_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            fp16 _inversed_query_3_y_0_to_fp16 = const()[name = string("_inversed_query_3_y_0_to_fp16"), val = fp16(0x1.6ap-4)];
            tensor<fp16, [1, 16, 1, 128]> var_177_cast_fp16 = transpose(perm = var_177_perm_0, x = var_176_cast_fp16)[name = string("transpose_139")];
            tensor<fp16, [1, 16, 1, 128]> _inversed_query_3_cast_fp16 = mul(x = var_177_cast_fp16, y = _inversed_query_3_y_0_to_fp16)[name = string("_inversed_query_3_cast_fp16")];
            tensor<fp16, [1, 1, 16, 128]> var_180_cast_fp16 = mul(x = value_7_cast_fp16, y = value_7_cast_fp16)[name = string("op_180_cast_fp16")];
            tensor<int32, [1]> var_182_axes_0 = const()[name = string("op_182_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_182_keep_dims_0 = const()[name = string("op_182_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 1, 16, 1]> var_182_cast_fp16 = reduce_sum(axes = var_182_axes_0, keep_dims = var_182_keep_dims_0, x = var_180_cast_fp16)[name = string("op_182_cast_fp16")];
            fp16 var_183_to_fp16 = const()[name = string("op_183_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 1, 16, 1]> var_184_cast_fp16 = add(x = var_182_cast_fp16, y = var_183_to_fp16)[name = string("op_184_cast_fp16")];
            fp32 var_185_epsilon_0 = const()[name = string("op_185_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 1, 16, 1]> var_185_cast_fp16 = rsqrt(epsilon = var_185_epsilon_0, x = var_184_cast_fp16)[name = string("op_185_cast_fp16")];
            tensor<fp16, [1, 1, 16, 128]> var_186_cast_fp16 = mul(x = value_7_cast_fp16, y = var_185_cast_fp16)[name = string("op_186_cast_fp16")];
            tensor<int32, [4]> key_3_perm_0 = const()[name = string("key_3_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<int32, [4]> value_11_perm_0 = const()[name = string("value_11_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<fp16, [16, 1024]> groups_0_0_in_proj_b_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(4850112))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(4862464))))[name = string("groups_0_0_in_proj_b_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16]> linear_1_bias_0_to_fp16 = const()[name = string("linear_1_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(4862656)))];
            tensor<fp16, [1, 16]> linear_1_cast_fp16 = linear(bias = linear_1_bias_0_to_fp16, weight = groups_0_0_in_proj_b_weight_promoted_to_fp16_palettized, x = input_1_cast_fp16)[name = string("linear_1_cast_fp16")];
            tensor<fp16, [1, 16]> var_191_cast_fp16 = sigmoid(x = linear_1_cast_fp16)[name = string("op_191_cast_fp16")];
            tensor<int32, [2]> var_192_perm_0 = const()[name = string("op_192_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> beta_1_axes_0 = const()[name = string("beta_1_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 1]> var_192_cast_fp16 = transpose(perm = var_192_perm_0, x = var_191_cast_fp16)[name = string("transpose_138")];
            tensor<fp16, [1, 16, 1]> beta_1_cast_fp16 = expand_dims(axes = beta_1_axes_0, x = var_192_cast_fp16)[name = string("beta_1_cast_fp16")];
            tensor<fp16, [16, 1024]> op_198_weight_0_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(4862784))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(4875136))))[name = string("op_198_weight_0_to_fp16_palettized")];
            tensor<fp16, [16]> var_198_bias_0_to_fp16 = const()[name = string("op_198_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(4875328)))];
            tensor<fp16, [1, 16]> var_198_cast_fp16 = linear(bias = var_198_bias_0_to_fp16, weight = op_198_weight_0_to_fp16_palettized, x = input_1_cast_fp16)[name = string("op_198_cast_fp16")];
            tensor<fp16, [1, 16]> var_199_cast_fp16 = softplus(x = var_198_cast_fp16)[name = string("op_199_cast_fp16")];
            tensor<fp16, [16]> var_195_promoted_to_fp16 = const()[name = string("op_195_promoted_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(4875456)))];
            tensor<fp16, [1, 16]> var_200_cast_fp16 = mul(x = var_195_promoted_to_fp16, y = var_199_cast_fp16)[name = string("op_200_cast_fp16")];
            tensor<int32, [2]> var_201_perm_0 = const()[name = string("op_201_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> decay_1_axes_0 = const()[name = string("decay_1_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 1]> var_201_cast_fp16 = transpose(perm = var_201_perm_0, x = var_200_cast_fp16)[name = string("transpose_137")];
            tensor<fp16, [1, 16, 1]> decay_1_cast_fp16 = expand_dims(axes = decay_1_axes_0, x = var_201_cast_fp16)[name = string("decay_1_cast_fp16")];
            tensor<fp16, [2048, 1024]> groups_0_0_in_proj_z_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [2048, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(4875584))), lut = tensor<fp16, [128, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(6448512))))[name = string("groups_0_0_in_proj_z_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [2048]> linear_3_bias_0_to_fp16 = const()[name = string("linear_3_bias_0_to_fp16"), val = tensor<fp16, [2048]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(6464960)))];
            tensor<fp16, [1, 2048]> linear_3_cast_fp16 = linear(bias = linear_3_bias_0_to_fp16, weight = groups_0_0_in_proj_z_weight_promoted_to_fp16_palettized, x = input_1_cast_fp16)[name = string("linear_3_cast_fp16")];
            tensor<int32, [3]> var_209_begin_0 = const()[name = string("op_209_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_209_end_0 = const()[name = string("op_209_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_209_end_mask_0 = const()[name = string("op_209_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_209_squeeze_mask_0 = const()[name = string("op_209_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_209_cast_fp16 = slice_by_index(begin = var_209_begin_0, end = var_209_end_0, end_mask = var_209_end_mask_0, squeeze_mask = var_209_squeeze_mask_0, x = decay_1_cast_fp16)[name = string("op_209_cast_fp16")];
            tensor<fp16, [1, 16]> var_210_cast_fp16 = exp(x = var_209_cast_fp16)[name = string("op_210_cast_fp16")];
            tensor<int32, [1]> var_211_axes_0 = const()[name = string("op_211_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_211_cast_fp16 = expand_dims(axes = var_211_axes_0, x = var_210_cast_fp16)[name = string("op_211_cast_fp16")];
            tensor<int32, [1]> var_212_axes_0 = const()[name = string("op_212_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_212_cast_fp16 = expand_dims(axes = var_212_axes_0, x = var_211_cast_fp16)[name = string("op_212_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_1_cast_fp16 = mul(x = rec0, y = var_212_cast_fp16)[name = string("state_1_cast_fp16")];
            tensor<int32, [4]> key_token_1_begin_0 = const()[name = string("key_token_1_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> key_token_1_end_0 = const()[name = string("key_token_1_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> key_token_1_end_mask_0 = const()[name = string("key_token_1_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_1_squeeze_mask_0 = const()[name = string("key_token_1_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 1, 128]> key_3_cast_fp16 = transpose(perm = key_3_perm_0, x = var_186_cast_fp16)[name = string("transpose_136")];
            tensor<fp16, [1, 16, 128]> key_token_1_cast_fp16 = slice_by_index(begin = key_token_1_begin_0, end = key_token_1_end_0, end_mask = key_token_1_end_mask_0, squeeze_mask = key_token_1_squeeze_mask_0, x = key_3_cast_fp16)[name = string("key_token_1_cast_fp16")];
            tensor<int32, [4]> value_token_1_begin_0 = const()[name = string("value_token_1_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> value_token_1_end_0 = const()[name = string("value_token_1_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> value_token_1_end_mask_0 = const()[name = string("value_token_1_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_1_squeeze_mask_0 = const()[name = string("value_token_1_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 1, 128]> value_11_cast_fp16 = transpose(perm = value_11_perm_0, x = value_9_cast_fp16)[name = string("transpose_135")];
            tensor<fp16, [1, 16, 128]> value_token_1_cast_fp16 = slice_by_index(begin = value_token_1_begin_0, end = value_token_1_end_0, end_mask = value_token_1_end_mask_0, squeeze_mask = value_token_1_squeeze_mask_0, x = value_11_cast_fp16)[name = string("value_token_1_cast_fp16")];
            tensor<int32, [1]> var_220_axes_0 = const()[name = string("op_220_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_220_cast_fp16 = expand_dims(axes = var_220_axes_0, x = key_token_1_cast_fp16)[name = string("op_220_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_221_cast_fp16 = mul(x = state_1_cast_fp16, y = var_220_cast_fp16)[name = string("op_221_cast_fp16")];
            tensor<int32, [1]> memory_1_axes_0 = const()[name = string("memory_1_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_1_keep_dims_0 = const()[name = string("memory_1_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_1_cast_fp16 = reduce_sum(axes = memory_1_axes_0, keep_dims = memory_1_keep_dims_0, x = var_221_cast_fp16)[name = string("memory_1_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_224_cast_fp16 = sub(x = value_token_1_cast_fp16, y = memory_1_cast_fp16)[name = string("op_224_cast_fp16")];
            tensor<int32, [3]> var_227_begin_0 = const()[name = string("op_227_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_227_end_0 = const()[name = string("op_227_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_227_end_mask_0 = const()[name = string("op_227_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_227_squeeze_mask_0 = const()[name = string("op_227_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_227_cast_fp16 = slice_by_index(begin = var_227_begin_0, end = var_227_end_0, end_mask = var_227_end_mask_0, squeeze_mask = var_227_squeeze_mask_0, x = beta_1_cast_fp16)[name = string("op_227_cast_fp16")];
            tensor<int32, [1]> var_228_axes_0 = const()[name = string("op_228_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_228_cast_fp16 = expand_dims(axes = var_228_axes_0, x = var_227_cast_fp16)[name = string("op_228_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_1_cast_fp16 = mul(x = var_224_cast_fp16, y = var_228_cast_fp16)[name = string("delta_1_cast_fp16")];
            tensor<int32, [1]> var_231_axes_0 = const()[name = string("op_231_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_231_cast_fp16 = expand_dims(axes = var_231_axes_0, x = delta_1_cast_fp16)[name = string("op_231_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_232_cast_fp16 = mul(x = var_220_cast_fp16, y = var_231_cast_fp16)[name = string("op_232_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> rec0_out = add(x = state_1_cast_fp16, y = var_232_cast_fp16)[name = string("state_3_cast_fp16")];
            tensor<int32, [4]> var_236_begin_0 = const()[name = string("op_236_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> var_236_end_0 = const()[name = string("op_236_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> var_236_end_mask_0 = const()[name = string("op_236_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_236_squeeze_mask_0 = const()[name = string("op_236_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_236_cast_fp16 = slice_by_index(begin = var_236_begin_0, end = var_236_end_0, end_mask = var_236_end_mask_0, squeeze_mask = var_236_squeeze_mask_0, x = _inversed_query_3_cast_fp16)[name = string("op_236_cast_fp16")];
            tensor<int32, [1]> var_237_axes_0 = const()[name = string("op_237_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_237_cast_fp16 = expand_dims(axes = var_237_axes_0, x = var_236_cast_fp16)[name = string("op_237_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_238_cast_fp16 = mul(x = rec0_out, y = var_237_cast_fp16)[name = string("op_238_cast_fp16")];
            tensor<int32, [1]> var_240_axes_0 = const()[name = string("op_240_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_240_keep_dims_0 = const()[name = string("op_240_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_240_cast_fp16 = reduce_sum(axes = var_240_axes_0, keep_dims = var_240_keep_dims_0, x = var_238_cast_fp16)[name = string("op_240_cast_fp16")];
            tensor<int32, [1]> attention_1_axes_0 = const()[name = string("attention_1_axes_0"), val = tensor<int32, [1]>([1])];
            tensor<fp16, [1, 1, 16, 128]> attention_1_cast_fp16 = expand_dims(axes = attention_1_axes_0, x = var_240_cast_fp16)[name = string("attention_1_cast_fp16")];
            tensor<int32, [2]> var_243 = const()[name = string("op_243"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [16, 128]> attention_3_cast_fp16 = reshape(shape = var_243, x = attention_1_cast_fp16)[name = string("attention_3_cast_fp16")];
            tensor<int32, [2]> var_245 = const()[name = string("op_245"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [16, 128]> input_5_cast_fp16 = reshape(shape = var_245, x = linear_3_cast_fp16)[name = string("input_5_cast_fp16")];
            tensor<fp16, [16, 128]> var_247_cast_fp16 = mul(x = attention_3_cast_fp16, y = attention_3_cast_fp16)[name = string("op_247_cast_fp16")];
            tensor<int32, [1]> variance_3_axes_0 = const()[name = string("variance_3_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_3_keep_dims_0 = const()[name = string("variance_3_keep_dims_0"), val = bool(true)];
            tensor<fp16, [16, 1]> variance_3_cast_fp16 = reduce_mean(axes = variance_3_axes_0, keep_dims = variance_3_keep_dims_0, x = var_247_cast_fp16)[name = string("variance_3_cast_fp16")];
            fp16 var_250_to_fp16 = const()[name = string("op_250_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [16, 1]> var_251_cast_fp16 = add(x = variance_3_cast_fp16, y = var_250_to_fp16)[name = string("op_251_cast_fp16")];
            fp32 var_252_epsilon_0 = const()[name = string("op_252_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [16, 1]> var_252_cast_fp16 = rsqrt(epsilon = var_252_epsilon_0, x = var_251_cast_fp16)[name = string("op_252_cast_fp16")];
            tensor<fp16, [16, 128]> attention_5_cast_fp16 = mul(x = attention_3_cast_fp16, y = var_252_cast_fp16)[name = string("attention_5_cast_fp16")];
            tensor<fp16, [128]> groups_0_0_gated_norm_promoted_to_fp16 = const()[name = string("groups_0_0_gated_norm_promoted_to_fp16"), val = tensor<fp16, [128]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(6469120)))];
            tensor<fp16, [16, 128]> attention_7_cast_fp16 = mul(x = attention_5_cast_fp16, y = groups_0_0_gated_norm_promoted_to_fp16)[name = string("attention_7_cast_fp16")];
            tensor<fp16, [16, 128]> var_255_cast_fp16 = silu(x = input_5_cast_fp16)[name = string("op_255_cast_fp16")];
            tensor<fp16, [16, 128]> attention_9_cast_fp16 = mul(x = attention_7_cast_fp16, y = var_255_cast_fp16)[name = string("attention_9_cast_fp16")];
            tensor<int32, [2]> var_257 = const()[name = string("op_257"), val = tensor<int32, [2]>([1, 2048])];
            tensor<fp16, [1, 2048]> input_7_cast_fp16 = reshape(shape = var_257, x = attention_9_cast_fp16)[name = string("input_7_cast_fp16")];
            tensor<fp16, [1024, 2048]> groups_0_0_out_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 2048]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(6469440))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(8042368))))[name = string("groups_0_0_out_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1024]> linear_4_bias_0_to_fp16 = const()[name = string("linear_4_bias_0_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(8050624)))];
            tensor<fp16, [1, 1024]> linear_4_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = groups_0_0_out_proj_weight_promoted_to_fp16_palettized, x = input_7_cast_fp16)[name = string("linear_4_cast_fp16")];
            tensor<fp16, [1, 1024]> value_13_cast_fp16 = add(x = hidden, y = linear_4_cast_fp16)[name = string("value_13_cast_fp16")];
            tensor<fp16, [1, 1024]> var_262_cast_fp16 = mul(x = value_13_cast_fp16, y = value_13_cast_fp16)[name = string("op_262_cast_fp16")];
            tensor<int32, [1]> variance_5_axes_0 = const()[name = string("variance_5_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_5_keep_dims_0 = const()[name = string("variance_5_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 1]> variance_5_cast_fp16 = reduce_mean(axes = variance_5_axes_0, keep_dims = variance_5_keep_dims_0, x = var_262_cast_fp16)[name = string("variance_5_cast_fp16")];
            fp16 var_265_to_fp16 = const()[name = string("op_265_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 1]> var_266_cast_fp16 = add(x = variance_5_cast_fp16, y = var_265_to_fp16)[name = string("op_266_cast_fp16")];
            fp32 var_267_epsilon_0 = const()[name = string("op_267_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 1]> var_267_cast_fp16 = rsqrt(epsilon = var_267_epsilon_0, x = var_266_cast_fp16)[name = string("op_267_cast_fp16")];
            tensor<fp16, [1, 1024]> var_268_cast_fp16 = mul(x = value_13_cast_fp16, y = var_267_cast_fp16)[name = string("op_268_cast_fp16")];
            tensor<fp16, [1024]> var_270_to_fp16 = const()[name = string("op_270_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(8052736)))];
            tensor<fp16, [1, 1024]> input_9_cast_fp16 = mul(x = var_268_cast_fp16, y = var_270_to_fp16)[name = string("input_9_cast_fp16")];
            tensor<fp16, [3584, 1024]> groups_0_0_gate_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(8054848))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(10807424))))[name = string("groups_0_0_gate_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [3584]> linear_5_bias_0_to_fp16 = const()[name = string("linear_5_bias_0_to_fp16"), val = tensor<fp16, [3584]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(10836160)))];
            tensor<fp16, [1, 3584]> linear_5_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = groups_0_0_gate_proj_weight_promoted_to_fp16_palettized, x = input_9_cast_fp16)[name = string("linear_5_cast_fp16")];
            tensor<fp16, [1, 3584]> var_274_cast_fp16 = silu(x = linear_5_cast_fp16)[name = string("op_274_cast_fp16")];
            tensor<fp16, [3584, 1024]> groups_0_0_up_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(10843392))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(13595968))))[name = string("groups_0_0_up_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 3584]> linear_6_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = groups_0_0_up_proj_weight_promoted_to_fp16_palettized, x = input_9_cast_fp16)[name = string("linear_6_cast_fp16")];
            tensor<fp16, [1, 3584]> input_13_cast_fp16 = mul(x = var_274_cast_fp16, y = linear_6_cast_fp16)[name = string("input_13_cast_fp16")];
            tensor<fp16, [1024, 3584]> groups_0_0_down_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 3584]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(13624704))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(16377280))))[name = string("groups_0_0_down_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 1024]> linear_7_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = groups_0_0_down_proj_weight_promoted_to_fp16_palettized, x = input_13_cast_fp16)[name = string("linear_7_cast_fp16")];
            tensor<fp16, [1, 1024]> value_15_cast_fp16 = add(x = value_13_cast_fp16, y = linear_7_cast_fp16)[name = string("value_15_cast_fp16")];
            int32 var_299 = const()[name = string("op_299"), val = int32(-1)];
            tensor<fp16, [1, 1024]> var_314_cast_fp16 = mul(x = value_15_cast_fp16, y = value_15_cast_fp16)[name = string("op_314_cast_fp16")];
            tensor<int32, [1]> variance_7_axes_0 = const()[name = string("variance_7_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_7_keep_dims_0 = const()[name = string("variance_7_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 1]> variance_7_cast_fp16 = reduce_mean(axes = variance_7_axes_0, keep_dims = variance_7_keep_dims_0, x = var_314_cast_fp16)[name = string("variance_7_cast_fp16")];
            fp16 var_317_to_fp16 = const()[name = string("op_317_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 1]> var_318_cast_fp16 = add(x = variance_7_cast_fp16, y = var_317_to_fp16)[name = string("op_318_cast_fp16")];
            fp32 var_319_epsilon_0 = const()[name = string("op_319_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 1]> var_319_cast_fp16 = rsqrt(epsilon = var_319_epsilon_0, x = var_318_cast_fp16)[name = string("op_319_cast_fp16")];
            tensor<fp16, [1, 1024]> var_320_cast_fp16 = mul(x = value_15_cast_fp16, y = var_319_cast_fp16)[name = string("op_320_cast_fp16")];
            tensor<fp16, [1024]> var_322_to_fp16 = const()[name = string("op_322_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(16385536)))];
            tensor<fp16, [1, 1024]> input_15_cast_fp16 = mul(x = var_320_cast_fp16, y = var_322_to_fp16)[name = string("input_15_cast_fp16")];
            tensor<fp16, [6144, 1024]> groups_0_1_in_proj_qkv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(16387648))), lut = tensor<fp16, [384, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(21106304))))[name = string("groups_0_1_in_proj_qkv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 6144]> linear_8_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = groups_0_1_in_proj_qkv_weight_promoted_to_fp16_palettized, x = input_15_cast_fp16)[name = string("linear_8_cast_fp16")];
            tensor<int32, [2]> var_326_perm_0 = const()[name = string("op_326_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> mixed_3_axes_0 = const()[name = string("mixed_3_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [6144, 1]> var_326_cast_fp16 = transpose(perm = var_326_perm_0, x = linear_8_cast_fp16)[name = string("transpose_134")];
            tensor<fp16, [1, 6144, 1]> mixed_3_cast_fp16 = expand_dims(axes = mixed_3_axes_0, x = var_326_cast_fp16)[name = string("mixed_3_cast_fp16")];
            bool convolution_input_3_interleave_0 = const()[name = string("convolution_input_3_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 6144, 4]> convolution_input_3_cast_fp16 = concat(axis = var_299, interleave = convolution_input_3_interleave_0, values = (conv1, mixed_3_cast_fp16))[name = string("convolution_input_3_cast_fp16")];
            string input_17_pad_type_0 = const()[name = string("input_17_pad_type_0"), val = string("valid")];
            int32 input_17_groups_0 = const()[name = string("input_17_groups_0"), val = int32(6144)];
            tensor<int32, [1]> input_17_strides_0 = const()[name = string("input_17_strides_0"), val = tensor<int32, [1]>([1])];
            tensor<int32, [2]> input_17_pad_0 = const()[name = string("input_17_pad_0"), val = tensor<int32, [2]>([0, 0])];
            tensor<int32, [1]> input_17_dilations_0 = const()[name = string("input_17_dilations_0"), val = tensor<int32, [1]>([1])];
            tensor<fp16, [6144, 1, 4]> groups_0_1_conv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1, 4]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(21155520))), lut = tensor<fp16, [384, 1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(21174016))))[name = string("groups_0_1_conv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 6144, 1]> input_17_cast_fp16 = conv(dilations = input_17_dilations_0, groups = input_17_groups_0, pad = input_17_pad_0, pad_type = input_17_pad_type_0, strides = input_17_strides_0, weight = groups_0_1_conv_weight_promoted_to_fp16_palettized, x = convolution_input_3_cast_fp16)[name = string("input_17_cast_fp16")];
            tensor<fp16, [1, 6144, 1]> var_335_cast_fp16 = silu(x = input_17_cast_fp16)[name = string("op_335_cast_fp16")];
            tensor<int32, [3]> var_336_perm_0 = const()[name = string("op_336_perm_0"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [3]> var_339_begin_0 = const()[name = string("op_339_begin_0"), val = tensor<int32, [3]>([0, 0, 1])];
            tensor<int32, [3]> var_339_end_0 = const()[name = string("op_339_end_0"), val = tensor<int32, [3]>([1, 6144, 4])];
            tensor<bool, [3]> var_339_end_mask_0 = const()[name = string("op_339_end_mask_0"), val = tensor<bool, [3]>([true, true, true])];
            tensor<fp16, [1, 6144, 3]> conv1_out = slice_by_index(begin = var_339_begin_0, end = var_339_end_0, end_mask = var_339_end_mask_0, x = convolution_input_3_cast_fp16)[name = string("op_339_cast_fp16")];
            tensor<int32, [3]> var_340 = const()[name = string("op_340"), val = tensor<int32, [3]>([2048, 2048, 2048])];
            int32 var_341_axis_0 = const()[name = string("op_341_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 1, 6144]> var_336_cast_fp16 = transpose(perm = var_336_perm_0, x = var_335_cast_fp16)[name = string("transpose_133")];
            tensor<fp16, [1, 1, 2048]> var_341_cast_fp16_0, tensor<fp16, [1, 1, 2048]> var_341_cast_fp16_1, tensor<fp16, [1, 1, 2048]> var_341_cast_fp16_2 = split(axis = var_341_axis_0, split_sizes = var_340, x = var_336_cast_fp16)[name = string("op_341_cast_fp16")];
            tensor<int32, [4]> var_345 = const()[name = string("op_345"), val = tensor<int32, [4]>([1, 1, 16, 128])];
            tensor<fp16, [1, 1, 16, 128]> value_19_cast_fp16 = reshape(shape = var_345, x = var_341_cast_fp16_0)[name = string("value_19_cast_fp16")];
            tensor<int32, [4]> var_347 = const()[name = string("op_347"), val = tensor<int32, [4]>([1, 1, 16, 128])];
            tensor<fp16, [1, 1, 16, 128]> value_21_cast_fp16 = reshape(shape = var_347, x = var_341_cast_fp16_1)[name = string("value_21_cast_fp16")];
            tensor<int32, [4]> var_349 = const()[name = string("op_349"), val = tensor<int32, [4]>([1, 1, 16, 128])];
            tensor<fp16, [1, 1, 16, 128]> value_23_cast_fp16 = reshape(shape = var_349, x = var_341_cast_fp16_2)[name = string("value_23_cast_fp16")];
            tensor<fp16, [1, 1, 16, 128]> var_351_cast_fp16 = mul(x = value_19_cast_fp16, y = value_19_cast_fp16)[name = string("op_351_cast_fp16")];
            tensor<int32, [1]> var_353_axes_0 = const()[name = string("op_353_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_353_keep_dims_0 = const()[name = string("op_353_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 1, 16, 1]> var_353_cast_fp16 = reduce_sum(axes = var_353_axes_0, keep_dims = var_353_keep_dims_0, x = var_351_cast_fp16)[name = string("op_353_cast_fp16")];
            fp16 var_354_to_fp16 = const()[name = string("op_354_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 1, 16, 1]> var_355_cast_fp16 = add(x = var_353_cast_fp16, y = var_354_to_fp16)[name = string("op_355_cast_fp16")];
            fp32 var_356_epsilon_0 = const()[name = string("op_356_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 1, 16, 1]> var_356_cast_fp16 = rsqrt(epsilon = var_356_epsilon_0, x = var_355_cast_fp16)[name = string("op_356_cast_fp16")];
            tensor<fp16, [1, 1, 16, 128]> var_357_cast_fp16 = mul(x = value_19_cast_fp16, y = var_356_cast_fp16)[name = string("op_357_cast_fp16")];
            tensor<int32, [4]> var_358_perm_0 = const()[name = string("op_358_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            fp16 _inversed_query_7_y_0_to_fp16 = const()[name = string("_inversed_query_7_y_0_to_fp16"), val = fp16(0x1.6ap-4)];
            tensor<fp16, [1, 16, 1, 128]> var_358_cast_fp16 = transpose(perm = var_358_perm_0, x = var_357_cast_fp16)[name = string("transpose_132")];
            tensor<fp16, [1, 16, 1, 128]> _inversed_query_7_cast_fp16 = mul(x = var_358_cast_fp16, y = _inversed_query_7_y_0_to_fp16)[name = string("_inversed_query_7_cast_fp16")];
            tensor<fp16, [1, 1, 16, 128]> var_361_cast_fp16 = mul(x = value_21_cast_fp16, y = value_21_cast_fp16)[name = string("op_361_cast_fp16")];
            tensor<int32, [1]> var_363_axes_0 = const()[name = string("op_363_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_363_keep_dims_0 = const()[name = string("op_363_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 1, 16, 1]> var_363_cast_fp16 = reduce_sum(axes = var_363_axes_0, keep_dims = var_363_keep_dims_0, x = var_361_cast_fp16)[name = string("op_363_cast_fp16")];
            fp16 var_364_to_fp16 = const()[name = string("op_364_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 1, 16, 1]> var_365_cast_fp16 = add(x = var_363_cast_fp16, y = var_364_to_fp16)[name = string("op_365_cast_fp16")];
            fp32 var_366_epsilon_0 = const()[name = string("op_366_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 1, 16, 1]> var_366_cast_fp16 = rsqrt(epsilon = var_366_epsilon_0, x = var_365_cast_fp16)[name = string("op_366_cast_fp16")];
            tensor<fp16, [1, 1, 16, 128]> var_367_cast_fp16 = mul(x = value_21_cast_fp16, y = var_366_cast_fp16)[name = string("op_367_cast_fp16")];
            tensor<int32, [4]> key_7_perm_0 = const()[name = string("key_7_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<int32, [4]> value_25_perm_0 = const()[name = string("value_25_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<fp16, [16, 1024]> groups_0_1_in_proj_b_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(21223232))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(21235584))))[name = string("groups_0_1_in_proj_b_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16]> linear_9_bias_0_to_fp16 = const()[name = string("linear_9_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(4862656)))];
            tensor<fp16, [1, 16]> linear_9_cast_fp16 = linear(bias = linear_9_bias_0_to_fp16, weight = groups_0_1_in_proj_b_weight_promoted_to_fp16_palettized, x = input_15_cast_fp16)[name = string("linear_9_cast_fp16")];
            tensor<fp16, [1, 16]> var_372_cast_fp16 = sigmoid(x = linear_9_cast_fp16)[name = string("op_372_cast_fp16")];
            tensor<int32, [2]> var_373_perm_0 = const()[name = string("op_373_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> beta_3_axes_0 = const()[name = string("beta_3_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 1]> var_373_cast_fp16 = transpose(perm = var_373_perm_0, x = var_372_cast_fp16)[name = string("transpose_131")];
            tensor<fp16, [1, 16, 1]> beta_3_cast_fp16 = expand_dims(axes = beta_3_axes_0, x = var_373_cast_fp16)[name = string("beta_3_cast_fp16")];
            tensor<fp16, [16, 1024]> op_379_weight_0_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(21235776))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(21248128))))[name = string("op_379_weight_0_to_fp16_palettized")];
            tensor<fp16, [16]> var_379_bias_0_to_fp16 = const()[name = string("op_379_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(21248320)))];
            tensor<fp16, [1, 16]> var_379_cast_fp16 = linear(bias = var_379_bias_0_to_fp16, weight = op_379_weight_0_to_fp16_palettized, x = input_15_cast_fp16)[name = string("op_379_cast_fp16")];
            tensor<fp16, [1, 16]> var_380_cast_fp16 = softplus(x = var_379_cast_fp16)[name = string("op_380_cast_fp16")];
            tensor<fp16, [16]> var_376_promoted_to_fp16 = const()[name = string("op_376_promoted_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(21248448)))];
            tensor<fp16, [1, 16]> var_381_cast_fp16 = mul(x = var_376_promoted_to_fp16, y = var_380_cast_fp16)[name = string("op_381_cast_fp16")];
            tensor<int32, [2]> var_382_perm_0 = const()[name = string("op_382_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> decay_3_axes_0 = const()[name = string("decay_3_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 1]> var_382_cast_fp16 = transpose(perm = var_382_perm_0, x = var_381_cast_fp16)[name = string("transpose_130")];
            tensor<fp16, [1, 16, 1]> decay_3_cast_fp16 = expand_dims(axes = decay_3_axes_0, x = var_382_cast_fp16)[name = string("decay_3_cast_fp16")];
            tensor<fp16, [2048, 1024]> groups_0_1_in_proj_z_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [2048, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(21248576))), lut = tensor<fp16, [128, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(22821504))))[name = string("groups_0_1_in_proj_z_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 2048]> linear_11_cast_fp16 = linear(bias = linear_3_bias_0_to_fp16, weight = groups_0_1_in_proj_z_weight_promoted_to_fp16_palettized, x = input_15_cast_fp16)[name = string("linear_11_cast_fp16")];
            tensor<int32, [3]> var_390_begin_0 = const()[name = string("op_390_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_390_end_0 = const()[name = string("op_390_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_390_end_mask_0 = const()[name = string("op_390_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_390_squeeze_mask_0 = const()[name = string("op_390_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_390_cast_fp16 = slice_by_index(begin = var_390_begin_0, end = var_390_end_0, end_mask = var_390_end_mask_0, squeeze_mask = var_390_squeeze_mask_0, x = decay_3_cast_fp16)[name = string("op_390_cast_fp16")];
            tensor<fp16, [1, 16]> var_391_cast_fp16 = exp(x = var_390_cast_fp16)[name = string("op_391_cast_fp16")];
            tensor<int32, [1]> var_392_axes_0 = const()[name = string("op_392_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_392_cast_fp16 = expand_dims(axes = var_392_axes_0, x = var_391_cast_fp16)[name = string("op_392_cast_fp16")];
            tensor<int32, [1]> var_393_axes_0 = const()[name = string("op_393_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_393_cast_fp16 = expand_dims(axes = var_393_axes_0, x = var_392_cast_fp16)[name = string("op_393_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_5_cast_fp16 = mul(x = rec1, y = var_393_cast_fp16)[name = string("state_5_cast_fp16")];
            tensor<int32, [4]> key_token_3_begin_0 = const()[name = string("key_token_3_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> key_token_3_end_0 = const()[name = string("key_token_3_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> key_token_3_end_mask_0 = const()[name = string("key_token_3_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_3_squeeze_mask_0 = const()[name = string("key_token_3_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 1, 128]> key_7_cast_fp16 = transpose(perm = key_7_perm_0, x = var_367_cast_fp16)[name = string("transpose_129")];
            tensor<fp16, [1, 16, 128]> key_token_3_cast_fp16 = slice_by_index(begin = key_token_3_begin_0, end = key_token_3_end_0, end_mask = key_token_3_end_mask_0, squeeze_mask = key_token_3_squeeze_mask_0, x = key_7_cast_fp16)[name = string("key_token_3_cast_fp16")];
            tensor<int32, [4]> value_token_3_begin_0 = const()[name = string("value_token_3_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> value_token_3_end_0 = const()[name = string("value_token_3_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> value_token_3_end_mask_0 = const()[name = string("value_token_3_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_3_squeeze_mask_0 = const()[name = string("value_token_3_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 1, 128]> value_25_cast_fp16 = transpose(perm = value_25_perm_0, x = value_23_cast_fp16)[name = string("transpose_128")];
            tensor<fp16, [1, 16, 128]> value_token_3_cast_fp16 = slice_by_index(begin = value_token_3_begin_0, end = value_token_3_end_0, end_mask = value_token_3_end_mask_0, squeeze_mask = value_token_3_squeeze_mask_0, x = value_25_cast_fp16)[name = string("value_token_3_cast_fp16")];
            tensor<int32, [1]> var_401_axes_0 = const()[name = string("op_401_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_401_cast_fp16 = expand_dims(axes = var_401_axes_0, x = key_token_3_cast_fp16)[name = string("op_401_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_402_cast_fp16 = mul(x = state_5_cast_fp16, y = var_401_cast_fp16)[name = string("op_402_cast_fp16")];
            tensor<int32, [1]> memory_3_axes_0 = const()[name = string("memory_3_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_3_keep_dims_0 = const()[name = string("memory_3_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_3_cast_fp16 = reduce_sum(axes = memory_3_axes_0, keep_dims = memory_3_keep_dims_0, x = var_402_cast_fp16)[name = string("memory_3_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_405_cast_fp16 = sub(x = value_token_3_cast_fp16, y = memory_3_cast_fp16)[name = string("op_405_cast_fp16")];
            tensor<int32, [3]> var_408_begin_0 = const()[name = string("op_408_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_408_end_0 = const()[name = string("op_408_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_408_end_mask_0 = const()[name = string("op_408_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_408_squeeze_mask_0 = const()[name = string("op_408_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_408_cast_fp16 = slice_by_index(begin = var_408_begin_0, end = var_408_end_0, end_mask = var_408_end_mask_0, squeeze_mask = var_408_squeeze_mask_0, x = beta_3_cast_fp16)[name = string("op_408_cast_fp16")];
            tensor<int32, [1]> var_409_axes_0 = const()[name = string("op_409_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_409_cast_fp16 = expand_dims(axes = var_409_axes_0, x = var_408_cast_fp16)[name = string("op_409_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_3_cast_fp16 = mul(x = var_405_cast_fp16, y = var_409_cast_fp16)[name = string("delta_3_cast_fp16")];
            tensor<int32, [1]> var_412_axes_0 = const()[name = string("op_412_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_412_cast_fp16 = expand_dims(axes = var_412_axes_0, x = delta_3_cast_fp16)[name = string("op_412_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_413_cast_fp16 = mul(x = var_401_cast_fp16, y = var_412_cast_fp16)[name = string("op_413_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> rec1_out = add(x = state_5_cast_fp16, y = var_413_cast_fp16)[name = string("state_7_cast_fp16")];
            tensor<int32, [4]> var_417_begin_0 = const()[name = string("op_417_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> var_417_end_0 = const()[name = string("op_417_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> var_417_end_mask_0 = const()[name = string("op_417_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_417_squeeze_mask_0 = const()[name = string("op_417_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_417_cast_fp16 = slice_by_index(begin = var_417_begin_0, end = var_417_end_0, end_mask = var_417_end_mask_0, squeeze_mask = var_417_squeeze_mask_0, x = _inversed_query_7_cast_fp16)[name = string("op_417_cast_fp16")];
            tensor<int32, [1]> var_418_axes_0 = const()[name = string("op_418_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_418_cast_fp16 = expand_dims(axes = var_418_axes_0, x = var_417_cast_fp16)[name = string("op_418_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_419_cast_fp16 = mul(x = rec1_out, y = var_418_cast_fp16)[name = string("op_419_cast_fp16")];
            tensor<int32, [1]> var_421_axes_0 = const()[name = string("op_421_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_421_keep_dims_0 = const()[name = string("op_421_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_421_cast_fp16 = reduce_sum(axes = var_421_axes_0, keep_dims = var_421_keep_dims_0, x = var_419_cast_fp16)[name = string("op_421_cast_fp16")];
            tensor<int32, [1]> attention_11_axes_0 = const()[name = string("attention_11_axes_0"), val = tensor<int32, [1]>([1])];
            tensor<fp16, [1, 1, 16, 128]> attention_11_cast_fp16 = expand_dims(axes = attention_11_axes_0, x = var_421_cast_fp16)[name = string("attention_11_cast_fp16")];
            tensor<int32, [2]> var_424 = const()[name = string("op_424"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [16, 128]> attention_13_cast_fp16 = reshape(shape = var_424, x = attention_11_cast_fp16)[name = string("attention_13_cast_fp16")];
            tensor<int32, [2]> var_426 = const()[name = string("op_426"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [16, 128]> input_19_cast_fp16 = reshape(shape = var_426, x = linear_11_cast_fp16)[name = string("input_19_cast_fp16")];
            tensor<fp16, [16, 128]> var_428_cast_fp16 = mul(x = attention_13_cast_fp16, y = attention_13_cast_fp16)[name = string("op_428_cast_fp16")];
            tensor<int32, [1]> variance_9_axes_0 = const()[name = string("variance_9_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_9_keep_dims_0 = const()[name = string("variance_9_keep_dims_0"), val = bool(true)];
            tensor<fp16, [16, 1]> variance_9_cast_fp16 = reduce_mean(axes = variance_9_axes_0, keep_dims = variance_9_keep_dims_0, x = var_428_cast_fp16)[name = string("variance_9_cast_fp16")];
            fp16 var_431_to_fp16 = const()[name = string("op_431_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [16, 1]> var_432_cast_fp16 = add(x = variance_9_cast_fp16, y = var_431_to_fp16)[name = string("op_432_cast_fp16")];
            fp32 var_433_epsilon_0 = const()[name = string("op_433_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [16, 1]> var_433_cast_fp16 = rsqrt(epsilon = var_433_epsilon_0, x = var_432_cast_fp16)[name = string("op_433_cast_fp16")];
            tensor<fp16, [16, 128]> attention_15_cast_fp16 = mul(x = attention_13_cast_fp16, y = var_433_cast_fp16)[name = string("attention_15_cast_fp16")];
            tensor<fp16, [128]> groups_0_1_gated_norm_promoted_to_fp16 = const()[name = string("groups_0_1_gated_norm_promoted_to_fp16"), val = tensor<fp16, [128]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(22837952)))];
            tensor<fp16, [16, 128]> attention_17_cast_fp16 = mul(x = attention_15_cast_fp16, y = groups_0_1_gated_norm_promoted_to_fp16)[name = string("attention_17_cast_fp16")];
            tensor<fp16, [16, 128]> var_436_cast_fp16 = silu(x = input_19_cast_fp16)[name = string("op_436_cast_fp16")];
            tensor<fp16, [16, 128]> attention_19_cast_fp16 = mul(x = attention_17_cast_fp16, y = var_436_cast_fp16)[name = string("attention_19_cast_fp16")];
            tensor<int32, [2]> var_438 = const()[name = string("op_438"), val = tensor<int32, [2]>([1, 2048])];
            tensor<fp16, [1, 2048]> input_21_cast_fp16 = reshape(shape = var_438, x = attention_19_cast_fp16)[name = string("input_21_cast_fp16")];
            tensor<fp16, [1024, 2048]> groups_0_1_out_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 2048]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(22838272))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(24411200))))[name = string("groups_0_1_out_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 1024]> linear_12_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = groups_0_1_out_proj_weight_promoted_to_fp16_palettized, x = input_21_cast_fp16)[name = string("linear_12_cast_fp16")];
            tensor<fp16, [1, 1024]> value_27_cast_fp16 = add(x = value_15_cast_fp16, y = linear_12_cast_fp16)[name = string("value_27_cast_fp16")];
            tensor<fp16, [1, 1024]> var_443_cast_fp16 = mul(x = value_27_cast_fp16, y = value_27_cast_fp16)[name = string("op_443_cast_fp16")];
            tensor<int32, [1]> variance_11_axes_0 = const()[name = string("variance_11_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_11_keep_dims_0 = const()[name = string("variance_11_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 1]> variance_11_cast_fp16 = reduce_mean(axes = variance_11_axes_0, keep_dims = variance_11_keep_dims_0, x = var_443_cast_fp16)[name = string("variance_11_cast_fp16")];
            fp16 var_446_to_fp16 = const()[name = string("op_446_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 1]> var_447_cast_fp16 = add(x = variance_11_cast_fp16, y = var_446_to_fp16)[name = string("op_447_cast_fp16")];
            fp32 var_448_epsilon_0 = const()[name = string("op_448_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 1]> var_448_cast_fp16 = rsqrt(epsilon = var_448_epsilon_0, x = var_447_cast_fp16)[name = string("op_448_cast_fp16")];
            tensor<fp16, [1, 1024]> var_449_cast_fp16 = mul(x = value_27_cast_fp16, y = var_448_cast_fp16)[name = string("op_449_cast_fp16")];
            tensor<fp16, [1024]> var_451_to_fp16 = const()[name = string("op_451_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(24419456)))];
            tensor<fp16, [1, 1024]> input_23_cast_fp16 = mul(x = var_449_cast_fp16, y = var_451_to_fp16)[name = string("input_23_cast_fp16")];
            tensor<fp16, [3584, 1024]> groups_0_1_gate_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(24421568))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(27174144))))[name = string("groups_0_1_gate_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 3584]> linear_13_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = groups_0_1_gate_proj_weight_promoted_to_fp16_palettized, x = input_23_cast_fp16)[name = string("linear_13_cast_fp16")];
            tensor<fp16, [1, 3584]> var_455_cast_fp16 = silu(x = linear_13_cast_fp16)[name = string("op_455_cast_fp16")];
            tensor<fp16, [3584, 1024]> groups_0_1_up_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(27202880))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(29955456))))[name = string("groups_0_1_up_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 3584]> linear_14_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = groups_0_1_up_proj_weight_promoted_to_fp16_palettized, x = input_23_cast_fp16)[name = string("linear_14_cast_fp16")];
            tensor<fp16, [1, 3584]> input_27_cast_fp16 = mul(x = var_455_cast_fp16, y = linear_14_cast_fp16)[name = string("input_27_cast_fp16")];
            tensor<fp16, [1024, 3584]> groups_0_1_down_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 3584]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(29984192))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(32736768))))[name = string("groups_0_1_down_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 1024]> linear_15_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = groups_0_1_down_proj_weight_promoted_to_fp16_palettized, x = input_27_cast_fp16)[name = string("linear_15_cast_fp16")];
            tensor<fp16, [1, 1024]> value_29_cast_fp16 = add(x = value_27_cast_fp16, y = linear_15_cast_fp16)[name = string("value_29_cast_fp16")];
            int32 var_480 = const()[name = string("op_480"), val = int32(-1)];
            tensor<fp16, [1, 1024]> var_495_cast_fp16 = mul(x = value_29_cast_fp16, y = value_29_cast_fp16)[name = string("op_495_cast_fp16")];
            tensor<int32, [1]> variance_13_axes_0 = const()[name = string("variance_13_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_13_keep_dims_0 = const()[name = string("variance_13_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 1]> variance_13_cast_fp16 = reduce_mean(axes = variance_13_axes_0, keep_dims = variance_13_keep_dims_0, x = var_495_cast_fp16)[name = string("variance_13_cast_fp16")];
            fp16 var_498_to_fp16 = const()[name = string("op_498_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 1]> var_499_cast_fp16 = add(x = variance_13_cast_fp16, y = var_498_to_fp16)[name = string("op_499_cast_fp16")];
            fp32 var_500_epsilon_0 = const()[name = string("op_500_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 1]> var_500_cast_fp16 = rsqrt(epsilon = var_500_epsilon_0, x = var_499_cast_fp16)[name = string("op_500_cast_fp16")];
            tensor<fp16, [1, 1024]> var_501_cast_fp16 = mul(x = value_29_cast_fp16, y = var_500_cast_fp16)[name = string("op_501_cast_fp16")];
            tensor<fp16, [1024]> var_503_to_fp16 = const()[name = string("op_503_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(32745024)))];
            tensor<fp16, [1, 1024]> input_29_cast_fp16 = mul(x = var_501_cast_fp16, y = var_503_to_fp16)[name = string("input_29_cast_fp16")];
            tensor<fp16, [6144, 1024]> groups_0_2_in_proj_qkv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(32747136))), lut = tensor<fp16, [384, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(37465792))))[name = string("groups_0_2_in_proj_qkv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 6144]> linear_16_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = groups_0_2_in_proj_qkv_weight_promoted_to_fp16_palettized, x = input_29_cast_fp16)[name = string("linear_16_cast_fp16")];
            tensor<int32, [2]> var_507_perm_0 = const()[name = string("op_507_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> mixed_5_axes_0 = const()[name = string("mixed_5_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [6144, 1]> var_507_cast_fp16 = transpose(perm = var_507_perm_0, x = linear_16_cast_fp16)[name = string("transpose_127")];
            tensor<fp16, [1, 6144, 1]> mixed_5_cast_fp16 = expand_dims(axes = mixed_5_axes_0, x = var_507_cast_fp16)[name = string("mixed_5_cast_fp16")];
            bool convolution_input_5_interleave_0 = const()[name = string("convolution_input_5_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 6144, 4]> convolution_input_5_cast_fp16 = concat(axis = var_480, interleave = convolution_input_5_interleave_0, values = (conv2, mixed_5_cast_fp16))[name = string("convolution_input_5_cast_fp16")];
            string input_31_pad_type_0 = const()[name = string("input_31_pad_type_0"), val = string("valid")];
            int32 input_31_groups_0 = const()[name = string("input_31_groups_0"), val = int32(6144)];
            tensor<int32, [1]> input_31_strides_0 = const()[name = string("input_31_strides_0"), val = tensor<int32, [1]>([1])];
            tensor<int32, [2]> input_31_pad_0 = const()[name = string("input_31_pad_0"), val = tensor<int32, [2]>([0, 0])];
            tensor<int32, [1]> input_31_dilations_0 = const()[name = string("input_31_dilations_0"), val = tensor<int32, [1]>([1])];
            tensor<fp16, [6144, 1, 4]> groups_0_2_conv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1, 4]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(37515008))), lut = tensor<fp16, [384, 1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(37533504))))[name = string("groups_0_2_conv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 6144, 1]> input_31_cast_fp16 = conv(dilations = input_31_dilations_0, groups = input_31_groups_0, pad = input_31_pad_0, pad_type = input_31_pad_type_0, strides = input_31_strides_0, weight = groups_0_2_conv_weight_promoted_to_fp16_palettized, x = convolution_input_5_cast_fp16)[name = string("input_31_cast_fp16")];
            tensor<fp16, [1, 6144, 1]> var_516_cast_fp16 = silu(x = input_31_cast_fp16)[name = string("op_516_cast_fp16")];
            tensor<int32, [3]> var_517_perm_0 = const()[name = string("op_517_perm_0"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [3]> var_520_begin_0 = const()[name = string("op_520_begin_0"), val = tensor<int32, [3]>([0, 0, 1])];
            tensor<int32, [3]> var_520_end_0 = const()[name = string("op_520_end_0"), val = tensor<int32, [3]>([1, 6144, 4])];
            tensor<bool, [3]> var_520_end_mask_0 = const()[name = string("op_520_end_mask_0"), val = tensor<bool, [3]>([true, true, true])];
            tensor<fp16, [1, 6144, 3]> conv2_out = slice_by_index(begin = var_520_begin_0, end = var_520_end_0, end_mask = var_520_end_mask_0, x = convolution_input_5_cast_fp16)[name = string("op_520_cast_fp16")];
            tensor<int32, [3]> var_521 = const()[name = string("op_521"), val = tensor<int32, [3]>([2048, 2048, 2048])];
            int32 var_522_axis_0 = const()[name = string("op_522_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 1, 6144]> var_517_cast_fp16 = transpose(perm = var_517_perm_0, x = var_516_cast_fp16)[name = string("transpose_126")];
            tensor<fp16, [1, 1, 2048]> var_522_cast_fp16_0, tensor<fp16, [1, 1, 2048]> var_522_cast_fp16_1, tensor<fp16, [1, 1, 2048]> var_522_cast_fp16_2 = split(axis = var_522_axis_0, split_sizes = var_521, x = var_517_cast_fp16)[name = string("op_522_cast_fp16")];
            tensor<int32, [4]> var_526 = const()[name = string("op_526"), val = tensor<int32, [4]>([1, 1, 16, 128])];
            tensor<fp16, [1, 1, 16, 128]> value_33_cast_fp16 = reshape(shape = var_526, x = var_522_cast_fp16_0)[name = string("value_33_cast_fp16")];
            tensor<int32, [4]> var_528 = const()[name = string("op_528"), val = tensor<int32, [4]>([1, 1, 16, 128])];
            tensor<fp16, [1, 1, 16, 128]> value_35_cast_fp16 = reshape(shape = var_528, x = var_522_cast_fp16_1)[name = string("value_35_cast_fp16")];
            tensor<int32, [4]> var_530 = const()[name = string("op_530"), val = tensor<int32, [4]>([1, 1, 16, 128])];
            tensor<fp16, [1, 1, 16, 128]> value_37_cast_fp16 = reshape(shape = var_530, x = var_522_cast_fp16_2)[name = string("value_37_cast_fp16")];
            tensor<fp16, [1, 1, 16, 128]> var_532_cast_fp16 = mul(x = value_33_cast_fp16, y = value_33_cast_fp16)[name = string("op_532_cast_fp16")];
            tensor<int32, [1]> var_534_axes_0 = const()[name = string("op_534_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_534_keep_dims_0 = const()[name = string("op_534_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 1, 16, 1]> var_534_cast_fp16 = reduce_sum(axes = var_534_axes_0, keep_dims = var_534_keep_dims_0, x = var_532_cast_fp16)[name = string("op_534_cast_fp16")];
            fp16 var_535_to_fp16 = const()[name = string("op_535_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 1, 16, 1]> var_536_cast_fp16 = add(x = var_534_cast_fp16, y = var_535_to_fp16)[name = string("op_536_cast_fp16")];
            fp32 var_537_epsilon_0 = const()[name = string("op_537_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 1, 16, 1]> var_537_cast_fp16 = rsqrt(epsilon = var_537_epsilon_0, x = var_536_cast_fp16)[name = string("op_537_cast_fp16")];
            tensor<fp16, [1, 1, 16, 128]> var_538_cast_fp16 = mul(x = value_33_cast_fp16, y = var_537_cast_fp16)[name = string("op_538_cast_fp16")];
            tensor<int32, [4]> var_539_perm_0 = const()[name = string("op_539_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            fp16 _inversed_query_11_y_0_to_fp16 = const()[name = string("_inversed_query_11_y_0_to_fp16"), val = fp16(0x1.6ap-4)];
            tensor<fp16, [1, 16, 1, 128]> var_539_cast_fp16 = transpose(perm = var_539_perm_0, x = var_538_cast_fp16)[name = string("transpose_125")];
            tensor<fp16, [1, 16, 1, 128]> _inversed_query_11_cast_fp16 = mul(x = var_539_cast_fp16, y = _inversed_query_11_y_0_to_fp16)[name = string("_inversed_query_11_cast_fp16")];
            tensor<fp16, [1, 1, 16, 128]> var_542_cast_fp16 = mul(x = value_35_cast_fp16, y = value_35_cast_fp16)[name = string("op_542_cast_fp16")];
            tensor<int32, [1]> var_544_axes_0 = const()[name = string("op_544_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_544_keep_dims_0 = const()[name = string("op_544_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 1, 16, 1]> var_544_cast_fp16 = reduce_sum(axes = var_544_axes_0, keep_dims = var_544_keep_dims_0, x = var_542_cast_fp16)[name = string("op_544_cast_fp16")];
            fp16 var_545_to_fp16 = const()[name = string("op_545_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 1, 16, 1]> var_546_cast_fp16 = add(x = var_544_cast_fp16, y = var_545_to_fp16)[name = string("op_546_cast_fp16")];
            fp32 var_547_epsilon_0 = const()[name = string("op_547_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 1, 16, 1]> var_547_cast_fp16 = rsqrt(epsilon = var_547_epsilon_0, x = var_546_cast_fp16)[name = string("op_547_cast_fp16")];
            tensor<fp16, [1, 1, 16, 128]> var_548_cast_fp16 = mul(x = value_35_cast_fp16, y = var_547_cast_fp16)[name = string("op_548_cast_fp16")];
            tensor<int32, [4]> key_11_perm_0 = const()[name = string("key_11_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<int32, [4]> value_39_perm_0 = const()[name = string("value_39_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<fp16, [16, 1024]> groups_0_2_in_proj_b_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(37582720))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(37595072))))[name = string("groups_0_2_in_proj_b_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16]> linear_17_bias_0_to_fp16 = const()[name = string("linear_17_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(4862656)))];
            tensor<fp16, [1, 16]> linear_17_cast_fp16 = linear(bias = linear_17_bias_0_to_fp16, weight = groups_0_2_in_proj_b_weight_promoted_to_fp16_palettized, x = input_29_cast_fp16)[name = string("linear_17_cast_fp16")];
            tensor<fp16, [1, 16]> var_553_cast_fp16 = sigmoid(x = linear_17_cast_fp16)[name = string("op_553_cast_fp16")];
            tensor<int32, [2]> var_554_perm_0 = const()[name = string("op_554_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> beta_5_axes_0 = const()[name = string("beta_5_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 1]> var_554_cast_fp16 = transpose(perm = var_554_perm_0, x = var_553_cast_fp16)[name = string("transpose_124")];
            tensor<fp16, [1, 16, 1]> beta_5_cast_fp16 = expand_dims(axes = beta_5_axes_0, x = var_554_cast_fp16)[name = string("beta_5_cast_fp16")];
            tensor<fp16, [16, 1024]> op_560_weight_0_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(37595264))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(37607616))))[name = string("op_560_weight_0_to_fp16_palettized")];
            tensor<fp16, [16]> var_560_bias_0_to_fp16 = const()[name = string("op_560_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(37607808)))];
            tensor<fp16, [1, 16]> var_560_cast_fp16 = linear(bias = var_560_bias_0_to_fp16, weight = op_560_weight_0_to_fp16_palettized, x = input_29_cast_fp16)[name = string("op_560_cast_fp16")];
            tensor<fp16, [1, 16]> var_561_cast_fp16 = softplus(x = var_560_cast_fp16)[name = string("op_561_cast_fp16")];
            tensor<fp16, [16]> var_557_promoted_to_fp16 = const()[name = string("op_557_promoted_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(37607936)))];
            tensor<fp16, [1, 16]> var_562_cast_fp16 = mul(x = var_557_promoted_to_fp16, y = var_561_cast_fp16)[name = string("op_562_cast_fp16")];
            tensor<int32, [2]> var_563_perm_0 = const()[name = string("op_563_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> decay_5_axes_0 = const()[name = string("decay_5_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 1]> var_563_cast_fp16 = transpose(perm = var_563_perm_0, x = var_562_cast_fp16)[name = string("transpose_123")];
            tensor<fp16, [1, 16, 1]> decay_5_cast_fp16 = expand_dims(axes = decay_5_axes_0, x = var_563_cast_fp16)[name = string("decay_5_cast_fp16")];
            tensor<fp16, [2048, 1024]> groups_0_2_in_proj_z_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [2048, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(37608064))), lut = tensor<fp16, [128, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(39180992))))[name = string("groups_0_2_in_proj_z_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 2048]> linear_19_cast_fp16 = linear(bias = linear_3_bias_0_to_fp16, weight = groups_0_2_in_proj_z_weight_promoted_to_fp16_palettized, x = input_29_cast_fp16)[name = string("linear_19_cast_fp16")];
            tensor<int32, [3]> var_571_begin_0 = const()[name = string("op_571_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_571_end_0 = const()[name = string("op_571_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_571_end_mask_0 = const()[name = string("op_571_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_571_squeeze_mask_0 = const()[name = string("op_571_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_571_cast_fp16 = slice_by_index(begin = var_571_begin_0, end = var_571_end_0, end_mask = var_571_end_mask_0, squeeze_mask = var_571_squeeze_mask_0, x = decay_5_cast_fp16)[name = string("op_571_cast_fp16")];
            tensor<fp16, [1, 16]> var_572_cast_fp16 = exp(x = var_571_cast_fp16)[name = string("op_572_cast_fp16")];
            tensor<int32, [1]> var_573_axes_0 = const()[name = string("op_573_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_573_cast_fp16 = expand_dims(axes = var_573_axes_0, x = var_572_cast_fp16)[name = string("op_573_cast_fp16")];
            tensor<int32, [1]> var_574_axes_0 = const()[name = string("op_574_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_574_cast_fp16 = expand_dims(axes = var_574_axes_0, x = var_573_cast_fp16)[name = string("op_574_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_9_cast_fp16 = mul(x = rec2, y = var_574_cast_fp16)[name = string("state_9_cast_fp16")];
            tensor<int32, [4]> key_token_5_begin_0 = const()[name = string("key_token_5_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> key_token_5_end_0 = const()[name = string("key_token_5_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> key_token_5_end_mask_0 = const()[name = string("key_token_5_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_5_squeeze_mask_0 = const()[name = string("key_token_5_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 1, 128]> key_11_cast_fp16 = transpose(perm = key_11_perm_0, x = var_548_cast_fp16)[name = string("transpose_122")];
            tensor<fp16, [1, 16, 128]> key_token_5_cast_fp16 = slice_by_index(begin = key_token_5_begin_0, end = key_token_5_end_0, end_mask = key_token_5_end_mask_0, squeeze_mask = key_token_5_squeeze_mask_0, x = key_11_cast_fp16)[name = string("key_token_5_cast_fp16")];
            tensor<int32, [4]> value_token_5_begin_0 = const()[name = string("value_token_5_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> value_token_5_end_0 = const()[name = string("value_token_5_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> value_token_5_end_mask_0 = const()[name = string("value_token_5_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_5_squeeze_mask_0 = const()[name = string("value_token_5_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 1, 128]> value_39_cast_fp16 = transpose(perm = value_39_perm_0, x = value_37_cast_fp16)[name = string("transpose_121")];
            tensor<fp16, [1, 16, 128]> value_token_5_cast_fp16 = slice_by_index(begin = value_token_5_begin_0, end = value_token_5_end_0, end_mask = value_token_5_end_mask_0, squeeze_mask = value_token_5_squeeze_mask_0, x = value_39_cast_fp16)[name = string("value_token_5_cast_fp16")];
            tensor<int32, [1]> var_582_axes_0 = const()[name = string("op_582_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_582_cast_fp16 = expand_dims(axes = var_582_axes_0, x = key_token_5_cast_fp16)[name = string("op_582_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_583_cast_fp16 = mul(x = state_9_cast_fp16, y = var_582_cast_fp16)[name = string("op_583_cast_fp16")];
            tensor<int32, [1]> memory_5_axes_0 = const()[name = string("memory_5_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_5_keep_dims_0 = const()[name = string("memory_5_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_5_cast_fp16 = reduce_sum(axes = memory_5_axes_0, keep_dims = memory_5_keep_dims_0, x = var_583_cast_fp16)[name = string("memory_5_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_586_cast_fp16 = sub(x = value_token_5_cast_fp16, y = memory_5_cast_fp16)[name = string("op_586_cast_fp16")];
            tensor<int32, [3]> var_589_begin_0 = const()[name = string("op_589_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_589_end_0 = const()[name = string("op_589_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_589_end_mask_0 = const()[name = string("op_589_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_589_squeeze_mask_0 = const()[name = string("op_589_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_589_cast_fp16 = slice_by_index(begin = var_589_begin_0, end = var_589_end_0, end_mask = var_589_end_mask_0, squeeze_mask = var_589_squeeze_mask_0, x = beta_5_cast_fp16)[name = string("op_589_cast_fp16")];
            tensor<int32, [1]> var_590_axes_0 = const()[name = string("op_590_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_590_cast_fp16 = expand_dims(axes = var_590_axes_0, x = var_589_cast_fp16)[name = string("op_590_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_5_cast_fp16 = mul(x = var_586_cast_fp16, y = var_590_cast_fp16)[name = string("delta_5_cast_fp16")];
            tensor<int32, [1]> var_593_axes_0 = const()[name = string("op_593_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_593_cast_fp16 = expand_dims(axes = var_593_axes_0, x = delta_5_cast_fp16)[name = string("op_593_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_594_cast_fp16 = mul(x = var_582_cast_fp16, y = var_593_cast_fp16)[name = string("op_594_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> rec2_out = add(x = state_9_cast_fp16, y = var_594_cast_fp16)[name = string("state_11_cast_fp16")];
            tensor<int32, [4]> var_598_begin_0 = const()[name = string("op_598_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> var_598_end_0 = const()[name = string("op_598_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> var_598_end_mask_0 = const()[name = string("op_598_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_598_squeeze_mask_0 = const()[name = string("op_598_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_598_cast_fp16 = slice_by_index(begin = var_598_begin_0, end = var_598_end_0, end_mask = var_598_end_mask_0, squeeze_mask = var_598_squeeze_mask_0, x = _inversed_query_11_cast_fp16)[name = string("op_598_cast_fp16")];
            tensor<int32, [1]> var_599_axes_0 = const()[name = string("op_599_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_599_cast_fp16 = expand_dims(axes = var_599_axes_0, x = var_598_cast_fp16)[name = string("op_599_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_600_cast_fp16 = mul(x = rec2_out, y = var_599_cast_fp16)[name = string("op_600_cast_fp16")];
            tensor<int32, [1]> var_602_axes_0 = const()[name = string("op_602_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_602_keep_dims_0 = const()[name = string("op_602_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_602_cast_fp16 = reduce_sum(axes = var_602_axes_0, keep_dims = var_602_keep_dims_0, x = var_600_cast_fp16)[name = string("op_602_cast_fp16")];
            tensor<int32, [1]> attention_21_axes_0 = const()[name = string("attention_21_axes_0"), val = tensor<int32, [1]>([1])];
            tensor<fp16, [1, 1, 16, 128]> attention_21_cast_fp16 = expand_dims(axes = attention_21_axes_0, x = var_602_cast_fp16)[name = string("attention_21_cast_fp16")];
            tensor<int32, [2]> var_605 = const()[name = string("op_605"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [16, 128]> attention_23_cast_fp16 = reshape(shape = var_605, x = attention_21_cast_fp16)[name = string("attention_23_cast_fp16")];
            tensor<int32, [2]> var_607 = const()[name = string("op_607"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [16, 128]> input_33_cast_fp16 = reshape(shape = var_607, x = linear_19_cast_fp16)[name = string("input_33_cast_fp16")];
            tensor<fp16, [16, 128]> var_609_cast_fp16 = mul(x = attention_23_cast_fp16, y = attention_23_cast_fp16)[name = string("op_609_cast_fp16")];
            tensor<int32, [1]> variance_15_axes_0 = const()[name = string("variance_15_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_15_keep_dims_0 = const()[name = string("variance_15_keep_dims_0"), val = bool(true)];
            tensor<fp16, [16, 1]> variance_15_cast_fp16 = reduce_mean(axes = variance_15_axes_0, keep_dims = variance_15_keep_dims_0, x = var_609_cast_fp16)[name = string("variance_15_cast_fp16")];
            fp16 var_612_to_fp16 = const()[name = string("op_612_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [16, 1]> var_613_cast_fp16 = add(x = variance_15_cast_fp16, y = var_612_to_fp16)[name = string("op_613_cast_fp16")];
            fp32 var_614_epsilon_0 = const()[name = string("op_614_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [16, 1]> var_614_cast_fp16 = rsqrt(epsilon = var_614_epsilon_0, x = var_613_cast_fp16)[name = string("op_614_cast_fp16")];
            tensor<fp16, [16, 128]> attention_25_cast_fp16 = mul(x = attention_23_cast_fp16, y = var_614_cast_fp16)[name = string("attention_25_cast_fp16")];
            tensor<fp16, [128]> groups_0_2_gated_norm_promoted_to_fp16 = const()[name = string("groups_0_2_gated_norm_promoted_to_fp16"), val = tensor<fp16, [128]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(39197440)))];
            tensor<fp16, [16, 128]> attention_27_cast_fp16 = mul(x = attention_25_cast_fp16, y = groups_0_2_gated_norm_promoted_to_fp16)[name = string("attention_27_cast_fp16")];
            tensor<fp16, [16, 128]> var_617_cast_fp16 = silu(x = input_33_cast_fp16)[name = string("op_617_cast_fp16")];
            tensor<fp16, [16, 128]> attention_29_cast_fp16 = mul(x = attention_27_cast_fp16, y = var_617_cast_fp16)[name = string("attention_29_cast_fp16")];
            tensor<int32, [2]> var_619 = const()[name = string("op_619"), val = tensor<int32, [2]>([1, 2048])];
            tensor<fp16, [1, 2048]> input_35_cast_fp16 = reshape(shape = var_619, x = attention_29_cast_fp16)[name = string("input_35_cast_fp16")];
            tensor<fp16, [1024, 2048]> groups_0_2_out_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 2048]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(39197760))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(40770688))))[name = string("groups_0_2_out_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 1024]> linear_20_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = groups_0_2_out_proj_weight_promoted_to_fp16_palettized, x = input_35_cast_fp16)[name = string("linear_20_cast_fp16")];
            tensor<fp16, [1, 1024]> value_41_cast_fp16 = add(x = value_29_cast_fp16, y = linear_20_cast_fp16)[name = string("value_41_cast_fp16")];
            tensor<fp16, [1, 1024]> var_624_cast_fp16 = mul(x = value_41_cast_fp16, y = value_41_cast_fp16)[name = string("op_624_cast_fp16")];
            tensor<int32, [1]> variance_17_axes_0 = const()[name = string("variance_17_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_17_keep_dims_0 = const()[name = string("variance_17_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 1]> variance_17_cast_fp16 = reduce_mean(axes = variance_17_axes_0, keep_dims = variance_17_keep_dims_0, x = var_624_cast_fp16)[name = string("variance_17_cast_fp16")];
            fp16 var_627_to_fp16 = const()[name = string("op_627_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 1]> var_628_cast_fp16 = add(x = variance_17_cast_fp16, y = var_627_to_fp16)[name = string("op_628_cast_fp16")];
            fp32 var_629_epsilon_0 = const()[name = string("op_629_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 1]> var_629_cast_fp16 = rsqrt(epsilon = var_629_epsilon_0, x = var_628_cast_fp16)[name = string("op_629_cast_fp16")];
            tensor<fp16, [1, 1024]> var_630_cast_fp16 = mul(x = value_41_cast_fp16, y = var_629_cast_fp16)[name = string("op_630_cast_fp16")];
            tensor<fp16, [1024]> var_632_to_fp16 = const()[name = string("op_632_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(40778944)))];
            tensor<fp16, [1, 1024]> input_37_cast_fp16 = mul(x = var_630_cast_fp16, y = var_632_to_fp16)[name = string("input_37_cast_fp16")];
            tensor<fp16, [3584, 1024]> groups_0_2_gate_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(40781056))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(43533632))))[name = string("groups_0_2_gate_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 3584]> linear_21_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = groups_0_2_gate_proj_weight_promoted_to_fp16_palettized, x = input_37_cast_fp16)[name = string("linear_21_cast_fp16")];
            tensor<fp16, [1, 3584]> var_636_cast_fp16 = silu(x = linear_21_cast_fp16)[name = string("op_636_cast_fp16")];
            tensor<fp16, [3584, 1024]> groups_0_2_up_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(43562368))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(46314944))))[name = string("groups_0_2_up_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 3584]> linear_22_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = groups_0_2_up_proj_weight_promoted_to_fp16_palettized, x = input_37_cast_fp16)[name = string("linear_22_cast_fp16")];
            tensor<fp16, [1, 3584]> input_41_cast_fp16 = mul(x = var_636_cast_fp16, y = linear_22_cast_fp16)[name = string("input_41_cast_fp16")];
            tensor<fp16, [1024, 3584]> groups_0_2_down_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 3584]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(46343680))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(49096256))))[name = string("groups_0_2_down_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 1024]> linear_23_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = groups_0_2_down_proj_weight_promoted_to_fp16_palettized, x = input_41_cast_fp16)[name = string("linear_23_cast_fp16")];
            tensor<fp16, [1, 1024]> value_43_cast_fp16 = add(x = value_41_cast_fp16, y = linear_23_cast_fp16)[name = string("value_43_cast_fp16")];
            int32 var_660 = const()[name = string("op_660"), val = int32(-1)];
            tensor<fp16, [1, 1024]> var_667_cast_fp16 = mul(x = value_43_cast_fp16, y = value_43_cast_fp16)[name = string("op_667_cast_fp16")];
            tensor<int32, [1]> variance_19_axes_0 = const()[name = string("variance_19_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_19_keep_dims_0 = const()[name = string("variance_19_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 1]> variance_19_cast_fp16 = reduce_mean(axes = variance_19_axes_0, keep_dims = variance_19_keep_dims_0, x = var_667_cast_fp16)[name = string("variance_19_cast_fp16")];
            fp16 var_670_to_fp16 = const()[name = string("op_670_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 1]> var_671_cast_fp16 = add(x = variance_19_cast_fp16, y = var_670_to_fp16)[name = string("op_671_cast_fp16")];
            fp32 var_672_epsilon_0 = const()[name = string("op_672_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 1]> var_672_cast_fp16 = rsqrt(epsilon = var_672_epsilon_0, x = var_671_cast_fp16)[name = string("op_672_cast_fp16")];
            tensor<fp16, [1, 1024]> var_673_cast_fp16 = mul(x = value_43_cast_fp16, y = var_672_cast_fp16)[name = string("op_673_cast_fp16")];
            tensor<fp16, [1024]> var_675_to_fp16 = const()[name = string("op_675_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(49104512)))];
            tensor<fp16, [1, 1024]> input_43_cast_fp16 = mul(x = var_673_cast_fp16, y = var_675_to_fp16)[name = string("input_43_cast_fp16")];
            tensor<fp16, [4096, 1024]> projections_0_q_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [4096, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(49106624))), lut = tensor<fp16, [256, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(52252416))))[name = string("projections_0_q_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4096]> linear_24_bias_0_to_fp16 = const()[name = string("linear_24_bias_0_to_fp16"), val = tensor<fp16, [4096]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(52285248)))];
            tensor<fp16, [1, 4096]> linear_24_cast_fp16 = linear(bias = linear_24_bias_0_to_fp16, weight = projections_0_q_proj_weight_promoted_to_fp16_palettized, x = input_43_cast_fp16)[name = string("linear_24_cast_fp16")];
            tensor<int32, [3]> var_679 = const()[name = string("op_679"), val = tensor<int32, [3]>([1, 8, 512])];
            tensor<fp16, [1, 8, 512]> query_and_gate_1_cast_fp16 = reshape(shape = var_679, x = linear_24_cast_fp16)[name = string("query_and_gate_1_cast_fp16")];
            tensor<int32, [2]> var_681_split_sizes_0 = const()[name = string("op_681_split_sizes_0"), val = tensor<int32, [2]>([256, 256])];
            int32 var_681_axis_0 = const()[name = string("op_681_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 8, 256]> var_681_cast_fp16_0, tensor<fp16, [1, 8, 256]> var_681_cast_fp16_1 = split(axis = var_681_axis_0, split_sizes = var_681_split_sizes_0, x = query_and_gate_1_cast_fp16)[name = string("op_681_cast_fp16")];
            tensor<fp16, [512, 1024]> projections_0_k_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [512, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(52293504))), lut = tensor<fp16, [32, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(52686784))))[name = string("projections_0_k_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [512]> linear_25_bias_0_to_fp16 = const()[name = string("linear_25_bias_0_to_fp16"), val = tensor<fp16, [512]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(52690944)))];
            tensor<fp16, [1, 512]> linear_25_cast_fp16 = linear(bias = linear_25_bias_0_to_fp16, weight = projections_0_k_proj_weight_promoted_to_fp16_palettized, x = input_43_cast_fp16)[name = string("linear_25_cast_fp16")];
            tensor<int32, [3]> var_685 = const()[name = string("op_685"), val = tensor<int32, [3]>([1, 2, 256])];
            tensor<fp16, [1, 2, 256]> value_47_cast_fp16 = reshape(shape = var_685, x = linear_25_cast_fp16)[name = string("value_47_cast_fp16")];
            tensor<fp16, [512, 1024]> projections_0_v_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [512, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(52692032))), lut = tensor<fp16, [32, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(53085312))))[name = string("projections_0_v_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 512]> linear_26_cast_fp16 = linear(bias = linear_25_bias_0_to_fp16, weight = projections_0_v_proj_weight_promoted_to_fp16_palettized, x = input_43_cast_fp16)[name = string("linear_26_cast_fp16")];
            tensor<int32, [3]> var_689 = const()[name = string("op_689"), val = tensor<int32, [3]>([1, 2, 256])];
            tensor<fp16, [1, 2, 256]> value_53_cast_fp16 = reshape(shape = var_689, x = linear_26_cast_fp16)[name = string("value_53_cast_fp16")];
            tensor<fp16, [1, 8, 256]> var_691_cast_fp16 = mul(x = var_681_cast_fp16_0, y = var_681_cast_fp16_0)[name = string("op_691_cast_fp16")];
            tensor<int32, [1]> variance_21_axes_0 = const()[name = string("variance_21_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_21_keep_dims_0 = const()[name = string("variance_21_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 8, 1]> variance_21_cast_fp16 = reduce_mean(axes = variance_21_axes_0, keep_dims = variance_21_keep_dims_0, x = var_691_cast_fp16)[name = string("variance_21_cast_fp16")];
            fp16 var_694_to_fp16 = const()[name = string("op_694_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 8, 1]> var_695_cast_fp16 = add(x = variance_21_cast_fp16, y = var_694_to_fp16)[name = string("op_695_cast_fp16")];
            fp32 var_696_epsilon_0 = const()[name = string("op_696_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 8, 1]> var_696_cast_fp16 = rsqrt(epsilon = var_696_epsilon_0, x = var_695_cast_fp16)[name = string("op_696_cast_fp16")];
            tensor<fp16, [1, 8, 256]> var_697_cast_fp16 = mul(x = var_681_cast_fp16_0, y = var_696_cast_fp16)[name = string("op_697_cast_fp16")];
            tensor<fp16, [256]> var_699_to_fp16 = const()[name = string("op_699_to_fp16"), val = tensor<fp16, [256]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(53089472)))];
            tensor<fp16, [1, 8, 256]> var_700_cast_fp16 = mul(x = var_697_cast_fp16, y = var_699_to_fp16)[name = string("op_700_cast_fp16")];
            tensor<int32, [3]> var_701 = const()[name = string("op_701"), val = tensor<int32, [3]>([1, 0, 2])];
            tensor<int32, [1]> value_49_axes_0 = const()[name = string("value_49_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [8, 1, 256]> var_702_cast_fp16 = transpose(perm = var_701, x = var_700_cast_fp16)[name = string("transpose_120")];
            tensor<fp16, [1, 8, 1, 256]> value_49_cast_fp16 = expand_dims(axes = value_49_axes_0, x = var_702_cast_fp16)[name = string("value_49_cast_fp16")];
            tensor<fp16, [1, 2, 256]> var_704_cast_fp16 = mul(x = value_47_cast_fp16, y = value_47_cast_fp16)[name = string("op_704_cast_fp16")];
            tensor<int32, [1]> variance_23_axes_0 = const()[name = string("variance_23_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_23_keep_dims_0 = const()[name = string("variance_23_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 2, 1]> variance_23_cast_fp16 = reduce_mean(axes = variance_23_axes_0, keep_dims = variance_23_keep_dims_0, x = var_704_cast_fp16)[name = string("variance_23_cast_fp16")];
            fp16 var_707_to_fp16 = const()[name = string("op_707_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 2, 1]> var_708_cast_fp16 = add(x = variance_23_cast_fp16, y = var_707_to_fp16)[name = string("op_708_cast_fp16")];
            fp32 var_709_epsilon_0 = const()[name = string("op_709_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 2, 1]> var_709_cast_fp16 = rsqrt(epsilon = var_709_epsilon_0, x = var_708_cast_fp16)[name = string("op_709_cast_fp16")];
            tensor<fp16, [1, 2, 256]> var_710_cast_fp16 = mul(x = value_47_cast_fp16, y = var_709_cast_fp16)[name = string("op_710_cast_fp16")];
            tensor<fp16, [256]> var_712_to_fp16 = const()[name = string("op_712_to_fp16"), val = tensor<fp16, [256]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(53090048)))];
            tensor<fp16, [1, 2, 256]> var_713_cast_fp16 = mul(x = var_710_cast_fp16, y = var_712_to_fp16)[name = string("op_713_cast_fp16")];
            tensor<int32, [3]> var_714 = const()[name = string("op_714"), val = tensor<int32, [3]>([1, 0, 2])];
            tensor<int32, [1]> value_51_axes_0 = const()[name = string("value_51_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [2, 1, 256]> var_715_cast_fp16 = transpose(perm = var_714, x = var_713_cast_fp16)[name = string("transpose_119")];
            tensor<fp16, [1, 2, 1, 256]> value_51_cast_fp16 = expand_dims(axes = value_51_axes_0, x = var_715_cast_fp16)[name = string("value_51_cast_fp16")];
            tensor<int32, [4]> rotated_1_begin_0 = const()[name = string("rotated_1_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> rotated_1_end_0 = const()[name = string("rotated_1_end_0"), val = tensor<int32, [4]>([1, 8, 1, 64])];
            tensor<bool, [4]> rotated_1_end_mask_0 = const()[name = string("rotated_1_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
            tensor<fp16, [1, 8, 1, 64]> rotated_1_cast_fp16 = slice_by_index(begin = rotated_1_begin_0, end = rotated_1_end_0, end_mask = rotated_1_end_mask_0, x = value_49_cast_fp16)[name = string("rotated_1_cast_fp16")];
            tensor<int32, [4]> passthrough_1_begin_0 = const()[name = string("passthrough_1_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 64])];
            tensor<int32, [4]> passthrough_1_end_0 = const()[name = string("passthrough_1_end_0"), val = tensor<int32, [4]>([1, 8, 1, 256])];
            tensor<bool, [4]> passthrough_1_end_mask_0 = const()[name = string("passthrough_1_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
            tensor<fp16, [1, 8, 1, 192]> passthrough_1_cast_fp16 = slice_by_index(begin = passthrough_1_begin_0, end = passthrough_1_end_0, end_mask = passthrough_1_end_mask_0, x = value_49_cast_fp16)[name = string("passthrough_1_cast_fp16")];
            tensor<int32, [2]> var_719_split_sizes_0 = const()[name = string("op_719_split_sizes_0"), val = tensor<int32, [2]>([32, 32])];
            int32 var_719_axis_0 = const()[name = string("op_719_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 8, 1, 32]> var_719_cast_fp16_0, tensor<fp16, [1, 8, 1, 32]> var_719_cast_fp16_1 = split(axis = var_719_axis_0, split_sizes = var_719_split_sizes_0, x = rotated_1_cast_fp16)[name = string("op_719_cast_fp16")];
            fp16 const_3_promoted_to_fp16 = const()[name = string("const_3_promoted_to_fp16"), val = fp16(-0x1p+0)];
            tensor<fp16, [1, 8, 1, 32]> var_721_cast_fp16 = mul(x = var_719_cast_fp16_1, y = const_3_promoted_to_fp16)[name = string("op_721_cast_fp16")];
            bool rotated_half_1_interleave_0 = const()[name = string("rotated_half_1_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 8, 1, 64]> rotated_half_1_cast_fp16 = concat(axis = var_660, interleave = rotated_half_1_interleave_0, values = (var_721_cast_fp16, var_719_cast_fp16_0))[name = string("rotated_half_1_cast_fp16")];
            tensor<fp16, [1, 8, 1, 64]> var_724_cast_fp16 = mul(x = rotated_1_cast_fp16, y = cos)[name = string("op_724_cast_fp16")];
            tensor<fp16, [1, 8, 1, 64]> var_725_cast_fp16 = mul(x = rotated_half_1_cast_fp16, y = sin)[name = string("op_725_cast_fp16")];
            tensor<fp16, [1, 8, 1, 64]> var_726_cast_fp16 = add(x = var_724_cast_fp16, y = var_725_cast_fp16)[name = string("op_726_cast_fp16")];
            bool query_13_interleave_0 = const()[name = string("query_13_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 8, 1, 256]> query_13_cast_fp16 = concat(axis = var_660, interleave = query_13_interleave_0, values = (var_726_cast_fp16, passthrough_1_cast_fp16))[name = string("query_13_cast_fp16")];
            tensor<int32, [4]> rotated_3_begin_0 = const()[name = string("rotated_3_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> rotated_3_end_0 = const()[name = string("rotated_3_end_0"), val = tensor<int32, [4]>([1, 2, 1, 64])];
            tensor<bool, [4]> rotated_3_end_mask_0 = const()[name = string("rotated_3_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
            tensor<fp16, [1, 2, 1, 64]> rotated_3_cast_fp16 = slice_by_index(begin = rotated_3_begin_0, end = rotated_3_end_0, end_mask = rotated_3_end_mask_0, x = value_51_cast_fp16)[name = string("rotated_3_cast_fp16")];
            tensor<int32, [4]> passthrough_3_begin_0 = const()[name = string("passthrough_3_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 64])];
            tensor<int32, [4]> passthrough_3_end_0 = const()[name = string("passthrough_3_end_0"), val = tensor<int32, [4]>([1, 2, 1, 256])];
            tensor<bool, [4]> passthrough_3_end_mask_0 = const()[name = string("passthrough_3_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
            tensor<fp16, [1, 2, 1, 192]> passthrough_3_cast_fp16 = slice_by_index(begin = passthrough_3_begin_0, end = passthrough_3_end_0, end_mask = passthrough_3_end_mask_0, x = value_51_cast_fp16)[name = string("passthrough_3_cast_fp16")];
            tensor<int32, [2]> var_731_split_sizes_0 = const()[name = string("op_731_split_sizes_0"), val = tensor<int32, [2]>([32, 32])];
            int32 var_731_axis_0 = const()[name = string("op_731_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 2, 1, 32]> var_731_cast_fp16_0, tensor<fp16, [1, 2, 1, 32]> var_731_cast_fp16_1 = split(axis = var_731_axis_0, split_sizes = var_731_split_sizes_0, x = rotated_3_cast_fp16)[name = string("op_731_cast_fp16")];
            fp16 const_4_promoted_to_fp16 = const()[name = string("const_4_promoted_to_fp16"), val = fp16(-0x1p+0)];
            tensor<fp16, [1, 2, 1, 32]> var_733_cast_fp16 = mul(x = var_731_cast_fp16_1, y = const_4_promoted_to_fp16)[name = string("op_733_cast_fp16")];
            bool rotated_half_3_interleave_0 = const()[name = string("rotated_half_3_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 2, 1, 64]> rotated_half_3_cast_fp16 = concat(axis = var_660, interleave = rotated_half_3_interleave_0, values = (var_733_cast_fp16, var_731_cast_fp16_0))[name = string("rotated_half_3_cast_fp16")];
            tensor<fp16, [1, 2, 1, 64]> var_736_cast_fp16 = mul(x = rotated_3_cast_fp16, y = cos)[name = string("op_736_cast_fp16")];
            tensor<fp16, [1, 2, 1, 64]> var_737_cast_fp16 = mul(x = rotated_half_3_cast_fp16, y = sin)[name = string("op_737_cast_fp16")];
            tensor<fp16, [1, 2, 1, 64]> var_738_cast_fp16 = add(x = var_736_cast_fp16, y = var_737_cast_fp16)[name = string("op_738_cast_fp16")];
            bool key_13_interleave_0 = const()[name = string("key_13_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 2, 1, 256]> key_13_cast_fp16 = concat(axis = var_660, interleave = key_13_interleave_0, values = (var_738_cast_fp16, passthrough_3_cast_fp16))[name = string("key_13_cast_fp16")];
            tensor<int32, [4]> var_741 = const()[name = string("op_741"), val = tensor<int32, [4]>([2, 4, 1, 256])];
            tensor<fp16, [2, 4, 1, 256]> var_742_cast_fp16 = reshape(shape = var_741, x = query_13_cast_fp16)[name = string("op_742_cast_fp16")];
            tensor<int32, [4]> transpose_0_perm_0 = const()[name = string("transpose_0_perm_0"), val = tensor<int32, [4]>([0, 1, 3, 2])];
            tensor<int32, [3]> var_745 = const()[name = string("op_745"), val = tensor<int32, [3]>([2, 1, 256])];
            tensor<fp16, [2, 1, 256]> key_15_cast_fp16 = reshape(shape = var_745, x = key_13_cast_fp16)[name = string("key_15_cast_fp16")];
            tensor<int32, [3]> var_747 = const()[name = string("op_747"), val = tensor<int32, [3]>([1, 0, 2])];
            tensor<int32, [2]> var_749 = const()[name = string("op_749"), val = tensor<int32, [2]>([1, 2048])];
            tensor<fp16, [1, 2048]> gate_3_cast_fp16 = reshape(shape = var_749, x = var_681_cast_fp16_1)[name = string("gate_3_cast_fp16")];
            tensor<int32, [1]> var_757_axes_0 = const()[name = string("op_757_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [1, 2, 1, 256]> var_757_cast_fp16 = expand_dims(axes = var_757_axes_0, x = key_15_cast_fp16)[name = string("op_757_cast_fp16")];
            tensor<int32, [1]> var_759_axes_0 = const()[name = string("op_759_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [1, 1, 2, 1, 256]> var_759_cast_fp16 = expand_dims(axes = var_759_axes_0, x = var_757_cast_fp16)[name = string("op_759_cast_fp16")];
            int32 var_761 = const()[name = string("op_761"), val = int32(1)];
            tensor<int32, [1]> var_762 = add(x = current_pos, y = var_761)[name = string("op_762")];
            tensor<fp16, [4, 2, 2, 2048, 256]> read_state_0 = read_state(input = kv_cache)[name = string("read_state_0")];
            tensor<int32, [1]> expand_dims_0 = const()[name = string("expand_dims_0"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_1 = const()[name = string("expand_dims_1"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_2 = const()[name = string("expand_dims_2"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_4 = const()[name = string("expand_dims_4"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_5 = const()[name = string("expand_dims_5"), val = tensor<int32, [1]>([1])];
            tensor<int32, [1]> expand_dims_6 = const()[name = string("expand_dims_6"), val = tensor<int32, [1]>([1])];
            int32 concat_2_axis_0 = const()[name = string("concat_2_axis_0"), val = int32(0)];
            bool concat_2_interleave_0 = const()[name = string("concat_2_interleave_0"), val = bool(false)];
            tensor<int32, [5]> concat_2 = concat(axis = concat_2_axis_0, interleave = concat_2_interleave_0, values = (expand_dims_0, expand_dims_1, expand_dims_2, current_pos, expand_dims_4))[name = string("concat_2")];
            tensor<int32, [1]> concat_3_values2_0 = const()[name = string("concat_3_values2_0"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> concat_3_values4_0 = const()[name = string("concat_3_values4_0"), val = tensor<int32, [1]>([0])];
            int32 concat_3_axis_0 = const()[name = string("concat_3_axis_0"), val = int32(0)];
            bool concat_3_interleave_0 = const()[name = string("concat_3_interleave_0"), val = bool(false)];
            tensor<int32, [5]> concat_3 = concat(axis = concat_3_axis_0, interleave = concat_3_interleave_0, values = (expand_dims_5, expand_dims_6, concat_3_values2_0, var_762, concat_3_values4_0))[name = string("concat_3")];
            tensor<int32, [5]> kv_cache_internal_tensor_assign_1_stride_0 = const()[name = string("kv_cache_internal_tensor_assign_1_stride_0"), val = tensor<int32, [5]>([1, 1, 1, 1, 1])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_1_begin_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_1_begin_mask_0"), val = tensor<bool, [5]>([false, false, false, false, false])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_1_end_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_1_end_mask_0"), val = tensor<bool, [5]>([false, false, true, false, true])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_1_squeeze_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_1_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, false, false, false])];
            tensor<fp16, [4, 2, 2, 2048, 256]> kv_cache_internal_tensor_assign_1_cast_fp16 = slice_update(begin = concat_2, begin_mask = kv_cache_internal_tensor_assign_1_begin_mask_0, end = concat_3, end_mask = kv_cache_internal_tensor_assign_1_end_mask_0, squeeze_mask = kv_cache_internal_tensor_assign_1_squeeze_mask_0, stride = kv_cache_internal_tensor_assign_1_stride_0, update = var_759_cast_fp16, x = read_state_0)[name = string("kv_cache_internal_tensor_assign_1_cast_fp16")];
            write_state(data = kv_cache_internal_tensor_assign_1_cast_fp16, input = kv_cache)[name = string("coreml_update_state_0_write_state")];
            tensor<fp16, [4, 2, 2, 2048, 256]> coreml_update_state_8 = read_state(input = kv_cache)[name = string("coreml_update_state_0")];
            tensor<int32, [1]> var_790_axes_0 = const()[name = string("op_790_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [2, 1, 256]> value_55_cast_fp16 = transpose(perm = var_747, x = value_53_cast_fp16)[name = string("transpose_118")];
            tensor<fp16, [1, 2, 1, 256]> var_790_cast_fp16 = expand_dims(axes = var_790_axes_0, x = value_55_cast_fp16)[name = string("op_790_cast_fp16")];
            tensor<int32, [1]> var_792_axes_0 = const()[name = string("op_792_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [1, 1, 2, 1, 256]> var_792_cast_fp16 = expand_dims(axes = var_792_axes_0, x = var_790_cast_fp16)[name = string("op_792_cast_fp16")];
            tensor<int32, [1]> expand_dims_8 = const()[name = string("expand_dims_8"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_9 = const()[name = string("expand_dims_9"), val = tensor<int32, [1]>([1])];
            tensor<int32, [1]> expand_dims_10 = const()[name = string("expand_dims_10"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_12 = const()[name = string("expand_dims_12"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_13 = const()[name = string("expand_dims_13"), val = tensor<int32, [1]>([1])];
            tensor<int32, [1]> expand_dims_14 = const()[name = string("expand_dims_14"), val = tensor<int32, [1]>([2])];
            int32 concat_6_axis_0 = const()[name = string("concat_6_axis_0"), val = int32(0)];
            bool concat_6_interleave_0 = const()[name = string("concat_6_interleave_0"), val = bool(false)];
            tensor<int32, [5]> concat_6 = concat(axis = concat_6_axis_0, interleave = concat_6_interleave_0, values = (expand_dims_8, expand_dims_9, expand_dims_10, current_pos, expand_dims_12))[name = string("concat_6")];
            tensor<int32, [1]> concat_7_values2_0 = const()[name = string("concat_7_values2_0"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> concat_7_values4_0 = const()[name = string("concat_7_values4_0"), val = tensor<int32, [1]>([0])];
            int32 concat_7_axis_0 = const()[name = string("concat_7_axis_0"), val = int32(0)];
            bool concat_7_interleave_0 = const()[name = string("concat_7_interleave_0"), val = bool(false)];
            tensor<int32, [5]> concat_7 = concat(axis = concat_7_axis_0, interleave = concat_7_interleave_0, values = (expand_dims_13, expand_dims_14, concat_7_values2_0, var_762, concat_7_values4_0))[name = string("concat_7")];
            tensor<int32, [5]> kv_cache_internal_tensor_assign_2_stride_0 = const()[name = string("kv_cache_internal_tensor_assign_2_stride_0"), val = tensor<int32, [5]>([1, 1, 1, 1, 1])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_2_begin_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_2_begin_mask_0"), val = tensor<bool, [5]>([false, false, false, false, false])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_2_end_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_2_end_mask_0"), val = tensor<bool, [5]>([false, false, true, false, true])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_2_squeeze_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_2_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, false, false, false])];
            tensor<fp16, [4, 2, 2, 2048, 256]> kv_cache_internal_tensor_assign_2_cast_fp16 = slice_update(begin = concat_6, begin_mask = kv_cache_internal_tensor_assign_2_begin_mask_0, end = concat_7, end_mask = kv_cache_internal_tensor_assign_2_end_mask_0, squeeze_mask = kv_cache_internal_tensor_assign_2_squeeze_mask_0, stride = kv_cache_internal_tensor_assign_2_stride_0, update = var_792_cast_fp16, x = coreml_update_state_8)[name = string("kv_cache_internal_tensor_assign_2_cast_fp16")];
            write_state(data = kv_cache_internal_tensor_assign_2_cast_fp16, input = kv_cache)[name = string("coreml_update_state_1_write_state")];
            tensor<fp16, [4, 2, 2, 2048, 256]> coreml_update_state_9 = read_state(input = kv_cache)[name = string("coreml_update_state_1")];
            tensor<int32, [5]> var_824_begin_0 = const()[name = string("op_824_begin_0"), val = tensor<int32, [5]>([0, 0, 0, 0, 0])];
            tensor<int32, [5]> var_824_end_0 = const()[name = string("op_824_end_0"), val = tensor<int32, [5]>([1, 2, 2, 2048, 256])];
            tensor<bool, [5]> var_824_end_mask_0 = const()[name = string("op_824_end_mask_0"), val = tensor<bool, [5]>([false, true, true, true, true])];
            tensor<bool, [5]> var_824_squeeze_mask_0 = const()[name = string("op_824_squeeze_mask_0"), val = tensor<bool, [5]>([true, false, false, false, false])];
            tensor<fp16, [2, 2, 2048, 256]> var_824_cast_fp16 = slice_by_index(begin = var_824_begin_0, end = var_824_end_0, end_mask = var_824_end_mask_0, squeeze_mask = var_824_squeeze_mask_0, x = coreml_update_state_9)[name = string("op_824_cast_fp16")];
            tensor<int32, [4]> keys_1_begin_0 = const()[name = string("keys_1_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> keys_1_end_0 = const()[name = string("keys_1_end_0"), val = tensor<int32, [4]>([1, 2, 2048, 256])];
            tensor<bool, [4]> keys_1_end_mask_0 = const()[name = string("keys_1_end_mask_0"), val = tensor<bool, [4]>([false, true, true, true])];
            tensor<bool, [4]> keys_1_squeeze_mask_0 = const()[name = string("keys_1_squeeze_mask_0"), val = tensor<bool, [4]>([true, false, false, false])];
            tensor<fp16, [2, 2048, 256]> keys_1_cast_fp16 = slice_by_index(begin = keys_1_begin_0, end = keys_1_end_0, end_mask = keys_1_end_mask_0, squeeze_mask = keys_1_squeeze_mask_0, x = var_824_cast_fp16)[name = string("keys_1_cast_fp16")];
            tensor<int32, [4]> values_1_begin_0 = const()[name = string("values_1_begin_0"), val = tensor<int32, [4]>([1, 0, 0, 0])];
            tensor<int32, [4]> values_1_end_0 = const()[name = string("values_1_end_0"), val = tensor<int32, [4]>([2, 2, 2048, 256])];
            tensor<bool, [4]> values_1_end_mask_0 = const()[name = string("values_1_end_mask_0"), val = tensor<bool, [4]>([false, true, true, true])];
            tensor<bool, [4]> values_1_squeeze_mask_0 = const()[name = string("values_1_squeeze_mask_0"), val = tensor<bool, [4]>([true, false, false, false])];
            tensor<fp16, [2, 2048, 256]> values_1_cast_fp16 = slice_by_index(begin = values_1_begin_0, end = values_1_end_0, end_mask = values_1_end_mask_0, squeeze_mask = values_1_squeeze_mask_0, x = var_824_cast_fp16)[name = string("values_1_cast_fp16")];
            int32 var_841 = const()[name = string("op_841"), val = int32(1)];
            tensor<int32, [4]> var_848 = const()[name = string("op_848"), val = tensor<int32, [4]>([1, 2048, 1, 1])];
            tensor<fp16, [2, 4, 256, 1]> transpose_0_cast_fp16 = transpose(perm = transpose_0_perm_0, x = var_742_cast_fp16)[name = string("transpose_117")];
            tensor<fp16, [1, 2048, 1, 1]> var_849_cast_fp16 = reshape(shape = var_848, x = transpose_0_cast_fp16)[name = string("op_849_cast_fp16")];
            tensor<int32, [3]> var_850 = const()[name = string("op_850"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [4]> var_853 = const()[name = string("op_853"), val = tensor<int32, [4]>([1, 512, 1, 2048])];
            tensor<fp16, [2, 256, 2048]> var_851_cast_fp16 = transpose(perm = var_850, x = keys_1_cast_fp16)[name = string("transpose_116")];
            tensor<fp16, [1, 512, 1, 2048]> key_native_1_cast_fp16 = reshape(shape = var_853, x = var_851_cast_fp16)[name = string("key_native_1_cast_fp16")];
            tensor<int32, [3]> var_855 = const()[name = string("op_855"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [4]> var_858 = const()[name = string("op_858"), val = tensor<int32, [4]>([1, 512, 1, 2048])];
            tensor<fp16, [2, 256, 2048]> var_856_cast_fp16 = transpose(perm = var_855, x = values_1_cast_fp16)[name = string("transpose_115")];
            tensor<fp16, [1, 512, 1, 2048]> var_859_cast_fp16 = reshape(shape = var_858, x = var_856_cast_fp16)[name = string("op_859_cast_fp16")];
            tensor<int32, [3]> var_860 = const()[name = string("op_860"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [1]> mask_native_1_axes_0 = const()[name = string("mask_native_1_axes_0"), val = tensor<int32, [1]>([2])];
            tensor<fp16, [1, 2048, 1]> var_861_cast_fp16 = transpose(perm = var_860, x = mask3)[name = string("transpose_114")];
            tensor<fp16, [1, 2048, 1, 1]> mask_native_1_cast_fp16 = expand_dims(axes = mask_native_1_axes_0, x = var_861_cast_fp16)[name = string("mask_native_1_cast_fp16")];
            tensor<int32, [8]> tile_0 = const()[name = string("tile_0"), val = tensor<int32, [8]>([256, 256, 256, 256, 256, 256, 256, 256])];
            int32 var_863_axis_0 = const()[name = string("op_863_axis_0"), val = int32(1)];
            tensor<fp16, [1, 256, 1, 1]> var_863_cast_fp16_0, tensor<fp16, [1, 256, 1, 1]> var_863_cast_fp16_1, tensor<fp16, [1, 256, 1, 1]> var_863_cast_fp16_2, tensor<fp16, [1, 256, 1, 1]> var_863_cast_fp16_3, tensor<fp16, [1, 256, 1, 1]> var_863_cast_fp16_4, tensor<fp16, [1, 256, 1, 1]> var_863_cast_fp16_5, tensor<fp16, [1, 256, 1, 1]> var_863_cast_fp16_6, tensor<fp16, [1, 256, 1, 1]> var_863_cast_fp16_7 = split(axis = var_863_axis_0, split_sizes = tile_0, x = var_849_cast_fp16)[name = string("op_863_cast_fp16")];
            tensor<int32, [4]> var_872_perm_0 = const()[name = string("op_872_perm_0"), val = tensor<int32, [4]>([0, 3, 2, 1])];
            tensor<int32, [2]> tile_1 = const()[name = string("tile_1"), val = tensor<int32, [2]>([256, 256])];
            int32 var_873_axis_0 = const()[name = string("op_873_axis_0"), val = int32(3)];
            tensor<fp16, [1, 2048, 1, 512]> var_872_cast_fp16 = transpose(perm = var_872_perm_0, x = key_native_1_cast_fp16)[name = string("transpose_113")];
            tensor<fp16, [1, 2048, 1, 256]> var_873_cast_fp16_0, tensor<fp16, [1, 2048, 1, 256]> var_873_cast_fp16_1 = split(axis = var_873_axis_0, split_sizes = tile_1, x = var_872_cast_fp16)[name = string("op_873_cast_fp16")];
            tensor<int32, [2]> tile_2 = const()[name = string("tile_2"), val = tensor<int32, [2]>([256, 256])];
            int32 var_876_axis_0 = const()[name = string("op_876_axis_0"), val = int32(1)];
            tensor<fp16, [1, 256, 1, 2048]> var_876_cast_fp16_0, tensor<fp16, [1, 256, 1, 2048]> var_876_cast_fp16_1 = split(axis = var_876_axis_0, split_sizes = tile_2, x = var_859_cast_fp16)[name = string("op_876_cast_fp16")];
            string scores_1_equation_0 = const()[name = string("scores_1_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 1]> scores_1_cast_fp16 = einsum(equation = scores_1_equation_0, values = (var_873_cast_fp16_0, var_863_cast_fp16_0))[name = string("scores_1_cast_fp16")];
            fp16 var_881_to_fp16 = const()[name = string("op_881_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 1]> var_882_cast_fp16 = mul(x = scores_1_cast_fp16, y = var_881_to_fp16)[name = string("op_882_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_883_cast_fp16 = add(x = var_882_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_883_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_884_cast_fp16 = softmax(axis = var_841, x = var_883_cast_fp16)[name = string("op_884_cast_fp16")];
            tensor<int32, [4]> tile_3 = const()[name = string("tile_3"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_885_axis_0 = const()[name = string("op_885_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 1]> var_885_cast_fp16_0, tensor<fp16, [1, 512, 1, 1]> var_885_cast_fp16_1, tensor<fp16, [1, 512, 1, 1]> var_885_cast_fp16_2, tensor<fp16, [1, 512, 1, 1]> var_885_cast_fp16_3 = split(axis = var_885_axis_0, split_sizes = tile_3, x = var_884_cast_fp16)[name = string("op_885_cast_fp16")];
            tensor<int32, [4]> tile_4 = const()[name = string("tile_4"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_890_axis_0 = const()[name = string("op_890_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_890_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_890_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_890_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_890_cast_fp16_3 = split(axis = var_890_axis_0, split_sizes = tile_4, x = var_876_cast_fp16_0)[name = string("op_890_cast_fp16")];
            fp16 var_895_to_fp16 = const()[name = string("op_895_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_896_cast_fp16 = mul(x = var_885_cast_fp16_0, y = var_895_to_fp16)[name = string("op_896_cast_fp16")];
            string var_898_equation_0 = const()[name = string("op_898_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_898_cast_fp16 = einsum(equation = var_898_equation_0, values = (var_890_cast_fp16_0, var_896_cast_fp16))[name = string("op_898_cast_fp16")];
            fp16 var_899_to_fp16 = const()[name = string("op_899_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_900_cast_fp16 = mul(x = var_885_cast_fp16_1, y = var_899_to_fp16)[name = string("op_900_cast_fp16")];
            string var_902_equation_0 = const()[name = string("op_902_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_902_cast_fp16 = einsum(equation = var_902_equation_0, values = (var_890_cast_fp16_1, var_900_cast_fp16))[name = string("op_902_cast_fp16")];
            fp16 var_903_to_fp16 = const()[name = string("op_903_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_904_cast_fp16 = mul(x = var_885_cast_fp16_2, y = var_903_to_fp16)[name = string("op_904_cast_fp16")];
            string var_906_equation_0 = const()[name = string("op_906_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_906_cast_fp16 = einsum(equation = var_906_equation_0, values = (var_890_cast_fp16_2, var_904_cast_fp16))[name = string("op_906_cast_fp16")];
            fp16 var_907_to_fp16 = const()[name = string("op_907_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_908_cast_fp16 = mul(x = var_885_cast_fp16_3, y = var_907_to_fp16)[name = string("op_908_cast_fp16")];
            string var_910_equation_0 = const()[name = string("op_910_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_910_cast_fp16 = einsum(equation = var_910_equation_0, values = (var_890_cast_fp16_3, var_908_cast_fp16))[name = string("op_910_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_911_cast_fp16 = add(x = var_898_cast_fp16, y = var_902_cast_fp16)[name = string("op_911_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_912_cast_fp16 = add(x = var_906_cast_fp16, y = var_910_cast_fp16)[name = string("op_912_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_913_cast_fp16 = add(x = var_911_cast_fp16, y = var_912_cast_fp16)[name = string("op_913_cast_fp16")];
            fp16 _inversed_915_y_0_to_fp16 = const()[name = string("_inversed_915_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 1]> _inversed_915_cast_fp16 = mul(x = var_913_cast_fp16, y = _inversed_915_y_0_to_fp16)[name = string("_inversed_915_cast_fp16")];
            string scores_3_equation_0 = const()[name = string("scores_3_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 1]> scores_3_cast_fp16 = einsum(equation = scores_3_equation_0, values = (var_873_cast_fp16_0, var_863_cast_fp16_1))[name = string("scores_3_cast_fp16")];
            fp16 var_918_to_fp16 = const()[name = string("op_918_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 1]> var_919_cast_fp16 = mul(x = scores_3_cast_fp16, y = var_918_to_fp16)[name = string("op_919_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_920_cast_fp16 = add(x = var_919_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_920_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_921_cast_fp16 = softmax(axis = var_841, x = var_920_cast_fp16)[name = string("op_921_cast_fp16")];
            tensor<int32, [4]> tile_5 = const()[name = string("tile_5"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_922_axis_0 = const()[name = string("op_922_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 1]> var_922_cast_fp16_0, tensor<fp16, [1, 512, 1, 1]> var_922_cast_fp16_1, tensor<fp16, [1, 512, 1, 1]> var_922_cast_fp16_2, tensor<fp16, [1, 512, 1, 1]> var_922_cast_fp16_3 = split(axis = var_922_axis_0, split_sizes = tile_5, x = var_921_cast_fp16)[name = string("op_922_cast_fp16")];
            tensor<int32, [4]> tile_6 = const()[name = string("tile_6"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_927_axis_0 = const()[name = string("op_927_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_927_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_927_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_927_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_927_cast_fp16_3 = split(axis = var_927_axis_0, split_sizes = tile_6, x = var_876_cast_fp16_0)[name = string("op_927_cast_fp16")];
            fp16 var_932_to_fp16 = const()[name = string("op_932_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_933_cast_fp16 = mul(x = var_922_cast_fp16_0, y = var_932_to_fp16)[name = string("op_933_cast_fp16")];
            string var_935_equation_0 = const()[name = string("op_935_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_935_cast_fp16 = einsum(equation = var_935_equation_0, values = (var_927_cast_fp16_0, var_933_cast_fp16))[name = string("op_935_cast_fp16")];
            fp16 var_936_to_fp16 = const()[name = string("op_936_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_937_cast_fp16 = mul(x = var_922_cast_fp16_1, y = var_936_to_fp16)[name = string("op_937_cast_fp16")];
            string var_939_equation_0 = const()[name = string("op_939_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_939_cast_fp16 = einsum(equation = var_939_equation_0, values = (var_927_cast_fp16_1, var_937_cast_fp16))[name = string("op_939_cast_fp16")];
            fp16 var_940_to_fp16 = const()[name = string("op_940_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_941_cast_fp16 = mul(x = var_922_cast_fp16_2, y = var_940_to_fp16)[name = string("op_941_cast_fp16")];
            string var_943_equation_0 = const()[name = string("op_943_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_943_cast_fp16 = einsum(equation = var_943_equation_0, values = (var_927_cast_fp16_2, var_941_cast_fp16))[name = string("op_943_cast_fp16")];
            fp16 var_944_to_fp16 = const()[name = string("op_944_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_945_cast_fp16 = mul(x = var_922_cast_fp16_3, y = var_944_to_fp16)[name = string("op_945_cast_fp16")];
            string var_947_equation_0 = const()[name = string("op_947_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_947_cast_fp16 = einsum(equation = var_947_equation_0, values = (var_927_cast_fp16_3, var_945_cast_fp16))[name = string("op_947_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_948_cast_fp16 = add(x = var_935_cast_fp16, y = var_939_cast_fp16)[name = string("op_948_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_949_cast_fp16 = add(x = var_943_cast_fp16, y = var_947_cast_fp16)[name = string("op_949_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_950_cast_fp16 = add(x = var_948_cast_fp16, y = var_949_cast_fp16)[name = string("op_950_cast_fp16")];
            fp16 _inversed_952_y_0_to_fp16 = const()[name = string("_inversed_952_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 1]> _inversed_952_cast_fp16 = mul(x = var_950_cast_fp16, y = _inversed_952_y_0_to_fp16)[name = string("_inversed_952_cast_fp16")];
            string scores_5_equation_0 = const()[name = string("scores_5_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 1]> scores_5_cast_fp16 = einsum(equation = scores_5_equation_0, values = (var_873_cast_fp16_0, var_863_cast_fp16_2))[name = string("scores_5_cast_fp16")];
            fp16 var_955_to_fp16 = const()[name = string("op_955_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 1]> var_956_cast_fp16 = mul(x = scores_5_cast_fp16, y = var_955_to_fp16)[name = string("op_956_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_957_cast_fp16 = add(x = var_956_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_957_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_958_cast_fp16 = softmax(axis = var_841, x = var_957_cast_fp16)[name = string("op_958_cast_fp16")];
            tensor<int32, [4]> tile_7 = const()[name = string("tile_7"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_959_axis_0 = const()[name = string("op_959_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 1]> var_959_cast_fp16_0, tensor<fp16, [1, 512, 1, 1]> var_959_cast_fp16_1, tensor<fp16, [1, 512, 1, 1]> var_959_cast_fp16_2, tensor<fp16, [1, 512, 1, 1]> var_959_cast_fp16_3 = split(axis = var_959_axis_0, split_sizes = tile_7, x = var_958_cast_fp16)[name = string("op_959_cast_fp16")];
            tensor<int32, [4]> tile_8 = const()[name = string("tile_8"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_964_axis_0 = const()[name = string("op_964_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_964_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_964_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_964_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_964_cast_fp16_3 = split(axis = var_964_axis_0, split_sizes = tile_8, x = var_876_cast_fp16_0)[name = string("op_964_cast_fp16")];
            fp16 var_969_to_fp16 = const()[name = string("op_969_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_970_cast_fp16 = mul(x = var_959_cast_fp16_0, y = var_969_to_fp16)[name = string("op_970_cast_fp16")];
            string var_972_equation_0 = const()[name = string("op_972_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_972_cast_fp16 = einsum(equation = var_972_equation_0, values = (var_964_cast_fp16_0, var_970_cast_fp16))[name = string("op_972_cast_fp16")];
            fp16 var_973_to_fp16 = const()[name = string("op_973_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_974_cast_fp16 = mul(x = var_959_cast_fp16_1, y = var_973_to_fp16)[name = string("op_974_cast_fp16")];
            string var_976_equation_0 = const()[name = string("op_976_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_976_cast_fp16 = einsum(equation = var_976_equation_0, values = (var_964_cast_fp16_1, var_974_cast_fp16))[name = string("op_976_cast_fp16")];
            fp16 var_977_to_fp16 = const()[name = string("op_977_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_978_cast_fp16 = mul(x = var_959_cast_fp16_2, y = var_977_to_fp16)[name = string("op_978_cast_fp16")];
            string var_980_equation_0 = const()[name = string("op_980_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_980_cast_fp16 = einsum(equation = var_980_equation_0, values = (var_964_cast_fp16_2, var_978_cast_fp16))[name = string("op_980_cast_fp16")];
            fp16 var_981_to_fp16 = const()[name = string("op_981_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_982_cast_fp16 = mul(x = var_959_cast_fp16_3, y = var_981_to_fp16)[name = string("op_982_cast_fp16")];
            string var_984_equation_0 = const()[name = string("op_984_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_984_cast_fp16 = einsum(equation = var_984_equation_0, values = (var_964_cast_fp16_3, var_982_cast_fp16))[name = string("op_984_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_985_cast_fp16 = add(x = var_972_cast_fp16, y = var_976_cast_fp16)[name = string("op_985_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_986_cast_fp16 = add(x = var_980_cast_fp16, y = var_984_cast_fp16)[name = string("op_986_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_987_cast_fp16 = add(x = var_985_cast_fp16, y = var_986_cast_fp16)[name = string("op_987_cast_fp16")];
            fp16 _inversed_989_y_0_to_fp16 = const()[name = string("_inversed_989_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 1]> _inversed_989_cast_fp16 = mul(x = var_987_cast_fp16, y = _inversed_989_y_0_to_fp16)[name = string("_inversed_989_cast_fp16")];
            string scores_7_equation_0 = const()[name = string("scores_7_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 1]> scores_7_cast_fp16 = einsum(equation = scores_7_equation_0, values = (var_873_cast_fp16_0, var_863_cast_fp16_3))[name = string("scores_7_cast_fp16")];
            fp16 var_992_to_fp16 = const()[name = string("op_992_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 1]> var_993_cast_fp16 = mul(x = scores_7_cast_fp16, y = var_992_to_fp16)[name = string("op_993_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_994_cast_fp16 = add(x = var_993_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_994_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_995_cast_fp16 = softmax(axis = var_841, x = var_994_cast_fp16)[name = string("op_995_cast_fp16")];
            tensor<int32, [4]> tile_9 = const()[name = string("tile_9"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_996_axis_0 = const()[name = string("op_996_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 1]> var_996_cast_fp16_0, tensor<fp16, [1, 512, 1, 1]> var_996_cast_fp16_1, tensor<fp16, [1, 512, 1, 1]> var_996_cast_fp16_2, tensor<fp16, [1, 512, 1, 1]> var_996_cast_fp16_3 = split(axis = var_996_axis_0, split_sizes = tile_9, x = var_995_cast_fp16)[name = string("op_996_cast_fp16")];
            tensor<int32, [4]> tile_10 = const()[name = string("tile_10"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_1001_axis_0 = const()[name = string("op_1001_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_1001_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_1001_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_1001_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_1001_cast_fp16_3 = split(axis = var_1001_axis_0, split_sizes = tile_10, x = var_876_cast_fp16_0)[name = string("op_1001_cast_fp16")];
            fp16 var_1006_to_fp16 = const()[name = string("op_1006_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_1007_cast_fp16 = mul(x = var_996_cast_fp16_0, y = var_1006_to_fp16)[name = string("op_1007_cast_fp16")];
            string var_1009_equation_0 = const()[name = string("op_1009_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_1009_cast_fp16 = einsum(equation = var_1009_equation_0, values = (var_1001_cast_fp16_0, var_1007_cast_fp16))[name = string("op_1009_cast_fp16")];
            fp16 var_1010_to_fp16 = const()[name = string("op_1010_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_1011_cast_fp16 = mul(x = var_996_cast_fp16_1, y = var_1010_to_fp16)[name = string("op_1011_cast_fp16")];
            string var_1013_equation_0 = const()[name = string("op_1013_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_1013_cast_fp16 = einsum(equation = var_1013_equation_0, values = (var_1001_cast_fp16_1, var_1011_cast_fp16))[name = string("op_1013_cast_fp16")];
            fp16 var_1014_to_fp16 = const()[name = string("op_1014_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_1015_cast_fp16 = mul(x = var_996_cast_fp16_2, y = var_1014_to_fp16)[name = string("op_1015_cast_fp16")];
            string var_1017_equation_0 = const()[name = string("op_1017_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_1017_cast_fp16 = einsum(equation = var_1017_equation_0, values = (var_1001_cast_fp16_2, var_1015_cast_fp16))[name = string("op_1017_cast_fp16")];
            fp16 var_1018_to_fp16 = const()[name = string("op_1018_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_1019_cast_fp16 = mul(x = var_996_cast_fp16_3, y = var_1018_to_fp16)[name = string("op_1019_cast_fp16")];
            string var_1021_equation_0 = const()[name = string("op_1021_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_1021_cast_fp16 = einsum(equation = var_1021_equation_0, values = (var_1001_cast_fp16_3, var_1019_cast_fp16))[name = string("op_1021_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_1022_cast_fp16 = add(x = var_1009_cast_fp16, y = var_1013_cast_fp16)[name = string("op_1022_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_1023_cast_fp16 = add(x = var_1017_cast_fp16, y = var_1021_cast_fp16)[name = string("op_1023_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_1024_cast_fp16 = add(x = var_1022_cast_fp16, y = var_1023_cast_fp16)[name = string("op_1024_cast_fp16")];
            fp16 _inversed_1026_y_0_to_fp16 = const()[name = string("_inversed_1026_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 1]> _inversed_1026_cast_fp16 = mul(x = var_1024_cast_fp16, y = _inversed_1026_y_0_to_fp16)[name = string("_inversed_1026_cast_fp16")];
            string scores_9_equation_0 = const()[name = string("scores_9_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 1]> scores_9_cast_fp16 = einsum(equation = scores_9_equation_0, values = (var_873_cast_fp16_1, var_863_cast_fp16_4))[name = string("scores_9_cast_fp16")];
            fp16 var_1029_to_fp16 = const()[name = string("op_1029_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 1]> var_1030_cast_fp16 = mul(x = scores_9_cast_fp16, y = var_1029_to_fp16)[name = string("op_1030_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_1031_cast_fp16 = add(x = var_1030_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_1031_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_1032_cast_fp16 = softmax(axis = var_841, x = var_1031_cast_fp16)[name = string("op_1032_cast_fp16")];
            tensor<int32, [4]> tile_11 = const()[name = string("tile_11"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_1033_axis_0 = const()[name = string("op_1033_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 1]> var_1033_cast_fp16_0, tensor<fp16, [1, 512, 1, 1]> var_1033_cast_fp16_1, tensor<fp16, [1, 512, 1, 1]> var_1033_cast_fp16_2, tensor<fp16, [1, 512, 1, 1]> var_1033_cast_fp16_3 = split(axis = var_1033_axis_0, split_sizes = tile_11, x = var_1032_cast_fp16)[name = string("op_1033_cast_fp16")];
            tensor<int32, [4]> tile_12 = const()[name = string("tile_12"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_1038_axis_0 = const()[name = string("op_1038_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_1038_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_1038_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_1038_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_1038_cast_fp16_3 = split(axis = var_1038_axis_0, split_sizes = tile_12, x = var_876_cast_fp16_1)[name = string("op_1038_cast_fp16")];
            fp16 var_1043_to_fp16 = const()[name = string("op_1043_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_1044_cast_fp16 = mul(x = var_1033_cast_fp16_0, y = var_1043_to_fp16)[name = string("op_1044_cast_fp16")];
            string var_1046_equation_0 = const()[name = string("op_1046_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_1046_cast_fp16 = einsum(equation = var_1046_equation_0, values = (var_1038_cast_fp16_0, var_1044_cast_fp16))[name = string("op_1046_cast_fp16")];
            fp16 var_1047_to_fp16 = const()[name = string("op_1047_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_1048_cast_fp16 = mul(x = var_1033_cast_fp16_1, y = var_1047_to_fp16)[name = string("op_1048_cast_fp16")];
            string var_1050_equation_0 = const()[name = string("op_1050_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_1050_cast_fp16 = einsum(equation = var_1050_equation_0, values = (var_1038_cast_fp16_1, var_1048_cast_fp16))[name = string("op_1050_cast_fp16")];
            fp16 var_1051_to_fp16 = const()[name = string("op_1051_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_1052_cast_fp16 = mul(x = var_1033_cast_fp16_2, y = var_1051_to_fp16)[name = string("op_1052_cast_fp16")];
            string var_1054_equation_0 = const()[name = string("op_1054_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_1054_cast_fp16 = einsum(equation = var_1054_equation_0, values = (var_1038_cast_fp16_2, var_1052_cast_fp16))[name = string("op_1054_cast_fp16")];
            fp16 var_1055_to_fp16 = const()[name = string("op_1055_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_1056_cast_fp16 = mul(x = var_1033_cast_fp16_3, y = var_1055_to_fp16)[name = string("op_1056_cast_fp16")];
            string var_1058_equation_0 = const()[name = string("op_1058_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_1058_cast_fp16 = einsum(equation = var_1058_equation_0, values = (var_1038_cast_fp16_3, var_1056_cast_fp16))[name = string("op_1058_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_1059_cast_fp16 = add(x = var_1046_cast_fp16, y = var_1050_cast_fp16)[name = string("op_1059_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_1060_cast_fp16 = add(x = var_1054_cast_fp16, y = var_1058_cast_fp16)[name = string("op_1060_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_1061_cast_fp16 = add(x = var_1059_cast_fp16, y = var_1060_cast_fp16)[name = string("op_1061_cast_fp16")];
            fp16 _inversed_1063_y_0_to_fp16 = const()[name = string("_inversed_1063_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 1]> _inversed_1063_cast_fp16 = mul(x = var_1061_cast_fp16, y = _inversed_1063_y_0_to_fp16)[name = string("_inversed_1063_cast_fp16")];
            string scores_11_equation_0 = const()[name = string("scores_11_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 1]> scores_11_cast_fp16 = einsum(equation = scores_11_equation_0, values = (var_873_cast_fp16_1, var_863_cast_fp16_5))[name = string("scores_11_cast_fp16")];
            fp16 var_1066_to_fp16 = const()[name = string("op_1066_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 1]> var_1067_cast_fp16 = mul(x = scores_11_cast_fp16, y = var_1066_to_fp16)[name = string("op_1067_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_1068_cast_fp16 = add(x = var_1067_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_1068_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_1069_cast_fp16 = softmax(axis = var_841, x = var_1068_cast_fp16)[name = string("op_1069_cast_fp16")];
            tensor<int32, [4]> tile_13 = const()[name = string("tile_13"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_1070_axis_0 = const()[name = string("op_1070_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 1]> var_1070_cast_fp16_0, tensor<fp16, [1, 512, 1, 1]> var_1070_cast_fp16_1, tensor<fp16, [1, 512, 1, 1]> var_1070_cast_fp16_2, tensor<fp16, [1, 512, 1, 1]> var_1070_cast_fp16_3 = split(axis = var_1070_axis_0, split_sizes = tile_13, x = var_1069_cast_fp16)[name = string("op_1070_cast_fp16")];
            tensor<int32, [4]> tile_14 = const()[name = string("tile_14"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_1075_axis_0 = const()[name = string("op_1075_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_1075_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_1075_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_1075_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_1075_cast_fp16_3 = split(axis = var_1075_axis_0, split_sizes = tile_14, x = var_876_cast_fp16_1)[name = string("op_1075_cast_fp16")];
            fp16 var_1080_to_fp16 = const()[name = string("op_1080_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_1081_cast_fp16 = mul(x = var_1070_cast_fp16_0, y = var_1080_to_fp16)[name = string("op_1081_cast_fp16")];
            string var_1083_equation_0 = const()[name = string("op_1083_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_1083_cast_fp16 = einsum(equation = var_1083_equation_0, values = (var_1075_cast_fp16_0, var_1081_cast_fp16))[name = string("op_1083_cast_fp16")];
            fp16 var_1084_to_fp16 = const()[name = string("op_1084_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_1085_cast_fp16 = mul(x = var_1070_cast_fp16_1, y = var_1084_to_fp16)[name = string("op_1085_cast_fp16")];
            string var_1087_equation_0 = const()[name = string("op_1087_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_1087_cast_fp16 = einsum(equation = var_1087_equation_0, values = (var_1075_cast_fp16_1, var_1085_cast_fp16))[name = string("op_1087_cast_fp16")];
            fp16 var_1088_to_fp16 = const()[name = string("op_1088_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_1089_cast_fp16 = mul(x = var_1070_cast_fp16_2, y = var_1088_to_fp16)[name = string("op_1089_cast_fp16")];
            string var_1091_equation_0 = const()[name = string("op_1091_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_1091_cast_fp16 = einsum(equation = var_1091_equation_0, values = (var_1075_cast_fp16_2, var_1089_cast_fp16))[name = string("op_1091_cast_fp16")];
            fp16 var_1092_to_fp16 = const()[name = string("op_1092_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_1093_cast_fp16 = mul(x = var_1070_cast_fp16_3, y = var_1092_to_fp16)[name = string("op_1093_cast_fp16")];
            string var_1095_equation_0 = const()[name = string("op_1095_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_1095_cast_fp16 = einsum(equation = var_1095_equation_0, values = (var_1075_cast_fp16_3, var_1093_cast_fp16))[name = string("op_1095_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_1096_cast_fp16 = add(x = var_1083_cast_fp16, y = var_1087_cast_fp16)[name = string("op_1096_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_1097_cast_fp16 = add(x = var_1091_cast_fp16, y = var_1095_cast_fp16)[name = string("op_1097_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_1098_cast_fp16 = add(x = var_1096_cast_fp16, y = var_1097_cast_fp16)[name = string("op_1098_cast_fp16")];
            fp16 _inversed_1100_y_0_to_fp16 = const()[name = string("_inversed_1100_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 1]> _inversed_1100_cast_fp16 = mul(x = var_1098_cast_fp16, y = _inversed_1100_y_0_to_fp16)[name = string("_inversed_1100_cast_fp16")];
            string scores_13_equation_0 = const()[name = string("scores_13_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 1]> scores_13_cast_fp16 = einsum(equation = scores_13_equation_0, values = (var_873_cast_fp16_1, var_863_cast_fp16_6))[name = string("scores_13_cast_fp16")];
            fp16 var_1103_to_fp16 = const()[name = string("op_1103_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 1]> var_1104_cast_fp16 = mul(x = scores_13_cast_fp16, y = var_1103_to_fp16)[name = string("op_1104_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_1105_cast_fp16 = add(x = var_1104_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_1105_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_1106_cast_fp16 = softmax(axis = var_841, x = var_1105_cast_fp16)[name = string("op_1106_cast_fp16")];
            tensor<int32, [4]> tile_15 = const()[name = string("tile_15"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_1107_axis_0 = const()[name = string("op_1107_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 1]> var_1107_cast_fp16_0, tensor<fp16, [1, 512, 1, 1]> var_1107_cast_fp16_1, tensor<fp16, [1, 512, 1, 1]> var_1107_cast_fp16_2, tensor<fp16, [1, 512, 1, 1]> var_1107_cast_fp16_3 = split(axis = var_1107_axis_0, split_sizes = tile_15, x = var_1106_cast_fp16)[name = string("op_1107_cast_fp16")];
            tensor<int32, [4]> tile_16 = const()[name = string("tile_16"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_1112_axis_0 = const()[name = string("op_1112_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_1112_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_1112_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_1112_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_1112_cast_fp16_3 = split(axis = var_1112_axis_0, split_sizes = tile_16, x = var_876_cast_fp16_1)[name = string("op_1112_cast_fp16")];
            fp16 var_1117_to_fp16 = const()[name = string("op_1117_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_1118_cast_fp16 = mul(x = var_1107_cast_fp16_0, y = var_1117_to_fp16)[name = string("op_1118_cast_fp16")];
            string var_1120_equation_0 = const()[name = string("op_1120_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_1120_cast_fp16 = einsum(equation = var_1120_equation_0, values = (var_1112_cast_fp16_0, var_1118_cast_fp16))[name = string("op_1120_cast_fp16")];
            fp16 var_1121_to_fp16 = const()[name = string("op_1121_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_1122_cast_fp16 = mul(x = var_1107_cast_fp16_1, y = var_1121_to_fp16)[name = string("op_1122_cast_fp16")];
            string var_1124_equation_0 = const()[name = string("op_1124_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_1124_cast_fp16 = einsum(equation = var_1124_equation_0, values = (var_1112_cast_fp16_1, var_1122_cast_fp16))[name = string("op_1124_cast_fp16")];
            fp16 var_1125_to_fp16 = const()[name = string("op_1125_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_1126_cast_fp16 = mul(x = var_1107_cast_fp16_2, y = var_1125_to_fp16)[name = string("op_1126_cast_fp16")];
            string var_1128_equation_0 = const()[name = string("op_1128_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_1128_cast_fp16 = einsum(equation = var_1128_equation_0, values = (var_1112_cast_fp16_2, var_1126_cast_fp16))[name = string("op_1128_cast_fp16")];
            fp16 var_1129_to_fp16 = const()[name = string("op_1129_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_1130_cast_fp16 = mul(x = var_1107_cast_fp16_3, y = var_1129_to_fp16)[name = string("op_1130_cast_fp16")];
            string var_1132_equation_0 = const()[name = string("op_1132_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_1132_cast_fp16 = einsum(equation = var_1132_equation_0, values = (var_1112_cast_fp16_3, var_1130_cast_fp16))[name = string("op_1132_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_1133_cast_fp16 = add(x = var_1120_cast_fp16, y = var_1124_cast_fp16)[name = string("op_1133_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_1134_cast_fp16 = add(x = var_1128_cast_fp16, y = var_1132_cast_fp16)[name = string("op_1134_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_1135_cast_fp16 = add(x = var_1133_cast_fp16, y = var_1134_cast_fp16)[name = string("op_1135_cast_fp16")];
            fp16 _inversed_1137_y_0_to_fp16 = const()[name = string("_inversed_1137_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 1]> _inversed_1137_cast_fp16 = mul(x = var_1135_cast_fp16, y = _inversed_1137_y_0_to_fp16)[name = string("_inversed_1137_cast_fp16")];
            string scores_15_equation_0 = const()[name = string("scores_15_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 1]> scores_15_cast_fp16 = einsum(equation = scores_15_equation_0, values = (var_873_cast_fp16_1, var_863_cast_fp16_7))[name = string("scores_15_cast_fp16")];
            fp16 var_1140_to_fp16 = const()[name = string("op_1140_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 1]> var_1141_cast_fp16 = mul(x = scores_15_cast_fp16, y = var_1140_to_fp16)[name = string("op_1141_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_1142_cast_fp16 = add(x = var_1141_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_1142_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_1143_cast_fp16 = softmax(axis = var_841, x = var_1142_cast_fp16)[name = string("op_1143_cast_fp16")];
            tensor<int32, [4]> tile_17 = const()[name = string("tile_17"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_1144_axis_0 = const()[name = string("op_1144_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 1]> var_1144_cast_fp16_0, tensor<fp16, [1, 512, 1, 1]> var_1144_cast_fp16_1, tensor<fp16, [1, 512, 1, 1]> var_1144_cast_fp16_2, tensor<fp16, [1, 512, 1, 1]> var_1144_cast_fp16_3 = split(axis = var_1144_axis_0, split_sizes = tile_17, x = var_1143_cast_fp16)[name = string("op_1144_cast_fp16")];
            tensor<int32, [4]> tile_18 = const()[name = string("tile_18"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_1149_axis_0 = const()[name = string("op_1149_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_1149_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_1149_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_1149_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_1149_cast_fp16_3 = split(axis = var_1149_axis_0, split_sizes = tile_18, x = var_876_cast_fp16_1)[name = string("op_1149_cast_fp16")];
            fp16 var_1154_to_fp16 = const()[name = string("op_1154_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_1155_cast_fp16 = mul(x = var_1144_cast_fp16_0, y = var_1154_to_fp16)[name = string("op_1155_cast_fp16")];
            string var_1157_equation_0 = const()[name = string("op_1157_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_1157_cast_fp16 = einsum(equation = var_1157_equation_0, values = (var_1149_cast_fp16_0, var_1155_cast_fp16))[name = string("op_1157_cast_fp16")];
            fp16 var_1158_to_fp16 = const()[name = string("op_1158_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_1159_cast_fp16 = mul(x = var_1144_cast_fp16_1, y = var_1158_to_fp16)[name = string("op_1159_cast_fp16")];
            string var_1161_equation_0 = const()[name = string("op_1161_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_1161_cast_fp16 = einsum(equation = var_1161_equation_0, values = (var_1149_cast_fp16_1, var_1159_cast_fp16))[name = string("op_1161_cast_fp16")];
            fp16 var_1162_to_fp16 = const()[name = string("op_1162_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_1163_cast_fp16 = mul(x = var_1144_cast_fp16_2, y = var_1162_to_fp16)[name = string("op_1163_cast_fp16")];
            string var_1165_equation_0 = const()[name = string("op_1165_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_1165_cast_fp16 = einsum(equation = var_1165_equation_0, values = (var_1149_cast_fp16_2, var_1163_cast_fp16))[name = string("op_1165_cast_fp16")];
            fp16 var_1166_to_fp16 = const()[name = string("op_1166_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_1167_cast_fp16 = mul(x = var_1144_cast_fp16_3, y = var_1166_to_fp16)[name = string("op_1167_cast_fp16")];
            string var_1169_equation_0 = const()[name = string("op_1169_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_1169_cast_fp16 = einsum(equation = var_1169_equation_0, values = (var_1149_cast_fp16_3, var_1167_cast_fp16))[name = string("op_1169_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_1170_cast_fp16 = add(x = var_1157_cast_fp16, y = var_1161_cast_fp16)[name = string("op_1170_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_1171_cast_fp16 = add(x = var_1165_cast_fp16, y = var_1169_cast_fp16)[name = string("op_1171_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_1172_cast_fp16 = add(x = var_1170_cast_fp16, y = var_1171_cast_fp16)[name = string("op_1172_cast_fp16")];
            fp16 _inversed_1174_y_0_to_fp16 = const()[name = string("_inversed_1174_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 1]> _inversed_1174_cast_fp16 = mul(x = var_1172_cast_fp16, y = _inversed_1174_y_0_to_fp16)[name = string("_inversed_1174_cast_fp16")];
            bool var_1176_interleave_0 = const()[name = string("op_1176_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 2048, 1, 1]> var_1176_cast_fp16 = concat(axis = var_841, interleave = var_1176_interleave_0, values = (_inversed_915_cast_fp16, _inversed_952_cast_fp16, _inversed_989_cast_fp16, _inversed_1026_cast_fp16, _inversed_1063_cast_fp16, _inversed_1100_cast_fp16, _inversed_1137_cast_fp16, _inversed_1174_cast_fp16))[name = string("op_1176_cast_fp16")];
            tensor<int32, [4]> var_1177 = const()[name = string("op_1177"), val = tensor<int32, [4]>([2, 4, 256, 1])];
            tensor<fp16, [2, 4, 256, 1]> var_1178_cast_fp16 = reshape(shape = var_1177, x = var_1176_cast_fp16)[name = string("op_1178_cast_fp16")];
            tensor<int32, [4]> transpose_1_perm_0 = const()[name = string("transpose_1_perm_0"), val = tensor<int32, [4]>([3, 0, 1, 2])];
            tensor<int32, [2]> var_1196 = const()[name = string("op_1196"), val = tensor<int32, [2]>([1, 2048])];
            tensor<fp16, [1, 2, 4, 256]> transpose_1_cast_fp16 = transpose(perm = transpose_1_perm_0, x = var_1178_cast_fp16)[name = string("transpose_112")];
            tensor<fp16, [1, 2048]> attention_output_3_cast_fp16 = reshape(shape = var_1196, x = transpose_1_cast_fp16)[name = string("attention_output_3_cast_fp16")];
            tensor<fp16, [1, 2048]> var_1198_cast_fp16 = sigmoid(x = gate_3_cast_fp16)[name = string("op_1198_cast_fp16")];
            tensor<fp16, [1, 2048]> input_45_cast_fp16 = mul(x = attention_output_3_cast_fp16, y = var_1198_cast_fp16)[name = string("input_45_cast_fp16")];
            tensor<fp16, [1024, 2048]> completions_0_o_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 2048]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(53090624))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(54663552))))[name = string("completions_0_o_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 1024]> linear_27_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = completions_0_o_proj_weight_promoted_to_fp16_palettized, x = input_45_cast_fp16)[name = string("linear_27_cast_fp16")];
            tensor<fp16, [1, 1024]> value_57_cast_fp16 = add(x = value_43_cast_fp16, y = linear_27_cast_fp16)[name = string("value_57_cast_fp16")];
            tensor<fp16, [1, 1024]> var_1203_cast_fp16 = mul(x = value_57_cast_fp16, y = value_57_cast_fp16)[name = string("op_1203_cast_fp16")];
            tensor<int32, [1]> variance_25_axes_0 = const()[name = string("variance_25_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_25_keep_dims_0 = const()[name = string("variance_25_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 1]> variance_25_cast_fp16 = reduce_mean(axes = variance_25_axes_0, keep_dims = variance_25_keep_dims_0, x = var_1203_cast_fp16)[name = string("variance_25_cast_fp16")];
            fp16 var_1206_to_fp16 = const()[name = string("op_1206_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 1]> var_1207_cast_fp16 = add(x = variance_25_cast_fp16, y = var_1206_to_fp16)[name = string("op_1207_cast_fp16")];
            fp32 var_1208_epsilon_0 = const()[name = string("op_1208_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 1]> var_1208_cast_fp16 = rsqrt(epsilon = var_1208_epsilon_0, x = var_1207_cast_fp16)[name = string("op_1208_cast_fp16")];
            tensor<fp16, [1, 1024]> var_1209_cast_fp16 = mul(x = value_57_cast_fp16, y = var_1208_cast_fp16)[name = string("op_1209_cast_fp16")];
            tensor<fp16, [1024]> var_1211_to_fp16 = const()[name = string("op_1211_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(54671808)))];
            tensor<fp16, [1, 1024]> input_47_cast_fp16 = mul(x = var_1209_cast_fp16, y = var_1211_to_fp16)[name = string("input_47_cast_fp16")];
            tensor<fp16, [3584, 1024]> completions_0_gate_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(54673920))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(57426496))))[name = string("completions_0_gate_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 3584]> linear_28_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = completions_0_gate_proj_weight_promoted_to_fp16_palettized, x = input_47_cast_fp16)[name = string("linear_28_cast_fp16")];
            tensor<fp16, [1, 3584]> var_1215_cast_fp16 = silu(x = linear_28_cast_fp16)[name = string("op_1215_cast_fp16")];
            tensor<fp16, [3584, 1024]> completions_0_up_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(57455232))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(60207808))))[name = string("completions_0_up_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 3584]> linear_29_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = completions_0_up_proj_weight_promoted_to_fp16_palettized, x = input_47_cast_fp16)[name = string("linear_29_cast_fp16")];
            tensor<fp16, [1, 3584]> input_51_cast_fp16 = mul(x = var_1215_cast_fp16, y = linear_29_cast_fp16)[name = string("input_51_cast_fp16")];
            tensor<fp16, [1024, 3584]> completions_0_down_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 3584]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(60236544))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(62989120))))[name = string("completions_0_down_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 1024]> linear_30_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = completions_0_down_proj_weight_promoted_to_fp16_palettized, x = input_51_cast_fp16)[name = string("linear_30_cast_fp16")];
            tensor<fp16, [1, 1024]> value_59_cast_fp16 = add(x = value_57_cast_fp16, y = linear_30_cast_fp16)[name = string("value_59_cast_fp16")];
            int32 var_1236 = const()[name = string("op_1236"), val = int32(-1)];
            tensor<fp16, [1, 1024]> var_1251_cast_fp16 = mul(x = value_59_cast_fp16, y = value_59_cast_fp16)[name = string("op_1251_cast_fp16")];
            tensor<int32, [1]> variance_27_axes_0 = const()[name = string("variance_27_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_27_keep_dims_0 = const()[name = string("variance_27_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 1]> variance_27_cast_fp16 = reduce_mean(axes = variance_27_axes_0, keep_dims = variance_27_keep_dims_0, x = var_1251_cast_fp16)[name = string("variance_27_cast_fp16")];
            fp16 var_1254_to_fp16 = const()[name = string("op_1254_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 1]> var_1255_cast_fp16 = add(x = variance_27_cast_fp16, y = var_1254_to_fp16)[name = string("op_1255_cast_fp16")];
            fp32 var_1256_epsilon_0 = const()[name = string("op_1256_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 1]> var_1256_cast_fp16 = rsqrt(epsilon = var_1256_epsilon_0, x = var_1255_cast_fp16)[name = string("op_1256_cast_fp16")];
            tensor<fp16, [1, 1024]> var_1257_cast_fp16 = mul(x = value_59_cast_fp16, y = var_1256_cast_fp16)[name = string("op_1257_cast_fp16")];
            tensor<fp16, [1024]> var_1259_to_fp16 = const()[name = string("op_1259_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(62997376)))];
            tensor<fp16, [1, 1024]> input_53_cast_fp16 = mul(x = var_1257_cast_fp16, y = var_1259_to_fp16)[name = string("input_53_cast_fp16")];
            tensor<fp16, [6144, 1024]> groups_1_0_in_proj_qkv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(62999488))), lut = tensor<fp16, [384, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(67718144))))[name = string("groups_1_0_in_proj_qkv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 6144]> linear_31_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = groups_1_0_in_proj_qkv_weight_promoted_to_fp16_palettized, x = input_53_cast_fp16)[name = string("linear_31_cast_fp16")];
            tensor<int32, [2]> var_1263_perm_0 = const()[name = string("op_1263_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> mixed_7_axes_0 = const()[name = string("mixed_7_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [6144, 1]> var_1263_cast_fp16 = transpose(perm = var_1263_perm_0, x = linear_31_cast_fp16)[name = string("transpose_111")];
            tensor<fp16, [1, 6144, 1]> mixed_7_cast_fp16 = expand_dims(axes = mixed_7_axes_0, x = var_1263_cast_fp16)[name = string("mixed_7_cast_fp16")];
            bool convolution_input_7_interleave_0 = const()[name = string("convolution_input_7_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 6144, 4]> convolution_input_7_cast_fp16 = concat(axis = var_1236, interleave = convolution_input_7_interleave_0, values = (conv4, mixed_7_cast_fp16))[name = string("convolution_input_7_cast_fp16")];
            string input_55_pad_type_0 = const()[name = string("input_55_pad_type_0"), val = string("valid")];
            int32 input_55_groups_0 = const()[name = string("input_55_groups_0"), val = int32(6144)];
            tensor<int32, [1]> input_55_strides_0 = const()[name = string("input_55_strides_0"), val = tensor<int32, [1]>([1])];
            tensor<int32, [2]> input_55_pad_0 = const()[name = string("input_55_pad_0"), val = tensor<int32, [2]>([0, 0])];
            tensor<int32, [1]> input_55_dilations_0 = const()[name = string("input_55_dilations_0"), val = tensor<int32, [1]>([1])];
            tensor<fp16, [6144, 1, 4]> groups_1_0_conv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1, 4]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(67767360))), lut = tensor<fp16, [384, 1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(67785856))))[name = string("groups_1_0_conv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 6144, 1]> input_55_cast_fp16 = conv(dilations = input_55_dilations_0, groups = input_55_groups_0, pad = input_55_pad_0, pad_type = input_55_pad_type_0, strides = input_55_strides_0, weight = groups_1_0_conv_weight_promoted_to_fp16_palettized, x = convolution_input_7_cast_fp16)[name = string("input_55_cast_fp16")];
            tensor<fp16, [1, 6144, 1]> var_1272_cast_fp16 = silu(x = input_55_cast_fp16)[name = string("op_1272_cast_fp16")];
            tensor<int32, [3]> var_1273_perm_0 = const()[name = string("op_1273_perm_0"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [3]> var_1276_begin_0 = const()[name = string("op_1276_begin_0"), val = tensor<int32, [3]>([0, 0, 1])];
            tensor<int32, [3]> var_1276_end_0 = const()[name = string("op_1276_end_0"), val = tensor<int32, [3]>([1, 6144, 4])];
            tensor<bool, [3]> var_1276_end_mask_0 = const()[name = string("op_1276_end_mask_0"), val = tensor<bool, [3]>([true, true, true])];
            tensor<fp16, [1, 6144, 3]> conv4_out = slice_by_index(begin = var_1276_begin_0, end = var_1276_end_0, end_mask = var_1276_end_mask_0, x = convolution_input_7_cast_fp16)[name = string("op_1276_cast_fp16")];
            tensor<int32, [3]> var_1277 = const()[name = string("op_1277"), val = tensor<int32, [3]>([2048, 2048, 2048])];
            int32 var_1278_axis_0 = const()[name = string("op_1278_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 1, 6144]> var_1273_cast_fp16 = transpose(perm = var_1273_perm_0, x = var_1272_cast_fp16)[name = string("transpose_110")];
            tensor<fp16, [1, 1, 2048]> var_1278_cast_fp16_0, tensor<fp16, [1, 1, 2048]> var_1278_cast_fp16_1, tensor<fp16, [1, 1, 2048]> var_1278_cast_fp16_2 = split(axis = var_1278_axis_0, split_sizes = var_1277, x = var_1273_cast_fp16)[name = string("op_1278_cast_fp16")];
            tensor<int32, [4]> var_1282 = const()[name = string("op_1282"), val = tensor<int32, [4]>([1, 1, 16, 128])];
            tensor<fp16, [1, 1, 16, 128]> value_63_cast_fp16 = reshape(shape = var_1282, x = var_1278_cast_fp16_0)[name = string("value_63_cast_fp16")];
            tensor<int32, [4]> var_1284 = const()[name = string("op_1284"), val = tensor<int32, [4]>([1, 1, 16, 128])];
            tensor<fp16, [1, 1, 16, 128]> value_65_cast_fp16 = reshape(shape = var_1284, x = var_1278_cast_fp16_1)[name = string("value_65_cast_fp16")];
            tensor<int32, [4]> var_1286 = const()[name = string("op_1286"), val = tensor<int32, [4]>([1, 1, 16, 128])];
            tensor<fp16, [1, 1, 16, 128]> value_67_cast_fp16 = reshape(shape = var_1286, x = var_1278_cast_fp16_2)[name = string("value_67_cast_fp16")];
            tensor<fp16, [1, 1, 16, 128]> var_1288_cast_fp16 = mul(x = value_63_cast_fp16, y = value_63_cast_fp16)[name = string("op_1288_cast_fp16")];
            tensor<int32, [1]> var_1290_axes_0 = const()[name = string("op_1290_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_1290_keep_dims_0 = const()[name = string("op_1290_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 1, 16, 1]> var_1290_cast_fp16 = reduce_sum(axes = var_1290_axes_0, keep_dims = var_1290_keep_dims_0, x = var_1288_cast_fp16)[name = string("op_1290_cast_fp16")];
            fp16 var_1291_to_fp16 = const()[name = string("op_1291_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 1, 16, 1]> var_1292_cast_fp16 = add(x = var_1290_cast_fp16, y = var_1291_to_fp16)[name = string("op_1292_cast_fp16")];
            fp32 var_1293_epsilon_0 = const()[name = string("op_1293_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 1, 16, 1]> var_1293_cast_fp16 = rsqrt(epsilon = var_1293_epsilon_0, x = var_1292_cast_fp16)[name = string("op_1293_cast_fp16")];
            tensor<fp16, [1, 1, 16, 128]> var_1294_cast_fp16 = mul(x = value_63_cast_fp16, y = var_1293_cast_fp16)[name = string("op_1294_cast_fp16")];
            tensor<int32, [4]> var_1295_perm_0 = const()[name = string("op_1295_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            fp16 _inversed_query_19_y_0_to_fp16 = const()[name = string("_inversed_query_19_y_0_to_fp16"), val = fp16(0x1.6ap-4)];
            tensor<fp16, [1, 16, 1, 128]> var_1295_cast_fp16 = transpose(perm = var_1295_perm_0, x = var_1294_cast_fp16)[name = string("transpose_109")];
            tensor<fp16, [1, 16, 1, 128]> _inversed_query_19_cast_fp16 = mul(x = var_1295_cast_fp16, y = _inversed_query_19_y_0_to_fp16)[name = string("_inversed_query_19_cast_fp16")];
            tensor<fp16, [1, 1, 16, 128]> var_1298_cast_fp16 = mul(x = value_65_cast_fp16, y = value_65_cast_fp16)[name = string("op_1298_cast_fp16")];
            tensor<int32, [1]> var_1300_axes_0 = const()[name = string("op_1300_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_1300_keep_dims_0 = const()[name = string("op_1300_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 1, 16, 1]> var_1300_cast_fp16 = reduce_sum(axes = var_1300_axes_0, keep_dims = var_1300_keep_dims_0, x = var_1298_cast_fp16)[name = string("op_1300_cast_fp16")];
            fp16 var_1301_to_fp16 = const()[name = string("op_1301_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 1, 16, 1]> var_1302_cast_fp16 = add(x = var_1300_cast_fp16, y = var_1301_to_fp16)[name = string("op_1302_cast_fp16")];
            fp32 var_1303_epsilon_0 = const()[name = string("op_1303_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 1, 16, 1]> var_1303_cast_fp16 = rsqrt(epsilon = var_1303_epsilon_0, x = var_1302_cast_fp16)[name = string("op_1303_cast_fp16")];
            tensor<fp16, [1, 1, 16, 128]> var_1304_cast_fp16 = mul(x = value_65_cast_fp16, y = var_1303_cast_fp16)[name = string("op_1304_cast_fp16")];
            tensor<int32, [4]> key_19_perm_0 = const()[name = string("key_19_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<int32, [4]> value_69_perm_0 = const()[name = string("value_69_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<fp16, [16, 1024]> groups_1_0_in_proj_b_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(67835072))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(67847424))))[name = string("groups_1_0_in_proj_b_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16]> linear_32_bias_0_to_fp16 = const()[name = string("linear_32_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(4862656)))];
            tensor<fp16, [1, 16]> linear_32_cast_fp16 = linear(bias = linear_32_bias_0_to_fp16, weight = groups_1_0_in_proj_b_weight_promoted_to_fp16_palettized, x = input_53_cast_fp16)[name = string("linear_32_cast_fp16")];
            tensor<fp16, [1, 16]> var_1309_cast_fp16 = sigmoid(x = linear_32_cast_fp16)[name = string("op_1309_cast_fp16")];
            tensor<int32, [2]> var_1310_perm_0 = const()[name = string("op_1310_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> beta_7_axes_0 = const()[name = string("beta_7_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 1]> var_1310_cast_fp16 = transpose(perm = var_1310_perm_0, x = var_1309_cast_fp16)[name = string("transpose_108")];
            tensor<fp16, [1, 16, 1]> beta_7_cast_fp16 = expand_dims(axes = beta_7_axes_0, x = var_1310_cast_fp16)[name = string("beta_7_cast_fp16")];
            tensor<fp16, [16, 1024]> op_1316_weight_0_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(67847616))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(67859968))))[name = string("op_1316_weight_0_to_fp16_palettized")];
            tensor<fp16, [16]> var_1316_bias_0_to_fp16 = const()[name = string("op_1316_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(67860160)))];
            tensor<fp16, [1, 16]> var_1316_cast_fp16 = linear(bias = var_1316_bias_0_to_fp16, weight = op_1316_weight_0_to_fp16_palettized, x = input_53_cast_fp16)[name = string("op_1316_cast_fp16")];
            tensor<fp16, [1, 16]> var_1317_cast_fp16 = softplus(x = var_1316_cast_fp16)[name = string("op_1317_cast_fp16")];
            tensor<fp16, [16]> var_1313_promoted_to_fp16 = const()[name = string("op_1313_promoted_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(67860288)))];
            tensor<fp16, [1, 16]> var_1318_cast_fp16 = mul(x = var_1313_promoted_to_fp16, y = var_1317_cast_fp16)[name = string("op_1318_cast_fp16")];
            tensor<int32, [2]> var_1319_perm_0 = const()[name = string("op_1319_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> decay_7_axes_0 = const()[name = string("decay_7_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 1]> var_1319_cast_fp16 = transpose(perm = var_1319_perm_0, x = var_1318_cast_fp16)[name = string("transpose_107")];
            tensor<fp16, [1, 16, 1]> decay_7_cast_fp16 = expand_dims(axes = decay_7_axes_0, x = var_1319_cast_fp16)[name = string("decay_7_cast_fp16")];
            tensor<fp16, [2048, 1024]> groups_1_0_in_proj_z_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [2048, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(67860416))), lut = tensor<fp16, [128, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(69433344))))[name = string("groups_1_0_in_proj_z_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 2048]> linear_34_cast_fp16 = linear(bias = linear_3_bias_0_to_fp16, weight = groups_1_0_in_proj_z_weight_promoted_to_fp16_palettized, x = input_53_cast_fp16)[name = string("linear_34_cast_fp16")];
            tensor<int32, [3]> var_1327_begin_0 = const()[name = string("op_1327_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_1327_end_0 = const()[name = string("op_1327_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_1327_end_mask_0 = const()[name = string("op_1327_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1327_squeeze_mask_0 = const()[name = string("op_1327_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1327_cast_fp16 = slice_by_index(begin = var_1327_begin_0, end = var_1327_end_0, end_mask = var_1327_end_mask_0, squeeze_mask = var_1327_squeeze_mask_0, x = decay_7_cast_fp16)[name = string("op_1327_cast_fp16")];
            tensor<fp16, [1, 16]> var_1328_cast_fp16 = exp(x = var_1327_cast_fp16)[name = string("op_1328_cast_fp16")];
            tensor<int32, [1]> var_1329_axes_0 = const()[name = string("op_1329_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1329_cast_fp16 = expand_dims(axes = var_1329_axes_0, x = var_1328_cast_fp16)[name = string("op_1329_cast_fp16")];
            tensor<int32, [1]> var_1330_axes_0 = const()[name = string("op_1330_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_1330_cast_fp16 = expand_dims(axes = var_1330_axes_0, x = var_1329_cast_fp16)[name = string("op_1330_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_13_cast_fp16 = mul(x = rec4, y = var_1330_cast_fp16)[name = string("state_13_cast_fp16")];
            tensor<int32, [4]> key_token_7_begin_0 = const()[name = string("key_token_7_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> key_token_7_end_0 = const()[name = string("key_token_7_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> key_token_7_end_mask_0 = const()[name = string("key_token_7_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_7_squeeze_mask_0 = const()[name = string("key_token_7_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 1, 128]> key_19_cast_fp16 = transpose(perm = key_19_perm_0, x = var_1304_cast_fp16)[name = string("transpose_106")];
            tensor<fp16, [1, 16, 128]> key_token_7_cast_fp16 = slice_by_index(begin = key_token_7_begin_0, end = key_token_7_end_0, end_mask = key_token_7_end_mask_0, squeeze_mask = key_token_7_squeeze_mask_0, x = key_19_cast_fp16)[name = string("key_token_7_cast_fp16")];
            tensor<int32, [4]> value_token_7_begin_0 = const()[name = string("value_token_7_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> value_token_7_end_0 = const()[name = string("value_token_7_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> value_token_7_end_mask_0 = const()[name = string("value_token_7_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_7_squeeze_mask_0 = const()[name = string("value_token_7_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 1, 128]> value_69_cast_fp16 = transpose(perm = value_69_perm_0, x = value_67_cast_fp16)[name = string("transpose_105")];
            tensor<fp16, [1, 16, 128]> value_token_7_cast_fp16 = slice_by_index(begin = value_token_7_begin_0, end = value_token_7_end_0, end_mask = value_token_7_end_mask_0, squeeze_mask = value_token_7_squeeze_mask_0, x = value_69_cast_fp16)[name = string("value_token_7_cast_fp16")];
            tensor<int32, [1]> var_1338_axes_0 = const()[name = string("op_1338_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1338_cast_fp16 = expand_dims(axes = var_1338_axes_0, x = key_token_7_cast_fp16)[name = string("op_1338_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1339_cast_fp16 = mul(x = state_13_cast_fp16, y = var_1338_cast_fp16)[name = string("op_1339_cast_fp16")];
            tensor<int32, [1]> memory_7_axes_0 = const()[name = string("memory_7_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_7_keep_dims_0 = const()[name = string("memory_7_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_7_cast_fp16 = reduce_sum(axes = memory_7_axes_0, keep_dims = memory_7_keep_dims_0, x = var_1339_cast_fp16)[name = string("memory_7_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_1342_cast_fp16 = sub(x = value_token_7_cast_fp16, y = memory_7_cast_fp16)[name = string("op_1342_cast_fp16")];
            tensor<int32, [3]> var_1345_begin_0 = const()[name = string("op_1345_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_1345_end_0 = const()[name = string("op_1345_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_1345_end_mask_0 = const()[name = string("op_1345_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1345_squeeze_mask_0 = const()[name = string("op_1345_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1345_cast_fp16 = slice_by_index(begin = var_1345_begin_0, end = var_1345_end_0, end_mask = var_1345_end_mask_0, squeeze_mask = var_1345_squeeze_mask_0, x = beta_7_cast_fp16)[name = string("op_1345_cast_fp16")];
            tensor<int32, [1]> var_1346_axes_0 = const()[name = string("op_1346_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1346_cast_fp16 = expand_dims(axes = var_1346_axes_0, x = var_1345_cast_fp16)[name = string("op_1346_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_7_cast_fp16 = mul(x = var_1342_cast_fp16, y = var_1346_cast_fp16)[name = string("delta_7_cast_fp16")];
            tensor<int32, [1]> var_1349_axes_0 = const()[name = string("op_1349_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_1349_cast_fp16 = expand_dims(axes = var_1349_axes_0, x = delta_7_cast_fp16)[name = string("op_1349_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1350_cast_fp16 = mul(x = var_1338_cast_fp16, y = var_1349_cast_fp16)[name = string("op_1350_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> rec4_out = add(x = state_13_cast_fp16, y = var_1350_cast_fp16)[name = string("state_15_cast_fp16")];
            tensor<int32, [4]> var_1354_begin_0 = const()[name = string("op_1354_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> var_1354_end_0 = const()[name = string("op_1354_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> var_1354_end_mask_0 = const()[name = string("op_1354_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_1354_squeeze_mask_0 = const()[name = string("op_1354_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_1354_cast_fp16 = slice_by_index(begin = var_1354_begin_0, end = var_1354_end_0, end_mask = var_1354_end_mask_0, squeeze_mask = var_1354_squeeze_mask_0, x = _inversed_query_19_cast_fp16)[name = string("op_1354_cast_fp16")];
            tensor<int32, [1]> var_1355_axes_0 = const()[name = string("op_1355_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1355_cast_fp16 = expand_dims(axes = var_1355_axes_0, x = var_1354_cast_fp16)[name = string("op_1355_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1356_cast_fp16 = mul(x = rec4_out, y = var_1355_cast_fp16)[name = string("op_1356_cast_fp16")];
            tensor<int32, [1]> var_1358_axes_0 = const()[name = string("op_1358_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_1358_keep_dims_0 = const()[name = string("op_1358_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_1358_cast_fp16 = reduce_sum(axes = var_1358_axes_0, keep_dims = var_1358_keep_dims_0, x = var_1356_cast_fp16)[name = string("op_1358_cast_fp16")];
            tensor<int32, [1]> attention_31_axes_0 = const()[name = string("attention_31_axes_0"), val = tensor<int32, [1]>([1])];
            tensor<fp16, [1, 1, 16, 128]> attention_31_cast_fp16 = expand_dims(axes = attention_31_axes_0, x = var_1358_cast_fp16)[name = string("attention_31_cast_fp16")];
            tensor<int32, [2]> var_1361 = const()[name = string("op_1361"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [16, 128]> attention_33_cast_fp16 = reshape(shape = var_1361, x = attention_31_cast_fp16)[name = string("attention_33_cast_fp16")];
            tensor<int32, [2]> var_1363 = const()[name = string("op_1363"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [16, 128]> input_57_cast_fp16 = reshape(shape = var_1363, x = linear_34_cast_fp16)[name = string("input_57_cast_fp16")];
            tensor<fp16, [16, 128]> var_1365_cast_fp16 = mul(x = attention_33_cast_fp16, y = attention_33_cast_fp16)[name = string("op_1365_cast_fp16")];
            tensor<int32, [1]> variance_29_axes_0 = const()[name = string("variance_29_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_29_keep_dims_0 = const()[name = string("variance_29_keep_dims_0"), val = bool(true)];
            tensor<fp16, [16, 1]> variance_29_cast_fp16 = reduce_mean(axes = variance_29_axes_0, keep_dims = variance_29_keep_dims_0, x = var_1365_cast_fp16)[name = string("variance_29_cast_fp16")];
            fp16 var_1368_to_fp16 = const()[name = string("op_1368_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [16, 1]> var_1369_cast_fp16 = add(x = variance_29_cast_fp16, y = var_1368_to_fp16)[name = string("op_1369_cast_fp16")];
            fp32 var_1370_epsilon_0 = const()[name = string("op_1370_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [16, 1]> var_1370_cast_fp16 = rsqrt(epsilon = var_1370_epsilon_0, x = var_1369_cast_fp16)[name = string("op_1370_cast_fp16")];
            tensor<fp16, [16, 128]> attention_35_cast_fp16 = mul(x = attention_33_cast_fp16, y = var_1370_cast_fp16)[name = string("attention_35_cast_fp16")];
            tensor<fp16, [128]> groups_1_0_gated_norm_promoted_to_fp16 = const()[name = string("groups_1_0_gated_norm_promoted_to_fp16"), val = tensor<fp16, [128]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(69449792)))];
            tensor<fp16, [16, 128]> attention_37_cast_fp16 = mul(x = attention_35_cast_fp16, y = groups_1_0_gated_norm_promoted_to_fp16)[name = string("attention_37_cast_fp16")];
            tensor<fp16, [16, 128]> var_1373_cast_fp16 = silu(x = input_57_cast_fp16)[name = string("op_1373_cast_fp16")];
            tensor<fp16, [16, 128]> attention_39_cast_fp16 = mul(x = attention_37_cast_fp16, y = var_1373_cast_fp16)[name = string("attention_39_cast_fp16")];
            tensor<int32, [2]> var_1375 = const()[name = string("op_1375"), val = tensor<int32, [2]>([1, 2048])];
            tensor<fp16, [1, 2048]> input_59_cast_fp16 = reshape(shape = var_1375, x = attention_39_cast_fp16)[name = string("input_59_cast_fp16")];
            tensor<fp16, [1024, 2048]> groups_1_0_out_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 2048]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(69450112))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(71023040))))[name = string("groups_1_0_out_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 1024]> linear_35_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = groups_1_0_out_proj_weight_promoted_to_fp16_palettized, x = input_59_cast_fp16)[name = string("linear_35_cast_fp16")];
            tensor<fp16, [1, 1024]> value_71_cast_fp16 = add(x = value_59_cast_fp16, y = linear_35_cast_fp16)[name = string("value_71_cast_fp16")];
            tensor<fp16, [1, 1024]> var_1380_cast_fp16 = mul(x = value_71_cast_fp16, y = value_71_cast_fp16)[name = string("op_1380_cast_fp16")];
            tensor<int32, [1]> variance_31_axes_0 = const()[name = string("variance_31_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_31_keep_dims_0 = const()[name = string("variance_31_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 1]> variance_31_cast_fp16 = reduce_mean(axes = variance_31_axes_0, keep_dims = variance_31_keep_dims_0, x = var_1380_cast_fp16)[name = string("variance_31_cast_fp16")];
            fp16 var_1383_to_fp16 = const()[name = string("op_1383_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 1]> var_1384_cast_fp16 = add(x = variance_31_cast_fp16, y = var_1383_to_fp16)[name = string("op_1384_cast_fp16")];
            fp32 var_1385_epsilon_0 = const()[name = string("op_1385_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 1]> var_1385_cast_fp16 = rsqrt(epsilon = var_1385_epsilon_0, x = var_1384_cast_fp16)[name = string("op_1385_cast_fp16")];
            tensor<fp16, [1, 1024]> var_1386_cast_fp16 = mul(x = value_71_cast_fp16, y = var_1385_cast_fp16)[name = string("op_1386_cast_fp16")];
            tensor<fp16, [1024]> var_1388_to_fp16 = const()[name = string("op_1388_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(71031296)))];
            tensor<fp16, [1, 1024]> input_61_cast_fp16 = mul(x = var_1386_cast_fp16, y = var_1388_to_fp16)[name = string("input_61_cast_fp16")];
            tensor<fp16, [3584, 1024]> groups_1_0_gate_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(71033408))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(73785984))))[name = string("groups_1_0_gate_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 3584]> linear_36_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = groups_1_0_gate_proj_weight_promoted_to_fp16_palettized, x = input_61_cast_fp16)[name = string("linear_36_cast_fp16")];
            tensor<fp16, [1, 3584]> var_1392_cast_fp16 = silu(x = linear_36_cast_fp16)[name = string("op_1392_cast_fp16")];
            tensor<fp16, [3584, 1024]> groups_1_0_up_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(73814720))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(76567296))))[name = string("groups_1_0_up_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 3584]> linear_37_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = groups_1_0_up_proj_weight_promoted_to_fp16_palettized, x = input_61_cast_fp16)[name = string("linear_37_cast_fp16")];
            tensor<fp16, [1, 3584]> input_65_cast_fp16 = mul(x = var_1392_cast_fp16, y = linear_37_cast_fp16)[name = string("input_65_cast_fp16")];
            tensor<fp16, [1024, 3584]> groups_1_0_down_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 3584]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(76596032))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(79348608))))[name = string("groups_1_0_down_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 1024]> linear_38_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = groups_1_0_down_proj_weight_promoted_to_fp16_palettized, x = input_65_cast_fp16)[name = string("linear_38_cast_fp16")];
            tensor<fp16, [1, 1024]> value_73_cast_fp16 = add(x = value_71_cast_fp16, y = linear_38_cast_fp16)[name = string("value_73_cast_fp16")];
            int32 var_1417 = const()[name = string("op_1417"), val = int32(-1)];
            tensor<fp16, [1, 1024]> var_1432_cast_fp16 = mul(x = value_73_cast_fp16, y = value_73_cast_fp16)[name = string("op_1432_cast_fp16")];
            tensor<int32, [1]> variance_33_axes_0 = const()[name = string("variance_33_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_33_keep_dims_0 = const()[name = string("variance_33_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 1]> variance_33_cast_fp16 = reduce_mean(axes = variance_33_axes_0, keep_dims = variance_33_keep_dims_0, x = var_1432_cast_fp16)[name = string("variance_33_cast_fp16")];
            fp16 var_1435_to_fp16 = const()[name = string("op_1435_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 1]> var_1436_cast_fp16 = add(x = variance_33_cast_fp16, y = var_1435_to_fp16)[name = string("op_1436_cast_fp16")];
            fp32 var_1437_epsilon_0 = const()[name = string("op_1437_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 1]> var_1437_cast_fp16 = rsqrt(epsilon = var_1437_epsilon_0, x = var_1436_cast_fp16)[name = string("op_1437_cast_fp16")];
            tensor<fp16, [1, 1024]> var_1438_cast_fp16 = mul(x = value_73_cast_fp16, y = var_1437_cast_fp16)[name = string("op_1438_cast_fp16")];
            tensor<fp16, [1024]> var_1440_to_fp16 = const()[name = string("op_1440_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(79356864)))];
            tensor<fp16, [1, 1024]> input_67_cast_fp16 = mul(x = var_1438_cast_fp16, y = var_1440_to_fp16)[name = string("input_67_cast_fp16")];
            tensor<fp16, [6144, 1024]> groups_1_1_in_proj_qkv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(79358976))), lut = tensor<fp16, [384, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(84077632))))[name = string("groups_1_1_in_proj_qkv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 6144]> linear_39_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = groups_1_1_in_proj_qkv_weight_promoted_to_fp16_palettized, x = input_67_cast_fp16)[name = string("linear_39_cast_fp16")];
            tensor<int32, [2]> var_1444_perm_0 = const()[name = string("op_1444_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> mixed_9_axes_0 = const()[name = string("mixed_9_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [6144, 1]> var_1444_cast_fp16 = transpose(perm = var_1444_perm_0, x = linear_39_cast_fp16)[name = string("transpose_104")];
            tensor<fp16, [1, 6144, 1]> mixed_9_cast_fp16 = expand_dims(axes = mixed_9_axes_0, x = var_1444_cast_fp16)[name = string("mixed_9_cast_fp16")];
            bool convolution_input_9_interleave_0 = const()[name = string("convolution_input_9_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 6144, 4]> convolution_input_9_cast_fp16 = concat(axis = var_1417, interleave = convolution_input_9_interleave_0, values = (conv5, mixed_9_cast_fp16))[name = string("convolution_input_9_cast_fp16")];
            string input_69_pad_type_0 = const()[name = string("input_69_pad_type_0"), val = string("valid")];
            int32 input_69_groups_0 = const()[name = string("input_69_groups_0"), val = int32(6144)];
            tensor<int32, [1]> input_69_strides_0 = const()[name = string("input_69_strides_0"), val = tensor<int32, [1]>([1])];
            tensor<int32, [2]> input_69_pad_0 = const()[name = string("input_69_pad_0"), val = tensor<int32, [2]>([0, 0])];
            tensor<int32, [1]> input_69_dilations_0 = const()[name = string("input_69_dilations_0"), val = tensor<int32, [1]>([1])];
            tensor<fp16, [6144, 1, 4]> groups_1_1_conv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1, 4]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(84126848))), lut = tensor<fp16, [384, 1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(84145344))))[name = string("groups_1_1_conv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 6144, 1]> input_69_cast_fp16 = conv(dilations = input_69_dilations_0, groups = input_69_groups_0, pad = input_69_pad_0, pad_type = input_69_pad_type_0, strides = input_69_strides_0, weight = groups_1_1_conv_weight_promoted_to_fp16_palettized, x = convolution_input_9_cast_fp16)[name = string("input_69_cast_fp16")];
            tensor<fp16, [1, 6144, 1]> var_1453_cast_fp16 = silu(x = input_69_cast_fp16)[name = string("op_1453_cast_fp16")];
            tensor<int32, [3]> var_1454_perm_0 = const()[name = string("op_1454_perm_0"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [3]> var_1457_begin_0 = const()[name = string("op_1457_begin_0"), val = tensor<int32, [3]>([0, 0, 1])];
            tensor<int32, [3]> var_1457_end_0 = const()[name = string("op_1457_end_0"), val = tensor<int32, [3]>([1, 6144, 4])];
            tensor<bool, [3]> var_1457_end_mask_0 = const()[name = string("op_1457_end_mask_0"), val = tensor<bool, [3]>([true, true, true])];
            tensor<fp16, [1, 6144, 3]> conv5_out = slice_by_index(begin = var_1457_begin_0, end = var_1457_end_0, end_mask = var_1457_end_mask_0, x = convolution_input_9_cast_fp16)[name = string("op_1457_cast_fp16")];
            tensor<int32, [3]> var_1458 = const()[name = string("op_1458"), val = tensor<int32, [3]>([2048, 2048, 2048])];
            int32 var_1459_axis_0 = const()[name = string("op_1459_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 1, 6144]> var_1454_cast_fp16 = transpose(perm = var_1454_perm_0, x = var_1453_cast_fp16)[name = string("transpose_103")];
            tensor<fp16, [1, 1, 2048]> var_1459_cast_fp16_0, tensor<fp16, [1, 1, 2048]> var_1459_cast_fp16_1, tensor<fp16, [1, 1, 2048]> var_1459_cast_fp16_2 = split(axis = var_1459_axis_0, split_sizes = var_1458, x = var_1454_cast_fp16)[name = string("op_1459_cast_fp16")];
            tensor<int32, [4]> var_1463 = const()[name = string("op_1463"), val = tensor<int32, [4]>([1, 1, 16, 128])];
            tensor<fp16, [1, 1, 16, 128]> value_77_cast_fp16 = reshape(shape = var_1463, x = var_1459_cast_fp16_0)[name = string("value_77_cast_fp16")];
            tensor<int32, [4]> var_1465 = const()[name = string("op_1465"), val = tensor<int32, [4]>([1, 1, 16, 128])];
            tensor<fp16, [1, 1, 16, 128]> value_79_cast_fp16 = reshape(shape = var_1465, x = var_1459_cast_fp16_1)[name = string("value_79_cast_fp16")];
            tensor<int32, [4]> var_1467 = const()[name = string("op_1467"), val = tensor<int32, [4]>([1, 1, 16, 128])];
            tensor<fp16, [1, 1, 16, 128]> value_81_cast_fp16 = reshape(shape = var_1467, x = var_1459_cast_fp16_2)[name = string("value_81_cast_fp16")];
            tensor<fp16, [1, 1, 16, 128]> var_1469_cast_fp16 = mul(x = value_77_cast_fp16, y = value_77_cast_fp16)[name = string("op_1469_cast_fp16")];
            tensor<int32, [1]> var_1471_axes_0 = const()[name = string("op_1471_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_1471_keep_dims_0 = const()[name = string("op_1471_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 1, 16, 1]> var_1471_cast_fp16 = reduce_sum(axes = var_1471_axes_0, keep_dims = var_1471_keep_dims_0, x = var_1469_cast_fp16)[name = string("op_1471_cast_fp16")];
            fp16 var_1472_to_fp16 = const()[name = string("op_1472_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 1, 16, 1]> var_1473_cast_fp16 = add(x = var_1471_cast_fp16, y = var_1472_to_fp16)[name = string("op_1473_cast_fp16")];
            fp32 var_1474_epsilon_0 = const()[name = string("op_1474_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 1, 16, 1]> var_1474_cast_fp16 = rsqrt(epsilon = var_1474_epsilon_0, x = var_1473_cast_fp16)[name = string("op_1474_cast_fp16")];
            tensor<fp16, [1, 1, 16, 128]> var_1475_cast_fp16 = mul(x = value_77_cast_fp16, y = var_1474_cast_fp16)[name = string("op_1475_cast_fp16")];
            tensor<int32, [4]> var_1476_perm_0 = const()[name = string("op_1476_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            fp16 _inversed_query_23_y_0_to_fp16 = const()[name = string("_inversed_query_23_y_0_to_fp16"), val = fp16(0x1.6ap-4)];
            tensor<fp16, [1, 16, 1, 128]> var_1476_cast_fp16 = transpose(perm = var_1476_perm_0, x = var_1475_cast_fp16)[name = string("transpose_102")];
            tensor<fp16, [1, 16, 1, 128]> _inversed_query_23_cast_fp16 = mul(x = var_1476_cast_fp16, y = _inversed_query_23_y_0_to_fp16)[name = string("_inversed_query_23_cast_fp16")];
            tensor<fp16, [1, 1, 16, 128]> var_1479_cast_fp16 = mul(x = value_79_cast_fp16, y = value_79_cast_fp16)[name = string("op_1479_cast_fp16")];
            tensor<int32, [1]> var_1481_axes_0 = const()[name = string("op_1481_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_1481_keep_dims_0 = const()[name = string("op_1481_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 1, 16, 1]> var_1481_cast_fp16 = reduce_sum(axes = var_1481_axes_0, keep_dims = var_1481_keep_dims_0, x = var_1479_cast_fp16)[name = string("op_1481_cast_fp16")];
            fp16 var_1482_to_fp16 = const()[name = string("op_1482_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 1, 16, 1]> var_1483_cast_fp16 = add(x = var_1481_cast_fp16, y = var_1482_to_fp16)[name = string("op_1483_cast_fp16")];
            fp32 var_1484_epsilon_0 = const()[name = string("op_1484_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 1, 16, 1]> var_1484_cast_fp16 = rsqrt(epsilon = var_1484_epsilon_0, x = var_1483_cast_fp16)[name = string("op_1484_cast_fp16")];
            tensor<fp16, [1, 1, 16, 128]> var_1485_cast_fp16 = mul(x = value_79_cast_fp16, y = var_1484_cast_fp16)[name = string("op_1485_cast_fp16")];
            tensor<int32, [4]> key_23_perm_0 = const()[name = string("key_23_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<int32, [4]> value_83_perm_0 = const()[name = string("value_83_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<fp16, [16, 1024]> groups_1_1_in_proj_b_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(84194560))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(84206912))))[name = string("groups_1_1_in_proj_b_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16]> linear_40_bias_0_to_fp16 = const()[name = string("linear_40_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(4862656)))];
            tensor<fp16, [1, 16]> linear_40_cast_fp16 = linear(bias = linear_40_bias_0_to_fp16, weight = groups_1_1_in_proj_b_weight_promoted_to_fp16_palettized, x = input_67_cast_fp16)[name = string("linear_40_cast_fp16")];
            tensor<fp16, [1, 16]> var_1490_cast_fp16 = sigmoid(x = linear_40_cast_fp16)[name = string("op_1490_cast_fp16")];
            tensor<int32, [2]> var_1491_perm_0 = const()[name = string("op_1491_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> beta_9_axes_0 = const()[name = string("beta_9_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 1]> var_1491_cast_fp16 = transpose(perm = var_1491_perm_0, x = var_1490_cast_fp16)[name = string("transpose_101")];
            tensor<fp16, [1, 16, 1]> beta_9_cast_fp16 = expand_dims(axes = beta_9_axes_0, x = var_1491_cast_fp16)[name = string("beta_9_cast_fp16")];
            tensor<fp16, [16, 1024]> op_1497_weight_0_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(84207104))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(84219456))))[name = string("op_1497_weight_0_to_fp16_palettized")];
            tensor<fp16, [16]> var_1497_bias_0_to_fp16 = const()[name = string("op_1497_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(84219648)))];
            tensor<fp16, [1, 16]> var_1497_cast_fp16 = linear(bias = var_1497_bias_0_to_fp16, weight = op_1497_weight_0_to_fp16_palettized, x = input_67_cast_fp16)[name = string("op_1497_cast_fp16")];
            tensor<fp16, [1, 16]> var_1498_cast_fp16 = softplus(x = var_1497_cast_fp16)[name = string("op_1498_cast_fp16")];
            tensor<fp16, [16]> var_1494_promoted_to_fp16 = const()[name = string("op_1494_promoted_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(84219776)))];
            tensor<fp16, [1, 16]> var_1499_cast_fp16 = mul(x = var_1494_promoted_to_fp16, y = var_1498_cast_fp16)[name = string("op_1499_cast_fp16")];
            tensor<int32, [2]> var_1500_perm_0 = const()[name = string("op_1500_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> decay_9_axes_0 = const()[name = string("decay_9_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 1]> var_1500_cast_fp16 = transpose(perm = var_1500_perm_0, x = var_1499_cast_fp16)[name = string("transpose_100")];
            tensor<fp16, [1, 16, 1]> decay_9_cast_fp16 = expand_dims(axes = decay_9_axes_0, x = var_1500_cast_fp16)[name = string("decay_9_cast_fp16")];
            tensor<fp16, [2048, 1024]> groups_1_1_in_proj_z_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [2048, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(84219904))), lut = tensor<fp16, [128, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(85792832))))[name = string("groups_1_1_in_proj_z_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 2048]> linear_42_cast_fp16 = linear(bias = linear_3_bias_0_to_fp16, weight = groups_1_1_in_proj_z_weight_promoted_to_fp16_palettized, x = input_67_cast_fp16)[name = string("linear_42_cast_fp16")];
            tensor<int32, [3]> var_1508_begin_0 = const()[name = string("op_1508_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_1508_end_0 = const()[name = string("op_1508_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_1508_end_mask_0 = const()[name = string("op_1508_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1508_squeeze_mask_0 = const()[name = string("op_1508_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1508_cast_fp16 = slice_by_index(begin = var_1508_begin_0, end = var_1508_end_0, end_mask = var_1508_end_mask_0, squeeze_mask = var_1508_squeeze_mask_0, x = decay_9_cast_fp16)[name = string("op_1508_cast_fp16")];
            tensor<fp16, [1, 16]> var_1509_cast_fp16 = exp(x = var_1508_cast_fp16)[name = string("op_1509_cast_fp16")];
            tensor<int32, [1]> var_1510_axes_0 = const()[name = string("op_1510_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1510_cast_fp16 = expand_dims(axes = var_1510_axes_0, x = var_1509_cast_fp16)[name = string("op_1510_cast_fp16")];
            tensor<int32, [1]> var_1511_axes_0 = const()[name = string("op_1511_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_1511_cast_fp16 = expand_dims(axes = var_1511_axes_0, x = var_1510_cast_fp16)[name = string("op_1511_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_17_cast_fp16 = mul(x = rec5, y = var_1511_cast_fp16)[name = string("state_17_cast_fp16")];
            tensor<int32, [4]> key_token_9_begin_0 = const()[name = string("key_token_9_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> key_token_9_end_0 = const()[name = string("key_token_9_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> key_token_9_end_mask_0 = const()[name = string("key_token_9_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_9_squeeze_mask_0 = const()[name = string("key_token_9_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 1, 128]> key_23_cast_fp16 = transpose(perm = key_23_perm_0, x = var_1485_cast_fp16)[name = string("transpose_99")];
            tensor<fp16, [1, 16, 128]> key_token_9_cast_fp16 = slice_by_index(begin = key_token_9_begin_0, end = key_token_9_end_0, end_mask = key_token_9_end_mask_0, squeeze_mask = key_token_9_squeeze_mask_0, x = key_23_cast_fp16)[name = string("key_token_9_cast_fp16")];
            tensor<int32, [4]> value_token_9_begin_0 = const()[name = string("value_token_9_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> value_token_9_end_0 = const()[name = string("value_token_9_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> value_token_9_end_mask_0 = const()[name = string("value_token_9_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_9_squeeze_mask_0 = const()[name = string("value_token_9_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 1, 128]> value_83_cast_fp16 = transpose(perm = value_83_perm_0, x = value_81_cast_fp16)[name = string("transpose_98")];
            tensor<fp16, [1, 16, 128]> value_token_9_cast_fp16 = slice_by_index(begin = value_token_9_begin_0, end = value_token_9_end_0, end_mask = value_token_9_end_mask_0, squeeze_mask = value_token_9_squeeze_mask_0, x = value_83_cast_fp16)[name = string("value_token_9_cast_fp16")];
            tensor<int32, [1]> var_1519_axes_0 = const()[name = string("op_1519_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1519_cast_fp16 = expand_dims(axes = var_1519_axes_0, x = key_token_9_cast_fp16)[name = string("op_1519_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1520_cast_fp16 = mul(x = state_17_cast_fp16, y = var_1519_cast_fp16)[name = string("op_1520_cast_fp16")];
            tensor<int32, [1]> memory_9_axes_0 = const()[name = string("memory_9_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_9_keep_dims_0 = const()[name = string("memory_9_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_9_cast_fp16 = reduce_sum(axes = memory_9_axes_0, keep_dims = memory_9_keep_dims_0, x = var_1520_cast_fp16)[name = string("memory_9_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_1523_cast_fp16 = sub(x = value_token_9_cast_fp16, y = memory_9_cast_fp16)[name = string("op_1523_cast_fp16")];
            tensor<int32, [3]> var_1526_begin_0 = const()[name = string("op_1526_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_1526_end_0 = const()[name = string("op_1526_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_1526_end_mask_0 = const()[name = string("op_1526_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1526_squeeze_mask_0 = const()[name = string("op_1526_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1526_cast_fp16 = slice_by_index(begin = var_1526_begin_0, end = var_1526_end_0, end_mask = var_1526_end_mask_0, squeeze_mask = var_1526_squeeze_mask_0, x = beta_9_cast_fp16)[name = string("op_1526_cast_fp16")];
            tensor<int32, [1]> var_1527_axes_0 = const()[name = string("op_1527_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1527_cast_fp16 = expand_dims(axes = var_1527_axes_0, x = var_1526_cast_fp16)[name = string("op_1527_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_9_cast_fp16 = mul(x = var_1523_cast_fp16, y = var_1527_cast_fp16)[name = string("delta_9_cast_fp16")];
            tensor<int32, [1]> var_1530_axes_0 = const()[name = string("op_1530_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_1530_cast_fp16 = expand_dims(axes = var_1530_axes_0, x = delta_9_cast_fp16)[name = string("op_1530_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1531_cast_fp16 = mul(x = var_1519_cast_fp16, y = var_1530_cast_fp16)[name = string("op_1531_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> rec5_out = add(x = state_17_cast_fp16, y = var_1531_cast_fp16)[name = string("state_19_cast_fp16")];
            tensor<int32, [4]> var_1535_begin_0 = const()[name = string("op_1535_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> var_1535_end_0 = const()[name = string("op_1535_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> var_1535_end_mask_0 = const()[name = string("op_1535_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_1535_squeeze_mask_0 = const()[name = string("op_1535_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_1535_cast_fp16 = slice_by_index(begin = var_1535_begin_0, end = var_1535_end_0, end_mask = var_1535_end_mask_0, squeeze_mask = var_1535_squeeze_mask_0, x = _inversed_query_23_cast_fp16)[name = string("op_1535_cast_fp16")];
            tensor<int32, [1]> var_1536_axes_0 = const()[name = string("op_1536_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1536_cast_fp16 = expand_dims(axes = var_1536_axes_0, x = var_1535_cast_fp16)[name = string("op_1536_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1537_cast_fp16 = mul(x = rec5_out, y = var_1536_cast_fp16)[name = string("op_1537_cast_fp16")];
            tensor<int32, [1]> var_1539_axes_0 = const()[name = string("op_1539_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_1539_keep_dims_0 = const()[name = string("op_1539_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_1539_cast_fp16 = reduce_sum(axes = var_1539_axes_0, keep_dims = var_1539_keep_dims_0, x = var_1537_cast_fp16)[name = string("op_1539_cast_fp16")];
            tensor<int32, [1]> attention_41_axes_0 = const()[name = string("attention_41_axes_0"), val = tensor<int32, [1]>([1])];
            tensor<fp16, [1, 1, 16, 128]> attention_41_cast_fp16 = expand_dims(axes = attention_41_axes_0, x = var_1539_cast_fp16)[name = string("attention_41_cast_fp16")];
            tensor<int32, [2]> var_1542 = const()[name = string("op_1542"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [16, 128]> attention_43_cast_fp16 = reshape(shape = var_1542, x = attention_41_cast_fp16)[name = string("attention_43_cast_fp16")];
            tensor<int32, [2]> var_1544 = const()[name = string("op_1544"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [16, 128]> input_71_cast_fp16 = reshape(shape = var_1544, x = linear_42_cast_fp16)[name = string("input_71_cast_fp16")];
            tensor<fp16, [16, 128]> var_1546_cast_fp16 = mul(x = attention_43_cast_fp16, y = attention_43_cast_fp16)[name = string("op_1546_cast_fp16")];
            tensor<int32, [1]> variance_35_axes_0 = const()[name = string("variance_35_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_35_keep_dims_0 = const()[name = string("variance_35_keep_dims_0"), val = bool(true)];
            tensor<fp16, [16, 1]> variance_35_cast_fp16 = reduce_mean(axes = variance_35_axes_0, keep_dims = variance_35_keep_dims_0, x = var_1546_cast_fp16)[name = string("variance_35_cast_fp16")];
            fp16 var_1549_to_fp16 = const()[name = string("op_1549_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [16, 1]> var_1550_cast_fp16 = add(x = variance_35_cast_fp16, y = var_1549_to_fp16)[name = string("op_1550_cast_fp16")];
            fp32 var_1551_epsilon_0 = const()[name = string("op_1551_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [16, 1]> var_1551_cast_fp16 = rsqrt(epsilon = var_1551_epsilon_0, x = var_1550_cast_fp16)[name = string("op_1551_cast_fp16")];
            tensor<fp16, [16, 128]> attention_45_cast_fp16 = mul(x = attention_43_cast_fp16, y = var_1551_cast_fp16)[name = string("attention_45_cast_fp16")];
            tensor<fp16, [128]> groups_1_1_gated_norm_promoted_to_fp16 = const()[name = string("groups_1_1_gated_norm_promoted_to_fp16"), val = tensor<fp16, [128]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(85809280)))];
            tensor<fp16, [16, 128]> attention_47_cast_fp16 = mul(x = attention_45_cast_fp16, y = groups_1_1_gated_norm_promoted_to_fp16)[name = string("attention_47_cast_fp16")];
            tensor<fp16, [16, 128]> var_1554_cast_fp16 = silu(x = input_71_cast_fp16)[name = string("op_1554_cast_fp16")];
            tensor<fp16, [16, 128]> attention_49_cast_fp16 = mul(x = attention_47_cast_fp16, y = var_1554_cast_fp16)[name = string("attention_49_cast_fp16")];
            tensor<int32, [2]> var_1556 = const()[name = string("op_1556"), val = tensor<int32, [2]>([1, 2048])];
            tensor<fp16, [1, 2048]> input_73_cast_fp16 = reshape(shape = var_1556, x = attention_49_cast_fp16)[name = string("input_73_cast_fp16")];
            tensor<fp16, [1024, 2048]> groups_1_1_out_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 2048]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(85809600))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(87382528))))[name = string("groups_1_1_out_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 1024]> linear_43_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = groups_1_1_out_proj_weight_promoted_to_fp16_palettized, x = input_73_cast_fp16)[name = string("linear_43_cast_fp16")];
            tensor<fp16, [1, 1024]> value_85_cast_fp16 = add(x = value_73_cast_fp16, y = linear_43_cast_fp16)[name = string("value_85_cast_fp16")];
            tensor<fp16, [1, 1024]> var_1561_cast_fp16 = mul(x = value_85_cast_fp16, y = value_85_cast_fp16)[name = string("op_1561_cast_fp16")];
            tensor<int32, [1]> variance_37_axes_0 = const()[name = string("variance_37_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_37_keep_dims_0 = const()[name = string("variance_37_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 1]> variance_37_cast_fp16 = reduce_mean(axes = variance_37_axes_0, keep_dims = variance_37_keep_dims_0, x = var_1561_cast_fp16)[name = string("variance_37_cast_fp16")];
            fp16 var_1564_to_fp16 = const()[name = string("op_1564_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 1]> var_1565_cast_fp16 = add(x = variance_37_cast_fp16, y = var_1564_to_fp16)[name = string("op_1565_cast_fp16")];
            fp32 var_1566_epsilon_0 = const()[name = string("op_1566_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 1]> var_1566_cast_fp16 = rsqrt(epsilon = var_1566_epsilon_0, x = var_1565_cast_fp16)[name = string("op_1566_cast_fp16")];
            tensor<fp16, [1, 1024]> var_1567_cast_fp16 = mul(x = value_85_cast_fp16, y = var_1566_cast_fp16)[name = string("op_1567_cast_fp16")];
            tensor<fp16, [1024]> var_1569_to_fp16 = const()[name = string("op_1569_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(87390784)))];
            tensor<fp16, [1, 1024]> input_75_cast_fp16 = mul(x = var_1567_cast_fp16, y = var_1569_to_fp16)[name = string("input_75_cast_fp16")];
            tensor<fp16, [3584, 1024]> groups_1_1_gate_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(87392896))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(90145472))))[name = string("groups_1_1_gate_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 3584]> linear_44_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = groups_1_1_gate_proj_weight_promoted_to_fp16_palettized, x = input_75_cast_fp16)[name = string("linear_44_cast_fp16")];
            tensor<fp16, [1, 3584]> var_1573_cast_fp16 = silu(x = linear_44_cast_fp16)[name = string("op_1573_cast_fp16")];
            tensor<fp16, [3584, 1024]> groups_1_1_up_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(90174208))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(92926784))))[name = string("groups_1_1_up_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 3584]> linear_45_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = groups_1_1_up_proj_weight_promoted_to_fp16_palettized, x = input_75_cast_fp16)[name = string("linear_45_cast_fp16")];
            tensor<fp16, [1, 3584]> input_79_cast_fp16 = mul(x = var_1573_cast_fp16, y = linear_45_cast_fp16)[name = string("input_79_cast_fp16")];
            tensor<fp16, [1024, 3584]> groups_1_1_down_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 3584]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(92955520))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(95708096))))[name = string("groups_1_1_down_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 1024]> linear_46_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = groups_1_1_down_proj_weight_promoted_to_fp16_palettized, x = input_79_cast_fp16)[name = string("linear_46_cast_fp16")];
            tensor<fp16, [1, 1024]> value_87_cast_fp16 = add(x = value_85_cast_fp16, y = linear_46_cast_fp16)[name = string("value_87_cast_fp16")];
            int32 var_1598 = const()[name = string("op_1598"), val = int32(-1)];
            tensor<fp16, [1, 1024]> var_1613_cast_fp16 = mul(x = value_87_cast_fp16, y = value_87_cast_fp16)[name = string("op_1613_cast_fp16")];
            tensor<int32, [1]> variance_39_axes_0 = const()[name = string("variance_39_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_39_keep_dims_0 = const()[name = string("variance_39_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 1]> variance_39_cast_fp16 = reduce_mean(axes = variance_39_axes_0, keep_dims = variance_39_keep_dims_0, x = var_1613_cast_fp16)[name = string("variance_39_cast_fp16")];
            fp16 var_1616_to_fp16 = const()[name = string("op_1616_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 1]> var_1617_cast_fp16 = add(x = variance_39_cast_fp16, y = var_1616_to_fp16)[name = string("op_1617_cast_fp16")];
            fp32 var_1618_epsilon_0 = const()[name = string("op_1618_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 1]> var_1618_cast_fp16 = rsqrt(epsilon = var_1618_epsilon_0, x = var_1617_cast_fp16)[name = string("op_1618_cast_fp16")];
            tensor<fp16, [1, 1024]> var_1619_cast_fp16 = mul(x = value_87_cast_fp16, y = var_1618_cast_fp16)[name = string("op_1619_cast_fp16")];
            tensor<fp16, [1024]> var_1621_to_fp16 = const()[name = string("op_1621_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(95716352)))];
            tensor<fp16, [1, 1024]> input_81_cast_fp16 = mul(x = var_1619_cast_fp16, y = var_1621_to_fp16)[name = string("input_81_cast_fp16")];
            tensor<fp16, [6144, 1024]> groups_1_2_in_proj_qkv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(95718464))), lut = tensor<fp16, [384, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(100437120))))[name = string("groups_1_2_in_proj_qkv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 6144]> linear_47_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = groups_1_2_in_proj_qkv_weight_promoted_to_fp16_palettized, x = input_81_cast_fp16)[name = string("linear_47_cast_fp16")];
            tensor<int32, [2]> var_1625_perm_0 = const()[name = string("op_1625_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> mixed_11_axes_0 = const()[name = string("mixed_11_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [6144, 1]> var_1625_cast_fp16 = transpose(perm = var_1625_perm_0, x = linear_47_cast_fp16)[name = string("transpose_97")];
            tensor<fp16, [1, 6144, 1]> mixed_11_cast_fp16 = expand_dims(axes = mixed_11_axes_0, x = var_1625_cast_fp16)[name = string("mixed_11_cast_fp16")];
            bool convolution_input_11_interleave_0 = const()[name = string("convolution_input_11_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 6144, 4]> convolution_input_11_cast_fp16 = concat(axis = var_1598, interleave = convolution_input_11_interleave_0, values = (conv6, mixed_11_cast_fp16))[name = string("convolution_input_11_cast_fp16")];
            string input_83_pad_type_0 = const()[name = string("input_83_pad_type_0"), val = string("valid")];
            int32 input_83_groups_0 = const()[name = string("input_83_groups_0"), val = int32(6144)];
            tensor<int32, [1]> input_83_strides_0 = const()[name = string("input_83_strides_0"), val = tensor<int32, [1]>([1])];
            tensor<int32, [2]> input_83_pad_0 = const()[name = string("input_83_pad_0"), val = tensor<int32, [2]>([0, 0])];
            tensor<int32, [1]> input_83_dilations_0 = const()[name = string("input_83_dilations_0"), val = tensor<int32, [1]>([1])];
            tensor<fp16, [6144, 1, 4]> groups_1_2_conv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1, 4]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(100486336))), lut = tensor<fp16, [384, 1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(100504832))))[name = string("groups_1_2_conv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 6144, 1]> input_83_cast_fp16 = conv(dilations = input_83_dilations_0, groups = input_83_groups_0, pad = input_83_pad_0, pad_type = input_83_pad_type_0, strides = input_83_strides_0, weight = groups_1_2_conv_weight_promoted_to_fp16_palettized, x = convolution_input_11_cast_fp16)[name = string("input_83_cast_fp16")];
            tensor<fp16, [1, 6144, 1]> var_1634_cast_fp16 = silu(x = input_83_cast_fp16)[name = string("op_1634_cast_fp16")];
            tensor<int32, [3]> var_1635_perm_0 = const()[name = string("op_1635_perm_0"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [3]> var_1638_begin_0 = const()[name = string("op_1638_begin_0"), val = tensor<int32, [3]>([0, 0, 1])];
            tensor<int32, [3]> var_1638_end_0 = const()[name = string("op_1638_end_0"), val = tensor<int32, [3]>([1, 6144, 4])];
            tensor<bool, [3]> var_1638_end_mask_0 = const()[name = string("op_1638_end_mask_0"), val = tensor<bool, [3]>([true, true, true])];
            tensor<fp16, [1, 6144, 3]> conv6_out = slice_by_index(begin = var_1638_begin_0, end = var_1638_end_0, end_mask = var_1638_end_mask_0, x = convolution_input_11_cast_fp16)[name = string("op_1638_cast_fp16")];
            tensor<int32, [3]> var_1639 = const()[name = string("op_1639"), val = tensor<int32, [3]>([2048, 2048, 2048])];
            int32 var_1640_axis_0 = const()[name = string("op_1640_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 1, 6144]> var_1635_cast_fp16 = transpose(perm = var_1635_perm_0, x = var_1634_cast_fp16)[name = string("transpose_96")];
            tensor<fp16, [1, 1, 2048]> var_1640_cast_fp16_0, tensor<fp16, [1, 1, 2048]> var_1640_cast_fp16_1, tensor<fp16, [1, 1, 2048]> var_1640_cast_fp16_2 = split(axis = var_1640_axis_0, split_sizes = var_1639, x = var_1635_cast_fp16)[name = string("op_1640_cast_fp16")];
            tensor<int32, [4]> var_1644 = const()[name = string("op_1644"), val = tensor<int32, [4]>([1, 1, 16, 128])];
            tensor<fp16, [1, 1, 16, 128]> value_91_cast_fp16 = reshape(shape = var_1644, x = var_1640_cast_fp16_0)[name = string("value_91_cast_fp16")];
            tensor<int32, [4]> var_1646 = const()[name = string("op_1646"), val = tensor<int32, [4]>([1, 1, 16, 128])];
            tensor<fp16, [1, 1, 16, 128]> value_93_cast_fp16 = reshape(shape = var_1646, x = var_1640_cast_fp16_1)[name = string("value_93_cast_fp16")];
            tensor<int32, [4]> var_1648 = const()[name = string("op_1648"), val = tensor<int32, [4]>([1, 1, 16, 128])];
            tensor<fp16, [1, 1, 16, 128]> value_95_cast_fp16 = reshape(shape = var_1648, x = var_1640_cast_fp16_2)[name = string("value_95_cast_fp16")];
            tensor<fp16, [1, 1, 16, 128]> var_1650_cast_fp16 = mul(x = value_91_cast_fp16, y = value_91_cast_fp16)[name = string("op_1650_cast_fp16")];
            tensor<int32, [1]> var_1652_axes_0 = const()[name = string("op_1652_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_1652_keep_dims_0 = const()[name = string("op_1652_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 1, 16, 1]> var_1652_cast_fp16 = reduce_sum(axes = var_1652_axes_0, keep_dims = var_1652_keep_dims_0, x = var_1650_cast_fp16)[name = string("op_1652_cast_fp16")];
            fp16 var_1653_to_fp16 = const()[name = string("op_1653_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 1, 16, 1]> var_1654_cast_fp16 = add(x = var_1652_cast_fp16, y = var_1653_to_fp16)[name = string("op_1654_cast_fp16")];
            fp32 var_1655_epsilon_0 = const()[name = string("op_1655_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 1, 16, 1]> var_1655_cast_fp16 = rsqrt(epsilon = var_1655_epsilon_0, x = var_1654_cast_fp16)[name = string("op_1655_cast_fp16")];
            tensor<fp16, [1, 1, 16, 128]> var_1656_cast_fp16 = mul(x = value_91_cast_fp16, y = var_1655_cast_fp16)[name = string("op_1656_cast_fp16")];
            tensor<int32, [4]> var_1657_perm_0 = const()[name = string("op_1657_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            fp16 _inversed_query_27_y_0_to_fp16 = const()[name = string("_inversed_query_27_y_0_to_fp16"), val = fp16(0x1.6ap-4)];
            tensor<fp16, [1, 16, 1, 128]> var_1657_cast_fp16 = transpose(perm = var_1657_perm_0, x = var_1656_cast_fp16)[name = string("transpose_95")];
            tensor<fp16, [1, 16, 1, 128]> _inversed_query_27_cast_fp16 = mul(x = var_1657_cast_fp16, y = _inversed_query_27_y_0_to_fp16)[name = string("_inversed_query_27_cast_fp16")];
            tensor<fp16, [1, 1, 16, 128]> var_1660_cast_fp16 = mul(x = value_93_cast_fp16, y = value_93_cast_fp16)[name = string("op_1660_cast_fp16")];
            tensor<int32, [1]> var_1662_axes_0 = const()[name = string("op_1662_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_1662_keep_dims_0 = const()[name = string("op_1662_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 1, 16, 1]> var_1662_cast_fp16 = reduce_sum(axes = var_1662_axes_0, keep_dims = var_1662_keep_dims_0, x = var_1660_cast_fp16)[name = string("op_1662_cast_fp16")];
            fp16 var_1663_to_fp16 = const()[name = string("op_1663_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 1, 16, 1]> var_1664_cast_fp16 = add(x = var_1662_cast_fp16, y = var_1663_to_fp16)[name = string("op_1664_cast_fp16")];
            fp32 var_1665_epsilon_0 = const()[name = string("op_1665_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 1, 16, 1]> var_1665_cast_fp16 = rsqrt(epsilon = var_1665_epsilon_0, x = var_1664_cast_fp16)[name = string("op_1665_cast_fp16")];
            tensor<fp16, [1, 1, 16, 128]> var_1666_cast_fp16 = mul(x = value_93_cast_fp16, y = var_1665_cast_fp16)[name = string("op_1666_cast_fp16")];
            tensor<int32, [4]> key_27_perm_0 = const()[name = string("key_27_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<int32, [4]> value_97_perm_0 = const()[name = string("value_97_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<fp16, [16, 1024]> groups_1_2_in_proj_b_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(100554048))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(100566400))))[name = string("groups_1_2_in_proj_b_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16]> linear_48_bias_0_to_fp16 = const()[name = string("linear_48_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(4862656)))];
            tensor<fp16, [1, 16]> linear_48_cast_fp16 = linear(bias = linear_48_bias_0_to_fp16, weight = groups_1_2_in_proj_b_weight_promoted_to_fp16_palettized, x = input_81_cast_fp16)[name = string("linear_48_cast_fp16")];
            tensor<fp16, [1, 16]> var_1671_cast_fp16 = sigmoid(x = linear_48_cast_fp16)[name = string("op_1671_cast_fp16")];
            tensor<int32, [2]> var_1672_perm_0 = const()[name = string("op_1672_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> beta_11_axes_0 = const()[name = string("beta_11_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 1]> var_1672_cast_fp16 = transpose(perm = var_1672_perm_0, x = var_1671_cast_fp16)[name = string("transpose_94")];
            tensor<fp16, [1, 16, 1]> beta_11_cast_fp16 = expand_dims(axes = beta_11_axes_0, x = var_1672_cast_fp16)[name = string("beta_11_cast_fp16")];
            tensor<fp16, [16, 1024]> op_1678_weight_0_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(100566592))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(100578944))))[name = string("op_1678_weight_0_to_fp16_palettized")];
            tensor<fp16, [16]> var_1678_bias_0_to_fp16 = const()[name = string("op_1678_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(100579136)))];
            tensor<fp16, [1, 16]> var_1678_cast_fp16 = linear(bias = var_1678_bias_0_to_fp16, weight = op_1678_weight_0_to_fp16_palettized, x = input_81_cast_fp16)[name = string("op_1678_cast_fp16")];
            tensor<fp16, [1, 16]> var_1679_cast_fp16 = softplus(x = var_1678_cast_fp16)[name = string("op_1679_cast_fp16")];
            tensor<fp16, [16]> var_1675_promoted_to_fp16 = const()[name = string("op_1675_promoted_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(100579264)))];
            tensor<fp16, [1, 16]> var_1680_cast_fp16 = mul(x = var_1675_promoted_to_fp16, y = var_1679_cast_fp16)[name = string("op_1680_cast_fp16")];
            tensor<int32, [2]> var_1681_perm_0 = const()[name = string("op_1681_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> decay_11_axes_0 = const()[name = string("decay_11_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 1]> var_1681_cast_fp16 = transpose(perm = var_1681_perm_0, x = var_1680_cast_fp16)[name = string("transpose_93")];
            tensor<fp16, [1, 16, 1]> decay_11_cast_fp16 = expand_dims(axes = decay_11_axes_0, x = var_1681_cast_fp16)[name = string("decay_11_cast_fp16")];
            tensor<fp16, [2048, 1024]> groups_1_2_in_proj_z_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [2048, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(100579392))), lut = tensor<fp16, [128, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(102152320))))[name = string("groups_1_2_in_proj_z_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 2048]> linear_50_cast_fp16 = linear(bias = linear_3_bias_0_to_fp16, weight = groups_1_2_in_proj_z_weight_promoted_to_fp16_palettized, x = input_81_cast_fp16)[name = string("linear_50_cast_fp16")];
            tensor<int32, [3]> var_1689_begin_0 = const()[name = string("op_1689_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_1689_end_0 = const()[name = string("op_1689_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_1689_end_mask_0 = const()[name = string("op_1689_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1689_squeeze_mask_0 = const()[name = string("op_1689_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1689_cast_fp16 = slice_by_index(begin = var_1689_begin_0, end = var_1689_end_0, end_mask = var_1689_end_mask_0, squeeze_mask = var_1689_squeeze_mask_0, x = decay_11_cast_fp16)[name = string("op_1689_cast_fp16")];
            tensor<fp16, [1, 16]> var_1690_cast_fp16 = exp(x = var_1689_cast_fp16)[name = string("op_1690_cast_fp16")];
            tensor<int32, [1]> var_1691_axes_0 = const()[name = string("op_1691_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1691_cast_fp16 = expand_dims(axes = var_1691_axes_0, x = var_1690_cast_fp16)[name = string("op_1691_cast_fp16")];
            tensor<int32, [1]> var_1692_axes_0 = const()[name = string("op_1692_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_1692_cast_fp16 = expand_dims(axes = var_1692_axes_0, x = var_1691_cast_fp16)[name = string("op_1692_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_21_cast_fp16 = mul(x = rec6, y = var_1692_cast_fp16)[name = string("state_21_cast_fp16")];
            tensor<int32, [4]> key_token_11_begin_0 = const()[name = string("key_token_11_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> key_token_11_end_0 = const()[name = string("key_token_11_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> key_token_11_end_mask_0 = const()[name = string("key_token_11_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_11_squeeze_mask_0 = const()[name = string("key_token_11_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 1, 128]> key_27_cast_fp16 = transpose(perm = key_27_perm_0, x = var_1666_cast_fp16)[name = string("transpose_92")];
            tensor<fp16, [1, 16, 128]> key_token_11_cast_fp16 = slice_by_index(begin = key_token_11_begin_0, end = key_token_11_end_0, end_mask = key_token_11_end_mask_0, squeeze_mask = key_token_11_squeeze_mask_0, x = key_27_cast_fp16)[name = string("key_token_11_cast_fp16")];
            tensor<int32, [4]> value_token_11_begin_0 = const()[name = string("value_token_11_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> value_token_11_end_0 = const()[name = string("value_token_11_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> value_token_11_end_mask_0 = const()[name = string("value_token_11_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_11_squeeze_mask_0 = const()[name = string("value_token_11_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 1, 128]> value_97_cast_fp16 = transpose(perm = value_97_perm_0, x = value_95_cast_fp16)[name = string("transpose_91")];
            tensor<fp16, [1, 16, 128]> value_token_11_cast_fp16 = slice_by_index(begin = value_token_11_begin_0, end = value_token_11_end_0, end_mask = value_token_11_end_mask_0, squeeze_mask = value_token_11_squeeze_mask_0, x = value_97_cast_fp16)[name = string("value_token_11_cast_fp16")];
            tensor<int32, [1]> var_1700_axes_0 = const()[name = string("op_1700_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1700_cast_fp16 = expand_dims(axes = var_1700_axes_0, x = key_token_11_cast_fp16)[name = string("op_1700_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1701_cast_fp16 = mul(x = state_21_cast_fp16, y = var_1700_cast_fp16)[name = string("op_1701_cast_fp16")];
            tensor<int32, [1]> memory_11_axes_0 = const()[name = string("memory_11_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_11_keep_dims_0 = const()[name = string("memory_11_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_11_cast_fp16 = reduce_sum(axes = memory_11_axes_0, keep_dims = memory_11_keep_dims_0, x = var_1701_cast_fp16)[name = string("memory_11_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_1704_cast_fp16 = sub(x = value_token_11_cast_fp16, y = memory_11_cast_fp16)[name = string("op_1704_cast_fp16")];
            tensor<int32, [3]> var_1707_begin_0 = const()[name = string("op_1707_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_1707_end_0 = const()[name = string("op_1707_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_1707_end_mask_0 = const()[name = string("op_1707_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1707_squeeze_mask_0 = const()[name = string("op_1707_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1707_cast_fp16 = slice_by_index(begin = var_1707_begin_0, end = var_1707_end_0, end_mask = var_1707_end_mask_0, squeeze_mask = var_1707_squeeze_mask_0, x = beta_11_cast_fp16)[name = string("op_1707_cast_fp16")];
            tensor<int32, [1]> var_1708_axes_0 = const()[name = string("op_1708_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1708_cast_fp16 = expand_dims(axes = var_1708_axes_0, x = var_1707_cast_fp16)[name = string("op_1708_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_11_cast_fp16 = mul(x = var_1704_cast_fp16, y = var_1708_cast_fp16)[name = string("delta_11_cast_fp16")];
            tensor<int32, [1]> var_1711_axes_0 = const()[name = string("op_1711_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_1711_cast_fp16 = expand_dims(axes = var_1711_axes_0, x = delta_11_cast_fp16)[name = string("op_1711_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1712_cast_fp16 = mul(x = var_1700_cast_fp16, y = var_1711_cast_fp16)[name = string("op_1712_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> rec6_out = add(x = state_21_cast_fp16, y = var_1712_cast_fp16)[name = string("state_23_cast_fp16")];
            tensor<int32, [4]> var_1716_begin_0 = const()[name = string("op_1716_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> var_1716_end_0 = const()[name = string("op_1716_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> var_1716_end_mask_0 = const()[name = string("op_1716_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_1716_squeeze_mask_0 = const()[name = string("op_1716_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_1716_cast_fp16 = slice_by_index(begin = var_1716_begin_0, end = var_1716_end_0, end_mask = var_1716_end_mask_0, squeeze_mask = var_1716_squeeze_mask_0, x = _inversed_query_27_cast_fp16)[name = string("op_1716_cast_fp16")];
            tensor<int32, [1]> var_1717_axes_0 = const()[name = string("op_1717_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1717_cast_fp16 = expand_dims(axes = var_1717_axes_0, x = var_1716_cast_fp16)[name = string("op_1717_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1718_cast_fp16 = mul(x = rec6_out, y = var_1717_cast_fp16)[name = string("op_1718_cast_fp16")];
            tensor<int32, [1]> var_1720_axes_0 = const()[name = string("op_1720_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_1720_keep_dims_0 = const()[name = string("op_1720_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_1720_cast_fp16 = reduce_sum(axes = var_1720_axes_0, keep_dims = var_1720_keep_dims_0, x = var_1718_cast_fp16)[name = string("op_1720_cast_fp16")];
            tensor<int32, [1]> attention_51_axes_0 = const()[name = string("attention_51_axes_0"), val = tensor<int32, [1]>([1])];
            tensor<fp16, [1, 1, 16, 128]> attention_51_cast_fp16 = expand_dims(axes = attention_51_axes_0, x = var_1720_cast_fp16)[name = string("attention_51_cast_fp16")];
            tensor<int32, [2]> var_1723 = const()[name = string("op_1723"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [16, 128]> attention_53_cast_fp16 = reshape(shape = var_1723, x = attention_51_cast_fp16)[name = string("attention_53_cast_fp16")];
            tensor<int32, [2]> var_1725 = const()[name = string("op_1725"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [16, 128]> input_85_cast_fp16 = reshape(shape = var_1725, x = linear_50_cast_fp16)[name = string("input_85_cast_fp16")];
            tensor<fp16, [16, 128]> var_1727_cast_fp16 = mul(x = attention_53_cast_fp16, y = attention_53_cast_fp16)[name = string("op_1727_cast_fp16")];
            tensor<int32, [1]> variance_41_axes_0 = const()[name = string("variance_41_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_41_keep_dims_0 = const()[name = string("variance_41_keep_dims_0"), val = bool(true)];
            tensor<fp16, [16, 1]> variance_41_cast_fp16 = reduce_mean(axes = variance_41_axes_0, keep_dims = variance_41_keep_dims_0, x = var_1727_cast_fp16)[name = string("variance_41_cast_fp16")];
            fp16 var_1730_to_fp16 = const()[name = string("op_1730_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [16, 1]> var_1731_cast_fp16 = add(x = variance_41_cast_fp16, y = var_1730_to_fp16)[name = string("op_1731_cast_fp16")];
            fp32 var_1732_epsilon_0 = const()[name = string("op_1732_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [16, 1]> var_1732_cast_fp16 = rsqrt(epsilon = var_1732_epsilon_0, x = var_1731_cast_fp16)[name = string("op_1732_cast_fp16")];
            tensor<fp16, [16, 128]> attention_55_cast_fp16 = mul(x = attention_53_cast_fp16, y = var_1732_cast_fp16)[name = string("attention_55_cast_fp16")];
            tensor<fp16, [128]> groups_1_2_gated_norm_promoted_to_fp16 = const()[name = string("groups_1_2_gated_norm_promoted_to_fp16"), val = tensor<fp16, [128]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(102168768)))];
            tensor<fp16, [16, 128]> attention_57_cast_fp16 = mul(x = attention_55_cast_fp16, y = groups_1_2_gated_norm_promoted_to_fp16)[name = string("attention_57_cast_fp16")];
            tensor<fp16, [16, 128]> var_1735_cast_fp16 = silu(x = input_85_cast_fp16)[name = string("op_1735_cast_fp16")];
            tensor<fp16, [16, 128]> attention_59_cast_fp16 = mul(x = attention_57_cast_fp16, y = var_1735_cast_fp16)[name = string("attention_59_cast_fp16")];
            tensor<int32, [2]> var_1737 = const()[name = string("op_1737"), val = tensor<int32, [2]>([1, 2048])];
            tensor<fp16, [1, 2048]> input_87_cast_fp16 = reshape(shape = var_1737, x = attention_59_cast_fp16)[name = string("input_87_cast_fp16")];
            tensor<fp16, [1024, 2048]> groups_1_2_out_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 2048]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(102169088))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(103742016))))[name = string("groups_1_2_out_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 1024]> linear_51_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = groups_1_2_out_proj_weight_promoted_to_fp16_palettized, x = input_87_cast_fp16)[name = string("linear_51_cast_fp16")];
            tensor<fp16, [1, 1024]> value_99_cast_fp16 = add(x = value_87_cast_fp16, y = linear_51_cast_fp16)[name = string("value_99_cast_fp16")];
            tensor<fp16, [1, 1024]> var_1742_cast_fp16 = mul(x = value_99_cast_fp16, y = value_99_cast_fp16)[name = string("op_1742_cast_fp16")];
            tensor<int32, [1]> variance_43_axes_0 = const()[name = string("variance_43_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_43_keep_dims_0 = const()[name = string("variance_43_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 1]> variance_43_cast_fp16 = reduce_mean(axes = variance_43_axes_0, keep_dims = variance_43_keep_dims_0, x = var_1742_cast_fp16)[name = string("variance_43_cast_fp16")];
            fp16 var_1745_to_fp16 = const()[name = string("op_1745_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 1]> var_1746_cast_fp16 = add(x = variance_43_cast_fp16, y = var_1745_to_fp16)[name = string("op_1746_cast_fp16")];
            fp32 var_1747_epsilon_0 = const()[name = string("op_1747_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 1]> var_1747_cast_fp16 = rsqrt(epsilon = var_1747_epsilon_0, x = var_1746_cast_fp16)[name = string("op_1747_cast_fp16")];
            tensor<fp16, [1, 1024]> var_1748_cast_fp16 = mul(x = value_99_cast_fp16, y = var_1747_cast_fp16)[name = string("op_1748_cast_fp16")];
            tensor<fp16, [1024]> var_1750_to_fp16 = const()[name = string("op_1750_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(103750272)))];
            tensor<fp16, [1, 1024]> input_89_cast_fp16 = mul(x = var_1748_cast_fp16, y = var_1750_to_fp16)[name = string("input_89_cast_fp16")];
            tensor<fp16, [3584, 1024]> groups_1_2_gate_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(103752384))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(106504960))))[name = string("groups_1_2_gate_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 3584]> linear_52_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = groups_1_2_gate_proj_weight_promoted_to_fp16_palettized, x = input_89_cast_fp16)[name = string("linear_52_cast_fp16")];
            tensor<fp16, [1, 3584]> var_1754_cast_fp16 = silu(x = linear_52_cast_fp16)[name = string("op_1754_cast_fp16")];
            tensor<fp16, [3584, 1024]> groups_1_2_up_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(106533696))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(109286272))))[name = string("groups_1_2_up_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 3584]> linear_53_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = groups_1_2_up_proj_weight_promoted_to_fp16_palettized, x = input_89_cast_fp16)[name = string("linear_53_cast_fp16")];
            tensor<fp16, [1, 3584]> input_93_cast_fp16 = mul(x = var_1754_cast_fp16, y = linear_53_cast_fp16)[name = string("input_93_cast_fp16")];
            tensor<fp16, [1024, 3584]> groups_1_2_down_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 3584]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(109315008))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(112067584))))[name = string("groups_1_2_down_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 1024]> linear_54_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = groups_1_2_down_proj_weight_promoted_to_fp16_palettized, x = input_93_cast_fp16)[name = string("linear_54_cast_fp16")];
            tensor<fp16, [1, 1024]> value_101_cast_fp16 = add(x = value_99_cast_fp16, y = linear_54_cast_fp16)[name = string("value_101_cast_fp16")];
            int32 var_1778 = const()[name = string("op_1778"), val = int32(-1)];
            tensor<fp16, [1, 1024]> var_1785_cast_fp16 = mul(x = value_101_cast_fp16, y = value_101_cast_fp16)[name = string("op_1785_cast_fp16")];
            tensor<int32, [1]> variance_45_axes_0 = const()[name = string("variance_45_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_45_keep_dims_0 = const()[name = string("variance_45_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 1]> variance_45_cast_fp16 = reduce_mean(axes = variance_45_axes_0, keep_dims = variance_45_keep_dims_0, x = var_1785_cast_fp16)[name = string("variance_45_cast_fp16")];
            fp16 var_1788_to_fp16 = const()[name = string("op_1788_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 1]> var_1789_cast_fp16 = add(x = variance_45_cast_fp16, y = var_1788_to_fp16)[name = string("op_1789_cast_fp16")];
            fp32 var_1790_epsilon_0 = const()[name = string("op_1790_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 1]> var_1790_cast_fp16 = rsqrt(epsilon = var_1790_epsilon_0, x = var_1789_cast_fp16)[name = string("op_1790_cast_fp16")];
            tensor<fp16, [1, 1024]> var_1791_cast_fp16 = mul(x = value_101_cast_fp16, y = var_1790_cast_fp16)[name = string("op_1791_cast_fp16")];
            tensor<fp16, [1024]> var_1793_to_fp16 = const()[name = string("op_1793_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(112075840)))];
            tensor<fp16, [1, 1024]> input_95_cast_fp16 = mul(x = var_1791_cast_fp16, y = var_1793_to_fp16)[name = string("input_95_cast_fp16")];
            tensor<fp16, [4096, 1024]> projections_1_q_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [4096, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(112077952))), lut = tensor<fp16, [256, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(115223744))))[name = string("projections_1_q_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 4096]> linear_55_cast_fp16 = linear(bias = linear_24_bias_0_to_fp16, weight = projections_1_q_proj_weight_promoted_to_fp16_palettized, x = input_95_cast_fp16)[name = string("linear_55_cast_fp16")];
            tensor<int32, [3]> var_1797 = const()[name = string("op_1797"), val = tensor<int32, [3]>([1, 8, 512])];
            tensor<fp16, [1, 8, 512]> query_and_gate_3_cast_fp16 = reshape(shape = var_1797, x = linear_55_cast_fp16)[name = string("query_and_gate_3_cast_fp16")];
            tensor<int32, [2]> var_1799_split_sizes_0 = const()[name = string("op_1799_split_sizes_0"), val = tensor<int32, [2]>([256, 256])];
            int32 var_1799_axis_0 = const()[name = string("op_1799_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 8, 256]> var_1799_cast_fp16_0, tensor<fp16, [1, 8, 256]> var_1799_cast_fp16_1 = split(axis = var_1799_axis_0, split_sizes = var_1799_split_sizes_0, x = query_and_gate_3_cast_fp16)[name = string("op_1799_cast_fp16")];
            tensor<fp16, [512, 1024]> projections_1_k_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [512, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(115256576))), lut = tensor<fp16, [32, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(115649856))))[name = string("projections_1_k_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 512]> linear_56_cast_fp16 = linear(bias = linear_25_bias_0_to_fp16, weight = projections_1_k_proj_weight_promoted_to_fp16_palettized, x = input_95_cast_fp16)[name = string("linear_56_cast_fp16")];
            tensor<int32, [3]> var_1803 = const()[name = string("op_1803"), val = tensor<int32, [3]>([1, 2, 256])];
            tensor<fp16, [1, 2, 256]> value_105_cast_fp16 = reshape(shape = var_1803, x = linear_56_cast_fp16)[name = string("value_105_cast_fp16")];
            tensor<fp16, [512, 1024]> projections_1_v_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [512, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(115654016))), lut = tensor<fp16, [32, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(116047296))))[name = string("projections_1_v_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 512]> linear_57_cast_fp16 = linear(bias = linear_25_bias_0_to_fp16, weight = projections_1_v_proj_weight_promoted_to_fp16_palettized, x = input_95_cast_fp16)[name = string("linear_57_cast_fp16")];
            tensor<int32, [3]> var_1807 = const()[name = string("op_1807"), val = tensor<int32, [3]>([1, 2, 256])];
            tensor<fp16, [1, 2, 256]> value_111_cast_fp16 = reshape(shape = var_1807, x = linear_57_cast_fp16)[name = string("value_111_cast_fp16")];
            tensor<fp16, [1, 8, 256]> var_1809_cast_fp16 = mul(x = var_1799_cast_fp16_0, y = var_1799_cast_fp16_0)[name = string("op_1809_cast_fp16")];
            tensor<int32, [1]> variance_47_axes_0 = const()[name = string("variance_47_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_47_keep_dims_0 = const()[name = string("variance_47_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 8, 1]> variance_47_cast_fp16 = reduce_mean(axes = variance_47_axes_0, keep_dims = variance_47_keep_dims_0, x = var_1809_cast_fp16)[name = string("variance_47_cast_fp16")];
            fp16 var_1812_to_fp16 = const()[name = string("op_1812_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 8, 1]> var_1813_cast_fp16 = add(x = variance_47_cast_fp16, y = var_1812_to_fp16)[name = string("op_1813_cast_fp16")];
            fp32 var_1814_epsilon_0 = const()[name = string("op_1814_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 8, 1]> var_1814_cast_fp16 = rsqrt(epsilon = var_1814_epsilon_0, x = var_1813_cast_fp16)[name = string("op_1814_cast_fp16")];
            tensor<fp16, [1, 8, 256]> var_1815_cast_fp16 = mul(x = var_1799_cast_fp16_0, y = var_1814_cast_fp16)[name = string("op_1815_cast_fp16")];
            tensor<fp16, [256]> var_1817_to_fp16 = const()[name = string("op_1817_to_fp16"), val = tensor<fp16, [256]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(116051456)))];
            tensor<fp16, [1, 8, 256]> var_1818_cast_fp16 = mul(x = var_1815_cast_fp16, y = var_1817_to_fp16)[name = string("op_1818_cast_fp16")];
            tensor<int32, [3]> var_1819 = const()[name = string("op_1819"), val = tensor<int32, [3]>([1, 0, 2])];
            tensor<int32, [1]> value_107_axes_0 = const()[name = string("value_107_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [8, 1, 256]> var_1820_cast_fp16 = transpose(perm = var_1819, x = var_1818_cast_fp16)[name = string("transpose_90")];
            tensor<fp16, [1, 8, 1, 256]> value_107_cast_fp16 = expand_dims(axes = value_107_axes_0, x = var_1820_cast_fp16)[name = string("value_107_cast_fp16")];
            tensor<fp16, [1, 2, 256]> var_1822_cast_fp16 = mul(x = value_105_cast_fp16, y = value_105_cast_fp16)[name = string("op_1822_cast_fp16")];
            tensor<int32, [1]> variance_49_axes_0 = const()[name = string("variance_49_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_49_keep_dims_0 = const()[name = string("variance_49_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 2, 1]> variance_49_cast_fp16 = reduce_mean(axes = variance_49_axes_0, keep_dims = variance_49_keep_dims_0, x = var_1822_cast_fp16)[name = string("variance_49_cast_fp16")];
            fp16 var_1825_to_fp16 = const()[name = string("op_1825_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 2, 1]> var_1826_cast_fp16 = add(x = variance_49_cast_fp16, y = var_1825_to_fp16)[name = string("op_1826_cast_fp16")];
            fp32 var_1827_epsilon_0 = const()[name = string("op_1827_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 2, 1]> var_1827_cast_fp16 = rsqrt(epsilon = var_1827_epsilon_0, x = var_1826_cast_fp16)[name = string("op_1827_cast_fp16")];
            tensor<fp16, [1, 2, 256]> var_1828_cast_fp16 = mul(x = value_105_cast_fp16, y = var_1827_cast_fp16)[name = string("op_1828_cast_fp16")];
            tensor<fp16, [256]> var_1830_to_fp16 = const()[name = string("op_1830_to_fp16"), val = tensor<fp16, [256]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(116052032)))];
            tensor<fp16, [1, 2, 256]> var_1831_cast_fp16 = mul(x = var_1828_cast_fp16, y = var_1830_to_fp16)[name = string("op_1831_cast_fp16")];
            tensor<int32, [3]> var_1832 = const()[name = string("op_1832"), val = tensor<int32, [3]>([1, 0, 2])];
            tensor<int32, [1]> value_109_axes_0 = const()[name = string("value_109_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [2, 1, 256]> var_1833_cast_fp16 = transpose(perm = var_1832, x = var_1831_cast_fp16)[name = string("transpose_89")];
            tensor<fp16, [1, 2, 1, 256]> value_109_cast_fp16 = expand_dims(axes = value_109_axes_0, x = var_1833_cast_fp16)[name = string("value_109_cast_fp16")];
            tensor<int32, [4]> rotated_5_begin_0 = const()[name = string("rotated_5_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> rotated_5_end_0 = const()[name = string("rotated_5_end_0"), val = tensor<int32, [4]>([1, 8, 1, 64])];
            tensor<bool, [4]> rotated_5_end_mask_0 = const()[name = string("rotated_5_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
            tensor<fp16, [1, 8, 1, 64]> rotated_5_cast_fp16 = slice_by_index(begin = rotated_5_begin_0, end = rotated_5_end_0, end_mask = rotated_5_end_mask_0, x = value_107_cast_fp16)[name = string("rotated_5_cast_fp16")];
            tensor<int32, [4]> passthrough_5_begin_0 = const()[name = string("passthrough_5_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 64])];
            tensor<int32, [4]> passthrough_5_end_0 = const()[name = string("passthrough_5_end_0"), val = tensor<int32, [4]>([1, 8, 1, 256])];
            tensor<bool, [4]> passthrough_5_end_mask_0 = const()[name = string("passthrough_5_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
            tensor<fp16, [1, 8, 1, 192]> passthrough_5_cast_fp16 = slice_by_index(begin = passthrough_5_begin_0, end = passthrough_5_end_0, end_mask = passthrough_5_end_mask_0, x = value_107_cast_fp16)[name = string("passthrough_5_cast_fp16")];
            tensor<int32, [2]> var_1837_split_sizes_0 = const()[name = string("op_1837_split_sizes_0"), val = tensor<int32, [2]>([32, 32])];
            int32 var_1837_axis_0 = const()[name = string("op_1837_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 8, 1, 32]> var_1837_cast_fp16_0, tensor<fp16, [1, 8, 1, 32]> var_1837_cast_fp16_1 = split(axis = var_1837_axis_0, split_sizes = var_1837_split_sizes_0, x = rotated_5_cast_fp16)[name = string("op_1837_cast_fp16")];
            fp16 const_46_promoted_to_fp16 = const()[name = string("const_46_promoted_to_fp16"), val = fp16(-0x1p+0)];
            tensor<fp16, [1, 8, 1, 32]> var_1839_cast_fp16 = mul(x = var_1837_cast_fp16_1, y = const_46_promoted_to_fp16)[name = string("op_1839_cast_fp16")];
            bool rotated_half_5_interleave_0 = const()[name = string("rotated_half_5_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 8, 1, 64]> rotated_half_5_cast_fp16 = concat(axis = var_1778, interleave = rotated_half_5_interleave_0, values = (var_1839_cast_fp16, var_1837_cast_fp16_0))[name = string("rotated_half_5_cast_fp16")];
            tensor<fp16, [1, 8, 1, 64]> var_1842_cast_fp16 = mul(x = rotated_5_cast_fp16, y = cos)[name = string("op_1842_cast_fp16")];
            tensor<fp16, [1, 8, 1, 64]> var_1843_cast_fp16 = mul(x = rotated_half_5_cast_fp16, y = sin)[name = string("op_1843_cast_fp16")];
            tensor<fp16, [1, 8, 1, 64]> var_1844_cast_fp16 = add(x = var_1842_cast_fp16, y = var_1843_cast_fp16)[name = string("op_1844_cast_fp16")];
            bool query_29_interleave_0 = const()[name = string("query_29_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 8, 1, 256]> query_29_cast_fp16 = concat(axis = var_1778, interleave = query_29_interleave_0, values = (var_1844_cast_fp16, passthrough_5_cast_fp16))[name = string("query_29_cast_fp16")];
            tensor<int32, [4]> rotated_7_begin_0 = const()[name = string("rotated_7_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> rotated_7_end_0 = const()[name = string("rotated_7_end_0"), val = tensor<int32, [4]>([1, 2, 1, 64])];
            tensor<bool, [4]> rotated_7_end_mask_0 = const()[name = string("rotated_7_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
            tensor<fp16, [1, 2, 1, 64]> rotated_7_cast_fp16 = slice_by_index(begin = rotated_7_begin_0, end = rotated_7_end_0, end_mask = rotated_7_end_mask_0, x = value_109_cast_fp16)[name = string("rotated_7_cast_fp16")];
            tensor<int32, [4]> passthrough_7_begin_0 = const()[name = string("passthrough_7_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 64])];
            tensor<int32, [4]> passthrough_7_end_0 = const()[name = string("passthrough_7_end_0"), val = tensor<int32, [4]>([1, 2, 1, 256])];
            tensor<bool, [4]> passthrough_7_end_mask_0 = const()[name = string("passthrough_7_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
            tensor<fp16, [1, 2, 1, 192]> passthrough_7_cast_fp16 = slice_by_index(begin = passthrough_7_begin_0, end = passthrough_7_end_0, end_mask = passthrough_7_end_mask_0, x = value_109_cast_fp16)[name = string("passthrough_7_cast_fp16")];
            tensor<int32, [2]> var_1849_split_sizes_0 = const()[name = string("op_1849_split_sizes_0"), val = tensor<int32, [2]>([32, 32])];
            int32 var_1849_axis_0 = const()[name = string("op_1849_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 2, 1, 32]> var_1849_cast_fp16_0, tensor<fp16, [1, 2, 1, 32]> var_1849_cast_fp16_1 = split(axis = var_1849_axis_0, split_sizes = var_1849_split_sizes_0, x = rotated_7_cast_fp16)[name = string("op_1849_cast_fp16")];
            fp16 const_47_promoted_to_fp16 = const()[name = string("const_47_promoted_to_fp16"), val = fp16(-0x1p+0)];
            tensor<fp16, [1, 2, 1, 32]> var_1851_cast_fp16 = mul(x = var_1849_cast_fp16_1, y = const_47_promoted_to_fp16)[name = string("op_1851_cast_fp16")];
            bool rotated_half_7_interleave_0 = const()[name = string("rotated_half_7_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 2, 1, 64]> rotated_half_7_cast_fp16 = concat(axis = var_1778, interleave = rotated_half_7_interleave_0, values = (var_1851_cast_fp16, var_1849_cast_fp16_0))[name = string("rotated_half_7_cast_fp16")];
            tensor<fp16, [1, 2, 1, 64]> var_1854_cast_fp16 = mul(x = rotated_7_cast_fp16, y = cos)[name = string("op_1854_cast_fp16")];
            tensor<fp16, [1, 2, 1, 64]> var_1855_cast_fp16 = mul(x = rotated_half_7_cast_fp16, y = sin)[name = string("op_1855_cast_fp16")];
            tensor<fp16, [1, 2, 1, 64]> var_1856_cast_fp16 = add(x = var_1854_cast_fp16, y = var_1855_cast_fp16)[name = string("op_1856_cast_fp16")];
            bool key_29_interleave_0 = const()[name = string("key_29_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 2, 1, 256]> key_29_cast_fp16 = concat(axis = var_1778, interleave = key_29_interleave_0, values = (var_1856_cast_fp16, passthrough_7_cast_fp16))[name = string("key_29_cast_fp16")];
            tensor<int32, [4]> var_1859 = const()[name = string("op_1859"), val = tensor<int32, [4]>([2, 4, 1, 256])];
            tensor<fp16, [2, 4, 1, 256]> var_1860_cast_fp16 = reshape(shape = var_1859, x = query_29_cast_fp16)[name = string("op_1860_cast_fp16")];
            tensor<int32, [4]> transpose_2_perm_0 = const()[name = string("transpose_2_perm_0"), val = tensor<int32, [4]>([0, 1, 3, 2])];
            tensor<int32, [3]> var_1863 = const()[name = string("op_1863"), val = tensor<int32, [3]>([2, 1, 256])];
            tensor<fp16, [2, 1, 256]> key_31_cast_fp16 = reshape(shape = var_1863, x = key_29_cast_fp16)[name = string("key_31_cast_fp16")];
            tensor<int32, [3]> var_1865 = const()[name = string("op_1865"), val = tensor<int32, [3]>([1, 0, 2])];
            tensor<int32, [2]> var_1867 = const()[name = string("op_1867"), val = tensor<int32, [2]>([1, 2048])];
            tensor<fp16, [1, 2048]> gate_7_cast_fp16 = reshape(shape = var_1867, x = var_1799_cast_fp16_1)[name = string("gate_7_cast_fp16")];
            tensor<int32, [1]> var_1875_axes_0 = const()[name = string("op_1875_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [1, 2, 1, 256]> var_1875_cast_fp16 = expand_dims(axes = var_1875_axes_0, x = key_31_cast_fp16)[name = string("op_1875_cast_fp16")];
            tensor<int32, [1]> var_1877_axes_0 = const()[name = string("op_1877_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [1, 1, 2, 1, 256]> var_1877_cast_fp16 = expand_dims(axes = var_1877_axes_0, x = var_1875_cast_fp16)[name = string("op_1877_cast_fp16")];
            tensor<int32, [1]> expand_dims_16 = const()[name = string("expand_dims_16"), val = tensor<int32, [1]>([1])];
            tensor<int32, [1]> expand_dims_17 = const()[name = string("expand_dims_17"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_18 = const()[name = string("expand_dims_18"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_20 = const()[name = string("expand_dims_20"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_21 = const()[name = string("expand_dims_21"), val = tensor<int32, [1]>([2])];
            tensor<int32, [1]> expand_dims_22 = const()[name = string("expand_dims_22"), val = tensor<int32, [1]>([1])];
            int32 concat_10_axis_0 = const()[name = string("concat_10_axis_0"), val = int32(0)];
            bool concat_10_interleave_0 = const()[name = string("concat_10_interleave_0"), val = bool(false)];
            tensor<int32, [5]> concat_10 = concat(axis = concat_10_axis_0, interleave = concat_10_interleave_0, values = (expand_dims_16, expand_dims_17, expand_dims_18, current_pos, expand_dims_20))[name = string("concat_10")];
            tensor<int32, [1]> concat_11_values2_0 = const()[name = string("concat_11_values2_0"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> concat_11_values4_0 = const()[name = string("concat_11_values4_0"), val = tensor<int32, [1]>([0])];
            int32 concat_11_axis_0 = const()[name = string("concat_11_axis_0"), val = int32(0)];
            bool concat_11_interleave_0 = const()[name = string("concat_11_interleave_0"), val = bool(false)];
            tensor<int32, [5]> concat_11 = concat(axis = concat_11_axis_0, interleave = concat_11_interleave_0, values = (expand_dims_21, expand_dims_22, concat_11_values2_0, var_762, concat_11_values4_0))[name = string("concat_11")];
            tensor<int32, [5]> kv_cache_internal_tensor_assign_3_stride_0 = const()[name = string("kv_cache_internal_tensor_assign_3_stride_0"), val = tensor<int32, [5]>([1, 1, 1, 1, 1])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_3_begin_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_3_begin_mask_0"), val = tensor<bool, [5]>([false, false, false, false, false])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_3_end_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_3_end_mask_0"), val = tensor<bool, [5]>([false, false, true, false, true])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_3_squeeze_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_3_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, false, false, false])];
            tensor<fp16, [4, 2, 2, 2048, 256]> kv_cache_internal_tensor_assign_3_cast_fp16 = slice_update(begin = concat_10, begin_mask = kv_cache_internal_tensor_assign_3_begin_mask_0, end = concat_11, end_mask = kv_cache_internal_tensor_assign_3_end_mask_0, squeeze_mask = kv_cache_internal_tensor_assign_3_squeeze_mask_0, stride = kv_cache_internal_tensor_assign_3_stride_0, update = var_1877_cast_fp16, x = coreml_update_state_9)[name = string("kv_cache_internal_tensor_assign_3_cast_fp16")];
            write_state(data = kv_cache_internal_tensor_assign_3_cast_fp16, input = kv_cache)[name = string("coreml_update_state_2_write_state")];
            tensor<fp16, [4, 2, 2, 2048, 256]> coreml_update_state_10 = read_state(input = kv_cache)[name = string("coreml_update_state_2")];
            tensor<int32, [1]> var_1908_axes_0 = const()[name = string("op_1908_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [2, 1, 256]> value_113_cast_fp16 = transpose(perm = var_1865, x = value_111_cast_fp16)[name = string("transpose_88")];
            tensor<fp16, [1, 2, 1, 256]> var_1908_cast_fp16 = expand_dims(axes = var_1908_axes_0, x = value_113_cast_fp16)[name = string("op_1908_cast_fp16")];
            tensor<int32, [1]> var_1910_axes_0 = const()[name = string("op_1910_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [1, 1, 2, 1, 256]> var_1910_cast_fp16 = expand_dims(axes = var_1910_axes_0, x = var_1908_cast_fp16)[name = string("op_1910_cast_fp16")];
            tensor<int32, [1]> expand_dims_24 = const()[name = string("expand_dims_24"), val = tensor<int32, [1]>([1])];
            tensor<int32, [1]> expand_dims_25 = const()[name = string("expand_dims_25"), val = tensor<int32, [1]>([1])];
            tensor<int32, [1]> expand_dims_26 = const()[name = string("expand_dims_26"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_28 = const()[name = string("expand_dims_28"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_29 = const()[name = string("expand_dims_29"), val = tensor<int32, [1]>([2])];
            tensor<int32, [1]> expand_dims_30 = const()[name = string("expand_dims_30"), val = tensor<int32, [1]>([2])];
            int32 concat_14_axis_0 = const()[name = string("concat_14_axis_0"), val = int32(0)];
            bool concat_14_interleave_0 = const()[name = string("concat_14_interleave_0"), val = bool(false)];
            tensor<int32, [5]> concat_14 = concat(axis = concat_14_axis_0, interleave = concat_14_interleave_0, values = (expand_dims_24, expand_dims_25, expand_dims_26, current_pos, expand_dims_28))[name = string("concat_14")];
            tensor<int32, [1]> concat_15_values2_0 = const()[name = string("concat_15_values2_0"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> concat_15_values4_0 = const()[name = string("concat_15_values4_0"), val = tensor<int32, [1]>([0])];
            int32 concat_15_axis_0 = const()[name = string("concat_15_axis_0"), val = int32(0)];
            bool concat_15_interleave_0 = const()[name = string("concat_15_interleave_0"), val = bool(false)];
            tensor<int32, [5]> concat_15 = concat(axis = concat_15_axis_0, interleave = concat_15_interleave_0, values = (expand_dims_29, expand_dims_30, concat_15_values2_0, var_762, concat_15_values4_0))[name = string("concat_15")];
            tensor<int32, [5]> kv_cache_internal_tensor_assign_4_stride_0 = const()[name = string("kv_cache_internal_tensor_assign_4_stride_0"), val = tensor<int32, [5]>([1, 1, 1, 1, 1])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_4_begin_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_4_begin_mask_0"), val = tensor<bool, [5]>([false, false, false, false, false])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_4_end_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_4_end_mask_0"), val = tensor<bool, [5]>([false, false, true, false, true])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_4_squeeze_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_4_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, false, false, false])];
            tensor<fp16, [4, 2, 2, 2048, 256]> kv_cache_internal_tensor_assign_4_cast_fp16 = slice_update(begin = concat_14, begin_mask = kv_cache_internal_tensor_assign_4_begin_mask_0, end = concat_15, end_mask = kv_cache_internal_tensor_assign_4_end_mask_0, squeeze_mask = kv_cache_internal_tensor_assign_4_squeeze_mask_0, stride = kv_cache_internal_tensor_assign_4_stride_0, update = var_1910_cast_fp16, x = coreml_update_state_10)[name = string("kv_cache_internal_tensor_assign_4_cast_fp16")];
            write_state(data = kv_cache_internal_tensor_assign_4_cast_fp16, input = kv_cache)[name = string("coreml_update_state_3_write_state")];
            tensor<fp16, [4, 2, 2, 2048, 256]> coreml_update_state_11 = read_state(input = kv_cache)[name = string("coreml_update_state_3")];
            tensor<int32, [5]> var_1942_begin_0 = const()[name = string("op_1942_begin_0"), val = tensor<int32, [5]>([1, 0, 0, 0, 0])];
            tensor<int32, [5]> var_1942_end_0 = const()[name = string("op_1942_end_0"), val = tensor<int32, [5]>([2, 2, 2, 2048, 256])];
            tensor<bool, [5]> var_1942_end_mask_0 = const()[name = string("op_1942_end_mask_0"), val = tensor<bool, [5]>([false, true, true, true, true])];
            tensor<bool, [5]> var_1942_squeeze_mask_0 = const()[name = string("op_1942_squeeze_mask_0"), val = tensor<bool, [5]>([true, false, false, false, false])];
            tensor<fp16, [2, 2, 2048, 256]> var_1942_cast_fp16 = slice_by_index(begin = var_1942_begin_0, end = var_1942_end_0, end_mask = var_1942_end_mask_0, squeeze_mask = var_1942_squeeze_mask_0, x = coreml_update_state_11)[name = string("op_1942_cast_fp16")];
            tensor<int32, [4]> keys_3_begin_0 = const()[name = string("keys_3_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> keys_3_end_0 = const()[name = string("keys_3_end_0"), val = tensor<int32, [4]>([1, 2, 2048, 256])];
            tensor<bool, [4]> keys_3_end_mask_0 = const()[name = string("keys_3_end_mask_0"), val = tensor<bool, [4]>([false, true, true, true])];
            tensor<bool, [4]> keys_3_squeeze_mask_0 = const()[name = string("keys_3_squeeze_mask_0"), val = tensor<bool, [4]>([true, false, false, false])];
            tensor<fp16, [2, 2048, 256]> keys_3_cast_fp16 = slice_by_index(begin = keys_3_begin_0, end = keys_3_end_0, end_mask = keys_3_end_mask_0, squeeze_mask = keys_3_squeeze_mask_0, x = var_1942_cast_fp16)[name = string("keys_3_cast_fp16")];
            tensor<int32, [4]> values_3_begin_0 = const()[name = string("values_3_begin_0"), val = tensor<int32, [4]>([1, 0, 0, 0])];
            tensor<int32, [4]> values_3_end_0 = const()[name = string("values_3_end_0"), val = tensor<int32, [4]>([2, 2, 2048, 256])];
            tensor<bool, [4]> values_3_end_mask_0 = const()[name = string("values_3_end_mask_0"), val = tensor<bool, [4]>([false, true, true, true])];
            tensor<bool, [4]> values_3_squeeze_mask_0 = const()[name = string("values_3_squeeze_mask_0"), val = tensor<bool, [4]>([true, false, false, false])];
            tensor<fp16, [2, 2048, 256]> values_3_cast_fp16 = slice_by_index(begin = values_3_begin_0, end = values_3_end_0, end_mask = values_3_end_mask_0, squeeze_mask = values_3_squeeze_mask_0, x = var_1942_cast_fp16)[name = string("values_3_cast_fp16")];
            int32 var_1959 = const()[name = string("op_1959"), val = int32(1)];
            tensor<int32, [4]> var_1966 = const()[name = string("op_1966"), val = tensor<int32, [4]>([1, 2048, 1, 1])];
            tensor<fp16, [2, 4, 256, 1]> transpose_2_cast_fp16 = transpose(perm = transpose_2_perm_0, x = var_1860_cast_fp16)[name = string("transpose_87")];
            tensor<fp16, [1, 2048, 1, 1]> var_1967_cast_fp16 = reshape(shape = var_1966, x = transpose_2_cast_fp16)[name = string("op_1967_cast_fp16")];
            tensor<int32, [3]> var_1968 = const()[name = string("op_1968"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [4]> var_1971 = const()[name = string("op_1971"), val = tensor<int32, [4]>([1, 512, 1, 2048])];
            tensor<fp16, [2, 256, 2048]> var_1969_cast_fp16 = transpose(perm = var_1968, x = keys_3_cast_fp16)[name = string("transpose_86")];
            tensor<fp16, [1, 512, 1, 2048]> key_native_3_cast_fp16 = reshape(shape = var_1971, x = var_1969_cast_fp16)[name = string("key_native_3_cast_fp16")];
            tensor<int32, [3]> var_1973 = const()[name = string("op_1973"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [4]> var_1976 = const()[name = string("op_1976"), val = tensor<int32, [4]>([1, 512, 1, 2048])];
            tensor<fp16, [2, 256, 2048]> var_1974_cast_fp16 = transpose(perm = var_1973, x = values_3_cast_fp16)[name = string("transpose_85")];
            tensor<fp16, [1, 512, 1, 2048]> var_1977_cast_fp16 = reshape(shape = var_1976, x = var_1974_cast_fp16)[name = string("op_1977_cast_fp16")];
            tensor<int32, [8]> tile_19 = const()[name = string("tile_19"), val = tensor<int32, [8]>([256, 256, 256, 256, 256, 256, 256, 256])];
            int32 var_1981_axis_0 = const()[name = string("op_1981_axis_0"), val = int32(1)];
            tensor<fp16, [1, 256, 1, 1]> var_1981_cast_fp16_0, tensor<fp16, [1, 256, 1, 1]> var_1981_cast_fp16_1, tensor<fp16, [1, 256, 1, 1]> var_1981_cast_fp16_2, tensor<fp16, [1, 256, 1, 1]> var_1981_cast_fp16_3, tensor<fp16, [1, 256, 1, 1]> var_1981_cast_fp16_4, tensor<fp16, [1, 256, 1, 1]> var_1981_cast_fp16_5, tensor<fp16, [1, 256, 1, 1]> var_1981_cast_fp16_6, tensor<fp16, [1, 256, 1, 1]> var_1981_cast_fp16_7 = split(axis = var_1981_axis_0, split_sizes = tile_19, x = var_1967_cast_fp16)[name = string("op_1981_cast_fp16")];
            tensor<int32, [4]> var_1990_perm_0 = const()[name = string("op_1990_perm_0"), val = tensor<int32, [4]>([0, 3, 2, 1])];
            tensor<int32, [2]> tile_20 = const()[name = string("tile_20"), val = tensor<int32, [2]>([256, 256])];
            int32 var_1991_axis_0 = const()[name = string("op_1991_axis_0"), val = int32(3)];
            tensor<fp16, [1, 2048, 1, 512]> var_1990_cast_fp16 = transpose(perm = var_1990_perm_0, x = key_native_3_cast_fp16)[name = string("transpose_84")];
            tensor<fp16, [1, 2048, 1, 256]> var_1991_cast_fp16_0, tensor<fp16, [1, 2048, 1, 256]> var_1991_cast_fp16_1 = split(axis = var_1991_axis_0, split_sizes = tile_20, x = var_1990_cast_fp16)[name = string("op_1991_cast_fp16")];
            tensor<int32, [2]> tile_21 = const()[name = string("tile_21"), val = tensor<int32, [2]>([256, 256])];
            int32 var_1994_axis_0 = const()[name = string("op_1994_axis_0"), val = int32(1)];
            tensor<fp16, [1, 256, 1, 2048]> var_1994_cast_fp16_0, tensor<fp16, [1, 256, 1, 2048]> var_1994_cast_fp16_1 = split(axis = var_1994_axis_0, split_sizes = tile_21, x = var_1977_cast_fp16)[name = string("op_1994_cast_fp16")];
            string scores_17_equation_0 = const()[name = string("scores_17_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 1]> scores_17_cast_fp16 = einsum(equation = scores_17_equation_0, values = (var_1991_cast_fp16_0, var_1981_cast_fp16_0))[name = string("scores_17_cast_fp16")];
            fp16 var_1999_to_fp16 = const()[name = string("op_1999_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 1]> var_2000_cast_fp16 = mul(x = scores_17_cast_fp16, y = var_1999_to_fp16)[name = string("op_2000_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_2001_cast_fp16 = add(x = var_2000_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_2001_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_2002_cast_fp16 = softmax(axis = var_1959, x = var_2001_cast_fp16)[name = string("op_2002_cast_fp16")];
            tensor<int32, [4]> tile_22 = const()[name = string("tile_22"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_2003_axis_0 = const()[name = string("op_2003_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 1]> var_2003_cast_fp16_0, tensor<fp16, [1, 512, 1, 1]> var_2003_cast_fp16_1, tensor<fp16, [1, 512, 1, 1]> var_2003_cast_fp16_2, tensor<fp16, [1, 512, 1, 1]> var_2003_cast_fp16_3 = split(axis = var_2003_axis_0, split_sizes = tile_22, x = var_2002_cast_fp16)[name = string("op_2003_cast_fp16")];
            tensor<int32, [4]> tile_23 = const()[name = string("tile_23"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_2008_axis_0 = const()[name = string("op_2008_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_2008_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_2008_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_2008_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_2008_cast_fp16_3 = split(axis = var_2008_axis_0, split_sizes = tile_23, x = var_1994_cast_fp16_0)[name = string("op_2008_cast_fp16")];
            fp16 var_2013_to_fp16 = const()[name = string("op_2013_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_2014_cast_fp16 = mul(x = var_2003_cast_fp16_0, y = var_2013_to_fp16)[name = string("op_2014_cast_fp16")];
            string var_2016_equation_0 = const()[name = string("op_2016_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_2016_cast_fp16 = einsum(equation = var_2016_equation_0, values = (var_2008_cast_fp16_0, var_2014_cast_fp16))[name = string("op_2016_cast_fp16")];
            fp16 var_2017_to_fp16 = const()[name = string("op_2017_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_2018_cast_fp16 = mul(x = var_2003_cast_fp16_1, y = var_2017_to_fp16)[name = string("op_2018_cast_fp16")];
            string var_2020_equation_0 = const()[name = string("op_2020_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_2020_cast_fp16 = einsum(equation = var_2020_equation_0, values = (var_2008_cast_fp16_1, var_2018_cast_fp16))[name = string("op_2020_cast_fp16")];
            fp16 var_2021_to_fp16 = const()[name = string("op_2021_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_2022_cast_fp16 = mul(x = var_2003_cast_fp16_2, y = var_2021_to_fp16)[name = string("op_2022_cast_fp16")];
            string var_2024_equation_0 = const()[name = string("op_2024_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_2024_cast_fp16 = einsum(equation = var_2024_equation_0, values = (var_2008_cast_fp16_2, var_2022_cast_fp16))[name = string("op_2024_cast_fp16")];
            fp16 var_2025_to_fp16 = const()[name = string("op_2025_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_2026_cast_fp16 = mul(x = var_2003_cast_fp16_3, y = var_2025_to_fp16)[name = string("op_2026_cast_fp16")];
            string var_2028_equation_0 = const()[name = string("op_2028_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_2028_cast_fp16 = einsum(equation = var_2028_equation_0, values = (var_2008_cast_fp16_3, var_2026_cast_fp16))[name = string("op_2028_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_2029_cast_fp16 = add(x = var_2016_cast_fp16, y = var_2020_cast_fp16)[name = string("op_2029_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_2030_cast_fp16 = add(x = var_2024_cast_fp16, y = var_2028_cast_fp16)[name = string("op_2030_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_2031_cast_fp16 = add(x = var_2029_cast_fp16, y = var_2030_cast_fp16)[name = string("op_2031_cast_fp16")];
            fp16 _inversed_2033_y_0_to_fp16 = const()[name = string("_inversed_2033_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 1]> _inversed_2033_cast_fp16 = mul(x = var_2031_cast_fp16, y = _inversed_2033_y_0_to_fp16)[name = string("_inversed_2033_cast_fp16")];
            string scores_19_equation_0 = const()[name = string("scores_19_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 1]> scores_19_cast_fp16 = einsum(equation = scores_19_equation_0, values = (var_1991_cast_fp16_0, var_1981_cast_fp16_1))[name = string("scores_19_cast_fp16")];
            fp16 var_2036_to_fp16 = const()[name = string("op_2036_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 1]> var_2037_cast_fp16 = mul(x = scores_19_cast_fp16, y = var_2036_to_fp16)[name = string("op_2037_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_2038_cast_fp16 = add(x = var_2037_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_2038_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_2039_cast_fp16 = softmax(axis = var_1959, x = var_2038_cast_fp16)[name = string("op_2039_cast_fp16")];
            tensor<int32, [4]> tile_24 = const()[name = string("tile_24"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_2040_axis_0 = const()[name = string("op_2040_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 1]> var_2040_cast_fp16_0, tensor<fp16, [1, 512, 1, 1]> var_2040_cast_fp16_1, tensor<fp16, [1, 512, 1, 1]> var_2040_cast_fp16_2, tensor<fp16, [1, 512, 1, 1]> var_2040_cast_fp16_3 = split(axis = var_2040_axis_0, split_sizes = tile_24, x = var_2039_cast_fp16)[name = string("op_2040_cast_fp16")];
            tensor<int32, [4]> tile_25 = const()[name = string("tile_25"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_2045_axis_0 = const()[name = string("op_2045_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_2045_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_2045_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_2045_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_2045_cast_fp16_3 = split(axis = var_2045_axis_0, split_sizes = tile_25, x = var_1994_cast_fp16_0)[name = string("op_2045_cast_fp16")];
            fp16 var_2050_to_fp16 = const()[name = string("op_2050_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_2051_cast_fp16 = mul(x = var_2040_cast_fp16_0, y = var_2050_to_fp16)[name = string("op_2051_cast_fp16")];
            string var_2053_equation_0 = const()[name = string("op_2053_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_2053_cast_fp16 = einsum(equation = var_2053_equation_0, values = (var_2045_cast_fp16_0, var_2051_cast_fp16))[name = string("op_2053_cast_fp16")];
            fp16 var_2054_to_fp16 = const()[name = string("op_2054_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_2055_cast_fp16 = mul(x = var_2040_cast_fp16_1, y = var_2054_to_fp16)[name = string("op_2055_cast_fp16")];
            string var_2057_equation_0 = const()[name = string("op_2057_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_2057_cast_fp16 = einsum(equation = var_2057_equation_0, values = (var_2045_cast_fp16_1, var_2055_cast_fp16))[name = string("op_2057_cast_fp16")];
            fp16 var_2058_to_fp16 = const()[name = string("op_2058_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_2059_cast_fp16 = mul(x = var_2040_cast_fp16_2, y = var_2058_to_fp16)[name = string("op_2059_cast_fp16")];
            string var_2061_equation_0 = const()[name = string("op_2061_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_2061_cast_fp16 = einsum(equation = var_2061_equation_0, values = (var_2045_cast_fp16_2, var_2059_cast_fp16))[name = string("op_2061_cast_fp16")];
            fp16 var_2062_to_fp16 = const()[name = string("op_2062_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_2063_cast_fp16 = mul(x = var_2040_cast_fp16_3, y = var_2062_to_fp16)[name = string("op_2063_cast_fp16")];
            string var_2065_equation_0 = const()[name = string("op_2065_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_2065_cast_fp16 = einsum(equation = var_2065_equation_0, values = (var_2045_cast_fp16_3, var_2063_cast_fp16))[name = string("op_2065_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_2066_cast_fp16 = add(x = var_2053_cast_fp16, y = var_2057_cast_fp16)[name = string("op_2066_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_2067_cast_fp16 = add(x = var_2061_cast_fp16, y = var_2065_cast_fp16)[name = string("op_2067_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_2068_cast_fp16 = add(x = var_2066_cast_fp16, y = var_2067_cast_fp16)[name = string("op_2068_cast_fp16")];
            fp16 _inversed_2070_y_0_to_fp16 = const()[name = string("_inversed_2070_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 1]> _inversed_2070_cast_fp16 = mul(x = var_2068_cast_fp16, y = _inversed_2070_y_0_to_fp16)[name = string("_inversed_2070_cast_fp16")];
            string scores_21_equation_0 = const()[name = string("scores_21_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 1]> scores_21_cast_fp16 = einsum(equation = scores_21_equation_0, values = (var_1991_cast_fp16_0, var_1981_cast_fp16_2))[name = string("scores_21_cast_fp16")];
            fp16 var_2073_to_fp16 = const()[name = string("op_2073_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 1]> var_2074_cast_fp16 = mul(x = scores_21_cast_fp16, y = var_2073_to_fp16)[name = string("op_2074_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_2075_cast_fp16 = add(x = var_2074_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_2075_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_2076_cast_fp16 = softmax(axis = var_1959, x = var_2075_cast_fp16)[name = string("op_2076_cast_fp16")];
            tensor<int32, [4]> tile_26 = const()[name = string("tile_26"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_2077_axis_0 = const()[name = string("op_2077_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 1]> var_2077_cast_fp16_0, tensor<fp16, [1, 512, 1, 1]> var_2077_cast_fp16_1, tensor<fp16, [1, 512, 1, 1]> var_2077_cast_fp16_2, tensor<fp16, [1, 512, 1, 1]> var_2077_cast_fp16_3 = split(axis = var_2077_axis_0, split_sizes = tile_26, x = var_2076_cast_fp16)[name = string("op_2077_cast_fp16")];
            tensor<int32, [4]> tile_27 = const()[name = string("tile_27"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_2082_axis_0 = const()[name = string("op_2082_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_2082_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_2082_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_2082_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_2082_cast_fp16_3 = split(axis = var_2082_axis_0, split_sizes = tile_27, x = var_1994_cast_fp16_0)[name = string("op_2082_cast_fp16")];
            fp16 var_2087_to_fp16 = const()[name = string("op_2087_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_2088_cast_fp16 = mul(x = var_2077_cast_fp16_0, y = var_2087_to_fp16)[name = string("op_2088_cast_fp16")];
            string var_2090_equation_0 = const()[name = string("op_2090_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_2090_cast_fp16 = einsum(equation = var_2090_equation_0, values = (var_2082_cast_fp16_0, var_2088_cast_fp16))[name = string("op_2090_cast_fp16")];
            fp16 var_2091_to_fp16 = const()[name = string("op_2091_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_2092_cast_fp16 = mul(x = var_2077_cast_fp16_1, y = var_2091_to_fp16)[name = string("op_2092_cast_fp16")];
            string var_2094_equation_0 = const()[name = string("op_2094_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_2094_cast_fp16 = einsum(equation = var_2094_equation_0, values = (var_2082_cast_fp16_1, var_2092_cast_fp16))[name = string("op_2094_cast_fp16")];
            fp16 var_2095_to_fp16 = const()[name = string("op_2095_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_2096_cast_fp16 = mul(x = var_2077_cast_fp16_2, y = var_2095_to_fp16)[name = string("op_2096_cast_fp16")];
            string var_2098_equation_0 = const()[name = string("op_2098_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_2098_cast_fp16 = einsum(equation = var_2098_equation_0, values = (var_2082_cast_fp16_2, var_2096_cast_fp16))[name = string("op_2098_cast_fp16")];
            fp16 var_2099_to_fp16 = const()[name = string("op_2099_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_2100_cast_fp16 = mul(x = var_2077_cast_fp16_3, y = var_2099_to_fp16)[name = string("op_2100_cast_fp16")];
            string var_2102_equation_0 = const()[name = string("op_2102_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_2102_cast_fp16 = einsum(equation = var_2102_equation_0, values = (var_2082_cast_fp16_3, var_2100_cast_fp16))[name = string("op_2102_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_2103_cast_fp16 = add(x = var_2090_cast_fp16, y = var_2094_cast_fp16)[name = string("op_2103_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_2104_cast_fp16 = add(x = var_2098_cast_fp16, y = var_2102_cast_fp16)[name = string("op_2104_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_2105_cast_fp16 = add(x = var_2103_cast_fp16, y = var_2104_cast_fp16)[name = string("op_2105_cast_fp16")];
            fp16 _inversed_2107_y_0_to_fp16 = const()[name = string("_inversed_2107_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 1]> _inversed_2107_cast_fp16 = mul(x = var_2105_cast_fp16, y = _inversed_2107_y_0_to_fp16)[name = string("_inversed_2107_cast_fp16")];
            string scores_23_equation_0 = const()[name = string("scores_23_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 1]> scores_23_cast_fp16 = einsum(equation = scores_23_equation_0, values = (var_1991_cast_fp16_0, var_1981_cast_fp16_3))[name = string("scores_23_cast_fp16")];
            fp16 var_2110_to_fp16 = const()[name = string("op_2110_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 1]> var_2111_cast_fp16 = mul(x = scores_23_cast_fp16, y = var_2110_to_fp16)[name = string("op_2111_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_2112_cast_fp16 = add(x = var_2111_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_2112_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_2113_cast_fp16 = softmax(axis = var_1959, x = var_2112_cast_fp16)[name = string("op_2113_cast_fp16")];
            tensor<int32, [4]> tile_28 = const()[name = string("tile_28"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_2114_axis_0 = const()[name = string("op_2114_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 1]> var_2114_cast_fp16_0, tensor<fp16, [1, 512, 1, 1]> var_2114_cast_fp16_1, tensor<fp16, [1, 512, 1, 1]> var_2114_cast_fp16_2, tensor<fp16, [1, 512, 1, 1]> var_2114_cast_fp16_3 = split(axis = var_2114_axis_0, split_sizes = tile_28, x = var_2113_cast_fp16)[name = string("op_2114_cast_fp16")];
            tensor<int32, [4]> tile_29 = const()[name = string("tile_29"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_2119_axis_0 = const()[name = string("op_2119_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_2119_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_2119_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_2119_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_2119_cast_fp16_3 = split(axis = var_2119_axis_0, split_sizes = tile_29, x = var_1994_cast_fp16_0)[name = string("op_2119_cast_fp16")];
            fp16 var_2124_to_fp16 = const()[name = string("op_2124_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_2125_cast_fp16 = mul(x = var_2114_cast_fp16_0, y = var_2124_to_fp16)[name = string("op_2125_cast_fp16")];
            string var_2127_equation_0 = const()[name = string("op_2127_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_2127_cast_fp16 = einsum(equation = var_2127_equation_0, values = (var_2119_cast_fp16_0, var_2125_cast_fp16))[name = string("op_2127_cast_fp16")];
            fp16 var_2128_to_fp16 = const()[name = string("op_2128_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_2129_cast_fp16 = mul(x = var_2114_cast_fp16_1, y = var_2128_to_fp16)[name = string("op_2129_cast_fp16")];
            string var_2131_equation_0 = const()[name = string("op_2131_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_2131_cast_fp16 = einsum(equation = var_2131_equation_0, values = (var_2119_cast_fp16_1, var_2129_cast_fp16))[name = string("op_2131_cast_fp16")];
            fp16 var_2132_to_fp16 = const()[name = string("op_2132_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_2133_cast_fp16 = mul(x = var_2114_cast_fp16_2, y = var_2132_to_fp16)[name = string("op_2133_cast_fp16")];
            string var_2135_equation_0 = const()[name = string("op_2135_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_2135_cast_fp16 = einsum(equation = var_2135_equation_0, values = (var_2119_cast_fp16_2, var_2133_cast_fp16))[name = string("op_2135_cast_fp16")];
            fp16 var_2136_to_fp16 = const()[name = string("op_2136_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_2137_cast_fp16 = mul(x = var_2114_cast_fp16_3, y = var_2136_to_fp16)[name = string("op_2137_cast_fp16")];
            string var_2139_equation_0 = const()[name = string("op_2139_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_2139_cast_fp16 = einsum(equation = var_2139_equation_0, values = (var_2119_cast_fp16_3, var_2137_cast_fp16))[name = string("op_2139_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_2140_cast_fp16 = add(x = var_2127_cast_fp16, y = var_2131_cast_fp16)[name = string("op_2140_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_2141_cast_fp16 = add(x = var_2135_cast_fp16, y = var_2139_cast_fp16)[name = string("op_2141_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_2142_cast_fp16 = add(x = var_2140_cast_fp16, y = var_2141_cast_fp16)[name = string("op_2142_cast_fp16")];
            fp16 _inversed_2144_y_0_to_fp16 = const()[name = string("_inversed_2144_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 1]> _inversed_2144_cast_fp16 = mul(x = var_2142_cast_fp16, y = _inversed_2144_y_0_to_fp16)[name = string("_inversed_2144_cast_fp16")];
            string scores_25_equation_0 = const()[name = string("scores_25_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 1]> scores_25_cast_fp16 = einsum(equation = scores_25_equation_0, values = (var_1991_cast_fp16_1, var_1981_cast_fp16_4))[name = string("scores_25_cast_fp16")];
            fp16 var_2147_to_fp16 = const()[name = string("op_2147_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 1]> var_2148_cast_fp16 = mul(x = scores_25_cast_fp16, y = var_2147_to_fp16)[name = string("op_2148_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_2149_cast_fp16 = add(x = var_2148_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_2149_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_2150_cast_fp16 = softmax(axis = var_1959, x = var_2149_cast_fp16)[name = string("op_2150_cast_fp16")];
            tensor<int32, [4]> tile_30 = const()[name = string("tile_30"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_2151_axis_0 = const()[name = string("op_2151_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 1]> var_2151_cast_fp16_0, tensor<fp16, [1, 512, 1, 1]> var_2151_cast_fp16_1, tensor<fp16, [1, 512, 1, 1]> var_2151_cast_fp16_2, tensor<fp16, [1, 512, 1, 1]> var_2151_cast_fp16_3 = split(axis = var_2151_axis_0, split_sizes = tile_30, x = var_2150_cast_fp16)[name = string("op_2151_cast_fp16")];
            tensor<int32, [4]> tile_31 = const()[name = string("tile_31"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_2156_axis_0 = const()[name = string("op_2156_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_2156_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_2156_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_2156_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_2156_cast_fp16_3 = split(axis = var_2156_axis_0, split_sizes = tile_31, x = var_1994_cast_fp16_1)[name = string("op_2156_cast_fp16")];
            fp16 var_2161_to_fp16 = const()[name = string("op_2161_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_2162_cast_fp16 = mul(x = var_2151_cast_fp16_0, y = var_2161_to_fp16)[name = string("op_2162_cast_fp16")];
            string var_2164_equation_0 = const()[name = string("op_2164_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_2164_cast_fp16 = einsum(equation = var_2164_equation_0, values = (var_2156_cast_fp16_0, var_2162_cast_fp16))[name = string("op_2164_cast_fp16")];
            fp16 var_2165_to_fp16 = const()[name = string("op_2165_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_2166_cast_fp16 = mul(x = var_2151_cast_fp16_1, y = var_2165_to_fp16)[name = string("op_2166_cast_fp16")];
            string var_2168_equation_0 = const()[name = string("op_2168_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_2168_cast_fp16 = einsum(equation = var_2168_equation_0, values = (var_2156_cast_fp16_1, var_2166_cast_fp16))[name = string("op_2168_cast_fp16")];
            fp16 var_2169_to_fp16 = const()[name = string("op_2169_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_2170_cast_fp16 = mul(x = var_2151_cast_fp16_2, y = var_2169_to_fp16)[name = string("op_2170_cast_fp16")];
            string var_2172_equation_0 = const()[name = string("op_2172_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_2172_cast_fp16 = einsum(equation = var_2172_equation_0, values = (var_2156_cast_fp16_2, var_2170_cast_fp16))[name = string("op_2172_cast_fp16")];
            fp16 var_2173_to_fp16 = const()[name = string("op_2173_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_2174_cast_fp16 = mul(x = var_2151_cast_fp16_3, y = var_2173_to_fp16)[name = string("op_2174_cast_fp16")];
            string var_2176_equation_0 = const()[name = string("op_2176_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_2176_cast_fp16 = einsum(equation = var_2176_equation_0, values = (var_2156_cast_fp16_3, var_2174_cast_fp16))[name = string("op_2176_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_2177_cast_fp16 = add(x = var_2164_cast_fp16, y = var_2168_cast_fp16)[name = string("op_2177_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_2178_cast_fp16 = add(x = var_2172_cast_fp16, y = var_2176_cast_fp16)[name = string("op_2178_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_2179_cast_fp16 = add(x = var_2177_cast_fp16, y = var_2178_cast_fp16)[name = string("op_2179_cast_fp16")];
            fp16 _inversed_2181_y_0_to_fp16 = const()[name = string("_inversed_2181_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 1]> _inversed_2181_cast_fp16 = mul(x = var_2179_cast_fp16, y = _inversed_2181_y_0_to_fp16)[name = string("_inversed_2181_cast_fp16")];
            string scores_27_equation_0 = const()[name = string("scores_27_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 1]> scores_27_cast_fp16 = einsum(equation = scores_27_equation_0, values = (var_1991_cast_fp16_1, var_1981_cast_fp16_5))[name = string("scores_27_cast_fp16")];
            fp16 var_2184_to_fp16 = const()[name = string("op_2184_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 1]> var_2185_cast_fp16 = mul(x = scores_27_cast_fp16, y = var_2184_to_fp16)[name = string("op_2185_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_2186_cast_fp16 = add(x = var_2185_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_2186_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_2187_cast_fp16 = softmax(axis = var_1959, x = var_2186_cast_fp16)[name = string("op_2187_cast_fp16")];
            tensor<int32, [4]> tile_32 = const()[name = string("tile_32"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_2188_axis_0 = const()[name = string("op_2188_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 1]> var_2188_cast_fp16_0, tensor<fp16, [1, 512, 1, 1]> var_2188_cast_fp16_1, tensor<fp16, [1, 512, 1, 1]> var_2188_cast_fp16_2, tensor<fp16, [1, 512, 1, 1]> var_2188_cast_fp16_3 = split(axis = var_2188_axis_0, split_sizes = tile_32, x = var_2187_cast_fp16)[name = string("op_2188_cast_fp16")];
            tensor<int32, [4]> tile_33 = const()[name = string("tile_33"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_2193_axis_0 = const()[name = string("op_2193_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_2193_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_2193_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_2193_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_2193_cast_fp16_3 = split(axis = var_2193_axis_0, split_sizes = tile_33, x = var_1994_cast_fp16_1)[name = string("op_2193_cast_fp16")];
            fp16 var_2198_to_fp16 = const()[name = string("op_2198_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_2199_cast_fp16 = mul(x = var_2188_cast_fp16_0, y = var_2198_to_fp16)[name = string("op_2199_cast_fp16")];
            string var_2201_equation_0 = const()[name = string("op_2201_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_2201_cast_fp16 = einsum(equation = var_2201_equation_0, values = (var_2193_cast_fp16_0, var_2199_cast_fp16))[name = string("op_2201_cast_fp16")];
            fp16 var_2202_to_fp16 = const()[name = string("op_2202_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_2203_cast_fp16 = mul(x = var_2188_cast_fp16_1, y = var_2202_to_fp16)[name = string("op_2203_cast_fp16")];
            string var_2205_equation_0 = const()[name = string("op_2205_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_2205_cast_fp16 = einsum(equation = var_2205_equation_0, values = (var_2193_cast_fp16_1, var_2203_cast_fp16))[name = string("op_2205_cast_fp16")];
            fp16 var_2206_to_fp16 = const()[name = string("op_2206_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_2207_cast_fp16 = mul(x = var_2188_cast_fp16_2, y = var_2206_to_fp16)[name = string("op_2207_cast_fp16")];
            string var_2209_equation_0 = const()[name = string("op_2209_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_2209_cast_fp16 = einsum(equation = var_2209_equation_0, values = (var_2193_cast_fp16_2, var_2207_cast_fp16))[name = string("op_2209_cast_fp16")];
            fp16 var_2210_to_fp16 = const()[name = string("op_2210_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_2211_cast_fp16 = mul(x = var_2188_cast_fp16_3, y = var_2210_to_fp16)[name = string("op_2211_cast_fp16")];
            string var_2213_equation_0 = const()[name = string("op_2213_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_2213_cast_fp16 = einsum(equation = var_2213_equation_0, values = (var_2193_cast_fp16_3, var_2211_cast_fp16))[name = string("op_2213_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_2214_cast_fp16 = add(x = var_2201_cast_fp16, y = var_2205_cast_fp16)[name = string("op_2214_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_2215_cast_fp16 = add(x = var_2209_cast_fp16, y = var_2213_cast_fp16)[name = string("op_2215_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_2216_cast_fp16 = add(x = var_2214_cast_fp16, y = var_2215_cast_fp16)[name = string("op_2216_cast_fp16")];
            fp16 _inversed_2218_y_0_to_fp16 = const()[name = string("_inversed_2218_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 1]> _inversed_2218_cast_fp16 = mul(x = var_2216_cast_fp16, y = _inversed_2218_y_0_to_fp16)[name = string("_inversed_2218_cast_fp16")];
            string scores_29_equation_0 = const()[name = string("scores_29_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 1]> scores_29_cast_fp16 = einsum(equation = scores_29_equation_0, values = (var_1991_cast_fp16_1, var_1981_cast_fp16_6))[name = string("scores_29_cast_fp16")];
            fp16 var_2221_to_fp16 = const()[name = string("op_2221_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 1]> var_2222_cast_fp16 = mul(x = scores_29_cast_fp16, y = var_2221_to_fp16)[name = string("op_2222_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_2223_cast_fp16 = add(x = var_2222_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_2223_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_2224_cast_fp16 = softmax(axis = var_1959, x = var_2223_cast_fp16)[name = string("op_2224_cast_fp16")];
            tensor<int32, [4]> tile_34 = const()[name = string("tile_34"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_2225_axis_0 = const()[name = string("op_2225_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 1]> var_2225_cast_fp16_0, tensor<fp16, [1, 512, 1, 1]> var_2225_cast_fp16_1, tensor<fp16, [1, 512, 1, 1]> var_2225_cast_fp16_2, tensor<fp16, [1, 512, 1, 1]> var_2225_cast_fp16_3 = split(axis = var_2225_axis_0, split_sizes = tile_34, x = var_2224_cast_fp16)[name = string("op_2225_cast_fp16")];
            tensor<int32, [4]> tile_35 = const()[name = string("tile_35"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_2230_axis_0 = const()[name = string("op_2230_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_2230_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_2230_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_2230_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_2230_cast_fp16_3 = split(axis = var_2230_axis_0, split_sizes = tile_35, x = var_1994_cast_fp16_1)[name = string("op_2230_cast_fp16")];
            fp16 var_2235_to_fp16 = const()[name = string("op_2235_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_2236_cast_fp16 = mul(x = var_2225_cast_fp16_0, y = var_2235_to_fp16)[name = string("op_2236_cast_fp16")];
            string var_2238_equation_0 = const()[name = string("op_2238_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_2238_cast_fp16 = einsum(equation = var_2238_equation_0, values = (var_2230_cast_fp16_0, var_2236_cast_fp16))[name = string("op_2238_cast_fp16")];
            fp16 var_2239_to_fp16 = const()[name = string("op_2239_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_2240_cast_fp16 = mul(x = var_2225_cast_fp16_1, y = var_2239_to_fp16)[name = string("op_2240_cast_fp16")];
            string var_2242_equation_0 = const()[name = string("op_2242_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_2242_cast_fp16 = einsum(equation = var_2242_equation_0, values = (var_2230_cast_fp16_1, var_2240_cast_fp16))[name = string("op_2242_cast_fp16")];
            fp16 var_2243_to_fp16 = const()[name = string("op_2243_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_2244_cast_fp16 = mul(x = var_2225_cast_fp16_2, y = var_2243_to_fp16)[name = string("op_2244_cast_fp16")];
            string var_2246_equation_0 = const()[name = string("op_2246_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_2246_cast_fp16 = einsum(equation = var_2246_equation_0, values = (var_2230_cast_fp16_2, var_2244_cast_fp16))[name = string("op_2246_cast_fp16")];
            fp16 var_2247_to_fp16 = const()[name = string("op_2247_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_2248_cast_fp16 = mul(x = var_2225_cast_fp16_3, y = var_2247_to_fp16)[name = string("op_2248_cast_fp16")];
            string var_2250_equation_0 = const()[name = string("op_2250_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_2250_cast_fp16 = einsum(equation = var_2250_equation_0, values = (var_2230_cast_fp16_3, var_2248_cast_fp16))[name = string("op_2250_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_2251_cast_fp16 = add(x = var_2238_cast_fp16, y = var_2242_cast_fp16)[name = string("op_2251_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_2252_cast_fp16 = add(x = var_2246_cast_fp16, y = var_2250_cast_fp16)[name = string("op_2252_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_2253_cast_fp16 = add(x = var_2251_cast_fp16, y = var_2252_cast_fp16)[name = string("op_2253_cast_fp16")];
            fp16 _inversed_2255_y_0_to_fp16 = const()[name = string("_inversed_2255_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 1]> _inversed_2255_cast_fp16 = mul(x = var_2253_cast_fp16, y = _inversed_2255_y_0_to_fp16)[name = string("_inversed_2255_cast_fp16")];
            string scores_31_equation_0 = const()[name = string("scores_31_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 1]> scores_31_cast_fp16 = einsum(equation = scores_31_equation_0, values = (var_1991_cast_fp16_1, var_1981_cast_fp16_7))[name = string("scores_31_cast_fp16")];
            fp16 var_2258_to_fp16 = const()[name = string("op_2258_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 1]> var_2259_cast_fp16 = mul(x = scores_31_cast_fp16, y = var_2258_to_fp16)[name = string("op_2259_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_2260_cast_fp16 = add(x = var_2259_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_2260_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_2261_cast_fp16 = softmax(axis = var_1959, x = var_2260_cast_fp16)[name = string("op_2261_cast_fp16")];
            tensor<int32, [4]> tile_36 = const()[name = string("tile_36"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_2262_axis_0 = const()[name = string("op_2262_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 1]> var_2262_cast_fp16_0, tensor<fp16, [1, 512, 1, 1]> var_2262_cast_fp16_1, tensor<fp16, [1, 512, 1, 1]> var_2262_cast_fp16_2, tensor<fp16, [1, 512, 1, 1]> var_2262_cast_fp16_3 = split(axis = var_2262_axis_0, split_sizes = tile_36, x = var_2261_cast_fp16)[name = string("op_2262_cast_fp16")];
            tensor<int32, [4]> tile_37 = const()[name = string("tile_37"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_2267_axis_0 = const()[name = string("op_2267_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_2267_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_2267_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_2267_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_2267_cast_fp16_3 = split(axis = var_2267_axis_0, split_sizes = tile_37, x = var_1994_cast_fp16_1)[name = string("op_2267_cast_fp16")];
            fp16 var_2272_to_fp16 = const()[name = string("op_2272_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_2273_cast_fp16 = mul(x = var_2262_cast_fp16_0, y = var_2272_to_fp16)[name = string("op_2273_cast_fp16")];
            string var_2275_equation_0 = const()[name = string("op_2275_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_2275_cast_fp16 = einsum(equation = var_2275_equation_0, values = (var_2267_cast_fp16_0, var_2273_cast_fp16))[name = string("op_2275_cast_fp16")];
            fp16 var_2276_to_fp16 = const()[name = string("op_2276_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_2277_cast_fp16 = mul(x = var_2262_cast_fp16_1, y = var_2276_to_fp16)[name = string("op_2277_cast_fp16")];
            string var_2279_equation_0 = const()[name = string("op_2279_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_2279_cast_fp16 = einsum(equation = var_2279_equation_0, values = (var_2267_cast_fp16_1, var_2277_cast_fp16))[name = string("op_2279_cast_fp16")];
            fp16 var_2280_to_fp16 = const()[name = string("op_2280_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_2281_cast_fp16 = mul(x = var_2262_cast_fp16_2, y = var_2280_to_fp16)[name = string("op_2281_cast_fp16")];
            string var_2283_equation_0 = const()[name = string("op_2283_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_2283_cast_fp16 = einsum(equation = var_2283_equation_0, values = (var_2267_cast_fp16_2, var_2281_cast_fp16))[name = string("op_2283_cast_fp16")];
            fp16 var_2284_to_fp16 = const()[name = string("op_2284_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_2285_cast_fp16 = mul(x = var_2262_cast_fp16_3, y = var_2284_to_fp16)[name = string("op_2285_cast_fp16")];
            string var_2287_equation_0 = const()[name = string("op_2287_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_2287_cast_fp16 = einsum(equation = var_2287_equation_0, values = (var_2267_cast_fp16_3, var_2285_cast_fp16))[name = string("op_2287_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_2288_cast_fp16 = add(x = var_2275_cast_fp16, y = var_2279_cast_fp16)[name = string("op_2288_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_2289_cast_fp16 = add(x = var_2283_cast_fp16, y = var_2287_cast_fp16)[name = string("op_2289_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_2290_cast_fp16 = add(x = var_2288_cast_fp16, y = var_2289_cast_fp16)[name = string("op_2290_cast_fp16")];
            fp16 _inversed_2292_y_0_to_fp16 = const()[name = string("_inversed_2292_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 1]> _inversed_2292_cast_fp16 = mul(x = var_2290_cast_fp16, y = _inversed_2292_y_0_to_fp16)[name = string("_inversed_2292_cast_fp16")];
            bool var_2294_interleave_0 = const()[name = string("op_2294_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 2048, 1, 1]> var_2294_cast_fp16 = concat(axis = var_1959, interleave = var_2294_interleave_0, values = (_inversed_2033_cast_fp16, _inversed_2070_cast_fp16, _inversed_2107_cast_fp16, _inversed_2144_cast_fp16, _inversed_2181_cast_fp16, _inversed_2218_cast_fp16, _inversed_2255_cast_fp16, _inversed_2292_cast_fp16))[name = string("op_2294_cast_fp16")];
            tensor<int32, [4]> var_2295 = const()[name = string("op_2295"), val = tensor<int32, [4]>([2, 4, 256, 1])];
            tensor<fp16, [2, 4, 256, 1]> var_2296_cast_fp16 = reshape(shape = var_2295, x = var_2294_cast_fp16)[name = string("op_2296_cast_fp16")];
            tensor<int32, [4]> transpose_3_perm_0 = const()[name = string("transpose_3_perm_0"), val = tensor<int32, [4]>([3, 0, 1, 2])];
            tensor<int32, [2]> var_2314 = const()[name = string("op_2314"), val = tensor<int32, [2]>([1, 2048])];
            tensor<fp16, [1, 2, 4, 256]> transpose_3_cast_fp16 = transpose(perm = transpose_3_perm_0, x = var_2296_cast_fp16)[name = string("transpose_83")];
            tensor<fp16, [1, 2048]> attention_output_7_cast_fp16 = reshape(shape = var_2314, x = transpose_3_cast_fp16)[name = string("attention_output_7_cast_fp16")];
            tensor<fp16, [1, 2048]> var_2316_cast_fp16 = sigmoid(x = gate_7_cast_fp16)[name = string("op_2316_cast_fp16")];
            tensor<fp16, [1, 2048]> input_97_cast_fp16 = mul(x = attention_output_7_cast_fp16, y = var_2316_cast_fp16)[name = string("input_97_cast_fp16")];
            tensor<fp16, [1024, 2048]> completions_1_o_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 2048]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(116052608))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(117625536))))[name = string("completions_1_o_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 1024]> linear_58_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = completions_1_o_proj_weight_promoted_to_fp16_palettized, x = input_97_cast_fp16)[name = string("linear_58_cast_fp16")];
            tensor<fp16, [1, 1024]> value_115_cast_fp16 = add(x = value_101_cast_fp16, y = linear_58_cast_fp16)[name = string("value_115_cast_fp16")];
            tensor<fp16, [1, 1024]> var_2321_cast_fp16 = mul(x = value_115_cast_fp16, y = value_115_cast_fp16)[name = string("op_2321_cast_fp16")];
            tensor<int32, [1]> variance_51_axes_0 = const()[name = string("variance_51_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_51_keep_dims_0 = const()[name = string("variance_51_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 1]> variance_51_cast_fp16 = reduce_mean(axes = variance_51_axes_0, keep_dims = variance_51_keep_dims_0, x = var_2321_cast_fp16)[name = string("variance_51_cast_fp16")];
            fp16 var_2324_to_fp16 = const()[name = string("op_2324_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 1]> var_2325_cast_fp16 = add(x = variance_51_cast_fp16, y = var_2324_to_fp16)[name = string("op_2325_cast_fp16")];
            fp32 var_2326_epsilon_0 = const()[name = string("op_2326_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 1]> var_2326_cast_fp16 = rsqrt(epsilon = var_2326_epsilon_0, x = var_2325_cast_fp16)[name = string("op_2326_cast_fp16")];
            tensor<fp16, [1, 1024]> var_2327_cast_fp16 = mul(x = value_115_cast_fp16, y = var_2326_cast_fp16)[name = string("op_2327_cast_fp16")];
            tensor<fp16, [1024]> var_2329_to_fp16 = const()[name = string("op_2329_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(117633792)))];
            tensor<fp16, [1, 1024]> input_99_cast_fp16 = mul(x = var_2327_cast_fp16, y = var_2329_to_fp16)[name = string("input_99_cast_fp16")];
            tensor<fp16, [3584, 1024]> completions_1_gate_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(117635904))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(120388480))))[name = string("completions_1_gate_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 3584]> linear_59_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = completions_1_gate_proj_weight_promoted_to_fp16_palettized, x = input_99_cast_fp16)[name = string("linear_59_cast_fp16")];
            tensor<fp16, [1, 3584]> var_2333_cast_fp16 = silu(x = linear_59_cast_fp16)[name = string("op_2333_cast_fp16")];
            tensor<fp16, [3584, 1024]> completions_1_up_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(120417216))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(123169792))))[name = string("completions_1_up_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 3584]> linear_60_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = completions_1_up_proj_weight_promoted_to_fp16_palettized, x = input_99_cast_fp16)[name = string("linear_60_cast_fp16")];
            tensor<fp16, [1, 3584]> input_103_cast_fp16 = mul(x = var_2333_cast_fp16, y = linear_60_cast_fp16)[name = string("input_103_cast_fp16")];
            tensor<fp16, [1024, 3584]> completions_1_down_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 3584]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(123198528))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(125951104))))[name = string("completions_1_down_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 1024]> linear_61_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = completions_1_down_proj_weight_promoted_to_fp16_palettized, x = input_103_cast_fp16)[name = string("linear_61_cast_fp16")];
            tensor<fp16, [1, 1024]> value_117_cast_fp16 = add(x = value_115_cast_fp16, y = linear_61_cast_fp16)[name = string("value_117_cast_fp16")];
            int32 var_2354 = const()[name = string("op_2354"), val = int32(-1)];
            tensor<fp16, [1, 1024]> var_2369_cast_fp16 = mul(x = value_117_cast_fp16, y = value_117_cast_fp16)[name = string("op_2369_cast_fp16")];
            tensor<int32, [1]> variance_53_axes_0 = const()[name = string("variance_53_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_53_keep_dims_0 = const()[name = string("variance_53_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 1]> variance_53_cast_fp16 = reduce_mean(axes = variance_53_axes_0, keep_dims = variance_53_keep_dims_0, x = var_2369_cast_fp16)[name = string("variance_53_cast_fp16")];
            fp16 var_2372_to_fp16 = const()[name = string("op_2372_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 1]> var_2373_cast_fp16 = add(x = variance_53_cast_fp16, y = var_2372_to_fp16)[name = string("op_2373_cast_fp16")];
            fp32 var_2374_epsilon_0 = const()[name = string("op_2374_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 1]> var_2374_cast_fp16 = rsqrt(epsilon = var_2374_epsilon_0, x = var_2373_cast_fp16)[name = string("op_2374_cast_fp16")];
            tensor<fp16, [1, 1024]> var_2375_cast_fp16 = mul(x = value_117_cast_fp16, y = var_2374_cast_fp16)[name = string("op_2375_cast_fp16")];
            tensor<fp16, [1024]> var_2377_to_fp16 = const()[name = string("op_2377_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(125959360)))];
            tensor<fp16, [1, 1024]> input_105_cast_fp16 = mul(x = var_2375_cast_fp16, y = var_2377_to_fp16)[name = string("input_105_cast_fp16")];
            tensor<fp16, [6144, 1024]> groups_2_0_in_proj_qkv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(125961472))), lut = tensor<fp16, [384, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(130680128))))[name = string("groups_2_0_in_proj_qkv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 6144]> linear_62_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = groups_2_0_in_proj_qkv_weight_promoted_to_fp16_palettized, x = input_105_cast_fp16)[name = string("linear_62_cast_fp16")];
            tensor<int32, [2]> var_2381_perm_0 = const()[name = string("op_2381_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> mixed_13_axes_0 = const()[name = string("mixed_13_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [6144, 1]> var_2381_cast_fp16 = transpose(perm = var_2381_perm_0, x = linear_62_cast_fp16)[name = string("transpose_82")];
            tensor<fp16, [1, 6144, 1]> mixed_13_cast_fp16 = expand_dims(axes = mixed_13_axes_0, x = var_2381_cast_fp16)[name = string("mixed_13_cast_fp16")];
            bool convolution_input_13_interleave_0 = const()[name = string("convolution_input_13_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 6144, 4]> convolution_input_13_cast_fp16 = concat(axis = var_2354, interleave = convolution_input_13_interleave_0, values = (conv8, mixed_13_cast_fp16))[name = string("convolution_input_13_cast_fp16")];
            string input_107_pad_type_0 = const()[name = string("input_107_pad_type_0"), val = string("valid")];
            int32 input_107_groups_0 = const()[name = string("input_107_groups_0"), val = int32(6144)];
            tensor<int32, [1]> input_107_strides_0 = const()[name = string("input_107_strides_0"), val = tensor<int32, [1]>([1])];
            tensor<int32, [2]> input_107_pad_0 = const()[name = string("input_107_pad_0"), val = tensor<int32, [2]>([0, 0])];
            tensor<int32, [1]> input_107_dilations_0 = const()[name = string("input_107_dilations_0"), val = tensor<int32, [1]>([1])];
            tensor<fp16, [6144, 1, 4]> groups_2_0_conv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1, 4]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(130729344))), lut = tensor<fp16, [384, 1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(130747840))))[name = string("groups_2_0_conv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 6144, 1]> input_107_cast_fp16 = conv(dilations = input_107_dilations_0, groups = input_107_groups_0, pad = input_107_pad_0, pad_type = input_107_pad_type_0, strides = input_107_strides_0, weight = groups_2_0_conv_weight_promoted_to_fp16_palettized, x = convolution_input_13_cast_fp16)[name = string("input_107_cast_fp16")];
            tensor<fp16, [1, 6144, 1]> var_2390_cast_fp16 = silu(x = input_107_cast_fp16)[name = string("op_2390_cast_fp16")];
            tensor<int32, [3]> var_2391_perm_0 = const()[name = string("op_2391_perm_0"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [3]> var_2394_begin_0 = const()[name = string("op_2394_begin_0"), val = tensor<int32, [3]>([0, 0, 1])];
            tensor<int32, [3]> var_2394_end_0 = const()[name = string("op_2394_end_0"), val = tensor<int32, [3]>([1, 6144, 4])];
            tensor<bool, [3]> var_2394_end_mask_0 = const()[name = string("op_2394_end_mask_0"), val = tensor<bool, [3]>([true, true, true])];
            tensor<fp16, [1, 6144, 3]> conv8_out = slice_by_index(begin = var_2394_begin_0, end = var_2394_end_0, end_mask = var_2394_end_mask_0, x = convolution_input_13_cast_fp16)[name = string("op_2394_cast_fp16")];
            tensor<int32, [3]> var_2395 = const()[name = string("op_2395"), val = tensor<int32, [3]>([2048, 2048, 2048])];
            int32 var_2396_axis_0 = const()[name = string("op_2396_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 1, 6144]> var_2391_cast_fp16 = transpose(perm = var_2391_perm_0, x = var_2390_cast_fp16)[name = string("transpose_81")];
            tensor<fp16, [1, 1, 2048]> var_2396_cast_fp16_0, tensor<fp16, [1, 1, 2048]> var_2396_cast_fp16_1, tensor<fp16, [1, 1, 2048]> var_2396_cast_fp16_2 = split(axis = var_2396_axis_0, split_sizes = var_2395, x = var_2391_cast_fp16)[name = string("op_2396_cast_fp16")];
            tensor<int32, [4]> var_2400 = const()[name = string("op_2400"), val = tensor<int32, [4]>([1, 1, 16, 128])];
            tensor<fp16, [1, 1, 16, 128]> value_121_cast_fp16 = reshape(shape = var_2400, x = var_2396_cast_fp16_0)[name = string("value_121_cast_fp16")];
            tensor<int32, [4]> var_2402 = const()[name = string("op_2402"), val = tensor<int32, [4]>([1, 1, 16, 128])];
            tensor<fp16, [1, 1, 16, 128]> value_123_cast_fp16 = reshape(shape = var_2402, x = var_2396_cast_fp16_1)[name = string("value_123_cast_fp16")];
            tensor<int32, [4]> var_2404 = const()[name = string("op_2404"), val = tensor<int32, [4]>([1, 1, 16, 128])];
            tensor<fp16, [1, 1, 16, 128]> value_125_cast_fp16 = reshape(shape = var_2404, x = var_2396_cast_fp16_2)[name = string("value_125_cast_fp16")];
            tensor<fp16, [1, 1, 16, 128]> var_2406_cast_fp16 = mul(x = value_121_cast_fp16, y = value_121_cast_fp16)[name = string("op_2406_cast_fp16")];
            tensor<int32, [1]> var_2408_axes_0 = const()[name = string("op_2408_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_2408_keep_dims_0 = const()[name = string("op_2408_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 1, 16, 1]> var_2408_cast_fp16 = reduce_sum(axes = var_2408_axes_0, keep_dims = var_2408_keep_dims_0, x = var_2406_cast_fp16)[name = string("op_2408_cast_fp16")];
            fp16 var_2409_to_fp16 = const()[name = string("op_2409_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 1, 16, 1]> var_2410_cast_fp16 = add(x = var_2408_cast_fp16, y = var_2409_to_fp16)[name = string("op_2410_cast_fp16")];
            fp32 var_2411_epsilon_0 = const()[name = string("op_2411_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 1, 16, 1]> var_2411_cast_fp16 = rsqrt(epsilon = var_2411_epsilon_0, x = var_2410_cast_fp16)[name = string("op_2411_cast_fp16")];
            tensor<fp16, [1, 1, 16, 128]> var_2412_cast_fp16 = mul(x = value_121_cast_fp16, y = var_2411_cast_fp16)[name = string("op_2412_cast_fp16")];
            tensor<int32, [4]> var_2413_perm_0 = const()[name = string("op_2413_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            fp16 _inversed_query_35_y_0_to_fp16 = const()[name = string("_inversed_query_35_y_0_to_fp16"), val = fp16(0x1.6ap-4)];
            tensor<fp16, [1, 16, 1, 128]> var_2413_cast_fp16 = transpose(perm = var_2413_perm_0, x = var_2412_cast_fp16)[name = string("transpose_80")];
            tensor<fp16, [1, 16, 1, 128]> _inversed_query_35_cast_fp16 = mul(x = var_2413_cast_fp16, y = _inversed_query_35_y_0_to_fp16)[name = string("_inversed_query_35_cast_fp16")];
            tensor<fp16, [1, 1, 16, 128]> var_2416_cast_fp16 = mul(x = value_123_cast_fp16, y = value_123_cast_fp16)[name = string("op_2416_cast_fp16")];
            tensor<int32, [1]> var_2418_axes_0 = const()[name = string("op_2418_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_2418_keep_dims_0 = const()[name = string("op_2418_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 1, 16, 1]> var_2418_cast_fp16 = reduce_sum(axes = var_2418_axes_0, keep_dims = var_2418_keep_dims_0, x = var_2416_cast_fp16)[name = string("op_2418_cast_fp16")];
            fp16 var_2419_to_fp16 = const()[name = string("op_2419_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 1, 16, 1]> var_2420_cast_fp16 = add(x = var_2418_cast_fp16, y = var_2419_to_fp16)[name = string("op_2420_cast_fp16")];
            fp32 var_2421_epsilon_0 = const()[name = string("op_2421_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 1, 16, 1]> var_2421_cast_fp16 = rsqrt(epsilon = var_2421_epsilon_0, x = var_2420_cast_fp16)[name = string("op_2421_cast_fp16")];
            tensor<fp16, [1, 1, 16, 128]> var_2422_cast_fp16 = mul(x = value_123_cast_fp16, y = var_2421_cast_fp16)[name = string("op_2422_cast_fp16")];
            tensor<int32, [4]> key_35_perm_0 = const()[name = string("key_35_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<int32, [4]> value_127_perm_0 = const()[name = string("value_127_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<fp16, [16, 1024]> groups_2_0_in_proj_b_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(130797056))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(130809408))))[name = string("groups_2_0_in_proj_b_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16]> linear_63_bias_0_to_fp16 = const()[name = string("linear_63_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(4862656)))];
            tensor<fp16, [1, 16]> linear_63_cast_fp16 = linear(bias = linear_63_bias_0_to_fp16, weight = groups_2_0_in_proj_b_weight_promoted_to_fp16_palettized, x = input_105_cast_fp16)[name = string("linear_63_cast_fp16")];
            tensor<fp16, [1, 16]> var_2427_cast_fp16 = sigmoid(x = linear_63_cast_fp16)[name = string("op_2427_cast_fp16")];
            tensor<int32, [2]> var_2428_perm_0 = const()[name = string("op_2428_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> beta_13_axes_0 = const()[name = string("beta_13_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 1]> var_2428_cast_fp16 = transpose(perm = var_2428_perm_0, x = var_2427_cast_fp16)[name = string("transpose_79")];
            tensor<fp16, [1, 16, 1]> beta_13_cast_fp16 = expand_dims(axes = beta_13_axes_0, x = var_2428_cast_fp16)[name = string("beta_13_cast_fp16")];
            tensor<fp16, [16, 1024]> op_2434_weight_0_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(130809600))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(130821952))))[name = string("op_2434_weight_0_to_fp16_palettized")];
            tensor<fp16, [16]> var_2434_bias_0_to_fp16 = const()[name = string("op_2434_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(130822144)))];
            tensor<fp16, [1, 16]> var_2434_cast_fp16 = linear(bias = var_2434_bias_0_to_fp16, weight = op_2434_weight_0_to_fp16_palettized, x = input_105_cast_fp16)[name = string("op_2434_cast_fp16")];
            tensor<fp16, [1, 16]> var_2435_cast_fp16 = softplus(x = var_2434_cast_fp16)[name = string("op_2435_cast_fp16")];
            tensor<fp16, [16]> var_2431_promoted_to_fp16 = const()[name = string("op_2431_promoted_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(130822272)))];
            tensor<fp16, [1, 16]> var_2436_cast_fp16 = mul(x = var_2431_promoted_to_fp16, y = var_2435_cast_fp16)[name = string("op_2436_cast_fp16")];
            tensor<int32, [2]> var_2437_perm_0 = const()[name = string("op_2437_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> decay_13_axes_0 = const()[name = string("decay_13_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 1]> var_2437_cast_fp16 = transpose(perm = var_2437_perm_0, x = var_2436_cast_fp16)[name = string("transpose_78")];
            tensor<fp16, [1, 16, 1]> decay_13_cast_fp16 = expand_dims(axes = decay_13_axes_0, x = var_2437_cast_fp16)[name = string("decay_13_cast_fp16")];
            tensor<fp16, [2048, 1024]> groups_2_0_in_proj_z_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [2048, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(130822400))), lut = tensor<fp16, [128, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(132395328))))[name = string("groups_2_0_in_proj_z_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 2048]> linear_65_cast_fp16 = linear(bias = linear_3_bias_0_to_fp16, weight = groups_2_0_in_proj_z_weight_promoted_to_fp16_palettized, x = input_105_cast_fp16)[name = string("linear_65_cast_fp16")];
            tensor<int32, [3]> var_2445_begin_0 = const()[name = string("op_2445_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_2445_end_0 = const()[name = string("op_2445_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_2445_end_mask_0 = const()[name = string("op_2445_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_2445_squeeze_mask_0 = const()[name = string("op_2445_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_2445_cast_fp16 = slice_by_index(begin = var_2445_begin_0, end = var_2445_end_0, end_mask = var_2445_end_mask_0, squeeze_mask = var_2445_squeeze_mask_0, x = decay_13_cast_fp16)[name = string("op_2445_cast_fp16")];
            tensor<fp16, [1, 16]> var_2446_cast_fp16 = exp(x = var_2445_cast_fp16)[name = string("op_2446_cast_fp16")];
            tensor<int32, [1]> var_2447_axes_0 = const()[name = string("op_2447_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_2447_cast_fp16 = expand_dims(axes = var_2447_axes_0, x = var_2446_cast_fp16)[name = string("op_2447_cast_fp16")];
            tensor<int32, [1]> var_2448_axes_0 = const()[name = string("op_2448_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_2448_cast_fp16 = expand_dims(axes = var_2448_axes_0, x = var_2447_cast_fp16)[name = string("op_2448_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_25_cast_fp16 = mul(x = rec8, y = var_2448_cast_fp16)[name = string("state_25_cast_fp16")];
            tensor<int32, [4]> key_token_13_begin_0 = const()[name = string("key_token_13_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> key_token_13_end_0 = const()[name = string("key_token_13_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> key_token_13_end_mask_0 = const()[name = string("key_token_13_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_13_squeeze_mask_0 = const()[name = string("key_token_13_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 1, 128]> key_35_cast_fp16 = transpose(perm = key_35_perm_0, x = var_2422_cast_fp16)[name = string("transpose_77")];
            tensor<fp16, [1, 16, 128]> key_token_13_cast_fp16 = slice_by_index(begin = key_token_13_begin_0, end = key_token_13_end_0, end_mask = key_token_13_end_mask_0, squeeze_mask = key_token_13_squeeze_mask_0, x = key_35_cast_fp16)[name = string("key_token_13_cast_fp16")];
            tensor<int32, [4]> value_token_13_begin_0 = const()[name = string("value_token_13_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> value_token_13_end_0 = const()[name = string("value_token_13_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> value_token_13_end_mask_0 = const()[name = string("value_token_13_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_13_squeeze_mask_0 = const()[name = string("value_token_13_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 1, 128]> value_127_cast_fp16 = transpose(perm = value_127_perm_0, x = value_125_cast_fp16)[name = string("transpose_76")];
            tensor<fp16, [1, 16, 128]> value_token_13_cast_fp16 = slice_by_index(begin = value_token_13_begin_0, end = value_token_13_end_0, end_mask = value_token_13_end_mask_0, squeeze_mask = value_token_13_squeeze_mask_0, x = value_127_cast_fp16)[name = string("value_token_13_cast_fp16")];
            tensor<int32, [1]> var_2456_axes_0 = const()[name = string("op_2456_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_2456_cast_fp16 = expand_dims(axes = var_2456_axes_0, x = key_token_13_cast_fp16)[name = string("op_2456_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2457_cast_fp16 = mul(x = state_25_cast_fp16, y = var_2456_cast_fp16)[name = string("op_2457_cast_fp16")];
            tensor<int32, [1]> memory_13_axes_0 = const()[name = string("memory_13_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_13_keep_dims_0 = const()[name = string("memory_13_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_13_cast_fp16 = reduce_sum(axes = memory_13_axes_0, keep_dims = memory_13_keep_dims_0, x = var_2457_cast_fp16)[name = string("memory_13_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_2460_cast_fp16 = sub(x = value_token_13_cast_fp16, y = memory_13_cast_fp16)[name = string("op_2460_cast_fp16")];
            tensor<int32, [3]> var_2463_begin_0 = const()[name = string("op_2463_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_2463_end_0 = const()[name = string("op_2463_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_2463_end_mask_0 = const()[name = string("op_2463_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_2463_squeeze_mask_0 = const()[name = string("op_2463_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_2463_cast_fp16 = slice_by_index(begin = var_2463_begin_0, end = var_2463_end_0, end_mask = var_2463_end_mask_0, squeeze_mask = var_2463_squeeze_mask_0, x = beta_13_cast_fp16)[name = string("op_2463_cast_fp16")];
            tensor<int32, [1]> var_2464_axes_0 = const()[name = string("op_2464_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_2464_cast_fp16 = expand_dims(axes = var_2464_axes_0, x = var_2463_cast_fp16)[name = string("op_2464_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_13_cast_fp16 = mul(x = var_2460_cast_fp16, y = var_2464_cast_fp16)[name = string("delta_13_cast_fp16")];
            tensor<int32, [1]> var_2467_axes_0 = const()[name = string("op_2467_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_2467_cast_fp16 = expand_dims(axes = var_2467_axes_0, x = delta_13_cast_fp16)[name = string("op_2467_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2468_cast_fp16 = mul(x = var_2456_cast_fp16, y = var_2467_cast_fp16)[name = string("op_2468_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> rec8_out = add(x = state_25_cast_fp16, y = var_2468_cast_fp16)[name = string("state_27_cast_fp16")];
            tensor<int32, [4]> var_2472_begin_0 = const()[name = string("op_2472_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> var_2472_end_0 = const()[name = string("op_2472_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> var_2472_end_mask_0 = const()[name = string("op_2472_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_2472_squeeze_mask_0 = const()[name = string("op_2472_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_2472_cast_fp16 = slice_by_index(begin = var_2472_begin_0, end = var_2472_end_0, end_mask = var_2472_end_mask_0, squeeze_mask = var_2472_squeeze_mask_0, x = _inversed_query_35_cast_fp16)[name = string("op_2472_cast_fp16")];
            tensor<int32, [1]> var_2473_axes_0 = const()[name = string("op_2473_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_2473_cast_fp16 = expand_dims(axes = var_2473_axes_0, x = var_2472_cast_fp16)[name = string("op_2473_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2474_cast_fp16 = mul(x = rec8_out, y = var_2473_cast_fp16)[name = string("op_2474_cast_fp16")];
            tensor<int32, [1]> var_2476_axes_0 = const()[name = string("op_2476_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_2476_keep_dims_0 = const()[name = string("op_2476_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_2476_cast_fp16 = reduce_sum(axes = var_2476_axes_0, keep_dims = var_2476_keep_dims_0, x = var_2474_cast_fp16)[name = string("op_2476_cast_fp16")];
            tensor<int32, [1]> attention_61_axes_0 = const()[name = string("attention_61_axes_0"), val = tensor<int32, [1]>([1])];
            tensor<fp16, [1, 1, 16, 128]> attention_61_cast_fp16 = expand_dims(axes = attention_61_axes_0, x = var_2476_cast_fp16)[name = string("attention_61_cast_fp16")];
            tensor<int32, [2]> var_2479 = const()[name = string("op_2479"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [16, 128]> attention_63_cast_fp16 = reshape(shape = var_2479, x = attention_61_cast_fp16)[name = string("attention_63_cast_fp16")];
            tensor<int32, [2]> var_2481 = const()[name = string("op_2481"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [16, 128]> input_109_cast_fp16 = reshape(shape = var_2481, x = linear_65_cast_fp16)[name = string("input_109_cast_fp16")];
            tensor<fp16, [16, 128]> var_2483_cast_fp16 = mul(x = attention_63_cast_fp16, y = attention_63_cast_fp16)[name = string("op_2483_cast_fp16")];
            tensor<int32, [1]> variance_55_axes_0 = const()[name = string("variance_55_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_55_keep_dims_0 = const()[name = string("variance_55_keep_dims_0"), val = bool(true)];
            tensor<fp16, [16, 1]> variance_55_cast_fp16 = reduce_mean(axes = variance_55_axes_0, keep_dims = variance_55_keep_dims_0, x = var_2483_cast_fp16)[name = string("variance_55_cast_fp16")];
            fp16 var_2486_to_fp16 = const()[name = string("op_2486_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [16, 1]> var_2487_cast_fp16 = add(x = variance_55_cast_fp16, y = var_2486_to_fp16)[name = string("op_2487_cast_fp16")];
            fp32 var_2488_epsilon_0 = const()[name = string("op_2488_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [16, 1]> var_2488_cast_fp16 = rsqrt(epsilon = var_2488_epsilon_0, x = var_2487_cast_fp16)[name = string("op_2488_cast_fp16")];
            tensor<fp16, [16, 128]> attention_65_cast_fp16 = mul(x = attention_63_cast_fp16, y = var_2488_cast_fp16)[name = string("attention_65_cast_fp16")];
            tensor<fp16, [128]> groups_2_0_gated_norm_promoted_to_fp16 = const()[name = string("groups_2_0_gated_norm_promoted_to_fp16"), val = tensor<fp16, [128]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(132411776)))];
            tensor<fp16, [16, 128]> attention_67_cast_fp16 = mul(x = attention_65_cast_fp16, y = groups_2_0_gated_norm_promoted_to_fp16)[name = string("attention_67_cast_fp16")];
            tensor<fp16, [16, 128]> var_2491_cast_fp16 = silu(x = input_109_cast_fp16)[name = string("op_2491_cast_fp16")];
            tensor<fp16, [16, 128]> attention_69_cast_fp16 = mul(x = attention_67_cast_fp16, y = var_2491_cast_fp16)[name = string("attention_69_cast_fp16")];
            tensor<int32, [2]> var_2493 = const()[name = string("op_2493"), val = tensor<int32, [2]>([1, 2048])];
            tensor<fp16, [1, 2048]> input_111_cast_fp16 = reshape(shape = var_2493, x = attention_69_cast_fp16)[name = string("input_111_cast_fp16")];
            tensor<fp16, [1024, 2048]> groups_2_0_out_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 2048]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(132412096))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(133985024))))[name = string("groups_2_0_out_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 1024]> linear_66_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = groups_2_0_out_proj_weight_promoted_to_fp16_palettized, x = input_111_cast_fp16)[name = string("linear_66_cast_fp16")];
            tensor<fp16, [1, 1024]> value_129_cast_fp16 = add(x = value_117_cast_fp16, y = linear_66_cast_fp16)[name = string("value_129_cast_fp16")];
            tensor<fp16, [1, 1024]> var_2498_cast_fp16 = mul(x = value_129_cast_fp16, y = value_129_cast_fp16)[name = string("op_2498_cast_fp16")];
            tensor<int32, [1]> variance_57_axes_0 = const()[name = string("variance_57_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_57_keep_dims_0 = const()[name = string("variance_57_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 1]> variance_57_cast_fp16 = reduce_mean(axes = variance_57_axes_0, keep_dims = variance_57_keep_dims_0, x = var_2498_cast_fp16)[name = string("variance_57_cast_fp16")];
            fp16 var_2501_to_fp16 = const()[name = string("op_2501_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 1]> var_2502_cast_fp16 = add(x = variance_57_cast_fp16, y = var_2501_to_fp16)[name = string("op_2502_cast_fp16")];
            fp32 var_2503_epsilon_0 = const()[name = string("op_2503_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 1]> var_2503_cast_fp16 = rsqrt(epsilon = var_2503_epsilon_0, x = var_2502_cast_fp16)[name = string("op_2503_cast_fp16")];
            tensor<fp16, [1, 1024]> var_2504_cast_fp16 = mul(x = value_129_cast_fp16, y = var_2503_cast_fp16)[name = string("op_2504_cast_fp16")];
            tensor<fp16, [1024]> var_2506_to_fp16 = const()[name = string("op_2506_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(133993280)))];
            tensor<fp16, [1, 1024]> input_113_cast_fp16 = mul(x = var_2504_cast_fp16, y = var_2506_to_fp16)[name = string("input_113_cast_fp16")];
            tensor<fp16, [3584, 1024]> groups_2_0_gate_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(133995392))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(136747968))))[name = string("groups_2_0_gate_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 3584]> linear_67_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = groups_2_0_gate_proj_weight_promoted_to_fp16_palettized, x = input_113_cast_fp16)[name = string("linear_67_cast_fp16")];
            tensor<fp16, [1, 3584]> var_2510_cast_fp16 = silu(x = linear_67_cast_fp16)[name = string("op_2510_cast_fp16")];
            tensor<fp16, [3584, 1024]> groups_2_0_up_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(136776704))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(139529280))))[name = string("groups_2_0_up_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 3584]> linear_68_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = groups_2_0_up_proj_weight_promoted_to_fp16_palettized, x = input_113_cast_fp16)[name = string("linear_68_cast_fp16")];
            tensor<fp16, [1, 3584]> input_117_cast_fp16 = mul(x = var_2510_cast_fp16, y = linear_68_cast_fp16)[name = string("input_117_cast_fp16")];
            tensor<fp16, [1024, 3584]> groups_2_0_down_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 3584]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(139558016))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(142310592))))[name = string("groups_2_0_down_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 1024]> linear_69_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = groups_2_0_down_proj_weight_promoted_to_fp16_palettized, x = input_117_cast_fp16)[name = string("linear_69_cast_fp16")];
            tensor<fp16, [1, 1024]> value_131_cast_fp16 = add(x = value_129_cast_fp16, y = linear_69_cast_fp16)[name = string("value_131_cast_fp16")];
            int32 var_2535 = const()[name = string("op_2535"), val = int32(-1)];
            tensor<fp16, [1, 1024]> var_2550_cast_fp16 = mul(x = value_131_cast_fp16, y = value_131_cast_fp16)[name = string("op_2550_cast_fp16")];
            tensor<int32, [1]> variance_59_axes_0 = const()[name = string("variance_59_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_59_keep_dims_0 = const()[name = string("variance_59_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 1]> variance_59_cast_fp16 = reduce_mean(axes = variance_59_axes_0, keep_dims = variance_59_keep_dims_0, x = var_2550_cast_fp16)[name = string("variance_59_cast_fp16")];
            fp16 var_2553_to_fp16 = const()[name = string("op_2553_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 1]> var_2554_cast_fp16 = add(x = variance_59_cast_fp16, y = var_2553_to_fp16)[name = string("op_2554_cast_fp16")];
            fp32 var_2555_epsilon_0 = const()[name = string("op_2555_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 1]> var_2555_cast_fp16 = rsqrt(epsilon = var_2555_epsilon_0, x = var_2554_cast_fp16)[name = string("op_2555_cast_fp16")];
            tensor<fp16, [1, 1024]> var_2556_cast_fp16 = mul(x = value_131_cast_fp16, y = var_2555_cast_fp16)[name = string("op_2556_cast_fp16")];
            tensor<fp16, [1024]> var_2558_to_fp16 = const()[name = string("op_2558_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(142318848)))];
            tensor<fp16, [1, 1024]> input_119_cast_fp16 = mul(x = var_2556_cast_fp16, y = var_2558_to_fp16)[name = string("input_119_cast_fp16")];
            tensor<fp16, [6144, 1024]> groups_2_1_in_proj_qkv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(142320960))), lut = tensor<fp16, [384, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(147039616))))[name = string("groups_2_1_in_proj_qkv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 6144]> linear_70_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = groups_2_1_in_proj_qkv_weight_promoted_to_fp16_palettized, x = input_119_cast_fp16)[name = string("linear_70_cast_fp16")];
            tensor<int32, [2]> var_2562_perm_0 = const()[name = string("op_2562_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> mixed_15_axes_0 = const()[name = string("mixed_15_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [6144, 1]> var_2562_cast_fp16 = transpose(perm = var_2562_perm_0, x = linear_70_cast_fp16)[name = string("transpose_75")];
            tensor<fp16, [1, 6144, 1]> mixed_15_cast_fp16 = expand_dims(axes = mixed_15_axes_0, x = var_2562_cast_fp16)[name = string("mixed_15_cast_fp16")];
            bool convolution_input_15_interleave_0 = const()[name = string("convolution_input_15_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 6144, 4]> convolution_input_15_cast_fp16 = concat(axis = var_2535, interleave = convolution_input_15_interleave_0, values = (conv9, mixed_15_cast_fp16))[name = string("convolution_input_15_cast_fp16")];
            string input_121_pad_type_0 = const()[name = string("input_121_pad_type_0"), val = string("valid")];
            int32 input_121_groups_0 = const()[name = string("input_121_groups_0"), val = int32(6144)];
            tensor<int32, [1]> input_121_strides_0 = const()[name = string("input_121_strides_0"), val = tensor<int32, [1]>([1])];
            tensor<int32, [2]> input_121_pad_0 = const()[name = string("input_121_pad_0"), val = tensor<int32, [2]>([0, 0])];
            tensor<int32, [1]> input_121_dilations_0 = const()[name = string("input_121_dilations_0"), val = tensor<int32, [1]>([1])];
            tensor<fp16, [6144, 1, 4]> groups_2_1_conv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1, 4]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(147088832))), lut = tensor<fp16, [384, 1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(147107328))))[name = string("groups_2_1_conv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 6144, 1]> input_121_cast_fp16 = conv(dilations = input_121_dilations_0, groups = input_121_groups_0, pad = input_121_pad_0, pad_type = input_121_pad_type_0, strides = input_121_strides_0, weight = groups_2_1_conv_weight_promoted_to_fp16_palettized, x = convolution_input_15_cast_fp16)[name = string("input_121_cast_fp16")];
            tensor<fp16, [1, 6144, 1]> var_2571_cast_fp16 = silu(x = input_121_cast_fp16)[name = string("op_2571_cast_fp16")];
            tensor<int32, [3]> var_2572_perm_0 = const()[name = string("op_2572_perm_0"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [3]> var_2575_begin_0 = const()[name = string("op_2575_begin_0"), val = tensor<int32, [3]>([0, 0, 1])];
            tensor<int32, [3]> var_2575_end_0 = const()[name = string("op_2575_end_0"), val = tensor<int32, [3]>([1, 6144, 4])];
            tensor<bool, [3]> var_2575_end_mask_0 = const()[name = string("op_2575_end_mask_0"), val = tensor<bool, [3]>([true, true, true])];
            tensor<fp16, [1, 6144, 3]> conv9_out = slice_by_index(begin = var_2575_begin_0, end = var_2575_end_0, end_mask = var_2575_end_mask_0, x = convolution_input_15_cast_fp16)[name = string("op_2575_cast_fp16")];
            tensor<int32, [3]> var_2576 = const()[name = string("op_2576"), val = tensor<int32, [3]>([2048, 2048, 2048])];
            int32 var_2577_axis_0 = const()[name = string("op_2577_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 1, 6144]> var_2572_cast_fp16 = transpose(perm = var_2572_perm_0, x = var_2571_cast_fp16)[name = string("transpose_74")];
            tensor<fp16, [1, 1, 2048]> var_2577_cast_fp16_0, tensor<fp16, [1, 1, 2048]> var_2577_cast_fp16_1, tensor<fp16, [1, 1, 2048]> var_2577_cast_fp16_2 = split(axis = var_2577_axis_0, split_sizes = var_2576, x = var_2572_cast_fp16)[name = string("op_2577_cast_fp16")];
            tensor<int32, [4]> var_2581 = const()[name = string("op_2581"), val = tensor<int32, [4]>([1, 1, 16, 128])];
            tensor<fp16, [1, 1, 16, 128]> value_135_cast_fp16 = reshape(shape = var_2581, x = var_2577_cast_fp16_0)[name = string("value_135_cast_fp16")];
            tensor<int32, [4]> var_2583 = const()[name = string("op_2583"), val = tensor<int32, [4]>([1, 1, 16, 128])];
            tensor<fp16, [1, 1, 16, 128]> value_137_cast_fp16 = reshape(shape = var_2583, x = var_2577_cast_fp16_1)[name = string("value_137_cast_fp16")];
            tensor<int32, [4]> var_2585 = const()[name = string("op_2585"), val = tensor<int32, [4]>([1, 1, 16, 128])];
            tensor<fp16, [1, 1, 16, 128]> value_139_cast_fp16 = reshape(shape = var_2585, x = var_2577_cast_fp16_2)[name = string("value_139_cast_fp16")];
            tensor<fp16, [1, 1, 16, 128]> var_2587_cast_fp16 = mul(x = value_135_cast_fp16, y = value_135_cast_fp16)[name = string("op_2587_cast_fp16")];
            tensor<int32, [1]> var_2589_axes_0 = const()[name = string("op_2589_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_2589_keep_dims_0 = const()[name = string("op_2589_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 1, 16, 1]> var_2589_cast_fp16 = reduce_sum(axes = var_2589_axes_0, keep_dims = var_2589_keep_dims_0, x = var_2587_cast_fp16)[name = string("op_2589_cast_fp16")];
            fp16 var_2590_to_fp16 = const()[name = string("op_2590_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 1, 16, 1]> var_2591_cast_fp16 = add(x = var_2589_cast_fp16, y = var_2590_to_fp16)[name = string("op_2591_cast_fp16")];
            fp32 var_2592_epsilon_0 = const()[name = string("op_2592_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 1, 16, 1]> var_2592_cast_fp16 = rsqrt(epsilon = var_2592_epsilon_0, x = var_2591_cast_fp16)[name = string("op_2592_cast_fp16")];
            tensor<fp16, [1, 1, 16, 128]> var_2593_cast_fp16 = mul(x = value_135_cast_fp16, y = var_2592_cast_fp16)[name = string("op_2593_cast_fp16")];
            tensor<int32, [4]> var_2594_perm_0 = const()[name = string("op_2594_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            fp16 _inversed_query_39_y_0_to_fp16 = const()[name = string("_inversed_query_39_y_0_to_fp16"), val = fp16(0x1.6ap-4)];
            tensor<fp16, [1, 16, 1, 128]> var_2594_cast_fp16 = transpose(perm = var_2594_perm_0, x = var_2593_cast_fp16)[name = string("transpose_73")];
            tensor<fp16, [1, 16, 1, 128]> _inversed_query_39_cast_fp16 = mul(x = var_2594_cast_fp16, y = _inversed_query_39_y_0_to_fp16)[name = string("_inversed_query_39_cast_fp16")];
            tensor<fp16, [1, 1, 16, 128]> var_2597_cast_fp16 = mul(x = value_137_cast_fp16, y = value_137_cast_fp16)[name = string("op_2597_cast_fp16")];
            tensor<int32, [1]> var_2599_axes_0 = const()[name = string("op_2599_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_2599_keep_dims_0 = const()[name = string("op_2599_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 1, 16, 1]> var_2599_cast_fp16 = reduce_sum(axes = var_2599_axes_0, keep_dims = var_2599_keep_dims_0, x = var_2597_cast_fp16)[name = string("op_2599_cast_fp16")];
            fp16 var_2600_to_fp16 = const()[name = string("op_2600_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 1, 16, 1]> var_2601_cast_fp16 = add(x = var_2599_cast_fp16, y = var_2600_to_fp16)[name = string("op_2601_cast_fp16")];
            fp32 var_2602_epsilon_0 = const()[name = string("op_2602_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 1, 16, 1]> var_2602_cast_fp16 = rsqrt(epsilon = var_2602_epsilon_0, x = var_2601_cast_fp16)[name = string("op_2602_cast_fp16")];
            tensor<fp16, [1, 1, 16, 128]> var_2603_cast_fp16 = mul(x = value_137_cast_fp16, y = var_2602_cast_fp16)[name = string("op_2603_cast_fp16")];
            tensor<int32, [4]> key_39_perm_0 = const()[name = string("key_39_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<int32, [4]> value_141_perm_0 = const()[name = string("value_141_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<fp16, [16, 1024]> groups_2_1_in_proj_b_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(147156544))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(147168896))))[name = string("groups_2_1_in_proj_b_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16]> linear_71_bias_0_to_fp16 = const()[name = string("linear_71_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(4862656)))];
            tensor<fp16, [1, 16]> linear_71_cast_fp16 = linear(bias = linear_71_bias_0_to_fp16, weight = groups_2_1_in_proj_b_weight_promoted_to_fp16_palettized, x = input_119_cast_fp16)[name = string("linear_71_cast_fp16")];
            tensor<fp16, [1, 16]> var_2608_cast_fp16 = sigmoid(x = linear_71_cast_fp16)[name = string("op_2608_cast_fp16")];
            tensor<int32, [2]> var_2609_perm_0 = const()[name = string("op_2609_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> beta_15_axes_0 = const()[name = string("beta_15_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 1]> var_2609_cast_fp16 = transpose(perm = var_2609_perm_0, x = var_2608_cast_fp16)[name = string("transpose_72")];
            tensor<fp16, [1, 16, 1]> beta_15_cast_fp16 = expand_dims(axes = beta_15_axes_0, x = var_2609_cast_fp16)[name = string("beta_15_cast_fp16")];
            tensor<fp16, [16, 1024]> op_2615_weight_0_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(147169088))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(147181440))))[name = string("op_2615_weight_0_to_fp16_palettized")];
            tensor<fp16, [16]> var_2615_bias_0_to_fp16 = const()[name = string("op_2615_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(147181632)))];
            tensor<fp16, [1, 16]> var_2615_cast_fp16 = linear(bias = var_2615_bias_0_to_fp16, weight = op_2615_weight_0_to_fp16_palettized, x = input_119_cast_fp16)[name = string("op_2615_cast_fp16")];
            tensor<fp16, [1, 16]> var_2616_cast_fp16 = softplus(x = var_2615_cast_fp16)[name = string("op_2616_cast_fp16")];
            tensor<fp16, [16]> var_2612_promoted_to_fp16 = const()[name = string("op_2612_promoted_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(147181760)))];
            tensor<fp16, [1, 16]> var_2617_cast_fp16 = mul(x = var_2612_promoted_to_fp16, y = var_2616_cast_fp16)[name = string("op_2617_cast_fp16")];
            tensor<int32, [2]> var_2618_perm_0 = const()[name = string("op_2618_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> decay_15_axes_0 = const()[name = string("decay_15_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 1]> var_2618_cast_fp16 = transpose(perm = var_2618_perm_0, x = var_2617_cast_fp16)[name = string("transpose_71")];
            tensor<fp16, [1, 16, 1]> decay_15_cast_fp16 = expand_dims(axes = decay_15_axes_0, x = var_2618_cast_fp16)[name = string("decay_15_cast_fp16")];
            tensor<fp16, [2048, 1024]> groups_2_1_in_proj_z_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [2048, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(147181888))), lut = tensor<fp16, [128, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(148754816))))[name = string("groups_2_1_in_proj_z_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 2048]> linear_73_cast_fp16 = linear(bias = linear_3_bias_0_to_fp16, weight = groups_2_1_in_proj_z_weight_promoted_to_fp16_palettized, x = input_119_cast_fp16)[name = string("linear_73_cast_fp16")];
            tensor<int32, [3]> var_2626_begin_0 = const()[name = string("op_2626_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_2626_end_0 = const()[name = string("op_2626_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_2626_end_mask_0 = const()[name = string("op_2626_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_2626_squeeze_mask_0 = const()[name = string("op_2626_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_2626_cast_fp16 = slice_by_index(begin = var_2626_begin_0, end = var_2626_end_0, end_mask = var_2626_end_mask_0, squeeze_mask = var_2626_squeeze_mask_0, x = decay_15_cast_fp16)[name = string("op_2626_cast_fp16")];
            tensor<fp16, [1, 16]> var_2627_cast_fp16 = exp(x = var_2626_cast_fp16)[name = string("op_2627_cast_fp16")];
            tensor<int32, [1]> var_2628_axes_0 = const()[name = string("op_2628_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_2628_cast_fp16 = expand_dims(axes = var_2628_axes_0, x = var_2627_cast_fp16)[name = string("op_2628_cast_fp16")];
            tensor<int32, [1]> var_2629_axes_0 = const()[name = string("op_2629_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_2629_cast_fp16 = expand_dims(axes = var_2629_axes_0, x = var_2628_cast_fp16)[name = string("op_2629_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_29_cast_fp16 = mul(x = rec9, y = var_2629_cast_fp16)[name = string("state_29_cast_fp16")];
            tensor<int32, [4]> key_token_15_begin_0 = const()[name = string("key_token_15_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> key_token_15_end_0 = const()[name = string("key_token_15_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> key_token_15_end_mask_0 = const()[name = string("key_token_15_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_15_squeeze_mask_0 = const()[name = string("key_token_15_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 1, 128]> key_39_cast_fp16 = transpose(perm = key_39_perm_0, x = var_2603_cast_fp16)[name = string("transpose_70")];
            tensor<fp16, [1, 16, 128]> key_token_15_cast_fp16 = slice_by_index(begin = key_token_15_begin_0, end = key_token_15_end_0, end_mask = key_token_15_end_mask_0, squeeze_mask = key_token_15_squeeze_mask_0, x = key_39_cast_fp16)[name = string("key_token_15_cast_fp16")];
            tensor<int32, [4]> value_token_15_begin_0 = const()[name = string("value_token_15_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> value_token_15_end_0 = const()[name = string("value_token_15_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> value_token_15_end_mask_0 = const()[name = string("value_token_15_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_15_squeeze_mask_0 = const()[name = string("value_token_15_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 1, 128]> value_141_cast_fp16 = transpose(perm = value_141_perm_0, x = value_139_cast_fp16)[name = string("transpose_69")];
            tensor<fp16, [1, 16, 128]> value_token_15_cast_fp16 = slice_by_index(begin = value_token_15_begin_0, end = value_token_15_end_0, end_mask = value_token_15_end_mask_0, squeeze_mask = value_token_15_squeeze_mask_0, x = value_141_cast_fp16)[name = string("value_token_15_cast_fp16")];
            tensor<int32, [1]> var_2637_axes_0 = const()[name = string("op_2637_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_2637_cast_fp16 = expand_dims(axes = var_2637_axes_0, x = key_token_15_cast_fp16)[name = string("op_2637_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2638_cast_fp16 = mul(x = state_29_cast_fp16, y = var_2637_cast_fp16)[name = string("op_2638_cast_fp16")];
            tensor<int32, [1]> memory_15_axes_0 = const()[name = string("memory_15_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_15_keep_dims_0 = const()[name = string("memory_15_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_15_cast_fp16 = reduce_sum(axes = memory_15_axes_0, keep_dims = memory_15_keep_dims_0, x = var_2638_cast_fp16)[name = string("memory_15_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_2641_cast_fp16 = sub(x = value_token_15_cast_fp16, y = memory_15_cast_fp16)[name = string("op_2641_cast_fp16")];
            tensor<int32, [3]> var_2644_begin_0 = const()[name = string("op_2644_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_2644_end_0 = const()[name = string("op_2644_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_2644_end_mask_0 = const()[name = string("op_2644_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_2644_squeeze_mask_0 = const()[name = string("op_2644_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_2644_cast_fp16 = slice_by_index(begin = var_2644_begin_0, end = var_2644_end_0, end_mask = var_2644_end_mask_0, squeeze_mask = var_2644_squeeze_mask_0, x = beta_15_cast_fp16)[name = string("op_2644_cast_fp16")];
            tensor<int32, [1]> var_2645_axes_0 = const()[name = string("op_2645_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_2645_cast_fp16 = expand_dims(axes = var_2645_axes_0, x = var_2644_cast_fp16)[name = string("op_2645_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_15_cast_fp16 = mul(x = var_2641_cast_fp16, y = var_2645_cast_fp16)[name = string("delta_15_cast_fp16")];
            tensor<int32, [1]> var_2648_axes_0 = const()[name = string("op_2648_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_2648_cast_fp16 = expand_dims(axes = var_2648_axes_0, x = delta_15_cast_fp16)[name = string("op_2648_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2649_cast_fp16 = mul(x = var_2637_cast_fp16, y = var_2648_cast_fp16)[name = string("op_2649_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> rec9_out = add(x = state_29_cast_fp16, y = var_2649_cast_fp16)[name = string("state_31_cast_fp16")];
            tensor<int32, [4]> var_2653_begin_0 = const()[name = string("op_2653_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> var_2653_end_0 = const()[name = string("op_2653_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> var_2653_end_mask_0 = const()[name = string("op_2653_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_2653_squeeze_mask_0 = const()[name = string("op_2653_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_2653_cast_fp16 = slice_by_index(begin = var_2653_begin_0, end = var_2653_end_0, end_mask = var_2653_end_mask_0, squeeze_mask = var_2653_squeeze_mask_0, x = _inversed_query_39_cast_fp16)[name = string("op_2653_cast_fp16")];
            tensor<int32, [1]> var_2654_axes_0 = const()[name = string("op_2654_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_2654_cast_fp16 = expand_dims(axes = var_2654_axes_0, x = var_2653_cast_fp16)[name = string("op_2654_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2655_cast_fp16 = mul(x = rec9_out, y = var_2654_cast_fp16)[name = string("op_2655_cast_fp16")];
            tensor<int32, [1]> var_2657_axes_0 = const()[name = string("op_2657_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_2657_keep_dims_0 = const()[name = string("op_2657_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_2657_cast_fp16 = reduce_sum(axes = var_2657_axes_0, keep_dims = var_2657_keep_dims_0, x = var_2655_cast_fp16)[name = string("op_2657_cast_fp16")];
            tensor<int32, [1]> attention_71_axes_0 = const()[name = string("attention_71_axes_0"), val = tensor<int32, [1]>([1])];
            tensor<fp16, [1, 1, 16, 128]> attention_71_cast_fp16 = expand_dims(axes = attention_71_axes_0, x = var_2657_cast_fp16)[name = string("attention_71_cast_fp16")];
            tensor<int32, [2]> var_2660 = const()[name = string("op_2660"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [16, 128]> attention_73_cast_fp16 = reshape(shape = var_2660, x = attention_71_cast_fp16)[name = string("attention_73_cast_fp16")];
            tensor<int32, [2]> var_2662 = const()[name = string("op_2662"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [16, 128]> input_123_cast_fp16 = reshape(shape = var_2662, x = linear_73_cast_fp16)[name = string("input_123_cast_fp16")];
            tensor<fp16, [16, 128]> var_2664_cast_fp16 = mul(x = attention_73_cast_fp16, y = attention_73_cast_fp16)[name = string("op_2664_cast_fp16")];
            tensor<int32, [1]> variance_61_axes_0 = const()[name = string("variance_61_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_61_keep_dims_0 = const()[name = string("variance_61_keep_dims_0"), val = bool(true)];
            tensor<fp16, [16, 1]> variance_61_cast_fp16 = reduce_mean(axes = variance_61_axes_0, keep_dims = variance_61_keep_dims_0, x = var_2664_cast_fp16)[name = string("variance_61_cast_fp16")];
            fp16 var_2667_to_fp16 = const()[name = string("op_2667_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [16, 1]> var_2668_cast_fp16 = add(x = variance_61_cast_fp16, y = var_2667_to_fp16)[name = string("op_2668_cast_fp16")];
            fp32 var_2669_epsilon_0 = const()[name = string("op_2669_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [16, 1]> var_2669_cast_fp16 = rsqrt(epsilon = var_2669_epsilon_0, x = var_2668_cast_fp16)[name = string("op_2669_cast_fp16")];
            tensor<fp16, [16, 128]> attention_75_cast_fp16 = mul(x = attention_73_cast_fp16, y = var_2669_cast_fp16)[name = string("attention_75_cast_fp16")];
            tensor<fp16, [128]> groups_2_1_gated_norm_promoted_to_fp16 = const()[name = string("groups_2_1_gated_norm_promoted_to_fp16"), val = tensor<fp16, [128]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(148771264)))];
            tensor<fp16, [16, 128]> attention_77_cast_fp16 = mul(x = attention_75_cast_fp16, y = groups_2_1_gated_norm_promoted_to_fp16)[name = string("attention_77_cast_fp16")];
            tensor<fp16, [16, 128]> var_2672_cast_fp16 = silu(x = input_123_cast_fp16)[name = string("op_2672_cast_fp16")];
            tensor<fp16, [16, 128]> attention_79_cast_fp16 = mul(x = attention_77_cast_fp16, y = var_2672_cast_fp16)[name = string("attention_79_cast_fp16")];
            tensor<int32, [2]> var_2674 = const()[name = string("op_2674"), val = tensor<int32, [2]>([1, 2048])];
            tensor<fp16, [1, 2048]> input_125_cast_fp16 = reshape(shape = var_2674, x = attention_79_cast_fp16)[name = string("input_125_cast_fp16")];
            tensor<fp16, [1024, 2048]> groups_2_1_out_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 2048]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(148771584))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(150344512))))[name = string("groups_2_1_out_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 1024]> linear_74_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = groups_2_1_out_proj_weight_promoted_to_fp16_palettized, x = input_125_cast_fp16)[name = string("linear_74_cast_fp16")];
            tensor<fp16, [1, 1024]> value_143_cast_fp16 = add(x = value_131_cast_fp16, y = linear_74_cast_fp16)[name = string("value_143_cast_fp16")];
            tensor<fp16, [1, 1024]> var_2679_cast_fp16 = mul(x = value_143_cast_fp16, y = value_143_cast_fp16)[name = string("op_2679_cast_fp16")];
            tensor<int32, [1]> variance_63_axes_0 = const()[name = string("variance_63_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_63_keep_dims_0 = const()[name = string("variance_63_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 1]> variance_63_cast_fp16 = reduce_mean(axes = variance_63_axes_0, keep_dims = variance_63_keep_dims_0, x = var_2679_cast_fp16)[name = string("variance_63_cast_fp16")];
            fp16 var_2682_to_fp16 = const()[name = string("op_2682_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 1]> var_2683_cast_fp16 = add(x = variance_63_cast_fp16, y = var_2682_to_fp16)[name = string("op_2683_cast_fp16")];
            fp32 var_2684_epsilon_0 = const()[name = string("op_2684_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 1]> var_2684_cast_fp16 = rsqrt(epsilon = var_2684_epsilon_0, x = var_2683_cast_fp16)[name = string("op_2684_cast_fp16")];
            tensor<fp16, [1, 1024]> var_2685_cast_fp16 = mul(x = value_143_cast_fp16, y = var_2684_cast_fp16)[name = string("op_2685_cast_fp16")];
            tensor<fp16, [1024]> var_2687_to_fp16 = const()[name = string("op_2687_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(150352768)))];
            tensor<fp16, [1, 1024]> input_127_cast_fp16 = mul(x = var_2685_cast_fp16, y = var_2687_to_fp16)[name = string("input_127_cast_fp16")];
            tensor<fp16, [3584, 1024]> groups_2_1_gate_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(150354880))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(153107456))))[name = string("groups_2_1_gate_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 3584]> linear_75_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = groups_2_1_gate_proj_weight_promoted_to_fp16_palettized, x = input_127_cast_fp16)[name = string("linear_75_cast_fp16")];
            tensor<fp16, [1, 3584]> var_2691_cast_fp16 = silu(x = linear_75_cast_fp16)[name = string("op_2691_cast_fp16")];
            tensor<fp16, [3584, 1024]> groups_2_1_up_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(153136192))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(155888768))))[name = string("groups_2_1_up_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 3584]> linear_76_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = groups_2_1_up_proj_weight_promoted_to_fp16_palettized, x = input_127_cast_fp16)[name = string("linear_76_cast_fp16")];
            tensor<fp16, [1, 3584]> input_131_cast_fp16 = mul(x = var_2691_cast_fp16, y = linear_76_cast_fp16)[name = string("input_131_cast_fp16")];
            tensor<fp16, [1024, 3584]> groups_2_1_down_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 3584]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(155917504))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(158670080))))[name = string("groups_2_1_down_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 1024]> linear_77_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = groups_2_1_down_proj_weight_promoted_to_fp16_palettized, x = input_131_cast_fp16)[name = string("linear_77_cast_fp16")];
            tensor<fp16, [1, 1024]> value_145_cast_fp16 = add(x = value_143_cast_fp16, y = linear_77_cast_fp16)[name = string("value_145_cast_fp16")];
            int32 var_2716 = const()[name = string("op_2716"), val = int32(-1)];
            tensor<fp16, [1, 1024]> var_2731_cast_fp16 = mul(x = value_145_cast_fp16, y = value_145_cast_fp16)[name = string("op_2731_cast_fp16")];
            tensor<int32, [1]> variance_65_axes_0 = const()[name = string("variance_65_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_65_keep_dims_0 = const()[name = string("variance_65_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 1]> variance_65_cast_fp16 = reduce_mean(axes = variance_65_axes_0, keep_dims = variance_65_keep_dims_0, x = var_2731_cast_fp16)[name = string("variance_65_cast_fp16")];
            fp16 var_2734_to_fp16 = const()[name = string("op_2734_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 1]> var_2735_cast_fp16 = add(x = variance_65_cast_fp16, y = var_2734_to_fp16)[name = string("op_2735_cast_fp16")];
            fp32 var_2736_epsilon_0 = const()[name = string("op_2736_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 1]> var_2736_cast_fp16 = rsqrt(epsilon = var_2736_epsilon_0, x = var_2735_cast_fp16)[name = string("op_2736_cast_fp16")];
            tensor<fp16, [1, 1024]> var_2737_cast_fp16 = mul(x = value_145_cast_fp16, y = var_2736_cast_fp16)[name = string("op_2737_cast_fp16")];
            tensor<fp16, [1024]> var_2739_to_fp16 = const()[name = string("op_2739_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(158678336)))];
            tensor<fp16, [1, 1024]> input_133_cast_fp16 = mul(x = var_2737_cast_fp16, y = var_2739_to_fp16)[name = string("input_133_cast_fp16")];
            tensor<fp16, [6144, 1024]> groups_2_2_in_proj_qkv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(158680448))), lut = tensor<fp16, [384, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(163399104))))[name = string("groups_2_2_in_proj_qkv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 6144]> linear_78_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = groups_2_2_in_proj_qkv_weight_promoted_to_fp16_palettized, x = input_133_cast_fp16)[name = string("linear_78_cast_fp16")];
            tensor<int32, [2]> var_2743_perm_0 = const()[name = string("op_2743_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> mixed_17_axes_0 = const()[name = string("mixed_17_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [6144, 1]> var_2743_cast_fp16 = transpose(perm = var_2743_perm_0, x = linear_78_cast_fp16)[name = string("transpose_68")];
            tensor<fp16, [1, 6144, 1]> mixed_17_cast_fp16 = expand_dims(axes = mixed_17_axes_0, x = var_2743_cast_fp16)[name = string("mixed_17_cast_fp16")];
            bool convolution_input_17_interleave_0 = const()[name = string("convolution_input_17_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 6144, 4]> convolution_input_17_cast_fp16 = concat(axis = var_2716, interleave = convolution_input_17_interleave_0, values = (conv10, mixed_17_cast_fp16))[name = string("convolution_input_17_cast_fp16")];
            string input_135_pad_type_0 = const()[name = string("input_135_pad_type_0"), val = string("valid")];
            int32 input_135_groups_0 = const()[name = string("input_135_groups_0"), val = int32(6144)];
            tensor<int32, [1]> input_135_strides_0 = const()[name = string("input_135_strides_0"), val = tensor<int32, [1]>([1])];
            tensor<int32, [2]> input_135_pad_0 = const()[name = string("input_135_pad_0"), val = tensor<int32, [2]>([0, 0])];
            tensor<int32, [1]> input_135_dilations_0 = const()[name = string("input_135_dilations_0"), val = tensor<int32, [1]>([1])];
            tensor<fp16, [6144, 1, 4]> groups_2_2_conv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1, 4]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(163448320))), lut = tensor<fp16, [384, 1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(163466816))))[name = string("groups_2_2_conv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 6144, 1]> input_135_cast_fp16 = conv(dilations = input_135_dilations_0, groups = input_135_groups_0, pad = input_135_pad_0, pad_type = input_135_pad_type_0, strides = input_135_strides_0, weight = groups_2_2_conv_weight_promoted_to_fp16_palettized, x = convolution_input_17_cast_fp16)[name = string("input_135_cast_fp16")];
            tensor<fp16, [1, 6144, 1]> var_2752_cast_fp16 = silu(x = input_135_cast_fp16)[name = string("op_2752_cast_fp16")];
            tensor<int32, [3]> var_2753_perm_0 = const()[name = string("op_2753_perm_0"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [3]> var_2756_begin_0 = const()[name = string("op_2756_begin_0"), val = tensor<int32, [3]>([0, 0, 1])];
            tensor<int32, [3]> var_2756_end_0 = const()[name = string("op_2756_end_0"), val = tensor<int32, [3]>([1, 6144, 4])];
            tensor<bool, [3]> var_2756_end_mask_0 = const()[name = string("op_2756_end_mask_0"), val = tensor<bool, [3]>([true, true, true])];
            tensor<fp16, [1, 6144, 3]> conv10_out = slice_by_index(begin = var_2756_begin_0, end = var_2756_end_0, end_mask = var_2756_end_mask_0, x = convolution_input_17_cast_fp16)[name = string("op_2756_cast_fp16")];
            tensor<int32, [3]> var_2757 = const()[name = string("op_2757"), val = tensor<int32, [3]>([2048, 2048, 2048])];
            int32 var_2758_axis_0 = const()[name = string("op_2758_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 1, 6144]> var_2753_cast_fp16 = transpose(perm = var_2753_perm_0, x = var_2752_cast_fp16)[name = string("transpose_67")];
            tensor<fp16, [1, 1, 2048]> var_2758_cast_fp16_0, tensor<fp16, [1, 1, 2048]> var_2758_cast_fp16_1, tensor<fp16, [1, 1, 2048]> var_2758_cast_fp16_2 = split(axis = var_2758_axis_0, split_sizes = var_2757, x = var_2753_cast_fp16)[name = string("op_2758_cast_fp16")];
            tensor<int32, [4]> var_2762 = const()[name = string("op_2762"), val = tensor<int32, [4]>([1, 1, 16, 128])];
            tensor<fp16, [1, 1, 16, 128]> value_149_cast_fp16 = reshape(shape = var_2762, x = var_2758_cast_fp16_0)[name = string("value_149_cast_fp16")];
            tensor<int32, [4]> var_2764 = const()[name = string("op_2764"), val = tensor<int32, [4]>([1, 1, 16, 128])];
            tensor<fp16, [1, 1, 16, 128]> value_151_cast_fp16 = reshape(shape = var_2764, x = var_2758_cast_fp16_1)[name = string("value_151_cast_fp16")];
            tensor<int32, [4]> var_2766 = const()[name = string("op_2766"), val = tensor<int32, [4]>([1, 1, 16, 128])];
            tensor<fp16, [1, 1, 16, 128]> value_153_cast_fp16 = reshape(shape = var_2766, x = var_2758_cast_fp16_2)[name = string("value_153_cast_fp16")];
            tensor<fp16, [1, 1, 16, 128]> var_2768_cast_fp16 = mul(x = value_149_cast_fp16, y = value_149_cast_fp16)[name = string("op_2768_cast_fp16")];
            tensor<int32, [1]> var_2770_axes_0 = const()[name = string("op_2770_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_2770_keep_dims_0 = const()[name = string("op_2770_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 1, 16, 1]> var_2770_cast_fp16 = reduce_sum(axes = var_2770_axes_0, keep_dims = var_2770_keep_dims_0, x = var_2768_cast_fp16)[name = string("op_2770_cast_fp16")];
            fp16 var_2771_to_fp16 = const()[name = string("op_2771_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 1, 16, 1]> var_2772_cast_fp16 = add(x = var_2770_cast_fp16, y = var_2771_to_fp16)[name = string("op_2772_cast_fp16")];
            fp32 var_2773_epsilon_0 = const()[name = string("op_2773_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 1, 16, 1]> var_2773_cast_fp16 = rsqrt(epsilon = var_2773_epsilon_0, x = var_2772_cast_fp16)[name = string("op_2773_cast_fp16")];
            tensor<fp16, [1, 1, 16, 128]> var_2774_cast_fp16 = mul(x = value_149_cast_fp16, y = var_2773_cast_fp16)[name = string("op_2774_cast_fp16")];
            tensor<int32, [4]> var_2775_perm_0 = const()[name = string("op_2775_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            fp16 _inversed_query_43_y_0_to_fp16 = const()[name = string("_inversed_query_43_y_0_to_fp16"), val = fp16(0x1.6ap-4)];
            tensor<fp16, [1, 16, 1, 128]> var_2775_cast_fp16 = transpose(perm = var_2775_perm_0, x = var_2774_cast_fp16)[name = string("transpose_66")];
            tensor<fp16, [1, 16, 1, 128]> _inversed_query_43_cast_fp16 = mul(x = var_2775_cast_fp16, y = _inversed_query_43_y_0_to_fp16)[name = string("_inversed_query_43_cast_fp16")];
            tensor<fp16, [1, 1, 16, 128]> var_2778_cast_fp16 = mul(x = value_151_cast_fp16, y = value_151_cast_fp16)[name = string("op_2778_cast_fp16")];
            tensor<int32, [1]> var_2780_axes_0 = const()[name = string("op_2780_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_2780_keep_dims_0 = const()[name = string("op_2780_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 1, 16, 1]> var_2780_cast_fp16 = reduce_sum(axes = var_2780_axes_0, keep_dims = var_2780_keep_dims_0, x = var_2778_cast_fp16)[name = string("op_2780_cast_fp16")];
            fp16 var_2781_to_fp16 = const()[name = string("op_2781_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 1, 16, 1]> var_2782_cast_fp16 = add(x = var_2780_cast_fp16, y = var_2781_to_fp16)[name = string("op_2782_cast_fp16")];
            fp32 var_2783_epsilon_0 = const()[name = string("op_2783_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 1, 16, 1]> var_2783_cast_fp16 = rsqrt(epsilon = var_2783_epsilon_0, x = var_2782_cast_fp16)[name = string("op_2783_cast_fp16")];
            tensor<fp16, [1, 1, 16, 128]> var_2784_cast_fp16 = mul(x = value_151_cast_fp16, y = var_2783_cast_fp16)[name = string("op_2784_cast_fp16")];
            tensor<int32, [4]> key_43_perm_0 = const()[name = string("key_43_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<int32, [4]> value_155_perm_0 = const()[name = string("value_155_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<fp16, [16, 1024]> groups_2_2_in_proj_b_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(163516032))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(163528384))))[name = string("groups_2_2_in_proj_b_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16]> linear_79_bias_0_to_fp16 = const()[name = string("linear_79_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(4862656)))];
            tensor<fp16, [1, 16]> linear_79_cast_fp16 = linear(bias = linear_79_bias_0_to_fp16, weight = groups_2_2_in_proj_b_weight_promoted_to_fp16_palettized, x = input_133_cast_fp16)[name = string("linear_79_cast_fp16")];
            tensor<fp16, [1, 16]> var_2789_cast_fp16 = sigmoid(x = linear_79_cast_fp16)[name = string("op_2789_cast_fp16")];
            tensor<int32, [2]> var_2790_perm_0 = const()[name = string("op_2790_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> beta_17_axes_0 = const()[name = string("beta_17_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 1]> var_2790_cast_fp16 = transpose(perm = var_2790_perm_0, x = var_2789_cast_fp16)[name = string("transpose_65")];
            tensor<fp16, [1, 16, 1]> beta_17_cast_fp16 = expand_dims(axes = beta_17_axes_0, x = var_2790_cast_fp16)[name = string("beta_17_cast_fp16")];
            tensor<fp16, [16, 1024]> op_2796_weight_0_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(163528576))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(163540928))))[name = string("op_2796_weight_0_to_fp16_palettized")];
            tensor<fp16, [16]> var_2796_bias_0_to_fp16 = const()[name = string("op_2796_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(163541120)))];
            tensor<fp16, [1, 16]> var_2796_cast_fp16 = linear(bias = var_2796_bias_0_to_fp16, weight = op_2796_weight_0_to_fp16_palettized, x = input_133_cast_fp16)[name = string("op_2796_cast_fp16")];
            tensor<fp16, [1, 16]> var_2797_cast_fp16 = softplus(x = var_2796_cast_fp16)[name = string("op_2797_cast_fp16")];
            tensor<fp16, [16]> var_2793_promoted_to_fp16 = const()[name = string("op_2793_promoted_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(163541248)))];
            tensor<fp16, [1, 16]> var_2798_cast_fp16 = mul(x = var_2793_promoted_to_fp16, y = var_2797_cast_fp16)[name = string("op_2798_cast_fp16")];
            tensor<int32, [2]> var_2799_perm_0 = const()[name = string("op_2799_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> decay_17_axes_0 = const()[name = string("decay_17_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 1]> var_2799_cast_fp16 = transpose(perm = var_2799_perm_0, x = var_2798_cast_fp16)[name = string("transpose_64")];
            tensor<fp16, [1, 16, 1]> decay_17_cast_fp16 = expand_dims(axes = decay_17_axes_0, x = var_2799_cast_fp16)[name = string("decay_17_cast_fp16")];
            tensor<fp16, [2048, 1024]> groups_2_2_in_proj_z_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [2048, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(163541376))), lut = tensor<fp16, [128, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(165114304))))[name = string("groups_2_2_in_proj_z_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 2048]> linear_81_cast_fp16 = linear(bias = linear_3_bias_0_to_fp16, weight = groups_2_2_in_proj_z_weight_promoted_to_fp16_palettized, x = input_133_cast_fp16)[name = string("linear_81_cast_fp16")];
            tensor<int32, [3]> var_2807_begin_0 = const()[name = string("op_2807_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_2807_end_0 = const()[name = string("op_2807_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_2807_end_mask_0 = const()[name = string("op_2807_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_2807_squeeze_mask_0 = const()[name = string("op_2807_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_2807_cast_fp16 = slice_by_index(begin = var_2807_begin_0, end = var_2807_end_0, end_mask = var_2807_end_mask_0, squeeze_mask = var_2807_squeeze_mask_0, x = decay_17_cast_fp16)[name = string("op_2807_cast_fp16")];
            tensor<fp16, [1, 16]> var_2808_cast_fp16 = exp(x = var_2807_cast_fp16)[name = string("op_2808_cast_fp16")];
            tensor<int32, [1]> var_2809_axes_0 = const()[name = string("op_2809_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_2809_cast_fp16 = expand_dims(axes = var_2809_axes_0, x = var_2808_cast_fp16)[name = string("op_2809_cast_fp16")];
            tensor<int32, [1]> var_2810_axes_0 = const()[name = string("op_2810_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_2810_cast_fp16 = expand_dims(axes = var_2810_axes_0, x = var_2809_cast_fp16)[name = string("op_2810_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_33_cast_fp16 = mul(x = rec10, y = var_2810_cast_fp16)[name = string("state_33_cast_fp16")];
            tensor<int32, [4]> key_token_17_begin_0 = const()[name = string("key_token_17_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> key_token_17_end_0 = const()[name = string("key_token_17_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> key_token_17_end_mask_0 = const()[name = string("key_token_17_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_17_squeeze_mask_0 = const()[name = string("key_token_17_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 1, 128]> key_43_cast_fp16 = transpose(perm = key_43_perm_0, x = var_2784_cast_fp16)[name = string("transpose_63")];
            tensor<fp16, [1, 16, 128]> key_token_17_cast_fp16 = slice_by_index(begin = key_token_17_begin_0, end = key_token_17_end_0, end_mask = key_token_17_end_mask_0, squeeze_mask = key_token_17_squeeze_mask_0, x = key_43_cast_fp16)[name = string("key_token_17_cast_fp16")];
            tensor<int32, [4]> value_token_17_begin_0 = const()[name = string("value_token_17_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> value_token_17_end_0 = const()[name = string("value_token_17_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> value_token_17_end_mask_0 = const()[name = string("value_token_17_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_17_squeeze_mask_0 = const()[name = string("value_token_17_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 1, 128]> value_155_cast_fp16 = transpose(perm = value_155_perm_0, x = value_153_cast_fp16)[name = string("transpose_62")];
            tensor<fp16, [1, 16, 128]> value_token_17_cast_fp16 = slice_by_index(begin = value_token_17_begin_0, end = value_token_17_end_0, end_mask = value_token_17_end_mask_0, squeeze_mask = value_token_17_squeeze_mask_0, x = value_155_cast_fp16)[name = string("value_token_17_cast_fp16")];
            tensor<int32, [1]> var_2818_axes_0 = const()[name = string("op_2818_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_2818_cast_fp16 = expand_dims(axes = var_2818_axes_0, x = key_token_17_cast_fp16)[name = string("op_2818_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2819_cast_fp16 = mul(x = state_33_cast_fp16, y = var_2818_cast_fp16)[name = string("op_2819_cast_fp16")];
            tensor<int32, [1]> memory_17_axes_0 = const()[name = string("memory_17_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_17_keep_dims_0 = const()[name = string("memory_17_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_17_cast_fp16 = reduce_sum(axes = memory_17_axes_0, keep_dims = memory_17_keep_dims_0, x = var_2819_cast_fp16)[name = string("memory_17_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_2822_cast_fp16 = sub(x = value_token_17_cast_fp16, y = memory_17_cast_fp16)[name = string("op_2822_cast_fp16")];
            tensor<int32, [3]> var_2825_begin_0 = const()[name = string("op_2825_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_2825_end_0 = const()[name = string("op_2825_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_2825_end_mask_0 = const()[name = string("op_2825_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_2825_squeeze_mask_0 = const()[name = string("op_2825_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_2825_cast_fp16 = slice_by_index(begin = var_2825_begin_0, end = var_2825_end_0, end_mask = var_2825_end_mask_0, squeeze_mask = var_2825_squeeze_mask_0, x = beta_17_cast_fp16)[name = string("op_2825_cast_fp16")];
            tensor<int32, [1]> var_2826_axes_0 = const()[name = string("op_2826_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_2826_cast_fp16 = expand_dims(axes = var_2826_axes_0, x = var_2825_cast_fp16)[name = string("op_2826_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_17_cast_fp16 = mul(x = var_2822_cast_fp16, y = var_2826_cast_fp16)[name = string("delta_17_cast_fp16")];
            tensor<int32, [1]> var_2829_axes_0 = const()[name = string("op_2829_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_2829_cast_fp16 = expand_dims(axes = var_2829_axes_0, x = delta_17_cast_fp16)[name = string("op_2829_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2830_cast_fp16 = mul(x = var_2818_cast_fp16, y = var_2829_cast_fp16)[name = string("op_2830_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> rec10_out = add(x = state_33_cast_fp16, y = var_2830_cast_fp16)[name = string("state_35_cast_fp16")];
            tensor<int32, [4]> var_2834_begin_0 = const()[name = string("op_2834_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> var_2834_end_0 = const()[name = string("op_2834_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> var_2834_end_mask_0 = const()[name = string("op_2834_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_2834_squeeze_mask_0 = const()[name = string("op_2834_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_2834_cast_fp16 = slice_by_index(begin = var_2834_begin_0, end = var_2834_end_0, end_mask = var_2834_end_mask_0, squeeze_mask = var_2834_squeeze_mask_0, x = _inversed_query_43_cast_fp16)[name = string("op_2834_cast_fp16")];
            tensor<int32, [1]> var_2835_axes_0 = const()[name = string("op_2835_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_2835_cast_fp16 = expand_dims(axes = var_2835_axes_0, x = var_2834_cast_fp16)[name = string("op_2835_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2836_cast_fp16 = mul(x = rec10_out, y = var_2835_cast_fp16)[name = string("op_2836_cast_fp16")];
            tensor<int32, [1]> var_2838_axes_0 = const()[name = string("op_2838_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_2838_keep_dims_0 = const()[name = string("op_2838_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_2838_cast_fp16 = reduce_sum(axes = var_2838_axes_0, keep_dims = var_2838_keep_dims_0, x = var_2836_cast_fp16)[name = string("op_2838_cast_fp16")];
            tensor<int32, [1]> attention_81_axes_0 = const()[name = string("attention_81_axes_0"), val = tensor<int32, [1]>([1])];
            tensor<fp16, [1, 1, 16, 128]> attention_81_cast_fp16 = expand_dims(axes = attention_81_axes_0, x = var_2838_cast_fp16)[name = string("attention_81_cast_fp16")];
            tensor<int32, [2]> var_2841 = const()[name = string("op_2841"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [16, 128]> attention_83_cast_fp16 = reshape(shape = var_2841, x = attention_81_cast_fp16)[name = string("attention_83_cast_fp16")];
            tensor<int32, [2]> var_2843 = const()[name = string("op_2843"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [16, 128]> input_137_cast_fp16 = reshape(shape = var_2843, x = linear_81_cast_fp16)[name = string("input_137_cast_fp16")];
            tensor<fp16, [16, 128]> var_2845_cast_fp16 = mul(x = attention_83_cast_fp16, y = attention_83_cast_fp16)[name = string("op_2845_cast_fp16")];
            tensor<int32, [1]> variance_67_axes_0 = const()[name = string("variance_67_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_67_keep_dims_0 = const()[name = string("variance_67_keep_dims_0"), val = bool(true)];
            tensor<fp16, [16, 1]> variance_67_cast_fp16 = reduce_mean(axes = variance_67_axes_0, keep_dims = variance_67_keep_dims_0, x = var_2845_cast_fp16)[name = string("variance_67_cast_fp16")];
            fp16 var_2848_to_fp16 = const()[name = string("op_2848_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [16, 1]> var_2849_cast_fp16 = add(x = variance_67_cast_fp16, y = var_2848_to_fp16)[name = string("op_2849_cast_fp16")];
            fp32 var_2850_epsilon_0 = const()[name = string("op_2850_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [16, 1]> var_2850_cast_fp16 = rsqrt(epsilon = var_2850_epsilon_0, x = var_2849_cast_fp16)[name = string("op_2850_cast_fp16")];
            tensor<fp16, [16, 128]> attention_85_cast_fp16 = mul(x = attention_83_cast_fp16, y = var_2850_cast_fp16)[name = string("attention_85_cast_fp16")];
            tensor<fp16, [128]> groups_2_2_gated_norm_promoted_to_fp16 = const()[name = string("groups_2_2_gated_norm_promoted_to_fp16"), val = tensor<fp16, [128]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(165130752)))];
            tensor<fp16, [16, 128]> attention_87_cast_fp16 = mul(x = attention_85_cast_fp16, y = groups_2_2_gated_norm_promoted_to_fp16)[name = string("attention_87_cast_fp16")];
            tensor<fp16, [16, 128]> var_2853_cast_fp16 = silu(x = input_137_cast_fp16)[name = string("op_2853_cast_fp16")];
            tensor<fp16, [16, 128]> attention_89_cast_fp16 = mul(x = attention_87_cast_fp16, y = var_2853_cast_fp16)[name = string("attention_89_cast_fp16")];
            tensor<int32, [2]> var_2855 = const()[name = string("op_2855"), val = tensor<int32, [2]>([1, 2048])];
            tensor<fp16, [1, 2048]> input_139_cast_fp16 = reshape(shape = var_2855, x = attention_89_cast_fp16)[name = string("input_139_cast_fp16")];
            tensor<fp16, [1024, 2048]> groups_2_2_out_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 2048]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(165131072))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(166704000))))[name = string("groups_2_2_out_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 1024]> linear_82_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = groups_2_2_out_proj_weight_promoted_to_fp16_palettized, x = input_139_cast_fp16)[name = string("linear_82_cast_fp16")];
            tensor<fp16, [1, 1024]> value_157_cast_fp16 = add(x = value_145_cast_fp16, y = linear_82_cast_fp16)[name = string("value_157_cast_fp16")];
            tensor<fp16, [1, 1024]> var_2860_cast_fp16 = mul(x = value_157_cast_fp16, y = value_157_cast_fp16)[name = string("op_2860_cast_fp16")];
            tensor<int32, [1]> variance_69_axes_0 = const()[name = string("variance_69_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_69_keep_dims_0 = const()[name = string("variance_69_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 1]> variance_69_cast_fp16 = reduce_mean(axes = variance_69_axes_0, keep_dims = variance_69_keep_dims_0, x = var_2860_cast_fp16)[name = string("variance_69_cast_fp16")];
            fp16 var_2863_to_fp16 = const()[name = string("op_2863_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 1]> var_2864_cast_fp16 = add(x = variance_69_cast_fp16, y = var_2863_to_fp16)[name = string("op_2864_cast_fp16")];
            fp32 var_2865_epsilon_0 = const()[name = string("op_2865_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 1]> var_2865_cast_fp16 = rsqrt(epsilon = var_2865_epsilon_0, x = var_2864_cast_fp16)[name = string("op_2865_cast_fp16")];
            tensor<fp16, [1, 1024]> var_2866_cast_fp16 = mul(x = value_157_cast_fp16, y = var_2865_cast_fp16)[name = string("op_2866_cast_fp16")];
            tensor<fp16, [1024]> var_2868_to_fp16 = const()[name = string("op_2868_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(166712256)))];
            tensor<fp16, [1, 1024]> input_141_cast_fp16 = mul(x = var_2866_cast_fp16, y = var_2868_to_fp16)[name = string("input_141_cast_fp16")];
            tensor<fp16, [3584, 1024]> groups_2_2_gate_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(166714368))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(169466944))))[name = string("groups_2_2_gate_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 3584]> linear_83_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = groups_2_2_gate_proj_weight_promoted_to_fp16_palettized, x = input_141_cast_fp16)[name = string("linear_83_cast_fp16")];
            tensor<fp16, [1, 3584]> var_2872_cast_fp16 = silu(x = linear_83_cast_fp16)[name = string("op_2872_cast_fp16")];
            tensor<fp16, [3584, 1024]> groups_2_2_up_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(169495680))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(172248256))))[name = string("groups_2_2_up_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 3584]> linear_84_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = groups_2_2_up_proj_weight_promoted_to_fp16_palettized, x = input_141_cast_fp16)[name = string("linear_84_cast_fp16")];
            tensor<fp16, [1, 3584]> input_145_cast_fp16 = mul(x = var_2872_cast_fp16, y = linear_84_cast_fp16)[name = string("input_145_cast_fp16")];
            tensor<fp16, [1024, 3584]> groups_2_2_down_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 3584]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(172276992))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(175029568))))[name = string("groups_2_2_down_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 1024]> linear_85_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = groups_2_2_down_proj_weight_promoted_to_fp16_palettized, x = input_145_cast_fp16)[name = string("linear_85_cast_fp16")];
            tensor<fp16, [1, 1024]> value_159_cast_fp16 = add(x = value_157_cast_fp16, y = linear_85_cast_fp16)[name = string("value_159_cast_fp16")];
            int32 var_2896 = const()[name = string("op_2896"), val = int32(-1)];
            tensor<fp16, [1, 1024]> var_2903_cast_fp16 = mul(x = value_159_cast_fp16, y = value_159_cast_fp16)[name = string("op_2903_cast_fp16")];
            tensor<int32, [1]> variance_71_axes_0 = const()[name = string("variance_71_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_71_keep_dims_0 = const()[name = string("variance_71_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 1]> variance_71_cast_fp16 = reduce_mean(axes = variance_71_axes_0, keep_dims = variance_71_keep_dims_0, x = var_2903_cast_fp16)[name = string("variance_71_cast_fp16")];
            fp16 var_2906_to_fp16 = const()[name = string("op_2906_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 1]> var_2907_cast_fp16 = add(x = variance_71_cast_fp16, y = var_2906_to_fp16)[name = string("op_2907_cast_fp16")];
            fp32 var_2908_epsilon_0 = const()[name = string("op_2908_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 1]> var_2908_cast_fp16 = rsqrt(epsilon = var_2908_epsilon_0, x = var_2907_cast_fp16)[name = string("op_2908_cast_fp16")];
            tensor<fp16, [1, 1024]> var_2909_cast_fp16 = mul(x = value_159_cast_fp16, y = var_2908_cast_fp16)[name = string("op_2909_cast_fp16")];
            tensor<fp16, [1024]> var_2911_to_fp16 = const()[name = string("op_2911_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(175037824)))];
            tensor<fp16, [1, 1024]> input_147_cast_fp16 = mul(x = var_2909_cast_fp16, y = var_2911_to_fp16)[name = string("input_147_cast_fp16")];
            tensor<fp16, [4096, 1024]> projections_2_q_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [4096, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(175039936))), lut = tensor<fp16, [256, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(178185728))))[name = string("projections_2_q_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 4096]> linear_86_cast_fp16 = linear(bias = linear_24_bias_0_to_fp16, weight = projections_2_q_proj_weight_promoted_to_fp16_palettized, x = input_147_cast_fp16)[name = string("linear_86_cast_fp16")];
            tensor<int32, [3]> var_2915 = const()[name = string("op_2915"), val = tensor<int32, [3]>([1, 8, 512])];
            tensor<fp16, [1, 8, 512]> query_and_gate_5_cast_fp16 = reshape(shape = var_2915, x = linear_86_cast_fp16)[name = string("query_and_gate_5_cast_fp16")];
            tensor<int32, [2]> var_2917_split_sizes_0 = const()[name = string("op_2917_split_sizes_0"), val = tensor<int32, [2]>([256, 256])];
            int32 var_2917_axis_0 = const()[name = string("op_2917_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 8, 256]> var_2917_cast_fp16_0, tensor<fp16, [1, 8, 256]> var_2917_cast_fp16_1 = split(axis = var_2917_axis_0, split_sizes = var_2917_split_sizes_0, x = query_and_gate_5_cast_fp16)[name = string("op_2917_cast_fp16")];
            tensor<fp16, [512, 1024]> projections_2_k_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [512, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(178218560))), lut = tensor<fp16, [32, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(178611840))))[name = string("projections_2_k_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 512]> linear_87_cast_fp16 = linear(bias = linear_25_bias_0_to_fp16, weight = projections_2_k_proj_weight_promoted_to_fp16_palettized, x = input_147_cast_fp16)[name = string("linear_87_cast_fp16")];
            tensor<int32, [3]> var_2921 = const()[name = string("op_2921"), val = tensor<int32, [3]>([1, 2, 256])];
            tensor<fp16, [1, 2, 256]> value_163_cast_fp16 = reshape(shape = var_2921, x = linear_87_cast_fp16)[name = string("value_163_cast_fp16")];
            tensor<fp16, [512, 1024]> projections_2_v_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [512, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(178616000))), lut = tensor<fp16, [32, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(179009280))))[name = string("projections_2_v_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 512]> linear_88_cast_fp16 = linear(bias = linear_25_bias_0_to_fp16, weight = projections_2_v_proj_weight_promoted_to_fp16_palettized, x = input_147_cast_fp16)[name = string("linear_88_cast_fp16")];
            tensor<int32, [3]> var_2925 = const()[name = string("op_2925"), val = tensor<int32, [3]>([1, 2, 256])];
            tensor<fp16, [1, 2, 256]> value_169_cast_fp16 = reshape(shape = var_2925, x = linear_88_cast_fp16)[name = string("value_169_cast_fp16")];
            tensor<fp16, [1, 8, 256]> var_2927_cast_fp16 = mul(x = var_2917_cast_fp16_0, y = var_2917_cast_fp16_0)[name = string("op_2927_cast_fp16")];
            tensor<int32, [1]> variance_73_axes_0 = const()[name = string("variance_73_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_73_keep_dims_0 = const()[name = string("variance_73_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 8, 1]> variance_73_cast_fp16 = reduce_mean(axes = variance_73_axes_0, keep_dims = variance_73_keep_dims_0, x = var_2927_cast_fp16)[name = string("variance_73_cast_fp16")];
            fp16 var_2930_to_fp16 = const()[name = string("op_2930_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 8, 1]> var_2931_cast_fp16 = add(x = variance_73_cast_fp16, y = var_2930_to_fp16)[name = string("op_2931_cast_fp16")];
            fp32 var_2932_epsilon_0 = const()[name = string("op_2932_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 8, 1]> var_2932_cast_fp16 = rsqrt(epsilon = var_2932_epsilon_0, x = var_2931_cast_fp16)[name = string("op_2932_cast_fp16")];
            tensor<fp16, [1, 8, 256]> var_2933_cast_fp16 = mul(x = var_2917_cast_fp16_0, y = var_2932_cast_fp16)[name = string("op_2933_cast_fp16")];
            tensor<fp16, [256]> var_2935_to_fp16 = const()[name = string("op_2935_to_fp16"), val = tensor<fp16, [256]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(179013440)))];
            tensor<fp16, [1, 8, 256]> var_2936_cast_fp16 = mul(x = var_2933_cast_fp16, y = var_2935_to_fp16)[name = string("op_2936_cast_fp16")];
            tensor<int32, [3]> var_2937 = const()[name = string("op_2937"), val = tensor<int32, [3]>([1, 0, 2])];
            tensor<int32, [1]> value_165_axes_0 = const()[name = string("value_165_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [8, 1, 256]> var_2938_cast_fp16 = transpose(perm = var_2937, x = var_2936_cast_fp16)[name = string("transpose_61")];
            tensor<fp16, [1, 8, 1, 256]> value_165_cast_fp16 = expand_dims(axes = value_165_axes_0, x = var_2938_cast_fp16)[name = string("value_165_cast_fp16")];
            tensor<fp16, [1, 2, 256]> var_2940_cast_fp16 = mul(x = value_163_cast_fp16, y = value_163_cast_fp16)[name = string("op_2940_cast_fp16")];
            tensor<int32, [1]> variance_75_axes_0 = const()[name = string("variance_75_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_75_keep_dims_0 = const()[name = string("variance_75_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 2, 1]> variance_75_cast_fp16 = reduce_mean(axes = variance_75_axes_0, keep_dims = variance_75_keep_dims_0, x = var_2940_cast_fp16)[name = string("variance_75_cast_fp16")];
            fp16 var_2943_to_fp16 = const()[name = string("op_2943_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 2, 1]> var_2944_cast_fp16 = add(x = variance_75_cast_fp16, y = var_2943_to_fp16)[name = string("op_2944_cast_fp16")];
            fp32 var_2945_epsilon_0 = const()[name = string("op_2945_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 2, 1]> var_2945_cast_fp16 = rsqrt(epsilon = var_2945_epsilon_0, x = var_2944_cast_fp16)[name = string("op_2945_cast_fp16")];
            tensor<fp16, [1, 2, 256]> var_2946_cast_fp16 = mul(x = value_163_cast_fp16, y = var_2945_cast_fp16)[name = string("op_2946_cast_fp16")];
            tensor<fp16, [256]> var_2948_to_fp16 = const()[name = string("op_2948_to_fp16"), val = tensor<fp16, [256]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(179014016)))];
            tensor<fp16, [1, 2, 256]> var_2949_cast_fp16 = mul(x = var_2946_cast_fp16, y = var_2948_to_fp16)[name = string("op_2949_cast_fp16")];
            tensor<int32, [3]> var_2950 = const()[name = string("op_2950"), val = tensor<int32, [3]>([1, 0, 2])];
            tensor<int32, [1]> value_167_axes_0 = const()[name = string("value_167_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [2, 1, 256]> var_2951_cast_fp16 = transpose(perm = var_2950, x = var_2949_cast_fp16)[name = string("transpose_60")];
            tensor<fp16, [1, 2, 1, 256]> value_167_cast_fp16 = expand_dims(axes = value_167_axes_0, x = var_2951_cast_fp16)[name = string("value_167_cast_fp16")];
            tensor<int32, [4]> rotated_9_begin_0 = const()[name = string("rotated_9_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> rotated_9_end_0 = const()[name = string("rotated_9_end_0"), val = tensor<int32, [4]>([1, 8, 1, 64])];
            tensor<bool, [4]> rotated_9_end_mask_0 = const()[name = string("rotated_9_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
            tensor<fp16, [1, 8, 1, 64]> rotated_9_cast_fp16 = slice_by_index(begin = rotated_9_begin_0, end = rotated_9_end_0, end_mask = rotated_9_end_mask_0, x = value_165_cast_fp16)[name = string("rotated_9_cast_fp16")];
            tensor<int32, [4]> passthrough_9_begin_0 = const()[name = string("passthrough_9_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 64])];
            tensor<int32, [4]> passthrough_9_end_0 = const()[name = string("passthrough_9_end_0"), val = tensor<int32, [4]>([1, 8, 1, 256])];
            tensor<bool, [4]> passthrough_9_end_mask_0 = const()[name = string("passthrough_9_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
            tensor<fp16, [1, 8, 1, 192]> passthrough_9_cast_fp16 = slice_by_index(begin = passthrough_9_begin_0, end = passthrough_9_end_0, end_mask = passthrough_9_end_mask_0, x = value_165_cast_fp16)[name = string("passthrough_9_cast_fp16")];
            tensor<int32, [2]> var_2955_split_sizes_0 = const()[name = string("op_2955_split_sizes_0"), val = tensor<int32, [2]>([32, 32])];
            int32 var_2955_axis_0 = const()[name = string("op_2955_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 8, 1, 32]> var_2955_cast_fp16_0, tensor<fp16, [1, 8, 1, 32]> var_2955_cast_fp16_1 = split(axis = var_2955_axis_0, split_sizes = var_2955_split_sizes_0, x = rotated_9_cast_fp16)[name = string("op_2955_cast_fp16")];
            fp16 const_89_promoted_to_fp16 = const()[name = string("const_89_promoted_to_fp16"), val = fp16(-0x1p+0)];
            tensor<fp16, [1, 8, 1, 32]> var_2957_cast_fp16 = mul(x = var_2955_cast_fp16_1, y = const_89_promoted_to_fp16)[name = string("op_2957_cast_fp16")];
            bool rotated_half_9_interleave_0 = const()[name = string("rotated_half_9_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 8, 1, 64]> rotated_half_9_cast_fp16 = concat(axis = var_2896, interleave = rotated_half_9_interleave_0, values = (var_2957_cast_fp16, var_2955_cast_fp16_0))[name = string("rotated_half_9_cast_fp16")];
            tensor<fp16, [1, 8, 1, 64]> var_2960_cast_fp16 = mul(x = rotated_9_cast_fp16, y = cos)[name = string("op_2960_cast_fp16")];
            tensor<fp16, [1, 8, 1, 64]> var_2961_cast_fp16 = mul(x = rotated_half_9_cast_fp16, y = sin)[name = string("op_2961_cast_fp16")];
            tensor<fp16, [1, 8, 1, 64]> var_2962_cast_fp16 = add(x = var_2960_cast_fp16, y = var_2961_cast_fp16)[name = string("op_2962_cast_fp16")];
            bool query_45_interleave_0 = const()[name = string("query_45_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 8, 1, 256]> query_45_cast_fp16 = concat(axis = var_2896, interleave = query_45_interleave_0, values = (var_2962_cast_fp16, passthrough_9_cast_fp16))[name = string("query_45_cast_fp16")];
            tensor<int32, [4]> rotated_11_begin_0 = const()[name = string("rotated_11_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> rotated_11_end_0 = const()[name = string("rotated_11_end_0"), val = tensor<int32, [4]>([1, 2, 1, 64])];
            tensor<bool, [4]> rotated_11_end_mask_0 = const()[name = string("rotated_11_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
            tensor<fp16, [1, 2, 1, 64]> rotated_11_cast_fp16 = slice_by_index(begin = rotated_11_begin_0, end = rotated_11_end_0, end_mask = rotated_11_end_mask_0, x = value_167_cast_fp16)[name = string("rotated_11_cast_fp16")];
            tensor<int32, [4]> passthrough_11_begin_0 = const()[name = string("passthrough_11_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 64])];
            tensor<int32, [4]> passthrough_11_end_0 = const()[name = string("passthrough_11_end_0"), val = tensor<int32, [4]>([1, 2, 1, 256])];
            tensor<bool, [4]> passthrough_11_end_mask_0 = const()[name = string("passthrough_11_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
            tensor<fp16, [1, 2, 1, 192]> passthrough_11_cast_fp16 = slice_by_index(begin = passthrough_11_begin_0, end = passthrough_11_end_0, end_mask = passthrough_11_end_mask_0, x = value_167_cast_fp16)[name = string("passthrough_11_cast_fp16")];
            tensor<int32, [2]> var_2967_split_sizes_0 = const()[name = string("op_2967_split_sizes_0"), val = tensor<int32, [2]>([32, 32])];
            int32 var_2967_axis_0 = const()[name = string("op_2967_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 2, 1, 32]> var_2967_cast_fp16_0, tensor<fp16, [1, 2, 1, 32]> var_2967_cast_fp16_1 = split(axis = var_2967_axis_0, split_sizes = var_2967_split_sizes_0, x = rotated_11_cast_fp16)[name = string("op_2967_cast_fp16")];
            fp16 const_90_promoted_to_fp16 = const()[name = string("const_90_promoted_to_fp16"), val = fp16(-0x1p+0)];
            tensor<fp16, [1, 2, 1, 32]> var_2969_cast_fp16 = mul(x = var_2967_cast_fp16_1, y = const_90_promoted_to_fp16)[name = string("op_2969_cast_fp16")];
            bool rotated_half_11_interleave_0 = const()[name = string("rotated_half_11_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 2, 1, 64]> rotated_half_11_cast_fp16 = concat(axis = var_2896, interleave = rotated_half_11_interleave_0, values = (var_2969_cast_fp16, var_2967_cast_fp16_0))[name = string("rotated_half_11_cast_fp16")];
            tensor<fp16, [1, 2, 1, 64]> var_2972_cast_fp16 = mul(x = rotated_11_cast_fp16, y = cos)[name = string("op_2972_cast_fp16")];
            tensor<fp16, [1, 2, 1, 64]> var_2973_cast_fp16 = mul(x = rotated_half_11_cast_fp16, y = sin)[name = string("op_2973_cast_fp16")];
            tensor<fp16, [1, 2, 1, 64]> var_2974_cast_fp16 = add(x = var_2972_cast_fp16, y = var_2973_cast_fp16)[name = string("op_2974_cast_fp16")];
            bool key_45_interleave_0 = const()[name = string("key_45_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 2, 1, 256]> key_45_cast_fp16 = concat(axis = var_2896, interleave = key_45_interleave_0, values = (var_2974_cast_fp16, passthrough_11_cast_fp16))[name = string("key_45_cast_fp16")];
            tensor<int32, [4]> var_2977 = const()[name = string("op_2977"), val = tensor<int32, [4]>([2, 4, 1, 256])];
            tensor<fp16, [2, 4, 1, 256]> var_2978_cast_fp16 = reshape(shape = var_2977, x = query_45_cast_fp16)[name = string("op_2978_cast_fp16")];
            tensor<int32, [4]> transpose_4_perm_0 = const()[name = string("transpose_4_perm_0"), val = tensor<int32, [4]>([0, 1, 3, 2])];
            tensor<int32, [3]> var_2981 = const()[name = string("op_2981"), val = tensor<int32, [3]>([2, 1, 256])];
            tensor<fp16, [2, 1, 256]> key_47_cast_fp16 = reshape(shape = var_2981, x = key_45_cast_fp16)[name = string("key_47_cast_fp16")];
            tensor<int32, [3]> var_2983 = const()[name = string("op_2983"), val = tensor<int32, [3]>([1, 0, 2])];
            tensor<int32, [2]> var_2985 = const()[name = string("op_2985"), val = tensor<int32, [2]>([1, 2048])];
            tensor<fp16, [1, 2048]> gate_11_cast_fp16 = reshape(shape = var_2985, x = var_2917_cast_fp16_1)[name = string("gate_11_cast_fp16")];
            tensor<int32, [1]> var_2993_axes_0 = const()[name = string("op_2993_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [1, 2, 1, 256]> var_2993_cast_fp16 = expand_dims(axes = var_2993_axes_0, x = key_47_cast_fp16)[name = string("op_2993_cast_fp16")];
            tensor<int32, [1]> var_2995_axes_0 = const()[name = string("op_2995_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [1, 1, 2, 1, 256]> var_2995_cast_fp16 = expand_dims(axes = var_2995_axes_0, x = var_2993_cast_fp16)[name = string("op_2995_cast_fp16")];
            tensor<int32, [1]> expand_dims_32 = const()[name = string("expand_dims_32"), val = tensor<int32, [1]>([2])];
            tensor<int32, [1]> expand_dims_33 = const()[name = string("expand_dims_33"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_34 = const()[name = string("expand_dims_34"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_36 = const()[name = string("expand_dims_36"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_37 = const()[name = string("expand_dims_37"), val = tensor<int32, [1]>([3])];
            tensor<int32, [1]> expand_dims_38 = const()[name = string("expand_dims_38"), val = tensor<int32, [1]>([1])];
            int32 concat_18_axis_0 = const()[name = string("concat_18_axis_0"), val = int32(0)];
            bool concat_18_interleave_0 = const()[name = string("concat_18_interleave_0"), val = bool(false)];
            tensor<int32, [5]> concat_18 = concat(axis = concat_18_axis_0, interleave = concat_18_interleave_0, values = (expand_dims_32, expand_dims_33, expand_dims_34, current_pos, expand_dims_36))[name = string("concat_18")];
            tensor<int32, [1]> concat_19_values2_0 = const()[name = string("concat_19_values2_0"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> concat_19_values4_0 = const()[name = string("concat_19_values4_0"), val = tensor<int32, [1]>([0])];
            int32 concat_19_axis_0 = const()[name = string("concat_19_axis_0"), val = int32(0)];
            bool concat_19_interleave_0 = const()[name = string("concat_19_interleave_0"), val = bool(false)];
            tensor<int32, [5]> concat_19 = concat(axis = concat_19_axis_0, interleave = concat_19_interleave_0, values = (expand_dims_37, expand_dims_38, concat_19_values2_0, var_762, concat_19_values4_0))[name = string("concat_19")];
            tensor<int32, [5]> kv_cache_internal_tensor_assign_5_stride_0 = const()[name = string("kv_cache_internal_tensor_assign_5_stride_0"), val = tensor<int32, [5]>([1, 1, 1, 1, 1])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_5_begin_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_5_begin_mask_0"), val = tensor<bool, [5]>([false, false, false, false, false])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_5_end_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_5_end_mask_0"), val = tensor<bool, [5]>([false, false, true, false, true])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_5_squeeze_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_5_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, false, false, false])];
            tensor<fp16, [4, 2, 2, 2048, 256]> kv_cache_internal_tensor_assign_5_cast_fp16 = slice_update(begin = concat_18, begin_mask = kv_cache_internal_tensor_assign_5_begin_mask_0, end = concat_19, end_mask = kv_cache_internal_tensor_assign_5_end_mask_0, squeeze_mask = kv_cache_internal_tensor_assign_5_squeeze_mask_0, stride = kv_cache_internal_tensor_assign_5_stride_0, update = var_2995_cast_fp16, x = coreml_update_state_11)[name = string("kv_cache_internal_tensor_assign_5_cast_fp16")];
            write_state(data = kv_cache_internal_tensor_assign_5_cast_fp16, input = kv_cache)[name = string("coreml_update_state_4_write_state")];
            tensor<fp16, [4, 2, 2, 2048, 256]> coreml_update_state_12 = read_state(input = kv_cache)[name = string("coreml_update_state_4")];
            tensor<int32, [1]> var_3026_axes_0 = const()[name = string("op_3026_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [2, 1, 256]> value_171_cast_fp16 = transpose(perm = var_2983, x = value_169_cast_fp16)[name = string("transpose_59")];
            tensor<fp16, [1, 2, 1, 256]> var_3026_cast_fp16 = expand_dims(axes = var_3026_axes_0, x = value_171_cast_fp16)[name = string("op_3026_cast_fp16")];
            tensor<int32, [1]> var_3028_axes_0 = const()[name = string("op_3028_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [1, 1, 2, 1, 256]> var_3028_cast_fp16 = expand_dims(axes = var_3028_axes_0, x = var_3026_cast_fp16)[name = string("op_3028_cast_fp16")];
            tensor<int32, [1]> expand_dims_40 = const()[name = string("expand_dims_40"), val = tensor<int32, [1]>([2])];
            tensor<int32, [1]> expand_dims_41 = const()[name = string("expand_dims_41"), val = tensor<int32, [1]>([1])];
            tensor<int32, [1]> expand_dims_42 = const()[name = string("expand_dims_42"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_44 = const()[name = string("expand_dims_44"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_45 = const()[name = string("expand_dims_45"), val = tensor<int32, [1]>([3])];
            tensor<int32, [1]> expand_dims_46 = const()[name = string("expand_dims_46"), val = tensor<int32, [1]>([2])];
            int32 concat_22_axis_0 = const()[name = string("concat_22_axis_0"), val = int32(0)];
            bool concat_22_interleave_0 = const()[name = string("concat_22_interleave_0"), val = bool(false)];
            tensor<int32, [5]> concat_22 = concat(axis = concat_22_axis_0, interleave = concat_22_interleave_0, values = (expand_dims_40, expand_dims_41, expand_dims_42, current_pos, expand_dims_44))[name = string("concat_22")];
            tensor<int32, [1]> concat_23_values2_0 = const()[name = string("concat_23_values2_0"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> concat_23_values4_0 = const()[name = string("concat_23_values4_0"), val = tensor<int32, [1]>([0])];
            int32 concat_23_axis_0 = const()[name = string("concat_23_axis_0"), val = int32(0)];
            bool concat_23_interleave_0 = const()[name = string("concat_23_interleave_0"), val = bool(false)];
            tensor<int32, [5]> concat_23 = concat(axis = concat_23_axis_0, interleave = concat_23_interleave_0, values = (expand_dims_45, expand_dims_46, concat_23_values2_0, var_762, concat_23_values4_0))[name = string("concat_23")];
            tensor<int32, [5]> kv_cache_internal_tensor_assign_6_stride_0 = const()[name = string("kv_cache_internal_tensor_assign_6_stride_0"), val = tensor<int32, [5]>([1, 1, 1, 1, 1])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_6_begin_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_6_begin_mask_0"), val = tensor<bool, [5]>([false, false, false, false, false])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_6_end_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_6_end_mask_0"), val = tensor<bool, [5]>([false, false, true, false, true])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_6_squeeze_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_6_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, false, false, false])];
            tensor<fp16, [4, 2, 2, 2048, 256]> kv_cache_internal_tensor_assign_6_cast_fp16 = slice_update(begin = concat_22, begin_mask = kv_cache_internal_tensor_assign_6_begin_mask_0, end = concat_23, end_mask = kv_cache_internal_tensor_assign_6_end_mask_0, squeeze_mask = kv_cache_internal_tensor_assign_6_squeeze_mask_0, stride = kv_cache_internal_tensor_assign_6_stride_0, update = var_3028_cast_fp16, x = coreml_update_state_12)[name = string("kv_cache_internal_tensor_assign_6_cast_fp16")];
            write_state(data = kv_cache_internal_tensor_assign_6_cast_fp16, input = kv_cache)[name = string("coreml_update_state_5_write_state")];
            tensor<fp16, [4, 2, 2, 2048, 256]> coreml_update_state_13 = read_state(input = kv_cache)[name = string("coreml_update_state_5")];
            tensor<int32, [5]> var_3060_begin_0 = const()[name = string("op_3060_begin_0"), val = tensor<int32, [5]>([2, 0, 0, 0, 0])];
            tensor<int32, [5]> var_3060_end_0 = const()[name = string("op_3060_end_0"), val = tensor<int32, [5]>([3, 2, 2, 2048, 256])];
            tensor<bool, [5]> var_3060_end_mask_0 = const()[name = string("op_3060_end_mask_0"), val = tensor<bool, [5]>([false, true, true, true, true])];
            tensor<bool, [5]> var_3060_squeeze_mask_0 = const()[name = string("op_3060_squeeze_mask_0"), val = tensor<bool, [5]>([true, false, false, false, false])];
            tensor<fp16, [2, 2, 2048, 256]> var_3060_cast_fp16 = slice_by_index(begin = var_3060_begin_0, end = var_3060_end_0, end_mask = var_3060_end_mask_0, squeeze_mask = var_3060_squeeze_mask_0, x = coreml_update_state_13)[name = string("op_3060_cast_fp16")];
            tensor<int32, [4]> keys_5_begin_0 = const()[name = string("keys_5_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> keys_5_end_0 = const()[name = string("keys_5_end_0"), val = tensor<int32, [4]>([1, 2, 2048, 256])];
            tensor<bool, [4]> keys_5_end_mask_0 = const()[name = string("keys_5_end_mask_0"), val = tensor<bool, [4]>([false, true, true, true])];
            tensor<bool, [4]> keys_5_squeeze_mask_0 = const()[name = string("keys_5_squeeze_mask_0"), val = tensor<bool, [4]>([true, false, false, false])];
            tensor<fp16, [2, 2048, 256]> keys_5_cast_fp16 = slice_by_index(begin = keys_5_begin_0, end = keys_5_end_0, end_mask = keys_5_end_mask_0, squeeze_mask = keys_5_squeeze_mask_0, x = var_3060_cast_fp16)[name = string("keys_5_cast_fp16")];
            tensor<int32, [4]> values_5_begin_0 = const()[name = string("values_5_begin_0"), val = tensor<int32, [4]>([1, 0, 0, 0])];
            tensor<int32, [4]> values_5_end_0 = const()[name = string("values_5_end_0"), val = tensor<int32, [4]>([2, 2, 2048, 256])];
            tensor<bool, [4]> values_5_end_mask_0 = const()[name = string("values_5_end_mask_0"), val = tensor<bool, [4]>([false, true, true, true])];
            tensor<bool, [4]> values_5_squeeze_mask_0 = const()[name = string("values_5_squeeze_mask_0"), val = tensor<bool, [4]>([true, false, false, false])];
            tensor<fp16, [2, 2048, 256]> values_5_cast_fp16 = slice_by_index(begin = values_5_begin_0, end = values_5_end_0, end_mask = values_5_end_mask_0, squeeze_mask = values_5_squeeze_mask_0, x = var_3060_cast_fp16)[name = string("values_5_cast_fp16")];
            int32 var_3077 = const()[name = string("op_3077"), val = int32(1)];
            tensor<int32, [4]> var_3084 = const()[name = string("op_3084"), val = tensor<int32, [4]>([1, 2048, 1, 1])];
            tensor<fp16, [2, 4, 256, 1]> transpose_4_cast_fp16 = transpose(perm = transpose_4_perm_0, x = var_2978_cast_fp16)[name = string("transpose_58")];
            tensor<fp16, [1, 2048, 1, 1]> var_3085_cast_fp16 = reshape(shape = var_3084, x = transpose_4_cast_fp16)[name = string("op_3085_cast_fp16")];
            tensor<int32, [3]> var_3086 = const()[name = string("op_3086"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [4]> var_3089 = const()[name = string("op_3089"), val = tensor<int32, [4]>([1, 512, 1, 2048])];
            tensor<fp16, [2, 256, 2048]> var_3087_cast_fp16 = transpose(perm = var_3086, x = keys_5_cast_fp16)[name = string("transpose_57")];
            tensor<fp16, [1, 512, 1, 2048]> key_native_5_cast_fp16 = reshape(shape = var_3089, x = var_3087_cast_fp16)[name = string("key_native_5_cast_fp16")];
            tensor<int32, [3]> var_3091 = const()[name = string("op_3091"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [4]> var_3094 = const()[name = string("op_3094"), val = tensor<int32, [4]>([1, 512, 1, 2048])];
            tensor<fp16, [2, 256, 2048]> var_3092_cast_fp16 = transpose(perm = var_3091, x = values_5_cast_fp16)[name = string("transpose_56")];
            tensor<fp16, [1, 512, 1, 2048]> var_3095_cast_fp16 = reshape(shape = var_3094, x = var_3092_cast_fp16)[name = string("op_3095_cast_fp16")];
            tensor<int32, [8]> tile_38 = const()[name = string("tile_38"), val = tensor<int32, [8]>([256, 256, 256, 256, 256, 256, 256, 256])];
            int32 var_3099_axis_0 = const()[name = string("op_3099_axis_0"), val = int32(1)];
            tensor<fp16, [1, 256, 1, 1]> var_3099_cast_fp16_0, tensor<fp16, [1, 256, 1, 1]> var_3099_cast_fp16_1, tensor<fp16, [1, 256, 1, 1]> var_3099_cast_fp16_2, tensor<fp16, [1, 256, 1, 1]> var_3099_cast_fp16_3, tensor<fp16, [1, 256, 1, 1]> var_3099_cast_fp16_4, tensor<fp16, [1, 256, 1, 1]> var_3099_cast_fp16_5, tensor<fp16, [1, 256, 1, 1]> var_3099_cast_fp16_6, tensor<fp16, [1, 256, 1, 1]> var_3099_cast_fp16_7 = split(axis = var_3099_axis_0, split_sizes = tile_38, x = var_3085_cast_fp16)[name = string("op_3099_cast_fp16")];
            tensor<int32, [4]> var_3108_perm_0 = const()[name = string("op_3108_perm_0"), val = tensor<int32, [4]>([0, 3, 2, 1])];
            tensor<int32, [2]> tile_39 = const()[name = string("tile_39"), val = tensor<int32, [2]>([256, 256])];
            int32 var_3109_axis_0 = const()[name = string("op_3109_axis_0"), val = int32(3)];
            tensor<fp16, [1, 2048, 1, 512]> var_3108_cast_fp16 = transpose(perm = var_3108_perm_0, x = key_native_5_cast_fp16)[name = string("transpose_55")];
            tensor<fp16, [1, 2048, 1, 256]> var_3109_cast_fp16_0, tensor<fp16, [1, 2048, 1, 256]> var_3109_cast_fp16_1 = split(axis = var_3109_axis_0, split_sizes = tile_39, x = var_3108_cast_fp16)[name = string("op_3109_cast_fp16")];
            tensor<int32, [2]> tile_40 = const()[name = string("tile_40"), val = tensor<int32, [2]>([256, 256])];
            int32 var_3112_axis_0 = const()[name = string("op_3112_axis_0"), val = int32(1)];
            tensor<fp16, [1, 256, 1, 2048]> var_3112_cast_fp16_0, tensor<fp16, [1, 256, 1, 2048]> var_3112_cast_fp16_1 = split(axis = var_3112_axis_0, split_sizes = tile_40, x = var_3095_cast_fp16)[name = string("op_3112_cast_fp16")];
            string scores_33_equation_0 = const()[name = string("scores_33_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 1]> scores_33_cast_fp16 = einsum(equation = scores_33_equation_0, values = (var_3109_cast_fp16_0, var_3099_cast_fp16_0))[name = string("scores_33_cast_fp16")];
            fp16 var_3117_to_fp16 = const()[name = string("op_3117_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 1]> var_3118_cast_fp16 = mul(x = scores_33_cast_fp16, y = var_3117_to_fp16)[name = string("op_3118_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_3119_cast_fp16 = add(x = var_3118_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_3119_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_3120_cast_fp16 = softmax(axis = var_3077, x = var_3119_cast_fp16)[name = string("op_3120_cast_fp16")];
            tensor<int32, [4]> tile_41 = const()[name = string("tile_41"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_3121_axis_0 = const()[name = string("op_3121_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 1]> var_3121_cast_fp16_0, tensor<fp16, [1, 512, 1, 1]> var_3121_cast_fp16_1, tensor<fp16, [1, 512, 1, 1]> var_3121_cast_fp16_2, tensor<fp16, [1, 512, 1, 1]> var_3121_cast_fp16_3 = split(axis = var_3121_axis_0, split_sizes = tile_41, x = var_3120_cast_fp16)[name = string("op_3121_cast_fp16")];
            tensor<int32, [4]> tile_42 = const()[name = string("tile_42"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_3126_axis_0 = const()[name = string("op_3126_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_3126_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_3126_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_3126_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_3126_cast_fp16_3 = split(axis = var_3126_axis_0, split_sizes = tile_42, x = var_3112_cast_fp16_0)[name = string("op_3126_cast_fp16")];
            fp16 var_3131_to_fp16 = const()[name = string("op_3131_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_3132_cast_fp16 = mul(x = var_3121_cast_fp16_0, y = var_3131_to_fp16)[name = string("op_3132_cast_fp16")];
            string var_3134_equation_0 = const()[name = string("op_3134_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_3134_cast_fp16 = einsum(equation = var_3134_equation_0, values = (var_3126_cast_fp16_0, var_3132_cast_fp16))[name = string("op_3134_cast_fp16")];
            fp16 var_3135_to_fp16 = const()[name = string("op_3135_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_3136_cast_fp16 = mul(x = var_3121_cast_fp16_1, y = var_3135_to_fp16)[name = string("op_3136_cast_fp16")];
            string var_3138_equation_0 = const()[name = string("op_3138_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_3138_cast_fp16 = einsum(equation = var_3138_equation_0, values = (var_3126_cast_fp16_1, var_3136_cast_fp16))[name = string("op_3138_cast_fp16")];
            fp16 var_3139_to_fp16 = const()[name = string("op_3139_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_3140_cast_fp16 = mul(x = var_3121_cast_fp16_2, y = var_3139_to_fp16)[name = string("op_3140_cast_fp16")];
            string var_3142_equation_0 = const()[name = string("op_3142_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_3142_cast_fp16 = einsum(equation = var_3142_equation_0, values = (var_3126_cast_fp16_2, var_3140_cast_fp16))[name = string("op_3142_cast_fp16")];
            fp16 var_3143_to_fp16 = const()[name = string("op_3143_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_3144_cast_fp16 = mul(x = var_3121_cast_fp16_3, y = var_3143_to_fp16)[name = string("op_3144_cast_fp16")];
            string var_3146_equation_0 = const()[name = string("op_3146_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_3146_cast_fp16 = einsum(equation = var_3146_equation_0, values = (var_3126_cast_fp16_3, var_3144_cast_fp16))[name = string("op_3146_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_3147_cast_fp16 = add(x = var_3134_cast_fp16, y = var_3138_cast_fp16)[name = string("op_3147_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_3148_cast_fp16 = add(x = var_3142_cast_fp16, y = var_3146_cast_fp16)[name = string("op_3148_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_3149_cast_fp16 = add(x = var_3147_cast_fp16, y = var_3148_cast_fp16)[name = string("op_3149_cast_fp16")];
            fp16 _inversed_3151_y_0_to_fp16 = const()[name = string("_inversed_3151_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 1]> _inversed_3151_cast_fp16 = mul(x = var_3149_cast_fp16, y = _inversed_3151_y_0_to_fp16)[name = string("_inversed_3151_cast_fp16")];
            string scores_35_equation_0 = const()[name = string("scores_35_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 1]> scores_35_cast_fp16 = einsum(equation = scores_35_equation_0, values = (var_3109_cast_fp16_0, var_3099_cast_fp16_1))[name = string("scores_35_cast_fp16")];
            fp16 var_3154_to_fp16 = const()[name = string("op_3154_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 1]> var_3155_cast_fp16 = mul(x = scores_35_cast_fp16, y = var_3154_to_fp16)[name = string("op_3155_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_3156_cast_fp16 = add(x = var_3155_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_3156_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_3157_cast_fp16 = softmax(axis = var_3077, x = var_3156_cast_fp16)[name = string("op_3157_cast_fp16")];
            tensor<int32, [4]> tile_43 = const()[name = string("tile_43"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_3158_axis_0 = const()[name = string("op_3158_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 1]> var_3158_cast_fp16_0, tensor<fp16, [1, 512, 1, 1]> var_3158_cast_fp16_1, tensor<fp16, [1, 512, 1, 1]> var_3158_cast_fp16_2, tensor<fp16, [1, 512, 1, 1]> var_3158_cast_fp16_3 = split(axis = var_3158_axis_0, split_sizes = tile_43, x = var_3157_cast_fp16)[name = string("op_3158_cast_fp16")];
            tensor<int32, [4]> tile_44 = const()[name = string("tile_44"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_3163_axis_0 = const()[name = string("op_3163_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_3163_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_3163_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_3163_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_3163_cast_fp16_3 = split(axis = var_3163_axis_0, split_sizes = tile_44, x = var_3112_cast_fp16_0)[name = string("op_3163_cast_fp16")];
            fp16 var_3168_to_fp16 = const()[name = string("op_3168_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_3169_cast_fp16 = mul(x = var_3158_cast_fp16_0, y = var_3168_to_fp16)[name = string("op_3169_cast_fp16")];
            string var_3171_equation_0 = const()[name = string("op_3171_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_3171_cast_fp16 = einsum(equation = var_3171_equation_0, values = (var_3163_cast_fp16_0, var_3169_cast_fp16))[name = string("op_3171_cast_fp16")];
            fp16 var_3172_to_fp16 = const()[name = string("op_3172_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_3173_cast_fp16 = mul(x = var_3158_cast_fp16_1, y = var_3172_to_fp16)[name = string("op_3173_cast_fp16")];
            string var_3175_equation_0 = const()[name = string("op_3175_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_3175_cast_fp16 = einsum(equation = var_3175_equation_0, values = (var_3163_cast_fp16_1, var_3173_cast_fp16))[name = string("op_3175_cast_fp16")];
            fp16 var_3176_to_fp16 = const()[name = string("op_3176_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_3177_cast_fp16 = mul(x = var_3158_cast_fp16_2, y = var_3176_to_fp16)[name = string("op_3177_cast_fp16")];
            string var_3179_equation_0 = const()[name = string("op_3179_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_3179_cast_fp16 = einsum(equation = var_3179_equation_0, values = (var_3163_cast_fp16_2, var_3177_cast_fp16))[name = string("op_3179_cast_fp16")];
            fp16 var_3180_to_fp16 = const()[name = string("op_3180_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_3181_cast_fp16 = mul(x = var_3158_cast_fp16_3, y = var_3180_to_fp16)[name = string("op_3181_cast_fp16")];
            string var_3183_equation_0 = const()[name = string("op_3183_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_3183_cast_fp16 = einsum(equation = var_3183_equation_0, values = (var_3163_cast_fp16_3, var_3181_cast_fp16))[name = string("op_3183_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_3184_cast_fp16 = add(x = var_3171_cast_fp16, y = var_3175_cast_fp16)[name = string("op_3184_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_3185_cast_fp16 = add(x = var_3179_cast_fp16, y = var_3183_cast_fp16)[name = string("op_3185_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_3186_cast_fp16 = add(x = var_3184_cast_fp16, y = var_3185_cast_fp16)[name = string("op_3186_cast_fp16")];
            fp16 _inversed_3188_y_0_to_fp16 = const()[name = string("_inversed_3188_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 1]> _inversed_3188_cast_fp16 = mul(x = var_3186_cast_fp16, y = _inversed_3188_y_0_to_fp16)[name = string("_inversed_3188_cast_fp16")];
            string scores_37_equation_0 = const()[name = string("scores_37_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 1]> scores_37_cast_fp16 = einsum(equation = scores_37_equation_0, values = (var_3109_cast_fp16_0, var_3099_cast_fp16_2))[name = string("scores_37_cast_fp16")];
            fp16 var_3191_to_fp16 = const()[name = string("op_3191_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 1]> var_3192_cast_fp16 = mul(x = scores_37_cast_fp16, y = var_3191_to_fp16)[name = string("op_3192_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_3193_cast_fp16 = add(x = var_3192_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_3193_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_3194_cast_fp16 = softmax(axis = var_3077, x = var_3193_cast_fp16)[name = string("op_3194_cast_fp16")];
            tensor<int32, [4]> tile_45 = const()[name = string("tile_45"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_3195_axis_0 = const()[name = string("op_3195_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 1]> var_3195_cast_fp16_0, tensor<fp16, [1, 512, 1, 1]> var_3195_cast_fp16_1, tensor<fp16, [1, 512, 1, 1]> var_3195_cast_fp16_2, tensor<fp16, [1, 512, 1, 1]> var_3195_cast_fp16_3 = split(axis = var_3195_axis_0, split_sizes = tile_45, x = var_3194_cast_fp16)[name = string("op_3195_cast_fp16")];
            tensor<int32, [4]> tile_46 = const()[name = string("tile_46"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_3200_axis_0 = const()[name = string("op_3200_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_3200_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_3200_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_3200_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_3200_cast_fp16_3 = split(axis = var_3200_axis_0, split_sizes = tile_46, x = var_3112_cast_fp16_0)[name = string("op_3200_cast_fp16")];
            fp16 var_3205_to_fp16 = const()[name = string("op_3205_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_3206_cast_fp16 = mul(x = var_3195_cast_fp16_0, y = var_3205_to_fp16)[name = string("op_3206_cast_fp16")];
            string var_3208_equation_0 = const()[name = string("op_3208_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_3208_cast_fp16 = einsum(equation = var_3208_equation_0, values = (var_3200_cast_fp16_0, var_3206_cast_fp16))[name = string("op_3208_cast_fp16")];
            fp16 var_3209_to_fp16 = const()[name = string("op_3209_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_3210_cast_fp16 = mul(x = var_3195_cast_fp16_1, y = var_3209_to_fp16)[name = string("op_3210_cast_fp16")];
            string var_3212_equation_0 = const()[name = string("op_3212_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_3212_cast_fp16 = einsum(equation = var_3212_equation_0, values = (var_3200_cast_fp16_1, var_3210_cast_fp16))[name = string("op_3212_cast_fp16")];
            fp16 var_3213_to_fp16 = const()[name = string("op_3213_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_3214_cast_fp16 = mul(x = var_3195_cast_fp16_2, y = var_3213_to_fp16)[name = string("op_3214_cast_fp16")];
            string var_3216_equation_0 = const()[name = string("op_3216_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_3216_cast_fp16 = einsum(equation = var_3216_equation_0, values = (var_3200_cast_fp16_2, var_3214_cast_fp16))[name = string("op_3216_cast_fp16")];
            fp16 var_3217_to_fp16 = const()[name = string("op_3217_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_3218_cast_fp16 = mul(x = var_3195_cast_fp16_3, y = var_3217_to_fp16)[name = string("op_3218_cast_fp16")];
            string var_3220_equation_0 = const()[name = string("op_3220_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_3220_cast_fp16 = einsum(equation = var_3220_equation_0, values = (var_3200_cast_fp16_3, var_3218_cast_fp16))[name = string("op_3220_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_3221_cast_fp16 = add(x = var_3208_cast_fp16, y = var_3212_cast_fp16)[name = string("op_3221_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_3222_cast_fp16 = add(x = var_3216_cast_fp16, y = var_3220_cast_fp16)[name = string("op_3222_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_3223_cast_fp16 = add(x = var_3221_cast_fp16, y = var_3222_cast_fp16)[name = string("op_3223_cast_fp16")];
            fp16 _inversed_3225_y_0_to_fp16 = const()[name = string("_inversed_3225_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 1]> _inversed_3225_cast_fp16 = mul(x = var_3223_cast_fp16, y = _inversed_3225_y_0_to_fp16)[name = string("_inversed_3225_cast_fp16")];
            string scores_39_equation_0 = const()[name = string("scores_39_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 1]> scores_39_cast_fp16 = einsum(equation = scores_39_equation_0, values = (var_3109_cast_fp16_0, var_3099_cast_fp16_3))[name = string("scores_39_cast_fp16")];
            fp16 var_3228_to_fp16 = const()[name = string("op_3228_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 1]> var_3229_cast_fp16 = mul(x = scores_39_cast_fp16, y = var_3228_to_fp16)[name = string("op_3229_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_3230_cast_fp16 = add(x = var_3229_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_3230_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_3231_cast_fp16 = softmax(axis = var_3077, x = var_3230_cast_fp16)[name = string("op_3231_cast_fp16")];
            tensor<int32, [4]> tile_47 = const()[name = string("tile_47"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_3232_axis_0 = const()[name = string("op_3232_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 1]> var_3232_cast_fp16_0, tensor<fp16, [1, 512, 1, 1]> var_3232_cast_fp16_1, tensor<fp16, [1, 512, 1, 1]> var_3232_cast_fp16_2, tensor<fp16, [1, 512, 1, 1]> var_3232_cast_fp16_3 = split(axis = var_3232_axis_0, split_sizes = tile_47, x = var_3231_cast_fp16)[name = string("op_3232_cast_fp16")];
            tensor<int32, [4]> tile_48 = const()[name = string("tile_48"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_3237_axis_0 = const()[name = string("op_3237_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_3237_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_3237_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_3237_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_3237_cast_fp16_3 = split(axis = var_3237_axis_0, split_sizes = tile_48, x = var_3112_cast_fp16_0)[name = string("op_3237_cast_fp16")];
            fp16 var_3242_to_fp16 = const()[name = string("op_3242_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_3243_cast_fp16 = mul(x = var_3232_cast_fp16_0, y = var_3242_to_fp16)[name = string("op_3243_cast_fp16")];
            string var_3245_equation_0 = const()[name = string("op_3245_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_3245_cast_fp16 = einsum(equation = var_3245_equation_0, values = (var_3237_cast_fp16_0, var_3243_cast_fp16))[name = string("op_3245_cast_fp16")];
            fp16 var_3246_to_fp16 = const()[name = string("op_3246_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_3247_cast_fp16 = mul(x = var_3232_cast_fp16_1, y = var_3246_to_fp16)[name = string("op_3247_cast_fp16")];
            string var_3249_equation_0 = const()[name = string("op_3249_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_3249_cast_fp16 = einsum(equation = var_3249_equation_0, values = (var_3237_cast_fp16_1, var_3247_cast_fp16))[name = string("op_3249_cast_fp16")];
            fp16 var_3250_to_fp16 = const()[name = string("op_3250_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_3251_cast_fp16 = mul(x = var_3232_cast_fp16_2, y = var_3250_to_fp16)[name = string("op_3251_cast_fp16")];
            string var_3253_equation_0 = const()[name = string("op_3253_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_3253_cast_fp16 = einsum(equation = var_3253_equation_0, values = (var_3237_cast_fp16_2, var_3251_cast_fp16))[name = string("op_3253_cast_fp16")];
            fp16 var_3254_to_fp16 = const()[name = string("op_3254_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_3255_cast_fp16 = mul(x = var_3232_cast_fp16_3, y = var_3254_to_fp16)[name = string("op_3255_cast_fp16")];
            string var_3257_equation_0 = const()[name = string("op_3257_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_3257_cast_fp16 = einsum(equation = var_3257_equation_0, values = (var_3237_cast_fp16_3, var_3255_cast_fp16))[name = string("op_3257_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_3258_cast_fp16 = add(x = var_3245_cast_fp16, y = var_3249_cast_fp16)[name = string("op_3258_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_3259_cast_fp16 = add(x = var_3253_cast_fp16, y = var_3257_cast_fp16)[name = string("op_3259_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_3260_cast_fp16 = add(x = var_3258_cast_fp16, y = var_3259_cast_fp16)[name = string("op_3260_cast_fp16")];
            fp16 _inversed_3262_y_0_to_fp16 = const()[name = string("_inversed_3262_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 1]> _inversed_3262_cast_fp16 = mul(x = var_3260_cast_fp16, y = _inversed_3262_y_0_to_fp16)[name = string("_inversed_3262_cast_fp16")];
            string scores_41_equation_0 = const()[name = string("scores_41_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 1]> scores_41_cast_fp16 = einsum(equation = scores_41_equation_0, values = (var_3109_cast_fp16_1, var_3099_cast_fp16_4))[name = string("scores_41_cast_fp16")];
            fp16 var_3265_to_fp16 = const()[name = string("op_3265_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 1]> var_3266_cast_fp16 = mul(x = scores_41_cast_fp16, y = var_3265_to_fp16)[name = string("op_3266_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_3267_cast_fp16 = add(x = var_3266_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_3267_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_3268_cast_fp16 = softmax(axis = var_3077, x = var_3267_cast_fp16)[name = string("op_3268_cast_fp16")];
            tensor<int32, [4]> tile_49 = const()[name = string("tile_49"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_3269_axis_0 = const()[name = string("op_3269_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 1]> var_3269_cast_fp16_0, tensor<fp16, [1, 512, 1, 1]> var_3269_cast_fp16_1, tensor<fp16, [1, 512, 1, 1]> var_3269_cast_fp16_2, tensor<fp16, [1, 512, 1, 1]> var_3269_cast_fp16_3 = split(axis = var_3269_axis_0, split_sizes = tile_49, x = var_3268_cast_fp16)[name = string("op_3269_cast_fp16")];
            tensor<int32, [4]> tile_50 = const()[name = string("tile_50"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_3274_axis_0 = const()[name = string("op_3274_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_3274_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_3274_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_3274_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_3274_cast_fp16_3 = split(axis = var_3274_axis_0, split_sizes = tile_50, x = var_3112_cast_fp16_1)[name = string("op_3274_cast_fp16")];
            fp16 var_3279_to_fp16 = const()[name = string("op_3279_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_3280_cast_fp16 = mul(x = var_3269_cast_fp16_0, y = var_3279_to_fp16)[name = string("op_3280_cast_fp16")];
            string var_3282_equation_0 = const()[name = string("op_3282_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_3282_cast_fp16 = einsum(equation = var_3282_equation_0, values = (var_3274_cast_fp16_0, var_3280_cast_fp16))[name = string("op_3282_cast_fp16")];
            fp16 var_3283_to_fp16 = const()[name = string("op_3283_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_3284_cast_fp16 = mul(x = var_3269_cast_fp16_1, y = var_3283_to_fp16)[name = string("op_3284_cast_fp16")];
            string var_3286_equation_0 = const()[name = string("op_3286_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_3286_cast_fp16 = einsum(equation = var_3286_equation_0, values = (var_3274_cast_fp16_1, var_3284_cast_fp16))[name = string("op_3286_cast_fp16")];
            fp16 var_3287_to_fp16 = const()[name = string("op_3287_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_3288_cast_fp16 = mul(x = var_3269_cast_fp16_2, y = var_3287_to_fp16)[name = string("op_3288_cast_fp16")];
            string var_3290_equation_0 = const()[name = string("op_3290_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_3290_cast_fp16 = einsum(equation = var_3290_equation_0, values = (var_3274_cast_fp16_2, var_3288_cast_fp16))[name = string("op_3290_cast_fp16")];
            fp16 var_3291_to_fp16 = const()[name = string("op_3291_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_3292_cast_fp16 = mul(x = var_3269_cast_fp16_3, y = var_3291_to_fp16)[name = string("op_3292_cast_fp16")];
            string var_3294_equation_0 = const()[name = string("op_3294_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_3294_cast_fp16 = einsum(equation = var_3294_equation_0, values = (var_3274_cast_fp16_3, var_3292_cast_fp16))[name = string("op_3294_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_3295_cast_fp16 = add(x = var_3282_cast_fp16, y = var_3286_cast_fp16)[name = string("op_3295_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_3296_cast_fp16 = add(x = var_3290_cast_fp16, y = var_3294_cast_fp16)[name = string("op_3296_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_3297_cast_fp16 = add(x = var_3295_cast_fp16, y = var_3296_cast_fp16)[name = string("op_3297_cast_fp16")];
            fp16 _inversed_3299_y_0_to_fp16 = const()[name = string("_inversed_3299_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 1]> _inversed_3299_cast_fp16 = mul(x = var_3297_cast_fp16, y = _inversed_3299_y_0_to_fp16)[name = string("_inversed_3299_cast_fp16")];
            string scores_43_equation_0 = const()[name = string("scores_43_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 1]> scores_43_cast_fp16 = einsum(equation = scores_43_equation_0, values = (var_3109_cast_fp16_1, var_3099_cast_fp16_5))[name = string("scores_43_cast_fp16")];
            fp16 var_3302_to_fp16 = const()[name = string("op_3302_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 1]> var_3303_cast_fp16 = mul(x = scores_43_cast_fp16, y = var_3302_to_fp16)[name = string("op_3303_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_3304_cast_fp16 = add(x = var_3303_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_3304_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_3305_cast_fp16 = softmax(axis = var_3077, x = var_3304_cast_fp16)[name = string("op_3305_cast_fp16")];
            tensor<int32, [4]> tile_51 = const()[name = string("tile_51"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_3306_axis_0 = const()[name = string("op_3306_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 1]> var_3306_cast_fp16_0, tensor<fp16, [1, 512, 1, 1]> var_3306_cast_fp16_1, tensor<fp16, [1, 512, 1, 1]> var_3306_cast_fp16_2, tensor<fp16, [1, 512, 1, 1]> var_3306_cast_fp16_3 = split(axis = var_3306_axis_0, split_sizes = tile_51, x = var_3305_cast_fp16)[name = string("op_3306_cast_fp16")];
            tensor<int32, [4]> tile_52 = const()[name = string("tile_52"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_3311_axis_0 = const()[name = string("op_3311_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_3311_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_3311_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_3311_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_3311_cast_fp16_3 = split(axis = var_3311_axis_0, split_sizes = tile_52, x = var_3112_cast_fp16_1)[name = string("op_3311_cast_fp16")];
            fp16 var_3316_to_fp16 = const()[name = string("op_3316_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_3317_cast_fp16 = mul(x = var_3306_cast_fp16_0, y = var_3316_to_fp16)[name = string("op_3317_cast_fp16")];
            string var_3319_equation_0 = const()[name = string("op_3319_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_3319_cast_fp16 = einsum(equation = var_3319_equation_0, values = (var_3311_cast_fp16_0, var_3317_cast_fp16))[name = string("op_3319_cast_fp16")];
            fp16 var_3320_to_fp16 = const()[name = string("op_3320_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_3321_cast_fp16 = mul(x = var_3306_cast_fp16_1, y = var_3320_to_fp16)[name = string("op_3321_cast_fp16")];
            string var_3323_equation_0 = const()[name = string("op_3323_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_3323_cast_fp16 = einsum(equation = var_3323_equation_0, values = (var_3311_cast_fp16_1, var_3321_cast_fp16))[name = string("op_3323_cast_fp16")];
            fp16 var_3324_to_fp16 = const()[name = string("op_3324_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_3325_cast_fp16 = mul(x = var_3306_cast_fp16_2, y = var_3324_to_fp16)[name = string("op_3325_cast_fp16")];
            string var_3327_equation_0 = const()[name = string("op_3327_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_3327_cast_fp16 = einsum(equation = var_3327_equation_0, values = (var_3311_cast_fp16_2, var_3325_cast_fp16))[name = string("op_3327_cast_fp16")];
            fp16 var_3328_to_fp16 = const()[name = string("op_3328_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_3329_cast_fp16 = mul(x = var_3306_cast_fp16_3, y = var_3328_to_fp16)[name = string("op_3329_cast_fp16")];
            string var_3331_equation_0 = const()[name = string("op_3331_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_3331_cast_fp16 = einsum(equation = var_3331_equation_0, values = (var_3311_cast_fp16_3, var_3329_cast_fp16))[name = string("op_3331_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_3332_cast_fp16 = add(x = var_3319_cast_fp16, y = var_3323_cast_fp16)[name = string("op_3332_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_3333_cast_fp16 = add(x = var_3327_cast_fp16, y = var_3331_cast_fp16)[name = string("op_3333_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_3334_cast_fp16 = add(x = var_3332_cast_fp16, y = var_3333_cast_fp16)[name = string("op_3334_cast_fp16")];
            fp16 _inversed_3336_y_0_to_fp16 = const()[name = string("_inversed_3336_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 1]> _inversed_3336_cast_fp16 = mul(x = var_3334_cast_fp16, y = _inversed_3336_y_0_to_fp16)[name = string("_inversed_3336_cast_fp16")];
            string scores_45_equation_0 = const()[name = string("scores_45_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 1]> scores_45_cast_fp16 = einsum(equation = scores_45_equation_0, values = (var_3109_cast_fp16_1, var_3099_cast_fp16_6))[name = string("scores_45_cast_fp16")];
            fp16 var_3339_to_fp16 = const()[name = string("op_3339_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 1]> var_3340_cast_fp16 = mul(x = scores_45_cast_fp16, y = var_3339_to_fp16)[name = string("op_3340_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_3341_cast_fp16 = add(x = var_3340_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_3341_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_3342_cast_fp16 = softmax(axis = var_3077, x = var_3341_cast_fp16)[name = string("op_3342_cast_fp16")];
            tensor<int32, [4]> tile_53 = const()[name = string("tile_53"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_3343_axis_0 = const()[name = string("op_3343_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 1]> var_3343_cast_fp16_0, tensor<fp16, [1, 512, 1, 1]> var_3343_cast_fp16_1, tensor<fp16, [1, 512, 1, 1]> var_3343_cast_fp16_2, tensor<fp16, [1, 512, 1, 1]> var_3343_cast_fp16_3 = split(axis = var_3343_axis_0, split_sizes = tile_53, x = var_3342_cast_fp16)[name = string("op_3343_cast_fp16")];
            tensor<int32, [4]> tile_54 = const()[name = string("tile_54"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_3348_axis_0 = const()[name = string("op_3348_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_3348_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_3348_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_3348_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_3348_cast_fp16_3 = split(axis = var_3348_axis_0, split_sizes = tile_54, x = var_3112_cast_fp16_1)[name = string("op_3348_cast_fp16")];
            fp16 var_3353_to_fp16 = const()[name = string("op_3353_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_3354_cast_fp16 = mul(x = var_3343_cast_fp16_0, y = var_3353_to_fp16)[name = string("op_3354_cast_fp16")];
            string var_3356_equation_0 = const()[name = string("op_3356_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_3356_cast_fp16 = einsum(equation = var_3356_equation_0, values = (var_3348_cast_fp16_0, var_3354_cast_fp16))[name = string("op_3356_cast_fp16")];
            fp16 var_3357_to_fp16 = const()[name = string("op_3357_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_3358_cast_fp16 = mul(x = var_3343_cast_fp16_1, y = var_3357_to_fp16)[name = string("op_3358_cast_fp16")];
            string var_3360_equation_0 = const()[name = string("op_3360_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_3360_cast_fp16 = einsum(equation = var_3360_equation_0, values = (var_3348_cast_fp16_1, var_3358_cast_fp16))[name = string("op_3360_cast_fp16")];
            fp16 var_3361_to_fp16 = const()[name = string("op_3361_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_3362_cast_fp16 = mul(x = var_3343_cast_fp16_2, y = var_3361_to_fp16)[name = string("op_3362_cast_fp16")];
            string var_3364_equation_0 = const()[name = string("op_3364_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_3364_cast_fp16 = einsum(equation = var_3364_equation_0, values = (var_3348_cast_fp16_2, var_3362_cast_fp16))[name = string("op_3364_cast_fp16")];
            fp16 var_3365_to_fp16 = const()[name = string("op_3365_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_3366_cast_fp16 = mul(x = var_3343_cast_fp16_3, y = var_3365_to_fp16)[name = string("op_3366_cast_fp16")];
            string var_3368_equation_0 = const()[name = string("op_3368_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_3368_cast_fp16 = einsum(equation = var_3368_equation_0, values = (var_3348_cast_fp16_3, var_3366_cast_fp16))[name = string("op_3368_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_3369_cast_fp16 = add(x = var_3356_cast_fp16, y = var_3360_cast_fp16)[name = string("op_3369_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_3370_cast_fp16 = add(x = var_3364_cast_fp16, y = var_3368_cast_fp16)[name = string("op_3370_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_3371_cast_fp16 = add(x = var_3369_cast_fp16, y = var_3370_cast_fp16)[name = string("op_3371_cast_fp16")];
            fp16 _inversed_3373_y_0_to_fp16 = const()[name = string("_inversed_3373_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 1]> _inversed_3373_cast_fp16 = mul(x = var_3371_cast_fp16, y = _inversed_3373_y_0_to_fp16)[name = string("_inversed_3373_cast_fp16")];
            string scores_47_equation_0 = const()[name = string("scores_47_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 1]> scores_47_cast_fp16 = einsum(equation = scores_47_equation_0, values = (var_3109_cast_fp16_1, var_3099_cast_fp16_7))[name = string("scores_47_cast_fp16")];
            fp16 var_3376_to_fp16 = const()[name = string("op_3376_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 1]> var_3377_cast_fp16 = mul(x = scores_47_cast_fp16, y = var_3376_to_fp16)[name = string("op_3377_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_3378_cast_fp16 = add(x = var_3377_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_3378_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_3379_cast_fp16 = softmax(axis = var_3077, x = var_3378_cast_fp16)[name = string("op_3379_cast_fp16")];
            tensor<int32, [4]> tile_55 = const()[name = string("tile_55"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_3380_axis_0 = const()[name = string("op_3380_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 1]> var_3380_cast_fp16_0, tensor<fp16, [1, 512, 1, 1]> var_3380_cast_fp16_1, tensor<fp16, [1, 512, 1, 1]> var_3380_cast_fp16_2, tensor<fp16, [1, 512, 1, 1]> var_3380_cast_fp16_3 = split(axis = var_3380_axis_0, split_sizes = tile_55, x = var_3379_cast_fp16)[name = string("op_3380_cast_fp16")];
            tensor<int32, [4]> tile_56 = const()[name = string("tile_56"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_3385_axis_0 = const()[name = string("op_3385_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_3385_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_3385_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_3385_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_3385_cast_fp16_3 = split(axis = var_3385_axis_0, split_sizes = tile_56, x = var_3112_cast_fp16_1)[name = string("op_3385_cast_fp16")];
            fp16 var_3390_to_fp16 = const()[name = string("op_3390_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_3391_cast_fp16 = mul(x = var_3380_cast_fp16_0, y = var_3390_to_fp16)[name = string("op_3391_cast_fp16")];
            string var_3393_equation_0 = const()[name = string("op_3393_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_3393_cast_fp16 = einsum(equation = var_3393_equation_0, values = (var_3385_cast_fp16_0, var_3391_cast_fp16))[name = string("op_3393_cast_fp16")];
            fp16 var_3394_to_fp16 = const()[name = string("op_3394_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_3395_cast_fp16 = mul(x = var_3380_cast_fp16_1, y = var_3394_to_fp16)[name = string("op_3395_cast_fp16")];
            string var_3397_equation_0 = const()[name = string("op_3397_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_3397_cast_fp16 = einsum(equation = var_3397_equation_0, values = (var_3385_cast_fp16_1, var_3395_cast_fp16))[name = string("op_3397_cast_fp16")];
            fp16 var_3398_to_fp16 = const()[name = string("op_3398_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_3399_cast_fp16 = mul(x = var_3380_cast_fp16_2, y = var_3398_to_fp16)[name = string("op_3399_cast_fp16")];
            string var_3401_equation_0 = const()[name = string("op_3401_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_3401_cast_fp16 = einsum(equation = var_3401_equation_0, values = (var_3385_cast_fp16_2, var_3399_cast_fp16))[name = string("op_3401_cast_fp16")];
            fp16 var_3402_to_fp16 = const()[name = string("op_3402_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_3403_cast_fp16 = mul(x = var_3380_cast_fp16_3, y = var_3402_to_fp16)[name = string("op_3403_cast_fp16")];
            string var_3405_equation_0 = const()[name = string("op_3405_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_3405_cast_fp16 = einsum(equation = var_3405_equation_0, values = (var_3385_cast_fp16_3, var_3403_cast_fp16))[name = string("op_3405_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_3406_cast_fp16 = add(x = var_3393_cast_fp16, y = var_3397_cast_fp16)[name = string("op_3406_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_3407_cast_fp16 = add(x = var_3401_cast_fp16, y = var_3405_cast_fp16)[name = string("op_3407_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_3408_cast_fp16 = add(x = var_3406_cast_fp16, y = var_3407_cast_fp16)[name = string("op_3408_cast_fp16")];
            fp16 _inversed_3410_y_0_to_fp16 = const()[name = string("_inversed_3410_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 1]> _inversed_3410_cast_fp16 = mul(x = var_3408_cast_fp16, y = _inversed_3410_y_0_to_fp16)[name = string("_inversed_3410_cast_fp16")];
            bool var_3412_interleave_0 = const()[name = string("op_3412_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 2048, 1, 1]> var_3412_cast_fp16 = concat(axis = var_3077, interleave = var_3412_interleave_0, values = (_inversed_3151_cast_fp16, _inversed_3188_cast_fp16, _inversed_3225_cast_fp16, _inversed_3262_cast_fp16, _inversed_3299_cast_fp16, _inversed_3336_cast_fp16, _inversed_3373_cast_fp16, _inversed_3410_cast_fp16))[name = string("op_3412_cast_fp16")];
            tensor<int32, [4]> var_3413 = const()[name = string("op_3413"), val = tensor<int32, [4]>([2, 4, 256, 1])];
            tensor<fp16, [2, 4, 256, 1]> var_3414_cast_fp16 = reshape(shape = var_3413, x = var_3412_cast_fp16)[name = string("op_3414_cast_fp16")];
            tensor<int32, [4]> transpose_5_perm_0 = const()[name = string("transpose_5_perm_0"), val = tensor<int32, [4]>([3, 0, 1, 2])];
            tensor<int32, [2]> var_3432 = const()[name = string("op_3432"), val = tensor<int32, [2]>([1, 2048])];
            tensor<fp16, [1, 2, 4, 256]> transpose_5_cast_fp16 = transpose(perm = transpose_5_perm_0, x = var_3414_cast_fp16)[name = string("transpose_54")];
            tensor<fp16, [1, 2048]> attention_output_11_cast_fp16 = reshape(shape = var_3432, x = transpose_5_cast_fp16)[name = string("attention_output_11_cast_fp16")];
            tensor<fp16, [1, 2048]> var_3434_cast_fp16 = sigmoid(x = gate_11_cast_fp16)[name = string("op_3434_cast_fp16")];
            tensor<fp16, [1, 2048]> input_149_cast_fp16 = mul(x = attention_output_11_cast_fp16, y = var_3434_cast_fp16)[name = string("input_149_cast_fp16")];
            tensor<fp16, [1024, 2048]> completions_2_o_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 2048]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(179014592))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(180587520))))[name = string("completions_2_o_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 1024]> linear_89_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = completions_2_o_proj_weight_promoted_to_fp16_palettized, x = input_149_cast_fp16)[name = string("linear_89_cast_fp16")];
            tensor<fp16, [1, 1024]> value_173_cast_fp16 = add(x = value_159_cast_fp16, y = linear_89_cast_fp16)[name = string("value_173_cast_fp16")];
            tensor<fp16, [1, 1024]> var_3439_cast_fp16 = mul(x = value_173_cast_fp16, y = value_173_cast_fp16)[name = string("op_3439_cast_fp16")];
            tensor<int32, [1]> variance_77_axes_0 = const()[name = string("variance_77_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_77_keep_dims_0 = const()[name = string("variance_77_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 1]> variance_77_cast_fp16 = reduce_mean(axes = variance_77_axes_0, keep_dims = variance_77_keep_dims_0, x = var_3439_cast_fp16)[name = string("variance_77_cast_fp16")];
            fp16 var_3442_to_fp16 = const()[name = string("op_3442_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 1]> var_3443_cast_fp16 = add(x = variance_77_cast_fp16, y = var_3442_to_fp16)[name = string("op_3443_cast_fp16")];
            fp32 var_3444_epsilon_0 = const()[name = string("op_3444_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 1]> var_3444_cast_fp16 = rsqrt(epsilon = var_3444_epsilon_0, x = var_3443_cast_fp16)[name = string("op_3444_cast_fp16")];
            tensor<fp16, [1, 1024]> var_3445_cast_fp16 = mul(x = value_173_cast_fp16, y = var_3444_cast_fp16)[name = string("op_3445_cast_fp16")];
            tensor<fp16, [1024]> var_3447_to_fp16 = const()[name = string("op_3447_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(180595776)))];
            tensor<fp16, [1, 1024]> input_151_cast_fp16 = mul(x = var_3445_cast_fp16, y = var_3447_to_fp16)[name = string("input_151_cast_fp16")];
            tensor<fp16, [3584, 1024]> completions_2_gate_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(180597888))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(183350464))))[name = string("completions_2_gate_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 3584]> linear_90_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = completions_2_gate_proj_weight_promoted_to_fp16_palettized, x = input_151_cast_fp16)[name = string("linear_90_cast_fp16")];
            tensor<fp16, [1, 3584]> var_3451_cast_fp16 = silu(x = linear_90_cast_fp16)[name = string("op_3451_cast_fp16")];
            tensor<fp16, [3584, 1024]> completions_2_up_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(183379200))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(186131776))))[name = string("completions_2_up_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 3584]> linear_91_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = completions_2_up_proj_weight_promoted_to_fp16_palettized, x = input_151_cast_fp16)[name = string("linear_91_cast_fp16")];
            tensor<fp16, [1, 3584]> input_155_cast_fp16 = mul(x = var_3451_cast_fp16, y = linear_91_cast_fp16)[name = string("input_155_cast_fp16")];
            tensor<fp16, [1024, 3584]> completions_2_down_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 3584]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(186160512))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(188913088))))[name = string("completions_2_down_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 1024]> linear_92_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = completions_2_down_proj_weight_promoted_to_fp16_palettized, x = input_155_cast_fp16)[name = string("linear_92_cast_fp16")];
            tensor<fp16, [1, 1024]> value_175_cast_fp16 = add(x = value_173_cast_fp16, y = linear_92_cast_fp16)[name = string("value_175_cast_fp16")];
            int32 var_3472 = const()[name = string("op_3472"), val = int32(-1)];
            tensor<fp16, [1, 1024]> var_3487_cast_fp16 = mul(x = value_175_cast_fp16, y = value_175_cast_fp16)[name = string("op_3487_cast_fp16")];
            tensor<int32, [1]> variance_79_axes_0 = const()[name = string("variance_79_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_79_keep_dims_0 = const()[name = string("variance_79_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 1]> variance_79_cast_fp16 = reduce_mean(axes = variance_79_axes_0, keep_dims = variance_79_keep_dims_0, x = var_3487_cast_fp16)[name = string("variance_79_cast_fp16")];
            fp16 var_3490_to_fp16 = const()[name = string("op_3490_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 1]> var_3491_cast_fp16 = add(x = variance_79_cast_fp16, y = var_3490_to_fp16)[name = string("op_3491_cast_fp16")];
            fp32 var_3492_epsilon_0 = const()[name = string("op_3492_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 1]> var_3492_cast_fp16 = rsqrt(epsilon = var_3492_epsilon_0, x = var_3491_cast_fp16)[name = string("op_3492_cast_fp16")];
            tensor<fp16, [1, 1024]> var_3493_cast_fp16 = mul(x = value_175_cast_fp16, y = var_3492_cast_fp16)[name = string("op_3493_cast_fp16")];
            tensor<fp16, [1024]> var_3495_to_fp16 = const()[name = string("op_3495_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(188921344)))];
            tensor<fp16, [1, 1024]> input_157_cast_fp16 = mul(x = var_3493_cast_fp16, y = var_3495_to_fp16)[name = string("input_157_cast_fp16")];
            tensor<fp16, [6144, 1024]> groups_3_0_in_proj_qkv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(188923456))), lut = tensor<fp16, [384, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(193642112))))[name = string("groups_3_0_in_proj_qkv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 6144]> linear_93_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = groups_3_0_in_proj_qkv_weight_promoted_to_fp16_palettized, x = input_157_cast_fp16)[name = string("linear_93_cast_fp16")];
            tensor<int32, [2]> var_3499_perm_0 = const()[name = string("op_3499_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> mixed_19_axes_0 = const()[name = string("mixed_19_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [6144, 1]> var_3499_cast_fp16 = transpose(perm = var_3499_perm_0, x = linear_93_cast_fp16)[name = string("transpose_53")];
            tensor<fp16, [1, 6144, 1]> mixed_19_cast_fp16 = expand_dims(axes = mixed_19_axes_0, x = var_3499_cast_fp16)[name = string("mixed_19_cast_fp16")];
            bool convolution_input_19_interleave_0 = const()[name = string("convolution_input_19_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 6144, 4]> convolution_input_19_cast_fp16 = concat(axis = var_3472, interleave = convolution_input_19_interleave_0, values = (conv12, mixed_19_cast_fp16))[name = string("convolution_input_19_cast_fp16")];
            string input_159_pad_type_0 = const()[name = string("input_159_pad_type_0"), val = string("valid")];
            int32 input_159_groups_0 = const()[name = string("input_159_groups_0"), val = int32(6144)];
            tensor<int32, [1]> input_159_strides_0 = const()[name = string("input_159_strides_0"), val = tensor<int32, [1]>([1])];
            tensor<int32, [2]> input_159_pad_0 = const()[name = string("input_159_pad_0"), val = tensor<int32, [2]>([0, 0])];
            tensor<int32, [1]> input_159_dilations_0 = const()[name = string("input_159_dilations_0"), val = tensor<int32, [1]>([1])];
            tensor<fp16, [6144, 1, 4]> groups_3_0_conv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1, 4]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(193691328))), lut = tensor<fp16, [384, 1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(193709824))))[name = string("groups_3_0_conv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 6144, 1]> input_159_cast_fp16 = conv(dilations = input_159_dilations_0, groups = input_159_groups_0, pad = input_159_pad_0, pad_type = input_159_pad_type_0, strides = input_159_strides_0, weight = groups_3_0_conv_weight_promoted_to_fp16_palettized, x = convolution_input_19_cast_fp16)[name = string("input_159_cast_fp16")];
            tensor<fp16, [1, 6144, 1]> var_3508_cast_fp16 = silu(x = input_159_cast_fp16)[name = string("op_3508_cast_fp16")];
            tensor<int32, [3]> var_3509_perm_0 = const()[name = string("op_3509_perm_0"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [3]> var_3512_begin_0 = const()[name = string("op_3512_begin_0"), val = tensor<int32, [3]>([0, 0, 1])];
            tensor<int32, [3]> var_3512_end_0 = const()[name = string("op_3512_end_0"), val = tensor<int32, [3]>([1, 6144, 4])];
            tensor<bool, [3]> var_3512_end_mask_0 = const()[name = string("op_3512_end_mask_0"), val = tensor<bool, [3]>([true, true, true])];
            tensor<fp16, [1, 6144, 3]> conv12_out = slice_by_index(begin = var_3512_begin_0, end = var_3512_end_0, end_mask = var_3512_end_mask_0, x = convolution_input_19_cast_fp16)[name = string("op_3512_cast_fp16")];
            tensor<int32, [3]> var_3513 = const()[name = string("op_3513"), val = tensor<int32, [3]>([2048, 2048, 2048])];
            int32 var_3514_axis_0 = const()[name = string("op_3514_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 1, 6144]> var_3509_cast_fp16 = transpose(perm = var_3509_perm_0, x = var_3508_cast_fp16)[name = string("transpose_52")];
            tensor<fp16, [1, 1, 2048]> var_3514_cast_fp16_0, tensor<fp16, [1, 1, 2048]> var_3514_cast_fp16_1, tensor<fp16, [1, 1, 2048]> var_3514_cast_fp16_2 = split(axis = var_3514_axis_0, split_sizes = var_3513, x = var_3509_cast_fp16)[name = string("op_3514_cast_fp16")];
            tensor<int32, [4]> var_3518 = const()[name = string("op_3518"), val = tensor<int32, [4]>([1, 1, 16, 128])];
            tensor<fp16, [1, 1, 16, 128]> value_179_cast_fp16 = reshape(shape = var_3518, x = var_3514_cast_fp16_0)[name = string("value_179_cast_fp16")];
            tensor<int32, [4]> var_3520 = const()[name = string("op_3520"), val = tensor<int32, [4]>([1, 1, 16, 128])];
            tensor<fp16, [1, 1, 16, 128]> value_181_cast_fp16 = reshape(shape = var_3520, x = var_3514_cast_fp16_1)[name = string("value_181_cast_fp16")];
            tensor<int32, [4]> var_3522 = const()[name = string("op_3522"), val = tensor<int32, [4]>([1, 1, 16, 128])];
            tensor<fp16, [1, 1, 16, 128]> value_183_cast_fp16 = reshape(shape = var_3522, x = var_3514_cast_fp16_2)[name = string("value_183_cast_fp16")];
            tensor<fp16, [1, 1, 16, 128]> var_3524_cast_fp16 = mul(x = value_179_cast_fp16, y = value_179_cast_fp16)[name = string("op_3524_cast_fp16")];
            tensor<int32, [1]> var_3526_axes_0 = const()[name = string("op_3526_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_3526_keep_dims_0 = const()[name = string("op_3526_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 1, 16, 1]> var_3526_cast_fp16 = reduce_sum(axes = var_3526_axes_0, keep_dims = var_3526_keep_dims_0, x = var_3524_cast_fp16)[name = string("op_3526_cast_fp16")];
            fp16 var_3527_to_fp16 = const()[name = string("op_3527_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 1, 16, 1]> var_3528_cast_fp16 = add(x = var_3526_cast_fp16, y = var_3527_to_fp16)[name = string("op_3528_cast_fp16")];
            fp32 var_3529_epsilon_0 = const()[name = string("op_3529_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 1, 16, 1]> var_3529_cast_fp16 = rsqrt(epsilon = var_3529_epsilon_0, x = var_3528_cast_fp16)[name = string("op_3529_cast_fp16")];
            tensor<fp16, [1, 1, 16, 128]> var_3530_cast_fp16 = mul(x = value_179_cast_fp16, y = var_3529_cast_fp16)[name = string("op_3530_cast_fp16")];
            tensor<int32, [4]> var_3531_perm_0 = const()[name = string("op_3531_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            fp16 _inversed_query_51_y_0_to_fp16 = const()[name = string("_inversed_query_51_y_0_to_fp16"), val = fp16(0x1.6ap-4)];
            tensor<fp16, [1, 16, 1, 128]> var_3531_cast_fp16 = transpose(perm = var_3531_perm_0, x = var_3530_cast_fp16)[name = string("transpose_51")];
            tensor<fp16, [1, 16, 1, 128]> _inversed_query_51_cast_fp16 = mul(x = var_3531_cast_fp16, y = _inversed_query_51_y_0_to_fp16)[name = string("_inversed_query_51_cast_fp16")];
            tensor<fp16, [1, 1, 16, 128]> var_3534_cast_fp16 = mul(x = value_181_cast_fp16, y = value_181_cast_fp16)[name = string("op_3534_cast_fp16")];
            tensor<int32, [1]> var_3536_axes_0 = const()[name = string("op_3536_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_3536_keep_dims_0 = const()[name = string("op_3536_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 1, 16, 1]> var_3536_cast_fp16 = reduce_sum(axes = var_3536_axes_0, keep_dims = var_3536_keep_dims_0, x = var_3534_cast_fp16)[name = string("op_3536_cast_fp16")];
            fp16 var_3537_to_fp16 = const()[name = string("op_3537_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 1, 16, 1]> var_3538_cast_fp16 = add(x = var_3536_cast_fp16, y = var_3537_to_fp16)[name = string("op_3538_cast_fp16")];
            fp32 var_3539_epsilon_0 = const()[name = string("op_3539_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 1, 16, 1]> var_3539_cast_fp16 = rsqrt(epsilon = var_3539_epsilon_0, x = var_3538_cast_fp16)[name = string("op_3539_cast_fp16")];
            tensor<fp16, [1, 1, 16, 128]> var_3540_cast_fp16 = mul(x = value_181_cast_fp16, y = var_3539_cast_fp16)[name = string("op_3540_cast_fp16")];
            tensor<int32, [4]> key_51_perm_0 = const()[name = string("key_51_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<int32, [4]> value_185_perm_0 = const()[name = string("value_185_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<fp16, [16, 1024]> groups_3_0_in_proj_b_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(193759040))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(193771392))))[name = string("groups_3_0_in_proj_b_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16]> linear_94_bias_0_to_fp16 = const()[name = string("linear_94_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(4862656)))];
            tensor<fp16, [1, 16]> linear_94_cast_fp16 = linear(bias = linear_94_bias_0_to_fp16, weight = groups_3_0_in_proj_b_weight_promoted_to_fp16_palettized, x = input_157_cast_fp16)[name = string("linear_94_cast_fp16")];
            tensor<fp16, [1, 16]> var_3545_cast_fp16 = sigmoid(x = linear_94_cast_fp16)[name = string("op_3545_cast_fp16")];
            tensor<int32, [2]> var_3546_perm_0 = const()[name = string("op_3546_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> beta_19_axes_0 = const()[name = string("beta_19_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 1]> var_3546_cast_fp16 = transpose(perm = var_3546_perm_0, x = var_3545_cast_fp16)[name = string("transpose_50")];
            tensor<fp16, [1, 16, 1]> beta_19_cast_fp16 = expand_dims(axes = beta_19_axes_0, x = var_3546_cast_fp16)[name = string("beta_19_cast_fp16")];
            tensor<fp16, [16, 1024]> op_3552_weight_0_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(193771584))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(193783936))))[name = string("op_3552_weight_0_to_fp16_palettized")];
            tensor<fp16, [16]> var_3552_bias_0_to_fp16 = const()[name = string("op_3552_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(193784128)))];
            tensor<fp16, [1, 16]> var_3552_cast_fp16 = linear(bias = var_3552_bias_0_to_fp16, weight = op_3552_weight_0_to_fp16_palettized, x = input_157_cast_fp16)[name = string("op_3552_cast_fp16")];
            tensor<fp16, [1, 16]> var_3553_cast_fp16 = softplus(x = var_3552_cast_fp16)[name = string("op_3553_cast_fp16")];
            tensor<fp16, [16]> var_3549_promoted_to_fp16 = const()[name = string("op_3549_promoted_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(193784256)))];
            tensor<fp16, [1, 16]> var_3554_cast_fp16 = mul(x = var_3549_promoted_to_fp16, y = var_3553_cast_fp16)[name = string("op_3554_cast_fp16")];
            tensor<int32, [2]> var_3555_perm_0 = const()[name = string("op_3555_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> decay_19_axes_0 = const()[name = string("decay_19_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 1]> var_3555_cast_fp16 = transpose(perm = var_3555_perm_0, x = var_3554_cast_fp16)[name = string("transpose_49")];
            tensor<fp16, [1, 16, 1]> decay_19_cast_fp16 = expand_dims(axes = decay_19_axes_0, x = var_3555_cast_fp16)[name = string("decay_19_cast_fp16")];
            tensor<fp16, [2048, 1024]> groups_3_0_in_proj_z_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [2048, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(193784384))), lut = tensor<fp16, [128, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(195357312))))[name = string("groups_3_0_in_proj_z_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 2048]> linear_96_cast_fp16 = linear(bias = linear_3_bias_0_to_fp16, weight = groups_3_0_in_proj_z_weight_promoted_to_fp16_palettized, x = input_157_cast_fp16)[name = string("linear_96_cast_fp16")];
            tensor<int32, [3]> var_3563_begin_0 = const()[name = string("op_3563_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_3563_end_0 = const()[name = string("op_3563_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_3563_end_mask_0 = const()[name = string("op_3563_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_3563_squeeze_mask_0 = const()[name = string("op_3563_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_3563_cast_fp16 = slice_by_index(begin = var_3563_begin_0, end = var_3563_end_0, end_mask = var_3563_end_mask_0, squeeze_mask = var_3563_squeeze_mask_0, x = decay_19_cast_fp16)[name = string("op_3563_cast_fp16")];
            tensor<fp16, [1, 16]> var_3564_cast_fp16 = exp(x = var_3563_cast_fp16)[name = string("op_3564_cast_fp16")];
            tensor<int32, [1]> var_3565_axes_0 = const()[name = string("op_3565_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_3565_cast_fp16 = expand_dims(axes = var_3565_axes_0, x = var_3564_cast_fp16)[name = string("op_3565_cast_fp16")];
            tensor<int32, [1]> var_3566_axes_0 = const()[name = string("op_3566_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_3566_cast_fp16 = expand_dims(axes = var_3566_axes_0, x = var_3565_cast_fp16)[name = string("op_3566_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_37_cast_fp16 = mul(x = rec12, y = var_3566_cast_fp16)[name = string("state_37_cast_fp16")];
            tensor<int32, [4]> key_token_19_begin_0 = const()[name = string("key_token_19_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> key_token_19_end_0 = const()[name = string("key_token_19_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> key_token_19_end_mask_0 = const()[name = string("key_token_19_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_19_squeeze_mask_0 = const()[name = string("key_token_19_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 1, 128]> key_51_cast_fp16 = transpose(perm = key_51_perm_0, x = var_3540_cast_fp16)[name = string("transpose_48")];
            tensor<fp16, [1, 16, 128]> key_token_19_cast_fp16 = slice_by_index(begin = key_token_19_begin_0, end = key_token_19_end_0, end_mask = key_token_19_end_mask_0, squeeze_mask = key_token_19_squeeze_mask_0, x = key_51_cast_fp16)[name = string("key_token_19_cast_fp16")];
            tensor<int32, [4]> value_token_19_begin_0 = const()[name = string("value_token_19_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> value_token_19_end_0 = const()[name = string("value_token_19_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> value_token_19_end_mask_0 = const()[name = string("value_token_19_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_19_squeeze_mask_0 = const()[name = string("value_token_19_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 1, 128]> value_185_cast_fp16 = transpose(perm = value_185_perm_0, x = value_183_cast_fp16)[name = string("transpose_47")];
            tensor<fp16, [1, 16, 128]> value_token_19_cast_fp16 = slice_by_index(begin = value_token_19_begin_0, end = value_token_19_end_0, end_mask = value_token_19_end_mask_0, squeeze_mask = value_token_19_squeeze_mask_0, x = value_185_cast_fp16)[name = string("value_token_19_cast_fp16")];
            tensor<int32, [1]> var_3574_axes_0 = const()[name = string("op_3574_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_3574_cast_fp16 = expand_dims(axes = var_3574_axes_0, x = key_token_19_cast_fp16)[name = string("op_3574_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3575_cast_fp16 = mul(x = state_37_cast_fp16, y = var_3574_cast_fp16)[name = string("op_3575_cast_fp16")];
            tensor<int32, [1]> memory_19_axes_0 = const()[name = string("memory_19_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_19_keep_dims_0 = const()[name = string("memory_19_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_19_cast_fp16 = reduce_sum(axes = memory_19_axes_0, keep_dims = memory_19_keep_dims_0, x = var_3575_cast_fp16)[name = string("memory_19_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_3578_cast_fp16 = sub(x = value_token_19_cast_fp16, y = memory_19_cast_fp16)[name = string("op_3578_cast_fp16")];
            tensor<int32, [3]> var_3581_begin_0 = const()[name = string("op_3581_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_3581_end_0 = const()[name = string("op_3581_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_3581_end_mask_0 = const()[name = string("op_3581_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_3581_squeeze_mask_0 = const()[name = string("op_3581_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_3581_cast_fp16 = slice_by_index(begin = var_3581_begin_0, end = var_3581_end_0, end_mask = var_3581_end_mask_0, squeeze_mask = var_3581_squeeze_mask_0, x = beta_19_cast_fp16)[name = string("op_3581_cast_fp16")];
            tensor<int32, [1]> var_3582_axes_0 = const()[name = string("op_3582_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_3582_cast_fp16 = expand_dims(axes = var_3582_axes_0, x = var_3581_cast_fp16)[name = string("op_3582_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_19_cast_fp16 = mul(x = var_3578_cast_fp16, y = var_3582_cast_fp16)[name = string("delta_19_cast_fp16")];
            tensor<int32, [1]> var_3585_axes_0 = const()[name = string("op_3585_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_3585_cast_fp16 = expand_dims(axes = var_3585_axes_0, x = delta_19_cast_fp16)[name = string("op_3585_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3586_cast_fp16 = mul(x = var_3574_cast_fp16, y = var_3585_cast_fp16)[name = string("op_3586_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> rec12_out = add(x = state_37_cast_fp16, y = var_3586_cast_fp16)[name = string("state_39_cast_fp16")];
            tensor<int32, [4]> var_3590_begin_0 = const()[name = string("op_3590_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> var_3590_end_0 = const()[name = string("op_3590_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> var_3590_end_mask_0 = const()[name = string("op_3590_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_3590_squeeze_mask_0 = const()[name = string("op_3590_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_3590_cast_fp16 = slice_by_index(begin = var_3590_begin_0, end = var_3590_end_0, end_mask = var_3590_end_mask_0, squeeze_mask = var_3590_squeeze_mask_0, x = _inversed_query_51_cast_fp16)[name = string("op_3590_cast_fp16")];
            tensor<int32, [1]> var_3591_axes_0 = const()[name = string("op_3591_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_3591_cast_fp16 = expand_dims(axes = var_3591_axes_0, x = var_3590_cast_fp16)[name = string("op_3591_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3592_cast_fp16 = mul(x = rec12_out, y = var_3591_cast_fp16)[name = string("op_3592_cast_fp16")];
            tensor<int32, [1]> var_3594_axes_0 = const()[name = string("op_3594_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_3594_keep_dims_0 = const()[name = string("op_3594_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_3594_cast_fp16 = reduce_sum(axes = var_3594_axes_0, keep_dims = var_3594_keep_dims_0, x = var_3592_cast_fp16)[name = string("op_3594_cast_fp16")];
            tensor<int32, [1]> attention_91_axes_0 = const()[name = string("attention_91_axes_0"), val = tensor<int32, [1]>([1])];
            tensor<fp16, [1, 1, 16, 128]> attention_91_cast_fp16 = expand_dims(axes = attention_91_axes_0, x = var_3594_cast_fp16)[name = string("attention_91_cast_fp16")];
            tensor<int32, [2]> var_3597 = const()[name = string("op_3597"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [16, 128]> attention_93_cast_fp16 = reshape(shape = var_3597, x = attention_91_cast_fp16)[name = string("attention_93_cast_fp16")];
            tensor<int32, [2]> var_3599 = const()[name = string("op_3599"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [16, 128]> input_161_cast_fp16 = reshape(shape = var_3599, x = linear_96_cast_fp16)[name = string("input_161_cast_fp16")];
            tensor<fp16, [16, 128]> var_3601_cast_fp16 = mul(x = attention_93_cast_fp16, y = attention_93_cast_fp16)[name = string("op_3601_cast_fp16")];
            tensor<int32, [1]> variance_81_axes_0 = const()[name = string("variance_81_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_81_keep_dims_0 = const()[name = string("variance_81_keep_dims_0"), val = bool(true)];
            tensor<fp16, [16, 1]> variance_81_cast_fp16 = reduce_mean(axes = variance_81_axes_0, keep_dims = variance_81_keep_dims_0, x = var_3601_cast_fp16)[name = string("variance_81_cast_fp16")];
            fp16 var_3604_to_fp16 = const()[name = string("op_3604_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [16, 1]> var_3605_cast_fp16 = add(x = variance_81_cast_fp16, y = var_3604_to_fp16)[name = string("op_3605_cast_fp16")];
            fp32 var_3606_epsilon_0 = const()[name = string("op_3606_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [16, 1]> var_3606_cast_fp16 = rsqrt(epsilon = var_3606_epsilon_0, x = var_3605_cast_fp16)[name = string("op_3606_cast_fp16")];
            tensor<fp16, [16, 128]> attention_95_cast_fp16 = mul(x = attention_93_cast_fp16, y = var_3606_cast_fp16)[name = string("attention_95_cast_fp16")];
            tensor<fp16, [128]> groups_3_0_gated_norm_promoted_to_fp16 = const()[name = string("groups_3_0_gated_norm_promoted_to_fp16"), val = tensor<fp16, [128]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(195373760)))];
            tensor<fp16, [16, 128]> attention_97_cast_fp16 = mul(x = attention_95_cast_fp16, y = groups_3_0_gated_norm_promoted_to_fp16)[name = string("attention_97_cast_fp16")];
            tensor<fp16, [16, 128]> var_3609_cast_fp16 = silu(x = input_161_cast_fp16)[name = string("op_3609_cast_fp16")];
            tensor<fp16, [16, 128]> attention_99_cast_fp16 = mul(x = attention_97_cast_fp16, y = var_3609_cast_fp16)[name = string("attention_99_cast_fp16")];
            tensor<int32, [2]> var_3611 = const()[name = string("op_3611"), val = tensor<int32, [2]>([1, 2048])];
            tensor<fp16, [1, 2048]> input_163_cast_fp16 = reshape(shape = var_3611, x = attention_99_cast_fp16)[name = string("input_163_cast_fp16")];
            tensor<fp16, [1024, 2048]> groups_3_0_out_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 2048]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(195374080))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(196947008))))[name = string("groups_3_0_out_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 1024]> linear_97_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = groups_3_0_out_proj_weight_promoted_to_fp16_palettized, x = input_163_cast_fp16)[name = string("linear_97_cast_fp16")];
            tensor<fp16, [1, 1024]> value_187_cast_fp16 = add(x = value_175_cast_fp16, y = linear_97_cast_fp16)[name = string("value_187_cast_fp16")];
            tensor<fp16, [1, 1024]> var_3616_cast_fp16 = mul(x = value_187_cast_fp16, y = value_187_cast_fp16)[name = string("op_3616_cast_fp16")];
            tensor<int32, [1]> variance_83_axes_0 = const()[name = string("variance_83_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_83_keep_dims_0 = const()[name = string("variance_83_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 1]> variance_83_cast_fp16 = reduce_mean(axes = variance_83_axes_0, keep_dims = variance_83_keep_dims_0, x = var_3616_cast_fp16)[name = string("variance_83_cast_fp16")];
            fp16 var_3619_to_fp16 = const()[name = string("op_3619_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 1]> var_3620_cast_fp16 = add(x = variance_83_cast_fp16, y = var_3619_to_fp16)[name = string("op_3620_cast_fp16")];
            fp32 var_3621_epsilon_0 = const()[name = string("op_3621_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 1]> var_3621_cast_fp16 = rsqrt(epsilon = var_3621_epsilon_0, x = var_3620_cast_fp16)[name = string("op_3621_cast_fp16")];
            tensor<fp16, [1, 1024]> var_3622_cast_fp16 = mul(x = value_187_cast_fp16, y = var_3621_cast_fp16)[name = string("op_3622_cast_fp16")];
            tensor<fp16, [1024]> var_3624_to_fp16 = const()[name = string("op_3624_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(196955264)))];
            tensor<fp16, [1, 1024]> input_165_cast_fp16 = mul(x = var_3622_cast_fp16, y = var_3624_to_fp16)[name = string("input_165_cast_fp16")];
            tensor<fp16, [3584, 1024]> groups_3_0_gate_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(196957376))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(199709952))))[name = string("groups_3_0_gate_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 3584]> linear_98_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = groups_3_0_gate_proj_weight_promoted_to_fp16_palettized, x = input_165_cast_fp16)[name = string("linear_98_cast_fp16")];
            tensor<fp16, [1, 3584]> var_3628_cast_fp16 = silu(x = linear_98_cast_fp16)[name = string("op_3628_cast_fp16")];
            tensor<fp16, [3584, 1024]> groups_3_0_up_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(199738688))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(202491264))))[name = string("groups_3_0_up_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 3584]> linear_99_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = groups_3_0_up_proj_weight_promoted_to_fp16_palettized, x = input_165_cast_fp16)[name = string("linear_99_cast_fp16")];
            tensor<fp16, [1, 3584]> input_169_cast_fp16 = mul(x = var_3628_cast_fp16, y = linear_99_cast_fp16)[name = string("input_169_cast_fp16")];
            tensor<fp16, [1024, 3584]> groups_3_0_down_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 3584]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(202520000))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(205272576))))[name = string("groups_3_0_down_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 1024]> linear_100_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = groups_3_0_down_proj_weight_promoted_to_fp16_palettized, x = input_169_cast_fp16)[name = string("linear_100_cast_fp16")];
            tensor<fp16, [1, 1024]> value_189_cast_fp16 = add(x = value_187_cast_fp16, y = linear_100_cast_fp16)[name = string("value_189_cast_fp16")];
            int32 var_3653 = const()[name = string("op_3653"), val = int32(-1)];
            tensor<fp16, [1, 1024]> var_3668_cast_fp16 = mul(x = value_189_cast_fp16, y = value_189_cast_fp16)[name = string("op_3668_cast_fp16")];
            tensor<int32, [1]> variance_85_axes_0 = const()[name = string("variance_85_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_85_keep_dims_0 = const()[name = string("variance_85_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 1]> variance_85_cast_fp16 = reduce_mean(axes = variance_85_axes_0, keep_dims = variance_85_keep_dims_0, x = var_3668_cast_fp16)[name = string("variance_85_cast_fp16")];
            fp16 var_3671_to_fp16 = const()[name = string("op_3671_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 1]> var_3672_cast_fp16 = add(x = variance_85_cast_fp16, y = var_3671_to_fp16)[name = string("op_3672_cast_fp16")];
            fp32 var_3673_epsilon_0 = const()[name = string("op_3673_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 1]> var_3673_cast_fp16 = rsqrt(epsilon = var_3673_epsilon_0, x = var_3672_cast_fp16)[name = string("op_3673_cast_fp16")];
            tensor<fp16, [1, 1024]> var_3674_cast_fp16 = mul(x = value_189_cast_fp16, y = var_3673_cast_fp16)[name = string("op_3674_cast_fp16")];
            tensor<fp16, [1024]> var_3676_to_fp16 = const()[name = string("op_3676_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(205280832)))];
            tensor<fp16, [1, 1024]> input_171_cast_fp16 = mul(x = var_3674_cast_fp16, y = var_3676_to_fp16)[name = string("input_171_cast_fp16")];
            tensor<fp16, [6144, 1024]> groups_3_1_in_proj_qkv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(205282944))), lut = tensor<fp16, [384, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(210001600))))[name = string("groups_3_1_in_proj_qkv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 6144]> linear_101_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = groups_3_1_in_proj_qkv_weight_promoted_to_fp16_palettized, x = input_171_cast_fp16)[name = string("linear_101_cast_fp16")];
            tensor<int32, [2]> var_3680_perm_0 = const()[name = string("op_3680_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> mixed_21_axes_0 = const()[name = string("mixed_21_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [6144, 1]> var_3680_cast_fp16 = transpose(perm = var_3680_perm_0, x = linear_101_cast_fp16)[name = string("transpose_46")];
            tensor<fp16, [1, 6144, 1]> mixed_21_cast_fp16 = expand_dims(axes = mixed_21_axes_0, x = var_3680_cast_fp16)[name = string("mixed_21_cast_fp16")];
            bool convolution_input_21_interleave_0 = const()[name = string("convolution_input_21_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 6144, 4]> convolution_input_21_cast_fp16 = concat(axis = var_3653, interleave = convolution_input_21_interleave_0, values = (conv13, mixed_21_cast_fp16))[name = string("convolution_input_21_cast_fp16")];
            string input_173_pad_type_0 = const()[name = string("input_173_pad_type_0"), val = string("valid")];
            int32 input_173_groups_0 = const()[name = string("input_173_groups_0"), val = int32(6144)];
            tensor<int32, [1]> input_173_strides_0 = const()[name = string("input_173_strides_0"), val = tensor<int32, [1]>([1])];
            tensor<int32, [2]> input_173_pad_0 = const()[name = string("input_173_pad_0"), val = tensor<int32, [2]>([0, 0])];
            tensor<int32, [1]> input_173_dilations_0 = const()[name = string("input_173_dilations_0"), val = tensor<int32, [1]>([1])];
            tensor<fp16, [6144, 1, 4]> groups_3_1_conv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1, 4]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(210050816))), lut = tensor<fp16, [384, 1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(210069312))))[name = string("groups_3_1_conv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 6144, 1]> input_173_cast_fp16 = conv(dilations = input_173_dilations_0, groups = input_173_groups_0, pad = input_173_pad_0, pad_type = input_173_pad_type_0, strides = input_173_strides_0, weight = groups_3_1_conv_weight_promoted_to_fp16_palettized, x = convolution_input_21_cast_fp16)[name = string("input_173_cast_fp16")];
            tensor<fp16, [1, 6144, 1]> var_3689_cast_fp16 = silu(x = input_173_cast_fp16)[name = string("op_3689_cast_fp16")];
            tensor<int32, [3]> var_3690_perm_0 = const()[name = string("op_3690_perm_0"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [3]> var_3693_begin_0 = const()[name = string("op_3693_begin_0"), val = tensor<int32, [3]>([0, 0, 1])];
            tensor<int32, [3]> var_3693_end_0 = const()[name = string("op_3693_end_0"), val = tensor<int32, [3]>([1, 6144, 4])];
            tensor<bool, [3]> var_3693_end_mask_0 = const()[name = string("op_3693_end_mask_0"), val = tensor<bool, [3]>([true, true, true])];
            tensor<fp16, [1, 6144, 3]> conv13_out = slice_by_index(begin = var_3693_begin_0, end = var_3693_end_0, end_mask = var_3693_end_mask_0, x = convolution_input_21_cast_fp16)[name = string("op_3693_cast_fp16")];
            tensor<int32, [3]> var_3694 = const()[name = string("op_3694"), val = tensor<int32, [3]>([2048, 2048, 2048])];
            int32 var_3695_axis_0 = const()[name = string("op_3695_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 1, 6144]> var_3690_cast_fp16 = transpose(perm = var_3690_perm_0, x = var_3689_cast_fp16)[name = string("transpose_45")];
            tensor<fp16, [1, 1, 2048]> var_3695_cast_fp16_0, tensor<fp16, [1, 1, 2048]> var_3695_cast_fp16_1, tensor<fp16, [1, 1, 2048]> var_3695_cast_fp16_2 = split(axis = var_3695_axis_0, split_sizes = var_3694, x = var_3690_cast_fp16)[name = string("op_3695_cast_fp16")];
            tensor<int32, [4]> var_3699 = const()[name = string("op_3699"), val = tensor<int32, [4]>([1, 1, 16, 128])];
            tensor<fp16, [1, 1, 16, 128]> value_193_cast_fp16 = reshape(shape = var_3699, x = var_3695_cast_fp16_0)[name = string("value_193_cast_fp16")];
            tensor<int32, [4]> var_3701 = const()[name = string("op_3701"), val = tensor<int32, [4]>([1, 1, 16, 128])];
            tensor<fp16, [1, 1, 16, 128]> value_195_cast_fp16 = reshape(shape = var_3701, x = var_3695_cast_fp16_1)[name = string("value_195_cast_fp16")];
            tensor<int32, [4]> var_3703 = const()[name = string("op_3703"), val = tensor<int32, [4]>([1, 1, 16, 128])];
            tensor<fp16, [1, 1, 16, 128]> value_197_cast_fp16 = reshape(shape = var_3703, x = var_3695_cast_fp16_2)[name = string("value_197_cast_fp16")];
            tensor<fp16, [1, 1, 16, 128]> var_3705_cast_fp16 = mul(x = value_193_cast_fp16, y = value_193_cast_fp16)[name = string("op_3705_cast_fp16")];
            tensor<int32, [1]> var_3707_axes_0 = const()[name = string("op_3707_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_3707_keep_dims_0 = const()[name = string("op_3707_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 1, 16, 1]> var_3707_cast_fp16 = reduce_sum(axes = var_3707_axes_0, keep_dims = var_3707_keep_dims_0, x = var_3705_cast_fp16)[name = string("op_3707_cast_fp16")];
            fp16 var_3708_to_fp16 = const()[name = string("op_3708_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 1, 16, 1]> var_3709_cast_fp16 = add(x = var_3707_cast_fp16, y = var_3708_to_fp16)[name = string("op_3709_cast_fp16")];
            fp32 var_3710_epsilon_0 = const()[name = string("op_3710_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 1, 16, 1]> var_3710_cast_fp16 = rsqrt(epsilon = var_3710_epsilon_0, x = var_3709_cast_fp16)[name = string("op_3710_cast_fp16")];
            tensor<fp16, [1, 1, 16, 128]> var_3711_cast_fp16 = mul(x = value_193_cast_fp16, y = var_3710_cast_fp16)[name = string("op_3711_cast_fp16")];
            tensor<int32, [4]> var_3712_perm_0 = const()[name = string("op_3712_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            fp16 _inversed_query_55_y_0_to_fp16 = const()[name = string("_inversed_query_55_y_0_to_fp16"), val = fp16(0x1.6ap-4)];
            tensor<fp16, [1, 16, 1, 128]> var_3712_cast_fp16 = transpose(perm = var_3712_perm_0, x = var_3711_cast_fp16)[name = string("transpose_44")];
            tensor<fp16, [1, 16, 1, 128]> _inversed_query_55_cast_fp16 = mul(x = var_3712_cast_fp16, y = _inversed_query_55_y_0_to_fp16)[name = string("_inversed_query_55_cast_fp16")];
            tensor<fp16, [1, 1, 16, 128]> var_3715_cast_fp16 = mul(x = value_195_cast_fp16, y = value_195_cast_fp16)[name = string("op_3715_cast_fp16")];
            tensor<int32, [1]> var_3717_axes_0 = const()[name = string("op_3717_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_3717_keep_dims_0 = const()[name = string("op_3717_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 1, 16, 1]> var_3717_cast_fp16 = reduce_sum(axes = var_3717_axes_0, keep_dims = var_3717_keep_dims_0, x = var_3715_cast_fp16)[name = string("op_3717_cast_fp16")];
            fp16 var_3718_to_fp16 = const()[name = string("op_3718_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 1, 16, 1]> var_3719_cast_fp16 = add(x = var_3717_cast_fp16, y = var_3718_to_fp16)[name = string("op_3719_cast_fp16")];
            fp32 var_3720_epsilon_0 = const()[name = string("op_3720_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 1, 16, 1]> var_3720_cast_fp16 = rsqrt(epsilon = var_3720_epsilon_0, x = var_3719_cast_fp16)[name = string("op_3720_cast_fp16")];
            tensor<fp16, [1, 1, 16, 128]> var_3721_cast_fp16 = mul(x = value_195_cast_fp16, y = var_3720_cast_fp16)[name = string("op_3721_cast_fp16")];
            tensor<int32, [4]> key_55_perm_0 = const()[name = string("key_55_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<int32, [4]> value_199_perm_0 = const()[name = string("value_199_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<fp16, [16, 1024]> groups_3_1_in_proj_b_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(210118528))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(210130880))))[name = string("groups_3_1_in_proj_b_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16]> linear_102_bias_0_to_fp16 = const()[name = string("linear_102_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(4862656)))];
            tensor<fp16, [1, 16]> linear_102_cast_fp16 = linear(bias = linear_102_bias_0_to_fp16, weight = groups_3_1_in_proj_b_weight_promoted_to_fp16_palettized, x = input_171_cast_fp16)[name = string("linear_102_cast_fp16")];
            tensor<fp16, [1, 16]> var_3726_cast_fp16 = sigmoid(x = linear_102_cast_fp16)[name = string("op_3726_cast_fp16")];
            tensor<int32, [2]> var_3727_perm_0 = const()[name = string("op_3727_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> beta_21_axes_0 = const()[name = string("beta_21_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 1]> var_3727_cast_fp16 = transpose(perm = var_3727_perm_0, x = var_3726_cast_fp16)[name = string("transpose_43")];
            tensor<fp16, [1, 16, 1]> beta_21_cast_fp16 = expand_dims(axes = beta_21_axes_0, x = var_3727_cast_fp16)[name = string("beta_21_cast_fp16")];
            tensor<fp16, [16, 1024]> op_3733_weight_0_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(210131072))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(210143424))))[name = string("op_3733_weight_0_to_fp16_palettized")];
            tensor<fp16, [16]> var_3733_bias_0_to_fp16 = const()[name = string("op_3733_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(210143616)))];
            tensor<fp16, [1, 16]> var_3733_cast_fp16 = linear(bias = var_3733_bias_0_to_fp16, weight = op_3733_weight_0_to_fp16_palettized, x = input_171_cast_fp16)[name = string("op_3733_cast_fp16")];
            tensor<fp16, [1, 16]> var_3734_cast_fp16 = softplus(x = var_3733_cast_fp16)[name = string("op_3734_cast_fp16")];
            tensor<fp16, [16]> var_3730_promoted_to_fp16 = const()[name = string("op_3730_promoted_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(210143744)))];
            tensor<fp16, [1, 16]> var_3735_cast_fp16 = mul(x = var_3730_promoted_to_fp16, y = var_3734_cast_fp16)[name = string("op_3735_cast_fp16")];
            tensor<int32, [2]> var_3736_perm_0 = const()[name = string("op_3736_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> decay_21_axes_0 = const()[name = string("decay_21_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 1]> var_3736_cast_fp16 = transpose(perm = var_3736_perm_0, x = var_3735_cast_fp16)[name = string("transpose_42")];
            tensor<fp16, [1, 16, 1]> decay_21_cast_fp16 = expand_dims(axes = decay_21_axes_0, x = var_3736_cast_fp16)[name = string("decay_21_cast_fp16")];
            tensor<fp16, [2048, 1024]> groups_3_1_in_proj_z_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [2048, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(210143872))), lut = tensor<fp16, [128, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(211716800))))[name = string("groups_3_1_in_proj_z_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 2048]> linear_104_cast_fp16 = linear(bias = linear_3_bias_0_to_fp16, weight = groups_3_1_in_proj_z_weight_promoted_to_fp16_palettized, x = input_171_cast_fp16)[name = string("linear_104_cast_fp16")];
            tensor<int32, [3]> var_3744_begin_0 = const()[name = string("op_3744_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_3744_end_0 = const()[name = string("op_3744_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_3744_end_mask_0 = const()[name = string("op_3744_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_3744_squeeze_mask_0 = const()[name = string("op_3744_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_3744_cast_fp16 = slice_by_index(begin = var_3744_begin_0, end = var_3744_end_0, end_mask = var_3744_end_mask_0, squeeze_mask = var_3744_squeeze_mask_0, x = decay_21_cast_fp16)[name = string("op_3744_cast_fp16")];
            tensor<fp16, [1, 16]> var_3745_cast_fp16 = exp(x = var_3744_cast_fp16)[name = string("op_3745_cast_fp16")];
            tensor<int32, [1]> var_3746_axes_0 = const()[name = string("op_3746_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_3746_cast_fp16 = expand_dims(axes = var_3746_axes_0, x = var_3745_cast_fp16)[name = string("op_3746_cast_fp16")];
            tensor<int32, [1]> var_3747_axes_0 = const()[name = string("op_3747_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_3747_cast_fp16 = expand_dims(axes = var_3747_axes_0, x = var_3746_cast_fp16)[name = string("op_3747_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_41_cast_fp16 = mul(x = rec13, y = var_3747_cast_fp16)[name = string("state_41_cast_fp16")];
            tensor<int32, [4]> key_token_21_begin_0 = const()[name = string("key_token_21_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> key_token_21_end_0 = const()[name = string("key_token_21_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> key_token_21_end_mask_0 = const()[name = string("key_token_21_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_21_squeeze_mask_0 = const()[name = string("key_token_21_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 1, 128]> key_55_cast_fp16 = transpose(perm = key_55_perm_0, x = var_3721_cast_fp16)[name = string("transpose_41")];
            tensor<fp16, [1, 16, 128]> key_token_21_cast_fp16 = slice_by_index(begin = key_token_21_begin_0, end = key_token_21_end_0, end_mask = key_token_21_end_mask_0, squeeze_mask = key_token_21_squeeze_mask_0, x = key_55_cast_fp16)[name = string("key_token_21_cast_fp16")];
            tensor<int32, [4]> value_token_21_begin_0 = const()[name = string("value_token_21_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> value_token_21_end_0 = const()[name = string("value_token_21_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> value_token_21_end_mask_0 = const()[name = string("value_token_21_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_21_squeeze_mask_0 = const()[name = string("value_token_21_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 1, 128]> value_199_cast_fp16 = transpose(perm = value_199_perm_0, x = value_197_cast_fp16)[name = string("transpose_40")];
            tensor<fp16, [1, 16, 128]> value_token_21_cast_fp16 = slice_by_index(begin = value_token_21_begin_0, end = value_token_21_end_0, end_mask = value_token_21_end_mask_0, squeeze_mask = value_token_21_squeeze_mask_0, x = value_199_cast_fp16)[name = string("value_token_21_cast_fp16")];
            tensor<int32, [1]> var_3755_axes_0 = const()[name = string("op_3755_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_3755_cast_fp16 = expand_dims(axes = var_3755_axes_0, x = key_token_21_cast_fp16)[name = string("op_3755_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3756_cast_fp16 = mul(x = state_41_cast_fp16, y = var_3755_cast_fp16)[name = string("op_3756_cast_fp16")];
            tensor<int32, [1]> memory_21_axes_0 = const()[name = string("memory_21_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_21_keep_dims_0 = const()[name = string("memory_21_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_21_cast_fp16 = reduce_sum(axes = memory_21_axes_0, keep_dims = memory_21_keep_dims_0, x = var_3756_cast_fp16)[name = string("memory_21_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_3759_cast_fp16 = sub(x = value_token_21_cast_fp16, y = memory_21_cast_fp16)[name = string("op_3759_cast_fp16")];
            tensor<int32, [3]> var_3762_begin_0 = const()[name = string("op_3762_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_3762_end_0 = const()[name = string("op_3762_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_3762_end_mask_0 = const()[name = string("op_3762_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_3762_squeeze_mask_0 = const()[name = string("op_3762_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_3762_cast_fp16 = slice_by_index(begin = var_3762_begin_0, end = var_3762_end_0, end_mask = var_3762_end_mask_0, squeeze_mask = var_3762_squeeze_mask_0, x = beta_21_cast_fp16)[name = string("op_3762_cast_fp16")];
            tensor<int32, [1]> var_3763_axes_0 = const()[name = string("op_3763_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_3763_cast_fp16 = expand_dims(axes = var_3763_axes_0, x = var_3762_cast_fp16)[name = string("op_3763_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_21_cast_fp16 = mul(x = var_3759_cast_fp16, y = var_3763_cast_fp16)[name = string("delta_21_cast_fp16")];
            tensor<int32, [1]> var_3766_axes_0 = const()[name = string("op_3766_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_3766_cast_fp16 = expand_dims(axes = var_3766_axes_0, x = delta_21_cast_fp16)[name = string("op_3766_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3767_cast_fp16 = mul(x = var_3755_cast_fp16, y = var_3766_cast_fp16)[name = string("op_3767_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> rec13_out = add(x = state_41_cast_fp16, y = var_3767_cast_fp16)[name = string("state_43_cast_fp16")];
            tensor<int32, [4]> var_3771_begin_0 = const()[name = string("op_3771_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> var_3771_end_0 = const()[name = string("op_3771_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> var_3771_end_mask_0 = const()[name = string("op_3771_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_3771_squeeze_mask_0 = const()[name = string("op_3771_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_3771_cast_fp16 = slice_by_index(begin = var_3771_begin_0, end = var_3771_end_0, end_mask = var_3771_end_mask_0, squeeze_mask = var_3771_squeeze_mask_0, x = _inversed_query_55_cast_fp16)[name = string("op_3771_cast_fp16")];
            tensor<int32, [1]> var_3772_axes_0 = const()[name = string("op_3772_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_3772_cast_fp16 = expand_dims(axes = var_3772_axes_0, x = var_3771_cast_fp16)[name = string("op_3772_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3773_cast_fp16 = mul(x = rec13_out, y = var_3772_cast_fp16)[name = string("op_3773_cast_fp16")];
            tensor<int32, [1]> var_3775_axes_0 = const()[name = string("op_3775_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_3775_keep_dims_0 = const()[name = string("op_3775_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_3775_cast_fp16 = reduce_sum(axes = var_3775_axes_0, keep_dims = var_3775_keep_dims_0, x = var_3773_cast_fp16)[name = string("op_3775_cast_fp16")];
            tensor<int32, [1]> attention_101_axes_0 = const()[name = string("attention_101_axes_0"), val = tensor<int32, [1]>([1])];
            tensor<fp16, [1, 1, 16, 128]> attention_101_cast_fp16 = expand_dims(axes = attention_101_axes_0, x = var_3775_cast_fp16)[name = string("attention_101_cast_fp16")];
            tensor<int32, [2]> var_3778 = const()[name = string("op_3778"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [16, 128]> attention_103_cast_fp16 = reshape(shape = var_3778, x = attention_101_cast_fp16)[name = string("attention_103_cast_fp16")];
            tensor<int32, [2]> var_3780 = const()[name = string("op_3780"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [16, 128]> input_175_cast_fp16 = reshape(shape = var_3780, x = linear_104_cast_fp16)[name = string("input_175_cast_fp16")];
            tensor<fp16, [16, 128]> var_3782_cast_fp16 = mul(x = attention_103_cast_fp16, y = attention_103_cast_fp16)[name = string("op_3782_cast_fp16")];
            tensor<int32, [1]> variance_87_axes_0 = const()[name = string("variance_87_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_87_keep_dims_0 = const()[name = string("variance_87_keep_dims_0"), val = bool(true)];
            tensor<fp16, [16, 1]> variance_87_cast_fp16 = reduce_mean(axes = variance_87_axes_0, keep_dims = variance_87_keep_dims_0, x = var_3782_cast_fp16)[name = string("variance_87_cast_fp16")];
            fp16 var_3785_to_fp16 = const()[name = string("op_3785_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [16, 1]> var_3786_cast_fp16 = add(x = variance_87_cast_fp16, y = var_3785_to_fp16)[name = string("op_3786_cast_fp16")];
            fp32 var_3787_epsilon_0 = const()[name = string("op_3787_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [16, 1]> var_3787_cast_fp16 = rsqrt(epsilon = var_3787_epsilon_0, x = var_3786_cast_fp16)[name = string("op_3787_cast_fp16")];
            tensor<fp16, [16, 128]> attention_105_cast_fp16 = mul(x = attention_103_cast_fp16, y = var_3787_cast_fp16)[name = string("attention_105_cast_fp16")];
            tensor<fp16, [128]> groups_3_1_gated_norm_promoted_to_fp16 = const()[name = string("groups_3_1_gated_norm_promoted_to_fp16"), val = tensor<fp16, [128]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(211733248)))];
            tensor<fp16, [16, 128]> attention_107_cast_fp16 = mul(x = attention_105_cast_fp16, y = groups_3_1_gated_norm_promoted_to_fp16)[name = string("attention_107_cast_fp16")];
            tensor<fp16, [16, 128]> var_3790_cast_fp16 = silu(x = input_175_cast_fp16)[name = string("op_3790_cast_fp16")];
            tensor<fp16, [16, 128]> attention_109_cast_fp16 = mul(x = attention_107_cast_fp16, y = var_3790_cast_fp16)[name = string("attention_109_cast_fp16")];
            tensor<int32, [2]> var_3792 = const()[name = string("op_3792"), val = tensor<int32, [2]>([1, 2048])];
            tensor<fp16, [1, 2048]> input_177_cast_fp16 = reshape(shape = var_3792, x = attention_109_cast_fp16)[name = string("input_177_cast_fp16")];
            tensor<fp16, [1024, 2048]> groups_3_1_out_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 2048]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(211733568))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(213306496))))[name = string("groups_3_1_out_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 1024]> linear_105_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = groups_3_1_out_proj_weight_promoted_to_fp16_palettized, x = input_177_cast_fp16)[name = string("linear_105_cast_fp16")];
            tensor<fp16, [1, 1024]> value_201_cast_fp16 = add(x = value_189_cast_fp16, y = linear_105_cast_fp16)[name = string("value_201_cast_fp16")];
            tensor<fp16, [1, 1024]> var_3797_cast_fp16 = mul(x = value_201_cast_fp16, y = value_201_cast_fp16)[name = string("op_3797_cast_fp16")];
            tensor<int32, [1]> variance_89_axes_0 = const()[name = string("variance_89_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_89_keep_dims_0 = const()[name = string("variance_89_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 1]> variance_89_cast_fp16 = reduce_mean(axes = variance_89_axes_0, keep_dims = variance_89_keep_dims_0, x = var_3797_cast_fp16)[name = string("variance_89_cast_fp16")];
            fp16 var_3800_to_fp16 = const()[name = string("op_3800_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 1]> var_3801_cast_fp16 = add(x = variance_89_cast_fp16, y = var_3800_to_fp16)[name = string("op_3801_cast_fp16")];
            fp32 var_3802_epsilon_0 = const()[name = string("op_3802_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 1]> var_3802_cast_fp16 = rsqrt(epsilon = var_3802_epsilon_0, x = var_3801_cast_fp16)[name = string("op_3802_cast_fp16")];
            tensor<fp16, [1, 1024]> var_3803_cast_fp16 = mul(x = value_201_cast_fp16, y = var_3802_cast_fp16)[name = string("op_3803_cast_fp16")];
            tensor<fp16, [1024]> var_3805_to_fp16 = const()[name = string("op_3805_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(213314752)))];
            tensor<fp16, [1, 1024]> input_179_cast_fp16 = mul(x = var_3803_cast_fp16, y = var_3805_to_fp16)[name = string("input_179_cast_fp16")];
            tensor<fp16, [3584, 1024]> groups_3_1_gate_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(213316864))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(216069440))))[name = string("groups_3_1_gate_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 3584]> linear_106_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = groups_3_1_gate_proj_weight_promoted_to_fp16_palettized, x = input_179_cast_fp16)[name = string("linear_106_cast_fp16")];
            tensor<fp16, [1, 3584]> var_3809_cast_fp16 = silu(x = linear_106_cast_fp16)[name = string("op_3809_cast_fp16")];
            tensor<fp16, [3584, 1024]> groups_3_1_up_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(216098176))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(218850752))))[name = string("groups_3_1_up_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 3584]> linear_107_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = groups_3_1_up_proj_weight_promoted_to_fp16_palettized, x = input_179_cast_fp16)[name = string("linear_107_cast_fp16")];
            tensor<fp16, [1, 3584]> input_183_cast_fp16 = mul(x = var_3809_cast_fp16, y = linear_107_cast_fp16)[name = string("input_183_cast_fp16")];
            tensor<fp16, [1024, 3584]> groups_3_1_down_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 3584]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(218879488))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(221632064))))[name = string("groups_3_1_down_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 1024]> linear_108_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = groups_3_1_down_proj_weight_promoted_to_fp16_palettized, x = input_183_cast_fp16)[name = string("linear_108_cast_fp16")];
            tensor<fp16, [1, 1024]> value_203_cast_fp16 = add(x = value_201_cast_fp16, y = linear_108_cast_fp16)[name = string("value_203_cast_fp16")];
            int32 var_3834 = const()[name = string("op_3834"), val = int32(-1)];
            tensor<fp16, [1, 1024]> var_3849_cast_fp16 = mul(x = value_203_cast_fp16, y = value_203_cast_fp16)[name = string("op_3849_cast_fp16")];
            tensor<int32, [1]> variance_91_axes_0 = const()[name = string("variance_91_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_91_keep_dims_0 = const()[name = string("variance_91_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 1]> variance_91_cast_fp16 = reduce_mean(axes = variance_91_axes_0, keep_dims = variance_91_keep_dims_0, x = var_3849_cast_fp16)[name = string("variance_91_cast_fp16")];
            fp16 var_3852_to_fp16 = const()[name = string("op_3852_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 1]> var_3853_cast_fp16 = add(x = variance_91_cast_fp16, y = var_3852_to_fp16)[name = string("op_3853_cast_fp16")];
            fp32 var_3854_epsilon_0 = const()[name = string("op_3854_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 1]> var_3854_cast_fp16 = rsqrt(epsilon = var_3854_epsilon_0, x = var_3853_cast_fp16)[name = string("op_3854_cast_fp16")];
            tensor<fp16, [1, 1024]> var_3855_cast_fp16 = mul(x = value_203_cast_fp16, y = var_3854_cast_fp16)[name = string("op_3855_cast_fp16")];
            tensor<fp16, [1024]> var_3857_to_fp16 = const()[name = string("op_3857_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(221640320)))];
            tensor<fp16, [1, 1024]> input_185_cast_fp16 = mul(x = var_3855_cast_fp16, y = var_3857_to_fp16)[name = string("input_185_cast_fp16")];
            tensor<fp16, [6144, 1024]> groups_3_2_in_proj_qkv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(221642432))), lut = tensor<fp16, [384, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(226361088))))[name = string("groups_3_2_in_proj_qkv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 6144]> linear_109_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = groups_3_2_in_proj_qkv_weight_promoted_to_fp16_palettized, x = input_185_cast_fp16)[name = string("linear_109_cast_fp16")];
            tensor<int32, [2]> var_3861_perm_0 = const()[name = string("op_3861_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> mixed_23_axes_0 = const()[name = string("mixed_23_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [6144, 1]> var_3861_cast_fp16 = transpose(perm = var_3861_perm_0, x = linear_109_cast_fp16)[name = string("transpose_39")];
            tensor<fp16, [1, 6144, 1]> mixed_23_cast_fp16 = expand_dims(axes = mixed_23_axes_0, x = var_3861_cast_fp16)[name = string("mixed_23_cast_fp16")];
            bool convolution_input_23_interleave_0 = const()[name = string("convolution_input_23_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 6144, 4]> convolution_input_23_cast_fp16 = concat(axis = var_3834, interleave = convolution_input_23_interleave_0, values = (conv14, mixed_23_cast_fp16))[name = string("convolution_input_23_cast_fp16")];
            string input_187_pad_type_0 = const()[name = string("input_187_pad_type_0"), val = string("valid")];
            int32 input_187_groups_0 = const()[name = string("input_187_groups_0"), val = int32(6144)];
            tensor<int32, [1]> input_187_strides_0 = const()[name = string("input_187_strides_0"), val = tensor<int32, [1]>([1])];
            tensor<int32, [2]> input_187_pad_0 = const()[name = string("input_187_pad_0"), val = tensor<int32, [2]>([0, 0])];
            tensor<int32, [1]> input_187_dilations_0 = const()[name = string("input_187_dilations_0"), val = tensor<int32, [1]>([1])];
            tensor<fp16, [6144, 1, 4]> groups_3_2_conv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1, 4]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(226410304))), lut = tensor<fp16, [384, 1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(226428800))))[name = string("groups_3_2_conv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 6144, 1]> input_187_cast_fp16 = conv(dilations = input_187_dilations_0, groups = input_187_groups_0, pad = input_187_pad_0, pad_type = input_187_pad_type_0, strides = input_187_strides_0, weight = groups_3_2_conv_weight_promoted_to_fp16_palettized, x = convolution_input_23_cast_fp16)[name = string("input_187_cast_fp16")];
            tensor<fp16, [1, 6144, 1]> var_3870_cast_fp16 = silu(x = input_187_cast_fp16)[name = string("op_3870_cast_fp16")];
            tensor<int32, [3]> var_3871_perm_0 = const()[name = string("op_3871_perm_0"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [3]> var_3874_begin_0 = const()[name = string("op_3874_begin_0"), val = tensor<int32, [3]>([0, 0, 1])];
            tensor<int32, [3]> var_3874_end_0 = const()[name = string("op_3874_end_0"), val = tensor<int32, [3]>([1, 6144, 4])];
            tensor<bool, [3]> var_3874_end_mask_0 = const()[name = string("op_3874_end_mask_0"), val = tensor<bool, [3]>([true, true, true])];
            tensor<fp16, [1, 6144, 3]> conv14_out = slice_by_index(begin = var_3874_begin_0, end = var_3874_end_0, end_mask = var_3874_end_mask_0, x = convolution_input_23_cast_fp16)[name = string("op_3874_cast_fp16")];
            tensor<int32, [3]> var_3875 = const()[name = string("op_3875"), val = tensor<int32, [3]>([2048, 2048, 2048])];
            int32 var_3876_axis_0 = const()[name = string("op_3876_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 1, 6144]> var_3871_cast_fp16 = transpose(perm = var_3871_perm_0, x = var_3870_cast_fp16)[name = string("transpose_38")];
            tensor<fp16, [1, 1, 2048]> var_3876_cast_fp16_0, tensor<fp16, [1, 1, 2048]> var_3876_cast_fp16_1, tensor<fp16, [1, 1, 2048]> var_3876_cast_fp16_2 = split(axis = var_3876_axis_0, split_sizes = var_3875, x = var_3871_cast_fp16)[name = string("op_3876_cast_fp16")];
            tensor<int32, [4]> var_3880 = const()[name = string("op_3880"), val = tensor<int32, [4]>([1, 1, 16, 128])];
            tensor<fp16, [1, 1, 16, 128]> value_207_cast_fp16 = reshape(shape = var_3880, x = var_3876_cast_fp16_0)[name = string("value_207_cast_fp16")];
            tensor<int32, [4]> var_3882 = const()[name = string("op_3882"), val = tensor<int32, [4]>([1, 1, 16, 128])];
            tensor<fp16, [1, 1, 16, 128]> value_209_cast_fp16 = reshape(shape = var_3882, x = var_3876_cast_fp16_1)[name = string("value_209_cast_fp16")];
            tensor<int32, [4]> var_3884 = const()[name = string("op_3884"), val = tensor<int32, [4]>([1, 1, 16, 128])];
            tensor<fp16, [1, 1, 16, 128]> value_211_cast_fp16 = reshape(shape = var_3884, x = var_3876_cast_fp16_2)[name = string("value_211_cast_fp16")];
            tensor<fp16, [1, 1, 16, 128]> var_3886_cast_fp16 = mul(x = value_207_cast_fp16, y = value_207_cast_fp16)[name = string("op_3886_cast_fp16")];
            tensor<int32, [1]> var_3888_axes_0 = const()[name = string("op_3888_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_3888_keep_dims_0 = const()[name = string("op_3888_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 1, 16, 1]> var_3888_cast_fp16 = reduce_sum(axes = var_3888_axes_0, keep_dims = var_3888_keep_dims_0, x = var_3886_cast_fp16)[name = string("op_3888_cast_fp16")];
            fp16 var_3889_to_fp16 = const()[name = string("op_3889_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 1, 16, 1]> var_3890_cast_fp16 = add(x = var_3888_cast_fp16, y = var_3889_to_fp16)[name = string("op_3890_cast_fp16")];
            fp32 var_3891_epsilon_0 = const()[name = string("op_3891_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 1, 16, 1]> var_3891_cast_fp16 = rsqrt(epsilon = var_3891_epsilon_0, x = var_3890_cast_fp16)[name = string("op_3891_cast_fp16")];
            tensor<fp16, [1, 1, 16, 128]> var_3892_cast_fp16 = mul(x = value_207_cast_fp16, y = var_3891_cast_fp16)[name = string("op_3892_cast_fp16")];
            tensor<int32, [4]> var_3893_perm_0 = const()[name = string("op_3893_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            fp16 _inversed_query_59_y_0_to_fp16 = const()[name = string("_inversed_query_59_y_0_to_fp16"), val = fp16(0x1.6ap-4)];
            tensor<fp16, [1, 16, 1, 128]> var_3893_cast_fp16 = transpose(perm = var_3893_perm_0, x = var_3892_cast_fp16)[name = string("transpose_37")];
            tensor<fp16, [1, 16, 1, 128]> _inversed_query_59_cast_fp16 = mul(x = var_3893_cast_fp16, y = _inversed_query_59_y_0_to_fp16)[name = string("_inversed_query_59_cast_fp16")];
            tensor<fp16, [1, 1, 16, 128]> var_3896_cast_fp16 = mul(x = value_209_cast_fp16, y = value_209_cast_fp16)[name = string("op_3896_cast_fp16")];
            tensor<int32, [1]> var_3898_axes_0 = const()[name = string("op_3898_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_3898_keep_dims_0 = const()[name = string("op_3898_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 1, 16, 1]> var_3898_cast_fp16 = reduce_sum(axes = var_3898_axes_0, keep_dims = var_3898_keep_dims_0, x = var_3896_cast_fp16)[name = string("op_3898_cast_fp16")];
            fp16 var_3899_to_fp16 = const()[name = string("op_3899_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 1, 16, 1]> var_3900_cast_fp16 = add(x = var_3898_cast_fp16, y = var_3899_to_fp16)[name = string("op_3900_cast_fp16")];
            fp32 var_3901_epsilon_0 = const()[name = string("op_3901_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 1, 16, 1]> var_3901_cast_fp16 = rsqrt(epsilon = var_3901_epsilon_0, x = var_3900_cast_fp16)[name = string("op_3901_cast_fp16")];
            tensor<fp16, [1, 1, 16, 128]> var_3902_cast_fp16 = mul(x = value_209_cast_fp16, y = var_3901_cast_fp16)[name = string("op_3902_cast_fp16")];
            tensor<int32, [4]> key_59_perm_0 = const()[name = string("key_59_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<int32, [4]> value_213_perm_0 = const()[name = string("value_213_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<fp16, [16, 1024]> groups_3_2_in_proj_b_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(226478016))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(226490368))))[name = string("groups_3_2_in_proj_b_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16]> linear_110_bias_0_to_fp16 = const()[name = string("linear_110_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(4862656)))];
            tensor<fp16, [1, 16]> linear_110_cast_fp16 = linear(bias = linear_110_bias_0_to_fp16, weight = groups_3_2_in_proj_b_weight_promoted_to_fp16_palettized, x = input_185_cast_fp16)[name = string("linear_110_cast_fp16")];
            tensor<fp16, [1, 16]> var_3907_cast_fp16 = sigmoid(x = linear_110_cast_fp16)[name = string("op_3907_cast_fp16")];
            tensor<int32, [2]> var_3908_perm_0 = const()[name = string("op_3908_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> beta_23_axes_0 = const()[name = string("beta_23_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 1]> var_3908_cast_fp16 = transpose(perm = var_3908_perm_0, x = var_3907_cast_fp16)[name = string("transpose_36")];
            tensor<fp16, [1, 16, 1]> beta_23_cast_fp16 = expand_dims(axes = beta_23_axes_0, x = var_3908_cast_fp16)[name = string("beta_23_cast_fp16")];
            tensor<fp16, [16, 1024]> op_3914_weight_0_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(226490560))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(226502912))))[name = string("op_3914_weight_0_to_fp16_palettized")];
            tensor<fp16, [16]> var_3914_bias_0_to_fp16 = const()[name = string("op_3914_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(226503104)))];
            tensor<fp16, [1, 16]> var_3914_cast_fp16 = linear(bias = var_3914_bias_0_to_fp16, weight = op_3914_weight_0_to_fp16_palettized, x = input_185_cast_fp16)[name = string("op_3914_cast_fp16")];
            tensor<fp16, [1, 16]> var_3915_cast_fp16 = softplus(x = var_3914_cast_fp16)[name = string("op_3915_cast_fp16")];
            tensor<fp16, [16]> var_3911_promoted_to_fp16 = const()[name = string("op_3911_promoted_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(226503232)))];
            tensor<fp16, [1, 16]> var_3916_cast_fp16 = mul(x = var_3911_promoted_to_fp16, y = var_3915_cast_fp16)[name = string("op_3916_cast_fp16")];
            tensor<int32, [2]> var_3917_perm_0 = const()[name = string("op_3917_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> decay_23_axes_0 = const()[name = string("decay_23_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 1]> var_3917_cast_fp16 = transpose(perm = var_3917_perm_0, x = var_3916_cast_fp16)[name = string("transpose_35")];
            tensor<fp16, [1, 16, 1]> decay_23_cast_fp16 = expand_dims(axes = decay_23_axes_0, x = var_3917_cast_fp16)[name = string("decay_23_cast_fp16")];
            tensor<fp16, [2048, 1024]> groups_3_2_in_proj_z_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [2048, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(226503360))), lut = tensor<fp16, [128, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(228076288))))[name = string("groups_3_2_in_proj_z_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 2048]> linear_112_cast_fp16 = linear(bias = linear_3_bias_0_to_fp16, weight = groups_3_2_in_proj_z_weight_promoted_to_fp16_palettized, x = input_185_cast_fp16)[name = string("linear_112_cast_fp16")];
            tensor<int32, [3]> var_3925_begin_0 = const()[name = string("op_3925_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_3925_end_0 = const()[name = string("op_3925_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_3925_end_mask_0 = const()[name = string("op_3925_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_3925_squeeze_mask_0 = const()[name = string("op_3925_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_3925_cast_fp16 = slice_by_index(begin = var_3925_begin_0, end = var_3925_end_0, end_mask = var_3925_end_mask_0, squeeze_mask = var_3925_squeeze_mask_0, x = decay_23_cast_fp16)[name = string("op_3925_cast_fp16")];
            tensor<fp16, [1, 16]> var_3926_cast_fp16 = exp(x = var_3925_cast_fp16)[name = string("op_3926_cast_fp16")];
            tensor<int32, [1]> var_3927_axes_0 = const()[name = string("op_3927_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_3927_cast_fp16 = expand_dims(axes = var_3927_axes_0, x = var_3926_cast_fp16)[name = string("op_3927_cast_fp16")];
            tensor<int32, [1]> var_3928_axes_0 = const()[name = string("op_3928_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_3928_cast_fp16 = expand_dims(axes = var_3928_axes_0, x = var_3927_cast_fp16)[name = string("op_3928_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_45_cast_fp16 = mul(x = rec14, y = var_3928_cast_fp16)[name = string("state_45_cast_fp16")];
            tensor<int32, [4]> key_token_23_begin_0 = const()[name = string("key_token_23_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> key_token_23_end_0 = const()[name = string("key_token_23_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> key_token_23_end_mask_0 = const()[name = string("key_token_23_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_23_squeeze_mask_0 = const()[name = string("key_token_23_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 1, 128]> key_59_cast_fp16 = transpose(perm = key_59_perm_0, x = var_3902_cast_fp16)[name = string("transpose_34")];
            tensor<fp16, [1, 16, 128]> key_token_23_cast_fp16 = slice_by_index(begin = key_token_23_begin_0, end = key_token_23_end_0, end_mask = key_token_23_end_mask_0, squeeze_mask = key_token_23_squeeze_mask_0, x = key_59_cast_fp16)[name = string("key_token_23_cast_fp16")];
            tensor<int32, [4]> value_token_23_begin_0 = const()[name = string("value_token_23_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> value_token_23_end_0 = const()[name = string("value_token_23_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> value_token_23_end_mask_0 = const()[name = string("value_token_23_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_23_squeeze_mask_0 = const()[name = string("value_token_23_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 1, 128]> value_213_cast_fp16 = transpose(perm = value_213_perm_0, x = value_211_cast_fp16)[name = string("transpose_33")];
            tensor<fp16, [1, 16, 128]> value_token_23_cast_fp16 = slice_by_index(begin = value_token_23_begin_0, end = value_token_23_end_0, end_mask = value_token_23_end_mask_0, squeeze_mask = value_token_23_squeeze_mask_0, x = value_213_cast_fp16)[name = string("value_token_23_cast_fp16")];
            tensor<int32, [1]> var_3936_axes_0 = const()[name = string("op_3936_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_3936_cast_fp16 = expand_dims(axes = var_3936_axes_0, x = key_token_23_cast_fp16)[name = string("op_3936_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3937_cast_fp16 = mul(x = state_45_cast_fp16, y = var_3936_cast_fp16)[name = string("op_3937_cast_fp16")];
            tensor<int32, [1]> memory_23_axes_0 = const()[name = string("memory_23_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_23_keep_dims_0 = const()[name = string("memory_23_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_23_cast_fp16 = reduce_sum(axes = memory_23_axes_0, keep_dims = memory_23_keep_dims_0, x = var_3937_cast_fp16)[name = string("memory_23_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_3940_cast_fp16 = sub(x = value_token_23_cast_fp16, y = memory_23_cast_fp16)[name = string("op_3940_cast_fp16")];
            tensor<int32, [3]> var_3943_begin_0 = const()[name = string("op_3943_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_3943_end_0 = const()[name = string("op_3943_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_3943_end_mask_0 = const()[name = string("op_3943_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_3943_squeeze_mask_0 = const()[name = string("op_3943_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_3943_cast_fp16 = slice_by_index(begin = var_3943_begin_0, end = var_3943_end_0, end_mask = var_3943_end_mask_0, squeeze_mask = var_3943_squeeze_mask_0, x = beta_23_cast_fp16)[name = string("op_3943_cast_fp16")];
            tensor<int32, [1]> var_3944_axes_0 = const()[name = string("op_3944_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_3944_cast_fp16 = expand_dims(axes = var_3944_axes_0, x = var_3943_cast_fp16)[name = string("op_3944_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_23_cast_fp16 = mul(x = var_3940_cast_fp16, y = var_3944_cast_fp16)[name = string("delta_23_cast_fp16")];
            tensor<int32, [1]> var_3947_axes_0 = const()[name = string("op_3947_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_3947_cast_fp16 = expand_dims(axes = var_3947_axes_0, x = delta_23_cast_fp16)[name = string("op_3947_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3948_cast_fp16 = mul(x = var_3936_cast_fp16, y = var_3947_cast_fp16)[name = string("op_3948_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> rec14_out = add(x = state_45_cast_fp16, y = var_3948_cast_fp16)[name = string("state_47_cast_fp16")];
            tensor<int32, [4]> var_3952_begin_0 = const()[name = string("op_3952_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> var_3952_end_0 = const()[name = string("op_3952_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> var_3952_end_mask_0 = const()[name = string("op_3952_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_3952_squeeze_mask_0 = const()[name = string("op_3952_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_3952_cast_fp16 = slice_by_index(begin = var_3952_begin_0, end = var_3952_end_0, end_mask = var_3952_end_mask_0, squeeze_mask = var_3952_squeeze_mask_0, x = _inversed_query_59_cast_fp16)[name = string("op_3952_cast_fp16")];
            tensor<int32, [1]> var_3953_axes_0 = const()[name = string("op_3953_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_3953_cast_fp16 = expand_dims(axes = var_3953_axes_0, x = var_3952_cast_fp16)[name = string("op_3953_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3954_cast_fp16 = mul(x = rec14_out, y = var_3953_cast_fp16)[name = string("op_3954_cast_fp16")];
            tensor<int32, [1]> var_3956_axes_0 = const()[name = string("op_3956_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_3956_keep_dims_0 = const()[name = string("op_3956_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_3956_cast_fp16 = reduce_sum(axes = var_3956_axes_0, keep_dims = var_3956_keep_dims_0, x = var_3954_cast_fp16)[name = string("op_3956_cast_fp16")];
            tensor<int32, [1]> attention_111_axes_0 = const()[name = string("attention_111_axes_0"), val = tensor<int32, [1]>([1])];
            tensor<fp16, [1, 1, 16, 128]> attention_111_cast_fp16 = expand_dims(axes = attention_111_axes_0, x = var_3956_cast_fp16)[name = string("attention_111_cast_fp16")];
            tensor<int32, [2]> var_3959 = const()[name = string("op_3959"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [16, 128]> attention_113_cast_fp16 = reshape(shape = var_3959, x = attention_111_cast_fp16)[name = string("attention_113_cast_fp16")];
            tensor<int32, [2]> var_3961 = const()[name = string("op_3961"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [16, 128]> input_189_cast_fp16 = reshape(shape = var_3961, x = linear_112_cast_fp16)[name = string("input_189_cast_fp16")];
            tensor<fp16, [16, 128]> var_3963_cast_fp16 = mul(x = attention_113_cast_fp16, y = attention_113_cast_fp16)[name = string("op_3963_cast_fp16")];
            tensor<int32, [1]> variance_93_axes_0 = const()[name = string("variance_93_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_93_keep_dims_0 = const()[name = string("variance_93_keep_dims_0"), val = bool(true)];
            tensor<fp16, [16, 1]> variance_93_cast_fp16 = reduce_mean(axes = variance_93_axes_0, keep_dims = variance_93_keep_dims_0, x = var_3963_cast_fp16)[name = string("variance_93_cast_fp16")];
            fp16 var_3966_to_fp16 = const()[name = string("op_3966_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [16, 1]> var_3967_cast_fp16 = add(x = variance_93_cast_fp16, y = var_3966_to_fp16)[name = string("op_3967_cast_fp16")];
            fp32 var_3968_epsilon_0 = const()[name = string("op_3968_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [16, 1]> var_3968_cast_fp16 = rsqrt(epsilon = var_3968_epsilon_0, x = var_3967_cast_fp16)[name = string("op_3968_cast_fp16")];
            tensor<fp16, [16, 128]> attention_115_cast_fp16 = mul(x = attention_113_cast_fp16, y = var_3968_cast_fp16)[name = string("attention_115_cast_fp16")];
            tensor<fp16, [128]> groups_3_2_gated_norm_promoted_to_fp16 = const()[name = string("groups_3_2_gated_norm_promoted_to_fp16"), val = tensor<fp16, [128]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(228092736)))];
            tensor<fp16, [16, 128]> attention_117_cast_fp16 = mul(x = attention_115_cast_fp16, y = groups_3_2_gated_norm_promoted_to_fp16)[name = string("attention_117_cast_fp16")];
            tensor<fp16, [16, 128]> var_3971_cast_fp16 = silu(x = input_189_cast_fp16)[name = string("op_3971_cast_fp16")];
            tensor<fp16, [16, 128]> attention_119_cast_fp16 = mul(x = attention_117_cast_fp16, y = var_3971_cast_fp16)[name = string("attention_119_cast_fp16")];
            tensor<int32, [2]> var_3973 = const()[name = string("op_3973"), val = tensor<int32, [2]>([1, 2048])];
            tensor<fp16, [1, 2048]> input_191_cast_fp16 = reshape(shape = var_3973, x = attention_119_cast_fp16)[name = string("input_191_cast_fp16")];
            tensor<fp16, [1024, 2048]> groups_3_2_out_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 2048]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(228093056))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(229665984))))[name = string("groups_3_2_out_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 1024]> linear_113_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = groups_3_2_out_proj_weight_promoted_to_fp16_palettized, x = input_191_cast_fp16)[name = string("linear_113_cast_fp16")];
            tensor<fp16, [1, 1024]> value_215_cast_fp16 = add(x = value_203_cast_fp16, y = linear_113_cast_fp16)[name = string("value_215_cast_fp16")];
            tensor<fp16, [1, 1024]> var_3978_cast_fp16 = mul(x = value_215_cast_fp16, y = value_215_cast_fp16)[name = string("op_3978_cast_fp16")];
            tensor<int32, [1]> variance_95_axes_0 = const()[name = string("variance_95_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_95_keep_dims_0 = const()[name = string("variance_95_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 1]> variance_95_cast_fp16 = reduce_mean(axes = variance_95_axes_0, keep_dims = variance_95_keep_dims_0, x = var_3978_cast_fp16)[name = string("variance_95_cast_fp16")];
            fp16 var_3981_to_fp16 = const()[name = string("op_3981_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 1]> var_3982_cast_fp16 = add(x = variance_95_cast_fp16, y = var_3981_to_fp16)[name = string("op_3982_cast_fp16")];
            fp32 var_3983_epsilon_0 = const()[name = string("op_3983_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 1]> var_3983_cast_fp16 = rsqrt(epsilon = var_3983_epsilon_0, x = var_3982_cast_fp16)[name = string("op_3983_cast_fp16")];
            tensor<fp16, [1, 1024]> var_3984_cast_fp16 = mul(x = value_215_cast_fp16, y = var_3983_cast_fp16)[name = string("op_3984_cast_fp16")];
            tensor<fp16, [1024]> var_3986_to_fp16 = const()[name = string("op_3986_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(229674240)))];
            tensor<fp16, [1, 1024]> input_193_cast_fp16 = mul(x = var_3984_cast_fp16, y = var_3986_to_fp16)[name = string("input_193_cast_fp16")];
            tensor<fp16, [3584, 1024]> groups_3_2_gate_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(229676352))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(232428928))))[name = string("groups_3_2_gate_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 3584]> linear_114_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = groups_3_2_gate_proj_weight_promoted_to_fp16_palettized, x = input_193_cast_fp16)[name = string("linear_114_cast_fp16")];
            tensor<fp16, [1, 3584]> var_3990_cast_fp16 = silu(x = linear_114_cast_fp16)[name = string("op_3990_cast_fp16")];
            tensor<fp16, [3584, 1024]> groups_3_2_up_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(232457664))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(235210240))))[name = string("groups_3_2_up_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 3584]> linear_115_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = groups_3_2_up_proj_weight_promoted_to_fp16_palettized, x = input_193_cast_fp16)[name = string("linear_115_cast_fp16")];
            tensor<fp16, [1, 3584]> input_197_cast_fp16 = mul(x = var_3990_cast_fp16, y = linear_115_cast_fp16)[name = string("input_197_cast_fp16")];
            tensor<fp16, [1024, 3584]> groups_3_2_down_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 3584]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(235238976))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(237991552))))[name = string("groups_3_2_down_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 1024]> linear_116_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = groups_3_2_down_proj_weight_promoted_to_fp16_palettized, x = input_197_cast_fp16)[name = string("linear_116_cast_fp16")];
            tensor<fp16, [1, 1024]> value_217_cast_fp16 = add(x = value_215_cast_fp16, y = linear_116_cast_fp16)[name = string("value_217_cast_fp16")];
            int32 var_4014 = const()[name = string("op_4014"), val = int32(-1)];
            tensor<fp16, [1, 1024]> var_4021_cast_fp16 = mul(x = value_217_cast_fp16, y = value_217_cast_fp16)[name = string("op_4021_cast_fp16")];
            tensor<int32, [1]> variance_97_axes_0 = const()[name = string("variance_97_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_97_keep_dims_0 = const()[name = string("variance_97_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 1]> variance_97_cast_fp16 = reduce_mean(axes = variance_97_axes_0, keep_dims = variance_97_keep_dims_0, x = var_4021_cast_fp16)[name = string("variance_97_cast_fp16")];
            fp16 var_4024_to_fp16 = const()[name = string("op_4024_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 1]> var_4025_cast_fp16 = add(x = variance_97_cast_fp16, y = var_4024_to_fp16)[name = string("op_4025_cast_fp16")];
            fp32 var_4026_epsilon_0 = const()[name = string("op_4026_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 1]> var_4026_cast_fp16 = rsqrt(epsilon = var_4026_epsilon_0, x = var_4025_cast_fp16)[name = string("op_4026_cast_fp16")];
            tensor<fp16, [1, 1024]> var_4027_cast_fp16 = mul(x = value_217_cast_fp16, y = var_4026_cast_fp16)[name = string("op_4027_cast_fp16")];
            tensor<fp16, [1024]> var_4029_to_fp16 = const()[name = string("op_4029_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(237999808)))];
            tensor<fp16, [1, 1024]> input_199_cast_fp16 = mul(x = var_4027_cast_fp16, y = var_4029_to_fp16)[name = string("input_199_cast_fp16")];
            tensor<fp16, [4096, 1024]> projections_3_q_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [4096, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(238001920))), lut = tensor<fp16, [256, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(241147712))))[name = string("projections_3_q_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 4096]> linear_117_cast_fp16 = linear(bias = linear_24_bias_0_to_fp16, weight = projections_3_q_proj_weight_promoted_to_fp16_palettized, x = input_199_cast_fp16)[name = string("linear_117_cast_fp16")];
            tensor<int32, [3]> var_4033 = const()[name = string("op_4033"), val = tensor<int32, [3]>([1, 8, 512])];
            tensor<fp16, [1, 8, 512]> query_and_gate_7_cast_fp16 = reshape(shape = var_4033, x = linear_117_cast_fp16)[name = string("query_and_gate_7_cast_fp16")];
            tensor<int32, [2]> var_4035_split_sizes_0 = const()[name = string("op_4035_split_sizes_0"), val = tensor<int32, [2]>([256, 256])];
            int32 var_4035_axis_0 = const()[name = string("op_4035_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 8, 256]> var_4035_cast_fp16_0, tensor<fp16, [1, 8, 256]> var_4035_cast_fp16_1 = split(axis = var_4035_axis_0, split_sizes = var_4035_split_sizes_0, x = query_and_gate_7_cast_fp16)[name = string("op_4035_cast_fp16")];
            tensor<fp16, [512, 1024]> projections_3_k_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [512, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(241180544))), lut = tensor<fp16, [32, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(241573824))))[name = string("projections_3_k_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 512]> linear_118_cast_fp16 = linear(bias = linear_25_bias_0_to_fp16, weight = projections_3_k_proj_weight_promoted_to_fp16_palettized, x = input_199_cast_fp16)[name = string("linear_118_cast_fp16")];
            tensor<int32, [3]> var_4039 = const()[name = string("op_4039"), val = tensor<int32, [3]>([1, 2, 256])];
            tensor<fp16, [1, 2, 256]> value_221_cast_fp16 = reshape(shape = var_4039, x = linear_118_cast_fp16)[name = string("value_221_cast_fp16")];
            tensor<fp16, [512, 1024]> projections_3_v_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [512, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(241577984))), lut = tensor<fp16, [32, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(241971264))))[name = string("projections_3_v_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 512]> linear_119_cast_fp16 = linear(bias = linear_25_bias_0_to_fp16, weight = projections_3_v_proj_weight_promoted_to_fp16_palettized, x = input_199_cast_fp16)[name = string("linear_119_cast_fp16")];
            tensor<int32, [3]> var_4043 = const()[name = string("op_4043"), val = tensor<int32, [3]>([1, 2, 256])];
            tensor<fp16, [1, 2, 256]> value_227_cast_fp16 = reshape(shape = var_4043, x = linear_119_cast_fp16)[name = string("value_227_cast_fp16")];
            tensor<fp16, [1, 8, 256]> var_4045_cast_fp16 = mul(x = var_4035_cast_fp16_0, y = var_4035_cast_fp16_0)[name = string("op_4045_cast_fp16")];
            tensor<int32, [1]> variance_99_axes_0 = const()[name = string("variance_99_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_99_keep_dims_0 = const()[name = string("variance_99_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 8, 1]> variance_99_cast_fp16 = reduce_mean(axes = variance_99_axes_0, keep_dims = variance_99_keep_dims_0, x = var_4045_cast_fp16)[name = string("variance_99_cast_fp16")];
            fp16 var_4048_to_fp16 = const()[name = string("op_4048_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 8, 1]> var_4049_cast_fp16 = add(x = variance_99_cast_fp16, y = var_4048_to_fp16)[name = string("op_4049_cast_fp16")];
            fp32 var_4050_epsilon_0 = const()[name = string("op_4050_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 8, 1]> var_4050_cast_fp16 = rsqrt(epsilon = var_4050_epsilon_0, x = var_4049_cast_fp16)[name = string("op_4050_cast_fp16")];
            tensor<fp16, [1, 8, 256]> var_4051_cast_fp16 = mul(x = var_4035_cast_fp16_0, y = var_4050_cast_fp16)[name = string("op_4051_cast_fp16")];
            tensor<fp16, [256]> var_4053_to_fp16 = const()[name = string("op_4053_to_fp16"), val = tensor<fp16, [256]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(241975424)))];
            tensor<fp16, [1, 8, 256]> var_4054_cast_fp16 = mul(x = var_4051_cast_fp16, y = var_4053_to_fp16)[name = string("op_4054_cast_fp16")];
            tensor<int32, [3]> var_4055 = const()[name = string("op_4055"), val = tensor<int32, [3]>([1, 0, 2])];
            tensor<int32, [1]> value_223_axes_0 = const()[name = string("value_223_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [8, 1, 256]> var_4056_cast_fp16 = transpose(perm = var_4055, x = var_4054_cast_fp16)[name = string("transpose_32")];
            tensor<fp16, [1, 8, 1, 256]> value_223_cast_fp16 = expand_dims(axes = value_223_axes_0, x = var_4056_cast_fp16)[name = string("value_223_cast_fp16")];
            tensor<fp16, [1, 2, 256]> var_4058_cast_fp16 = mul(x = value_221_cast_fp16, y = value_221_cast_fp16)[name = string("op_4058_cast_fp16")];
            tensor<int32, [1]> variance_101_axes_0 = const()[name = string("variance_101_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_101_keep_dims_0 = const()[name = string("variance_101_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 2, 1]> variance_101_cast_fp16 = reduce_mean(axes = variance_101_axes_0, keep_dims = variance_101_keep_dims_0, x = var_4058_cast_fp16)[name = string("variance_101_cast_fp16")];
            fp16 var_4061_to_fp16 = const()[name = string("op_4061_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 2, 1]> var_4062_cast_fp16 = add(x = variance_101_cast_fp16, y = var_4061_to_fp16)[name = string("op_4062_cast_fp16")];
            fp32 var_4063_epsilon_0 = const()[name = string("op_4063_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 2, 1]> var_4063_cast_fp16 = rsqrt(epsilon = var_4063_epsilon_0, x = var_4062_cast_fp16)[name = string("op_4063_cast_fp16")];
            tensor<fp16, [1, 2, 256]> var_4064_cast_fp16 = mul(x = value_221_cast_fp16, y = var_4063_cast_fp16)[name = string("op_4064_cast_fp16")];
            tensor<fp16, [256]> var_4066_to_fp16 = const()[name = string("op_4066_to_fp16"), val = tensor<fp16, [256]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(241976000)))];
            tensor<fp16, [1, 2, 256]> var_4067_cast_fp16 = mul(x = var_4064_cast_fp16, y = var_4066_to_fp16)[name = string("op_4067_cast_fp16")];
            tensor<int32, [3]> var_4068 = const()[name = string("op_4068"), val = tensor<int32, [3]>([1, 0, 2])];
            tensor<int32, [1]> value_225_axes_0 = const()[name = string("value_225_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [2, 1, 256]> var_4069_cast_fp16 = transpose(perm = var_4068, x = var_4067_cast_fp16)[name = string("transpose_31")];
            tensor<fp16, [1, 2, 1, 256]> value_225_cast_fp16 = expand_dims(axes = value_225_axes_0, x = var_4069_cast_fp16)[name = string("value_225_cast_fp16")];
            tensor<int32, [4]> rotated_13_begin_0 = const()[name = string("rotated_13_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> rotated_13_end_0 = const()[name = string("rotated_13_end_0"), val = tensor<int32, [4]>([1, 8, 1, 64])];
            tensor<bool, [4]> rotated_13_end_mask_0 = const()[name = string("rotated_13_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
            tensor<fp16, [1, 8, 1, 64]> rotated_13_cast_fp16 = slice_by_index(begin = rotated_13_begin_0, end = rotated_13_end_0, end_mask = rotated_13_end_mask_0, x = value_223_cast_fp16)[name = string("rotated_13_cast_fp16")];
            tensor<int32, [4]> passthrough_13_begin_0 = const()[name = string("passthrough_13_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 64])];
            tensor<int32, [4]> passthrough_13_end_0 = const()[name = string("passthrough_13_end_0"), val = tensor<int32, [4]>([1, 8, 1, 256])];
            tensor<bool, [4]> passthrough_13_end_mask_0 = const()[name = string("passthrough_13_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
            tensor<fp16, [1, 8, 1, 192]> passthrough_13_cast_fp16 = slice_by_index(begin = passthrough_13_begin_0, end = passthrough_13_end_0, end_mask = passthrough_13_end_mask_0, x = value_223_cast_fp16)[name = string("passthrough_13_cast_fp16")];
            tensor<int32, [2]> var_4073_split_sizes_0 = const()[name = string("op_4073_split_sizes_0"), val = tensor<int32, [2]>([32, 32])];
            int32 var_4073_axis_0 = const()[name = string("op_4073_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 8, 1, 32]> var_4073_cast_fp16_0, tensor<fp16, [1, 8, 1, 32]> var_4073_cast_fp16_1 = split(axis = var_4073_axis_0, split_sizes = var_4073_split_sizes_0, x = rotated_13_cast_fp16)[name = string("op_4073_cast_fp16")];
            fp16 const_132_promoted_to_fp16 = const()[name = string("const_132_promoted_to_fp16"), val = fp16(-0x1p+0)];
            tensor<fp16, [1, 8, 1, 32]> var_4075_cast_fp16 = mul(x = var_4073_cast_fp16_1, y = const_132_promoted_to_fp16)[name = string("op_4075_cast_fp16")];
            bool rotated_half_13_interleave_0 = const()[name = string("rotated_half_13_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 8, 1, 64]> rotated_half_13_cast_fp16 = concat(axis = var_4014, interleave = rotated_half_13_interleave_0, values = (var_4075_cast_fp16, var_4073_cast_fp16_0))[name = string("rotated_half_13_cast_fp16")];
            tensor<fp16, [1, 8, 1, 64]> var_4078_cast_fp16 = mul(x = rotated_13_cast_fp16, y = cos)[name = string("op_4078_cast_fp16")];
            tensor<fp16, [1, 8, 1, 64]> var_4079_cast_fp16 = mul(x = rotated_half_13_cast_fp16, y = sin)[name = string("op_4079_cast_fp16")];
            tensor<fp16, [1, 8, 1, 64]> var_4080_cast_fp16 = add(x = var_4078_cast_fp16, y = var_4079_cast_fp16)[name = string("op_4080_cast_fp16")];
            bool query_61_interleave_0 = const()[name = string("query_61_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 8, 1, 256]> query_61_cast_fp16 = concat(axis = var_4014, interleave = query_61_interleave_0, values = (var_4080_cast_fp16, passthrough_13_cast_fp16))[name = string("query_61_cast_fp16")];
            tensor<int32, [4]> rotated_15_begin_0 = const()[name = string("rotated_15_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> rotated_15_end_0 = const()[name = string("rotated_15_end_0"), val = tensor<int32, [4]>([1, 2, 1, 64])];
            tensor<bool, [4]> rotated_15_end_mask_0 = const()[name = string("rotated_15_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
            tensor<fp16, [1, 2, 1, 64]> rotated_15_cast_fp16 = slice_by_index(begin = rotated_15_begin_0, end = rotated_15_end_0, end_mask = rotated_15_end_mask_0, x = value_225_cast_fp16)[name = string("rotated_15_cast_fp16")];
            tensor<int32, [4]> passthrough_15_begin_0 = const()[name = string("passthrough_15_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 64])];
            tensor<int32, [4]> passthrough_15_end_0 = const()[name = string("passthrough_15_end_0"), val = tensor<int32, [4]>([1, 2, 1, 256])];
            tensor<bool, [4]> passthrough_15_end_mask_0 = const()[name = string("passthrough_15_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
            tensor<fp16, [1, 2, 1, 192]> passthrough_15_cast_fp16 = slice_by_index(begin = passthrough_15_begin_0, end = passthrough_15_end_0, end_mask = passthrough_15_end_mask_0, x = value_225_cast_fp16)[name = string("passthrough_15_cast_fp16")];
            tensor<int32, [2]> var_4085_split_sizes_0 = const()[name = string("op_4085_split_sizes_0"), val = tensor<int32, [2]>([32, 32])];
            int32 var_4085_axis_0 = const()[name = string("op_4085_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 2, 1, 32]> var_4085_cast_fp16_0, tensor<fp16, [1, 2, 1, 32]> var_4085_cast_fp16_1 = split(axis = var_4085_axis_0, split_sizes = var_4085_split_sizes_0, x = rotated_15_cast_fp16)[name = string("op_4085_cast_fp16")];
            fp16 const_133_promoted_to_fp16 = const()[name = string("const_133_promoted_to_fp16"), val = fp16(-0x1p+0)];
            tensor<fp16, [1, 2, 1, 32]> var_4087_cast_fp16 = mul(x = var_4085_cast_fp16_1, y = const_133_promoted_to_fp16)[name = string("op_4087_cast_fp16")];
            bool rotated_half_15_interleave_0 = const()[name = string("rotated_half_15_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 2, 1, 64]> rotated_half_15_cast_fp16 = concat(axis = var_4014, interleave = rotated_half_15_interleave_0, values = (var_4087_cast_fp16, var_4085_cast_fp16_0))[name = string("rotated_half_15_cast_fp16")];
            tensor<fp16, [1, 2, 1, 64]> var_4090_cast_fp16 = mul(x = rotated_15_cast_fp16, y = cos)[name = string("op_4090_cast_fp16")];
            tensor<fp16, [1, 2, 1, 64]> var_4091_cast_fp16 = mul(x = rotated_half_15_cast_fp16, y = sin)[name = string("op_4091_cast_fp16")];
            tensor<fp16, [1, 2, 1, 64]> var_4092_cast_fp16 = add(x = var_4090_cast_fp16, y = var_4091_cast_fp16)[name = string("op_4092_cast_fp16")];
            bool key_61_interleave_0 = const()[name = string("key_61_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 2, 1, 256]> key_61_cast_fp16 = concat(axis = var_4014, interleave = key_61_interleave_0, values = (var_4092_cast_fp16, passthrough_15_cast_fp16))[name = string("key_61_cast_fp16")];
            tensor<int32, [4]> var_4095 = const()[name = string("op_4095"), val = tensor<int32, [4]>([2, 4, 1, 256])];
            tensor<fp16, [2, 4, 1, 256]> var_4096_cast_fp16 = reshape(shape = var_4095, x = query_61_cast_fp16)[name = string("op_4096_cast_fp16")];
            tensor<int32, [4]> transpose_6_perm_0 = const()[name = string("transpose_6_perm_0"), val = tensor<int32, [4]>([0, 1, 3, 2])];
            tensor<int32, [3]> var_4099 = const()[name = string("op_4099"), val = tensor<int32, [3]>([2, 1, 256])];
            tensor<fp16, [2, 1, 256]> key_63_cast_fp16 = reshape(shape = var_4099, x = key_61_cast_fp16)[name = string("key_63_cast_fp16")];
            tensor<int32, [3]> var_4101 = const()[name = string("op_4101"), val = tensor<int32, [3]>([1, 0, 2])];
            tensor<int32, [2]> var_4103 = const()[name = string("op_4103"), val = tensor<int32, [2]>([1, 2048])];
            tensor<fp16, [1, 2048]> gate_15_cast_fp16 = reshape(shape = var_4103, x = var_4035_cast_fp16_1)[name = string("gate_15_cast_fp16")];
            tensor<int32, [1]> var_4111_axes_0 = const()[name = string("op_4111_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [1, 2, 1, 256]> var_4111_cast_fp16 = expand_dims(axes = var_4111_axes_0, x = key_63_cast_fp16)[name = string("op_4111_cast_fp16")];
            tensor<int32, [1]> var_4113_axes_0 = const()[name = string("op_4113_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [1, 1, 2, 1, 256]> var_4113_cast_fp16 = expand_dims(axes = var_4113_axes_0, x = var_4111_cast_fp16)[name = string("op_4113_cast_fp16")];
            tensor<int32, [1]> expand_dims_48 = const()[name = string("expand_dims_48"), val = tensor<int32, [1]>([3])];
            tensor<int32, [1]> expand_dims_49 = const()[name = string("expand_dims_49"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_50 = const()[name = string("expand_dims_50"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_52 = const()[name = string("expand_dims_52"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_53 = const()[name = string("expand_dims_53"), val = tensor<int32, [1]>([4])];
            tensor<int32, [1]> expand_dims_54 = const()[name = string("expand_dims_54"), val = tensor<int32, [1]>([1])];
            int32 concat_26_axis_0 = const()[name = string("concat_26_axis_0"), val = int32(0)];
            bool concat_26_interleave_0 = const()[name = string("concat_26_interleave_0"), val = bool(false)];
            tensor<int32, [5]> concat_26 = concat(axis = concat_26_axis_0, interleave = concat_26_interleave_0, values = (expand_dims_48, expand_dims_49, expand_dims_50, current_pos, expand_dims_52))[name = string("concat_26")];
            tensor<int32, [1]> concat_27_values2_0 = const()[name = string("concat_27_values2_0"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> concat_27_values4_0 = const()[name = string("concat_27_values4_0"), val = tensor<int32, [1]>([0])];
            int32 concat_27_axis_0 = const()[name = string("concat_27_axis_0"), val = int32(0)];
            bool concat_27_interleave_0 = const()[name = string("concat_27_interleave_0"), val = bool(false)];
            tensor<int32, [5]> concat_27 = concat(axis = concat_27_axis_0, interleave = concat_27_interleave_0, values = (expand_dims_53, expand_dims_54, concat_27_values2_0, var_762, concat_27_values4_0))[name = string("concat_27")];
            tensor<int32, [5]> kv_cache_internal_tensor_assign_7_stride_0 = const()[name = string("kv_cache_internal_tensor_assign_7_stride_0"), val = tensor<int32, [5]>([1, 1, 1, 1, 1])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_7_begin_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_7_begin_mask_0"), val = tensor<bool, [5]>([false, false, false, false, false])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_7_end_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_7_end_mask_0"), val = tensor<bool, [5]>([false, false, true, false, true])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_7_squeeze_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_7_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, false, false, false])];
            tensor<fp16, [4, 2, 2, 2048, 256]> kv_cache_internal_tensor_assign_7_cast_fp16 = slice_update(begin = concat_26, begin_mask = kv_cache_internal_tensor_assign_7_begin_mask_0, end = concat_27, end_mask = kv_cache_internal_tensor_assign_7_end_mask_0, squeeze_mask = kv_cache_internal_tensor_assign_7_squeeze_mask_0, stride = kv_cache_internal_tensor_assign_7_stride_0, update = var_4113_cast_fp16, x = coreml_update_state_13)[name = string("kv_cache_internal_tensor_assign_7_cast_fp16")];
            write_state(data = kv_cache_internal_tensor_assign_7_cast_fp16, input = kv_cache)[name = string("coreml_update_state_6_write_state")];
            tensor<fp16, [4, 2, 2, 2048, 256]> coreml_update_state_14 = read_state(input = kv_cache)[name = string("coreml_update_state_6")];
            tensor<int32, [1]> var_4144_axes_0 = const()[name = string("op_4144_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [2, 1, 256]> value_229_cast_fp16 = transpose(perm = var_4101, x = value_227_cast_fp16)[name = string("transpose_30")];
            tensor<fp16, [1, 2, 1, 256]> var_4144_cast_fp16 = expand_dims(axes = var_4144_axes_0, x = value_229_cast_fp16)[name = string("op_4144_cast_fp16")];
            tensor<int32, [1]> var_4146_axes_0 = const()[name = string("op_4146_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [1, 1, 2, 1, 256]> var_4146_cast_fp16 = expand_dims(axes = var_4146_axes_0, x = var_4144_cast_fp16)[name = string("op_4146_cast_fp16")];
            tensor<int32, [1]> expand_dims_56 = const()[name = string("expand_dims_56"), val = tensor<int32, [1]>([3])];
            tensor<int32, [1]> expand_dims_57 = const()[name = string("expand_dims_57"), val = tensor<int32, [1]>([1])];
            tensor<int32, [1]> expand_dims_58 = const()[name = string("expand_dims_58"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_60 = const()[name = string("expand_dims_60"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_61 = const()[name = string("expand_dims_61"), val = tensor<int32, [1]>([4])];
            tensor<int32, [1]> expand_dims_62 = const()[name = string("expand_dims_62"), val = tensor<int32, [1]>([2])];
            int32 concat_30_axis_0 = const()[name = string("concat_30_axis_0"), val = int32(0)];
            bool concat_30_interleave_0 = const()[name = string("concat_30_interleave_0"), val = bool(false)];
            tensor<int32, [5]> concat_30 = concat(axis = concat_30_axis_0, interleave = concat_30_interleave_0, values = (expand_dims_56, expand_dims_57, expand_dims_58, current_pos, expand_dims_60))[name = string("concat_30")];
            tensor<int32, [1]> concat_31_values2_0 = const()[name = string("concat_31_values2_0"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> concat_31_values4_0 = const()[name = string("concat_31_values4_0"), val = tensor<int32, [1]>([0])];
            int32 concat_31_axis_0 = const()[name = string("concat_31_axis_0"), val = int32(0)];
            bool concat_31_interleave_0 = const()[name = string("concat_31_interleave_0"), val = bool(false)];
            tensor<int32, [5]> concat_31 = concat(axis = concat_31_axis_0, interleave = concat_31_interleave_0, values = (expand_dims_61, expand_dims_62, concat_31_values2_0, var_762, concat_31_values4_0))[name = string("concat_31")];
            tensor<int32, [5]> kv_cache_internal_tensor_assign_8_stride_0 = const()[name = string("kv_cache_internal_tensor_assign_8_stride_0"), val = tensor<int32, [5]>([1, 1, 1, 1, 1])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_8_begin_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_8_begin_mask_0"), val = tensor<bool, [5]>([false, false, false, false, false])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_8_end_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_8_end_mask_0"), val = tensor<bool, [5]>([false, false, true, false, true])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_8_squeeze_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_8_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, false, false, false])];
            tensor<fp16, [4, 2, 2, 2048, 256]> kv_cache_internal_tensor_assign_8_cast_fp16 = slice_update(begin = concat_30, begin_mask = kv_cache_internal_tensor_assign_8_begin_mask_0, end = concat_31, end_mask = kv_cache_internal_tensor_assign_8_end_mask_0, squeeze_mask = kv_cache_internal_tensor_assign_8_squeeze_mask_0, stride = kv_cache_internal_tensor_assign_8_stride_0, update = var_4146_cast_fp16, x = coreml_update_state_14)[name = string("kv_cache_internal_tensor_assign_8_cast_fp16")];
            write_state(data = kv_cache_internal_tensor_assign_8_cast_fp16, input = kv_cache)[name = string("coreml_update_state_7_write_state")];
            tensor<fp16, [4, 2, 2, 2048, 256]> coreml_update_state_15 = read_state(input = kv_cache)[name = string("coreml_update_state_7")];
            tensor<int32, [5]> var_4178_begin_0 = const()[name = string("op_4178_begin_0"), val = tensor<int32, [5]>([3, 0, 0, 0, 0])];
            tensor<int32, [5]> var_4178_end_0 = const()[name = string("op_4178_end_0"), val = tensor<int32, [5]>([4, 2, 2, 2048, 256])];
            tensor<bool, [5]> var_4178_end_mask_0 = const()[name = string("op_4178_end_mask_0"), val = tensor<bool, [5]>([false, true, true, true, true])];
            tensor<bool, [5]> var_4178_squeeze_mask_0 = const()[name = string("op_4178_squeeze_mask_0"), val = tensor<bool, [5]>([true, false, false, false, false])];
            tensor<fp16, [2, 2, 2048, 256]> var_4178_cast_fp16 = slice_by_index(begin = var_4178_begin_0, end = var_4178_end_0, end_mask = var_4178_end_mask_0, squeeze_mask = var_4178_squeeze_mask_0, x = coreml_update_state_15)[name = string("op_4178_cast_fp16")];
            tensor<int32, [4]> keys_begin_0 = const()[name = string("keys_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> keys_end_0 = const()[name = string("keys_end_0"), val = tensor<int32, [4]>([1, 2, 2048, 256])];
            tensor<bool, [4]> keys_end_mask_0 = const()[name = string("keys_end_mask_0"), val = tensor<bool, [4]>([false, true, true, true])];
            tensor<bool, [4]> keys_squeeze_mask_0 = const()[name = string("keys_squeeze_mask_0"), val = tensor<bool, [4]>([true, false, false, false])];
            tensor<fp16, [2, 2048, 256]> keys_cast_fp16 = slice_by_index(begin = keys_begin_0, end = keys_end_0, end_mask = keys_end_mask_0, squeeze_mask = keys_squeeze_mask_0, x = var_4178_cast_fp16)[name = string("keys_cast_fp16")];
            tensor<int32, [4]> values_begin_0 = const()[name = string("values_begin_0"), val = tensor<int32, [4]>([1, 0, 0, 0])];
            tensor<int32, [4]> values_end_0 = const()[name = string("values_end_0"), val = tensor<int32, [4]>([2, 2, 2048, 256])];
            tensor<bool, [4]> values_end_mask_0 = const()[name = string("values_end_mask_0"), val = tensor<bool, [4]>([false, true, true, true])];
            tensor<bool, [4]> values_squeeze_mask_0 = const()[name = string("values_squeeze_mask_0"), val = tensor<bool, [4]>([true, false, false, false])];
            tensor<fp16, [2, 2048, 256]> values_cast_fp16 = slice_by_index(begin = values_begin_0, end = values_end_0, end_mask = values_end_mask_0, squeeze_mask = values_squeeze_mask_0, x = var_4178_cast_fp16)[name = string("values_cast_fp16")];
            int32 var_4195 = const()[name = string("op_4195"), val = int32(1)];
            tensor<int32, [4]> var_4202 = const()[name = string("op_4202"), val = tensor<int32, [4]>([1, 2048, 1, 1])];
            tensor<fp16, [2, 4, 256, 1]> transpose_6_cast_fp16 = transpose(perm = transpose_6_perm_0, x = var_4096_cast_fp16)[name = string("transpose_29")];
            tensor<fp16, [1, 2048, 1, 1]> var_4203_cast_fp16 = reshape(shape = var_4202, x = transpose_6_cast_fp16)[name = string("op_4203_cast_fp16")];
            tensor<int32, [3]> var_4204 = const()[name = string("op_4204"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [4]> var_4207 = const()[name = string("op_4207"), val = tensor<int32, [4]>([1, 512, 1, 2048])];
            tensor<fp16, [2, 256, 2048]> var_4205_cast_fp16 = transpose(perm = var_4204, x = keys_cast_fp16)[name = string("transpose_28")];
            tensor<fp16, [1, 512, 1, 2048]> key_native_cast_fp16 = reshape(shape = var_4207, x = var_4205_cast_fp16)[name = string("key_native_cast_fp16")];
            tensor<int32, [3]> var_4209 = const()[name = string("op_4209"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [4]> var_4212 = const()[name = string("op_4212"), val = tensor<int32, [4]>([1, 512, 1, 2048])];
            tensor<fp16, [2, 256, 2048]> var_4210_cast_fp16 = transpose(perm = var_4209, x = values_cast_fp16)[name = string("transpose_27")];
            tensor<fp16, [1, 512, 1, 2048]> var_4213_cast_fp16 = reshape(shape = var_4212, x = var_4210_cast_fp16)[name = string("op_4213_cast_fp16")];
            tensor<int32, [8]> tile_57 = const()[name = string("tile_57"), val = tensor<int32, [8]>([256, 256, 256, 256, 256, 256, 256, 256])];
            int32 var_4217_axis_0 = const()[name = string("op_4217_axis_0"), val = int32(1)];
            tensor<fp16, [1, 256, 1, 1]> var_4217_cast_fp16_0, tensor<fp16, [1, 256, 1, 1]> var_4217_cast_fp16_1, tensor<fp16, [1, 256, 1, 1]> var_4217_cast_fp16_2, tensor<fp16, [1, 256, 1, 1]> var_4217_cast_fp16_3, tensor<fp16, [1, 256, 1, 1]> var_4217_cast_fp16_4, tensor<fp16, [1, 256, 1, 1]> var_4217_cast_fp16_5, tensor<fp16, [1, 256, 1, 1]> var_4217_cast_fp16_6, tensor<fp16, [1, 256, 1, 1]> var_4217_cast_fp16_7 = split(axis = var_4217_axis_0, split_sizes = tile_57, x = var_4203_cast_fp16)[name = string("op_4217_cast_fp16")];
            tensor<int32, [4]> var_4226_perm_0 = const()[name = string("op_4226_perm_0"), val = tensor<int32, [4]>([0, 3, 2, 1])];
            tensor<int32, [2]> tile_58 = const()[name = string("tile_58"), val = tensor<int32, [2]>([256, 256])];
            int32 var_4227_axis_0 = const()[name = string("op_4227_axis_0"), val = int32(3)];
            tensor<fp16, [1, 2048, 1, 512]> var_4226_cast_fp16 = transpose(perm = var_4226_perm_0, x = key_native_cast_fp16)[name = string("transpose_26")];
            tensor<fp16, [1, 2048, 1, 256]> var_4227_cast_fp16_0, tensor<fp16, [1, 2048, 1, 256]> var_4227_cast_fp16_1 = split(axis = var_4227_axis_0, split_sizes = tile_58, x = var_4226_cast_fp16)[name = string("op_4227_cast_fp16")];
            tensor<int32, [2]> tile_59 = const()[name = string("tile_59"), val = tensor<int32, [2]>([256, 256])];
            int32 var_4230_axis_0 = const()[name = string("op_4230_axis_0"), val = int32(1)];
            tensor<fp16, [1, 256, 1, 2048]> var_4230_cast_fp16_0, tensor<fp16, [1, 256, 1, 2048]> var_4230_cast_fp16_1 = split(axis = var_4230_axis_0, split_sizes = tile_59, x = var_4213_cast_fp16)[name = string("op_4230_cast_fp16")];
            string scores_49_equation_0 = const()[name = string("scores_49_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 1]> scores_49_cast_fp16 = einsum(equation = scores_49_equation_0, values = (var_4227_cast_fp16_0, var_4217_cast_fp16_0))[name = string("scores_49_cast_fp16")];
            fp16 var_4235_to_fp16 = const()[name = string("op_4235_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 1]> var_4236_cast_fp16 = mul(x = scores_49_cast_fp16, y = var_4235_to_fp16)[name = string("op_4236_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_4237_cast_fp16 = add(x = var_4236_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_4237_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_4238_cast_fp16 = softmax(axis = var_4195, x = var_4237_cast_fp16)[name = string("op_4238_cast_fp16")];
            tensor<int32, [4]> tile_60 = const()[name = string("tile_60"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_4239_axis_0 = const()[name = string("op_4239_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 1]> var_4239_cast_fp16_0, tensor<fp16, [1, 512, 1, 1]> var_4239_cast_fp16_1, tensor<fp16, [1, 512, 1, 1]> var_4239_cast_fp16_2, tensor<fp16, [1, 512, 1, 1]> var_4239_cast_fp16_3 = split(axis = var_4239_axis_0, split_sizes = tile_60, x = var_4238_cast_fp16)[name = string("op_4239_cast_fp16")];
            tensor<int32, [4]> tile_61 = const()[name = string("tile_61"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_4244_axis_0 = const()[name = string("op_4244_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_4244_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_4244_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_4244_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_4244_cast_fp16_3 = split(axis = var_4244_axis_0, split_sizes = tile_61, x = var_4230_cast_fp16_0)[name = string("op_4244_cast_fp16")];
            fp16 var_4249_to_fp16 = const()[name = string("op_4249_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_4250_cast_fp16 = mul(x = var_4239_cast_fp16_0, y = var_4249_to_fp16)[name = string("op_4250_cast_fp16")];
            string var_4252_equation_0 = const()[name = string("op_4252_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_4252_cast_fp16 = einsum(equation = var_4252_equation_0, values = (var_4244_cast_fp16_0, var_4250_cast_fp16))[name = string("op_4252_cast_fp16")];
            fp16 var_4253_to_fp16 = const()[name = string("op_4253_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_4254_cast_fp16 = mul(x = var_4239_cast_fp16_1, y = var_4253_to_fp16)[name = string("op_4254_cast_fp16")];
            string var_4256_equation_0 = const()[name = string("op_4256_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_4256_cast_fp16 = einsum(equation = var_4256_equation_0, values = (var_4244_cast_fp16_1, var_4254_cast_fp16))[name = string("op_4256_cast_fp16")];
            fp16 var_4257_to_fp16 = const()[name = string("op_4257_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_4258_cast_fp16 = mul(x = var_4239_cast_fp16_2, y = var_4257_to_fp16)[name = string("op_4258_cast_fp16")];
            string var_4260_equation_0 = const()[name = string("op_4260_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_4260_cast_fp16 = einsum(equation = var_4260_equation_0, values = (var_4244_cast_fp16_2, var_4258_cast_fp16))[name = string("op_4260_cast_fp16")];
            fp16 var_4261_to_fp16 = const()[name = string("op_4261_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_4262_cast_fp16 = mul(x = var_4239_cast_fp16_3, y = var_4261_to_fp16)[name = string("op_4262_cast_fp16")];
            string var_4264_equation_0 = const()[name = string("op_4264_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_4264_cast_fp16 = einsum(equation = var_4264_equation_0, values = (var_4244_cast_fp16_3, var_4262_cast_fp16))[name = string("op_4264_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_4265_cast_fp16 = add(x = var_4252_cast_fp16, y = var_4256_cast_fp16)[name = string("op_4265_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_4266_cast_fp16 = add(x = var_4260_cast_fp16, y = var_4264_cast_fp16)[name = string("op_4266_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_4267_cast_fp16 = add(x = var_4265_cast_fp16, y = var_4266_cast_fp16)[name = string("op_4267_cast_fp16")];
            fp16 _inversed_4269_y_0_to_fp16 = const()[name = string("_inversed_4269_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 1]> _inversed_4269_cast_fp16 = mul(x = var_4267_cast_fp16, y = _inversed_4269_y_0_to_fp16)[name = string("_inversed_4269_cast_fp16")];
            string scores_51_equation_0 = const()[name = string("scores_51_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 1]> scores_51_cast_fp16 = einsum(equation = scores_51_equation_0, values = (var_4227_cast_fp16_0, var_4217_cast_fp16_1))[name = string("scores_51_cast_fp16")];
            fp16 var_4272_to_fp16 = const()[name = string("op_4272_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 1]> var_4273_cast_fp16 = mul(x = scores_51_cast_fp16, y = var_4272_to_fp16)[name = string("op_4273_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_4274_cast_fp16 = add(x = var_4273_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_4274_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_4275_cast_fp16 = softmax(axis = var_4195, x = var_4274_cast_fp16)[name = string("op_4275_cast_fp16")];
            tensor<int32, [4]> tile_62 = const()[name = string("tile_62"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_4276_axis_0 = const()[name = string("op_4276_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 1]> var_4276_cast_fp16_0, tensor<fp16, [1, 512, 1, 1]> var_4276_cast_fp16_1, tensor<fp16, [1, 512, 1, 1]> var_4276_cast_fp16_2, tensor<fp16, [1, 512, 1, 1]> var_4276_cast_fp16_3 = split(axis = var_4276_axis_0, split_sizes = tile_62, x = var_4275_cast_fp16)[name = string("op_4276_cast_fp16")];
            tensor<int32, [4]> tile_63 = const()[name = string("tile_63"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_4281_axis_0 = const()[name = string("op_4281_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_4281_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_4281_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_4281_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_4281_cast_fp16_3 = split(axis = var_4281_axis_0, split_sizes = tile_63, x = var_4230_cast_fp16_0)[name = string("op_4281_cast_fp16")];
            fp16 var_4286_to_fp16 = const()[name = string("op_4286_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_4287_cast_fp16 = mul(x = var_4276_cast_fp16_0, y = var_4286_to_fp16)[name = string("op_4287_cast_fp16")];
            string var_4289_equation_0 = const()[name = string("op_4289_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_4289_cast_fp16 = einsum(equation = var_4289_equation_0, values = (var_4281_cast_fp16_0, var_4287_cast_fp16))[name = string("op_4289_cast_fp16")];
            fp16 var_4290_to_fp16 = const()[name = string("op_4290_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_4291_cast_fp16 = mul(x = var_4276_cast_fp16_1, y = var_4290_to_fp16)[name = string("op_4291_cast_fp16")];
            string var_4293_equation_0 = const()[name = string("op_4293_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_4293_cast_fp16 = einsum(equation = var_4293_equation_0, values = (var_4281_cast_fp16_1, var_4291_cast_fp16))[name = string("op_4293_cast_fp16")];
            fp16 var_4294_to_fp16 = const()[name = string("op_4294_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_4295_cast_fp16 = mul(x = var_4276_cast_fp16_2, y = var_4294_to_fp16)[name = string("op_4295_cast_fp16")];
            string var_4297_equation_0 = const()[name = string("op_4297_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_4297_cast_fp16 = einsum(equation = var_4297_equation_0, values = (var_4281_cast_fp16_2, var_4295_cast_fp16))[name = string("op_4297_cast_fp16")];
            fp16 var_4298_to_fp16 = const()[name = string("op_4298_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_4299_cast_fp16 = mul(x = var_4276_cast_fp16_3, y = var_4298_to_fp16)[name = string("op_4299_cast_fp16")];
            string var_4301_equation_0 = const()[name = string("op_4301_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_4301_cast_fp16 = einsum(equation = var_4301_equation_0, values = (var_4281_cast_fp16_3, var_4299_cast_fp16))[name = string("op_4301_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_4302_cast_fp16 = add(x = var_4289_cast_fp16, y = var_4293_cast_fp16)[name = string("op_4302_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_4303_cast_fp16 = add(x = var_4297_cast_fp16, y = var_4301_cast_fp16)[name = string("op_4303_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_4304_cast_fp16 = add(x = var_4302_cast_fp16, y = var_4303_cast_fp16)[name = string("op_4304_cast_fp16")];
            fp16 _inversed_4306_y_0_to_fp16 = const()[name = string("_inversed_4306_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 1]> _inversed_4306_cast_fp16 = mul(x = var_4304_cast_fp16, y = _inversed_4306_y_0_to_fp16)[name = string("_inversed_4306_cast_fp16")];
            string scores_53_equation_0 = const()[name = string("scores_53_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 1]> scores_53_cast_fp16 = einsum(equation = scores_53_equation_0, values = (var_4227_cast_fp16_0, var_4217_cast_fp16_2))[name = string("scores_53_cast_fp16")];
            fp16 var_4309_to_fp16 = const()[name = string("op_4309_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 1]> var_4310_cast_fp16 = mul(x = scores_53_cast_fp16, y = var_4309_to_fp16)[name = string("op_4310_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_4311_cast_fp16 = add(x = var_4310_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_4311_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_4312_cast_fp16 = softmax(axis = var_4195, x = var_4311_cast_fp16)[name = string("op_4312_cast_fp16")];
            tensor<int32, [4]> tile_64 = const()[name = string("tile_64"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_4313_axis_0 = const()[name = string("op_4313_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 1]> var_4313_cast_fp16_0, tensor<fp16, [1, 512, 1, 1]> var_4313_cast_fp16_1, tensor<fp16, [1, 512, 1, 1]> var_4313_cast_fp16_2, tensor<fp16, [1, 512, 1, 1]> var_4313_cast_fp16_3 = split(axis = var_4313_axis_0, split_sizes = tile_64, x = var_4312_cast_fp16)[name = string("op_4313_cast_fp16")];
            tensor<int32, [4]> tile_65 = const()[name = string("tile_65"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_4318_axis_0 = const()[name = string("op_4318_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_4318_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_4318_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_4318_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_4318_cast_fp16_3 = split(axis = var_4318_axis_0, split_sizes = tile_65, x = var_4230_cast_fp16_0)[name = string("op_4318_cast_fp16")];
            fp16 var_4323_to_fp16 = const()[name = string("op_4323_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_4324_cast_fp16 = mul(x = var_4313_cast_fp16_0, y = var_4323_to_fp16)[name = string("op_4324_cast_fp16")];
            string var_4326_equation_0 = const()[name = string("op_4326_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_4326_cast_fp16 = einsum(equation = var_4326_equation_0, values = (var_4318_cast_fp16_0, var_4324_cast_fp16))[name = string("op_4326_cast_fp16")];
            fp16 var_4327_to_fp16 = const()[name = string("op_4327_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_4328_cast_fp16 = mul(x = var_4313_cast_fp16_1, y = var_4327_to_fp16)[name = string("op_4328_cast_fp16")];
            string var_4330_equation_0 = const()[name = string("op_4330_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_4330_cast_fp16 = einsum(equation = var_4330_equation_0, values = (var_4318_cast_fp16_1, var_4328_cast_fp16))[name = string("op_4330_cast_fp16")];
            fp16 var_4331_to_fp16 = const()[name = string("op_4331_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_4332_cast_fp16 = mul(x = var_4313_cast_fp16_2, y = var_4331_to_fp16)[name = string("op_4332_cast_fp16")];
            string var_4334_equation_0 = const()[name = string("op_4334_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_4334_cast_fp16 = einsum(equation = var_4334_equation_0, values = (var_4318_cast_fp16_2, var_4332_cast_fp16))[name = string("op_4334_cast_fp16")];
            fp16 var_4335_to_fp16 = const()[name = string("op_4335_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_4336_cast_fp16 = mul(x = var_4313_cast_fp16_3, y = var_4335_to_fp16)[name = string("op_4336_cast_fp16")];
            string var_4338_equation_0 = const()[name = string("op_4338_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_4338_cast_fp16 = einsum(equation = var_4338_equation_0, values = (var_4318_cast_fp16_3, var_4336_cast_fp16))[name = string("op_4338_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_4339_cast_fp16 = add(x = var_4326_cast_fp16, y = var_4330_cast_fp16)[name = string("op_4339_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_4340_cast_fp16 = add(x = var_4334_cast_fp16, y = var_4338_cast_fp16)[name = string("op_4340_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_4341_cast_fp16 = add(x = var_4339_cast_fp16, y = var_4340_cast_fp16)[name = string("op_4341_cast_fp16")];
            fp16 _inversed_4343_y_0_to_fp16 = const()[name = string("_inversed_4343_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 1]> _inversed_4343_cast_fp16 = mul(x = var_4341_cast_fp16, y = _inversed_4343_y_0_to_fp16)[name = string("_inversed_4343_cast_fp16")];
            string scores_55_equation_0 = const()[name = string("scores_55_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 1]> scores_55_cast_fp16 = einsum(equation = scores_55_equation_0, values = (var_4227_cast_fp16_0, var_4217_cast_fp16_3))[name = string("scores_55_cast_fp16")];
            fp16 var_4346_to_fp16 = const()[name = string("op_4346_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 1]> var_4347_cast_fp16 = mul(x = scores_55_cast_fp16, y = var_4346_to_fp16)[name = string("op_4347_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_4348_cast_fp16 = add(x = var_4347_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_4348_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_4349_cast_fp16 = softmax(axis = var_4195, x = var_4348_cast_fp16)[name = string("op_4349_cast_fp16")];
            tensor<int32, [4]> tile_66 = const()[name = string("tile_66"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_4350_axis_0 = const()[name = string("op_4350_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 1]> var_4350_cast_fp16_0, tensor<fp16, [1, 512, 1, 1]> var_4350_cast_fp16_1, tensor<fp16, [1, 512, 1, 1]> var_4350_cast_fp16_2, tensor<fp16, [1, 512, 1, 1]> var_4350_cast_fp16_3 = split(axis = var_4350_axis_0, split_sizes = tile_66, x = var_4349_cast_fp16)[name = string("op_4350_cast_fp16")];
            tensor<int32, [4]> tile_67 = const()[name = string("tile_67"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_4355_axis_0 = const()[name = string("op_4355_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_4355_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_4355_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_4355_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_4355_cast_fp16_3 = split(axis = var_4355_axis_0, split_sizes = tile_67, x = var_4230_cast_fp16_0)[name = string("op_4355_cast_fp16")];
            fp16 var_4360_to_fp16 = const()[name = string("op_4360_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_4361_cast_fp16 = mul(x = var_4350_cast_fp16_0, y = var_4360_to_fp16)[name = string("op_4361_cast_fp16")];
            string var_4363_equation_0 = const()[name = string("op_4363_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_4363_cast_fp16 = einsum(equation = var_4363_equation_0, values = (var_4355_cast_fp16_0, var_4361_cast_fp16))[name = string("op_4363_cast_fp16")];
            fp16 var_4364_to_fp16 = const()[name = string("op_4364_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_4365_cast_fp16 = mul(x = var_4350_cast_fp16_1, y = var_4364_to_fp16)[name = string("op_4365_cast_fp16")];
            string var_4367_equation_0 = const()[name = string("op_4367_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_4367_cast_fp16 = einsum(equation = var_4367_equation_0, values = (var_4355_cast_fp16_1, var_4365_cast_fp16))[name = string("op_4367_cast_fp16")];
            fp16 var_4368_to_fp16 = const()[name = string("op_4368_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_4369_cast_fp16 = mul(x = var_4350_cast_fp16_2, y = var_4368_to_fp16)[name = string("op_4369_cast_fp16")];
            string var_4371_equation_0 = const()[name = string("op_4371_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_4371_cast_fp16 = einsum(equation = var_4371_equation_0, values = (var_4355_cast_fp16_2, var_4369_cast_fp16))[name = string("op_4371_cast_fp16")];
            fp16 var_4372_to_fp16 = const()[name = string("op_4372_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_4373_cast_fp16 = mul(x = var_4350_cast_fp16_3, y = var_4372_to_fp16)[name = string("op_4373_cast_fp16")];
            string var_4375_equation_0 = const()[name = string("op_4375_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_4375_cast_fp16 = einsum(equation = var_4375_equation_0, values = (var_4355_cast_fp16_3, var_4373_cast_fp16))[name = string("op_4375_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_4376_cast_fp16 = add(x = var_4363_cast_fp16, y = var_4367_cast_fp16)[name = string("op_4376_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_4377_cast_fp16 = add(x = var_4371_cast_fp16, y = var_4375_cast_fp16)[name = string("op_4377_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_4378_cast_fp16 = add(x = var_4376_cast_fp16, y = var_4377_cast_fp16)[name = string("op_4378_cast_fp16")];
            fp16 _inversed_4380_y_0_to_fp16 = const()[name = string("_inversed_4380_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 1]> _inversed_4380_cast_fp16 = mul(x = var_4378_cast_fp16, y = _inversed_4380_y_0_to_fp16)[name = string("_inversed_4380_cast_fp16")];
            string scores_57_equation_0 = const()[name = string("scores_57_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 1]> scores_57_cast_fp16 = einsum(equation = scores_57_equation_0, values = (var_4227_cast_fp16_1, var_4217_cast_fp16_4))[name = string("scores_57_cast_fp16")];
            fp16 var_4383_to_fp16 = const()[name = string("op_4383_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 1]> var_4384_cast_fp16 = mul(x = scores_57_cast_fp16, y = var_4383_to_fp16)[name = string("op_4384_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_4385_cast_fp16 = add(x = var_4384_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_4385_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_4386_cast_fp16 = softmax(axis = var_4195, x = var_4385_cast_fp16)[name = string("op_4386_cast_fp16")];
            tensor<int32, [4]> tile_68 = const()[name = string("tile_68"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_4387_axis_0 = const()[name = string("op_4387_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 1]> var_4387_cast_fp16_0, tensor<fp16, [1, 512, 1, 1]> var_4387_cast_fp16_1, tensor<fp16, [1, 512, 1, 1]> var_4387_cast_fp16_2, tensor<fp16, [1, 512, 1, 1]> var_4387_cast_fp16_3 = split(axis = var_4387_axis_0, split_sizes = tile_68, x = var_4386_cast_fp16)[name = string("op_4387_cast_fp16")];
            tensor<int32, [4]> tile_69 = const()[name = string("tile_69"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_4392_axis_0 = const()[name = string("op_4392_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_4392_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_4392_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_4392_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_4392_cast_fp16_3 = split(axis = var_4392_axis_0, split_sizes = tile_69, x = var_4230_cast_fp16_1)[name = string("op_4392_cast_fp16")];
            fp16 var_4397_to_fp16 = const()[name = string("op_4397_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_4398_cast_fp16 = mul(x = var_4387_cast_fp16_0, y = var_4397_to_fp16)[name = string("op_4398_cast_fp16")];
            string var_4400_equation_0 = const()[name = string("op_4400_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_4400_cast_fp16 = einsum(equation = var_4400_equation_0, values = (var_4392_cast_fp16_0, var_4398_cast_fp16))[name = string("op_4400_cast_fp16")];
            fp16 var_4401_to_fp16 = const()[name = string("op_4401_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_4402_cast_fp16 = mul(x = var_4387_cast_fp16_1, y = var_4401_to_fp16)[name = string("op_4402_cast_fp16")];
            string var_4404_equation_0 = const()[name = string("op_4404_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_4404_cast_fp16 = einsum(equation = var_4404_equation_0, values = (var_4392_cast_fp16_1, var_4402_cast_fp16))[name = string("op_4404_cast_fp16")];
            fp16 var_4405_to_fp16 = const()[name = string("op_4405_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_4406_cast_fp16 = mul(x = var_4387_cast_fp16_2, y = var_4405_to_fp16)[name = string("op_4406_cast_fp16")];
            string var_4408_equation_0 = const()[name = string("op_4408_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_4408_cast_fp16 = einsum(equation = var_4408_equation_0, values = (var_4392_cast_fp16_2, var_4406_cast_fp16))[name = string("op_4408_cast_fp16")];
            fp16 var_4409_to_fp16 = const()[name = string("op_4409_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_4410_cast_fp16 = mul(x = var_4387_cast_fp16_3, y = var_4409_to_fp16)[name = string("op_4410_cast_fp16")];
            string var_4412_equation_0 = const()[name = string("op_4412_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_4412_cast_fp16 = einsum(equation = var_4412_equation_0, values = (var_4392_cast_fp16_3, var_4410_cast_fp16))[name = string("op_4412_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_4413_cast_fp16 = add(x = var_4400_cast_fp16, y = var_4404_cast_fp16)[name = string("op_4413_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_4414_cast_fp16 = add(x = var_4408_cast_fp16, y = var_4412_cast_fp16)[name = string("op_4414_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_4415_cast_fp16 = add(x = var_4413_cast_fp16, y = var_4414_cast_fp16)[name = string("op_4415_cast_fp16")];
            fp16 _inversed_4417_y_0_to_fp16 = const()[name = string("_inversed_4417_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 1]> _inversed_4417_cast_fp16 = mul(x = var_4415_cast_fp16, y = _inversed_4417_y_0_to_fp16)[name = string("_inversed_4417_cast_fp16")];
            string scores_59_equation_0 = const()[name = string("scores_59_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 1]> scores_59_cast_fp16 = einsum(equation = scores_59_equation_0, values = (var_4227_cast_fp16_1, var_4217_cast_fp16_5))[name = string("scores_59_cast_fp16")];
            fp16 var_4420_to_fp16 = const()[name = string("op_4420_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 1]> var_4421_cast_fp16 = mul(x = scores_59_cast_fp16, y = var_4420_to_fp16)[name = string("op_4421_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_4422_cast_fp16 = add(x = var_4421_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_4422_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_4423_cast_fp16 = softmax(axis = var_4195, x = var_4422_cast_fp16)[name = string("op_4423_cast_fp16")];
            tensor<int32, [4]> tile_70 = const()[name = string("tile_70"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_4424_axis_0 = const()[name = string("op_4424_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 1]> var_4424_cast_fp16_0, tensor<fp16, [1, 512, 1, 1]> var_4424_cast_fp16_1, tensor<fp16, [1, 512, 1, 1]> var_4424_cast_fp16_2, tensor<fp16, [1, 512, 1, 1]> var_4424_cast_fp16_3 = split(axis = var_4424_axis_0, split_sizes = tile_70, x = var_4423_cast_fp16)[name = string("op_4424_cast_fp16")];
            tensor<int32, [4]> tile_71 = const()[name = string("tile_71"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_4429_axis_0 = const()[name = string("op_4429_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_4429_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_4429_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_4429_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_4429_cast_fp16_3 = split(axis = var_4429_axis_0, split_sizes = tile_71, x = var_4230_cast_fp16_1)[name = string("op_4429_cast_fp16")];
            fp16 var_4434_to_fp16 = const()[name = string("op_4434_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_4435_cast_fp16 = mul(x = var_4424_cast_fp16_0, y = var_4434_to_fp16)[name = string("op_4435_cast_fp16")];
            string var_4437_equation_0 = const()[name = string("op_4437_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_4437_cast_fp16 = einsum(equation = var_4437_equation_0, values = (var_4429_cast_fp16_0, var_4435_cast_fp16))[name = string("op_4437_cast_fp16")];
            fp16 var_4438_to_fp16 = const()[name = string("op_4438_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_4439_cast_fp16 = mul(x = var_4424_cast_fp16_1, y = var_4438_to_fp16)[name = string("op_4439_cast_fp16")];
            string var_4441_equation_0 = const()[name = string("op_4441_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_4441_cast_fp16 = einsum(equation = var_4441_equation_0, values = (var_4429_cast_fp16_1, var_4439_cast_fp16))[name = string("op_4441_cast_fp16")];
            fp16 var_4442_to_fp16 = const()[name = string("op_4442_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_4443_cast_fp16 = mul(x = var_4424_cast_fp16_2, y = var_4442_to_fp16)[name = string("op_4443_cast_fp16")];
            string var_4445_equation_0 = const()[name = string("op_4445_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_4445_cast_fp16 = einsum(equation = var_4445_equation_0, values = (var_4429_cast_fp16_2, var_4443_cast_fp16))[name = string("op_4445_cast_fp16")];
            fp16 var_4446_to_fp16 = const()[name = string("op_4446_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_4447_cast_fp16 = mul(x = var_4424_cast_fp16_3, y = var_4446_to_fp16)[name = string("op_4447_cast_fp16")];
            string var_4449_equation_0 = const()[name = string("op_4449_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_4449_cast_fp16 = einsum(equation = var_4449_equation_0, values = (var_4429_cast_fp16_3, var_4447_cast_fp16))[name = string("op_4449_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_4450_cast_fp16 = add(x = var_4437_cast_fp16, y = var_4441_cast_fp16)[name = string("op_4450_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_4451_cast_fp16 = add(x = var_4445_cast_fp16, y = var_4449_cast_fp16)[name = string("op_4451_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_4452_cast_fp16 = add(x = var_4450_cast_fp16, y = var_4451_cast_fp16)[name = string("op_4452_cast_fp16")];
            fp16 _inversed_4454_y_0_to_fp16 = const()[name = string("_inversed_4454_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 1]> _inversed_4454_cast_fp16 = mul(x = var_4452_cast_fp16, y = _inversed_4454_y_0_to_fp16)[name = string("_inversed_4454_cast_fp16")];
            string scores_61_equation_0 = const()[name = string("scores_61_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 1]> scores_61_cast_fp16 = einsum(equation = scores_61_equation_0, values = (var_4227_cast_fp16_1, var_4217_cast_fp16_6))[name = string("scores_61_cast_fp16")];
            fp16 var_4457_to_fp16 = const()[name = string("op_4457_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 1]> var_4458_cast_fp16 = mul(x = scores_61_cast_fp16, y = var_4457_to_fp16)[name = string("op_4458_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_4459_cast_fp16 = add(x = var_4458_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_4459_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_4460_cast_fp16 = softmax(axis = var_4195, x = var_4459_cast_fp16)[name = string("op_4460_cast_fp16")];
            tensor<int32, [4]> tile_72 = const()[name = string("tile_72"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_4461_axis_0 = const()[name = string("op_4461_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 1]> var_4461_cast_fp16_0, tensor<fp16, [1, 512, 1, 1]> var_4461_cast_fp16_1, tensor<fp16, [1, 512, 1, 1]> var_4461_cast_fp16_2, tensor<fp16, [1, 512, 1, 1]> var_4461_cast_fp16_3 = split(axis = var_4461_axis_0, split_sizes = tile_72, x = var_4460_cast_fp16)[name = string("op_4461_cast_fp16")];
            tensor<int32, [4]> tile_73 = const()[name = string("tile_73"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_4466_axis_0 = const()[name = string("op_4466_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_4466_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_4466_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_4466_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_4466_cast_fp16_3 = split(axis = var_4466_axis_0, split_sizes = tile_73, x = var_4230_cast_fp16_1)[name = string("op_4466_cast_fp16")];
            fp16 var_4471_to_fp16 = const()[name = string("op_4471_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_4472_cast_fp16 = mul(x = var_4461_cast_fp16_0, y = var_4471_to_fp16)[name = string("op_4472_cast_fp16")];
            string var_4474_equation_0 = const()[name = string("op_4474_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_4474_cast_fp16 = einsum(equation = var_4474_equation_0, values = (var_4466_cast_fp16_0, var_4472_cast_fp16))[name = string("op_4474_cast_fp16")];
            fp16 var_4475_to_fp16 = const()[name = string("op_4475_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_4476_cast_fp16 = mul(x = var_4461_cast_fp16_1, y = var_4475_to_fp16)[name = string("op_4476_cast_fp16")];
            string var_4478_equation_0 = const()[name = string("op_4478_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_4478_cast_fp16 = einsum(equation = var_4478_equation_0, values = (var_4466_cast_fp16_1, var_4476_cast_fp16))[name = string("op_4478_cast_fp16")];
            fp16 var_4479_to_fp16 = const()[name = string("op_4479_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_4480_cast_fp16 = mul(x = var_4461_cast_fp16_2, y = var_4479_to_fp16)[name = string("op_4480_cast_fp16")];
            string var_4482_equation_0 = const()[name = string("op_4482_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_4482_cast_fp16 = einsum(equation = var_4482_equation_0, values = (var_4466_cast_fp16_2, var_4480_cast_fp16))[name = string("op_4482_cast_fp16")];
            fp16 var_4483_to_fp16 = const()[name = string("op_4483_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_4484_cast_fp16 = mul(x = var_4461_cast_fp16_3, y = var_4483_to_fp16)[name = string("op_4484_cast_fp16")];
            string var_4486_equation_0 = const()[name = string("op_4486_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_4486_cast_fp16 = einsum(equation = var_4486_equation_0, values = (var_4466_cast_fp16_3, var_4484_cast_fp16))[name = string("op_4486_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_4487_cast_fp16 = add(x = var_4474_cast_fp16, y = var_4478_cast_fp16)[name = string("op_4487_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_4488_cast_fp16 = add(x = var_4482_cast_fp16, y = var_4486_cast_fp16)[name = string("op_4488_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_4489_cast_fp16 = add(x = var_4487_cast_fp16, y = var_4488_cast_fp16)[name = string("op_4489_cast_fp16")];
            fp16 _inversed_4491_y_0_to_fp16 = const()[name = string("_inversed_4491_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 1]> _inversed_4491_cast_fp16 = mul(x = var_4489_cast_fp16, y = _inversed_4491_y_0_to_fp16)[name = string("_inversed_4491_cast_fp16")];
            string scores_equation_0 = const()[name = string("scores_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 1]> scores_cast_fp16 = einsum(equation = scores_equation_0, values = (var_4227_cast_fp16_1, var_4217_cast_fp16_7))[name = string("scores_cast_fp16")];
            fp16 var_4494_to_fp16 = const()[name = string("op_4494_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 1]> var_4495_cast_fp16 = mul(x = scores_cast_fp16, y = var_4494_to_fp16)[name = string("op_4495_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_4496_cast_fp16 = add(x = var_4495_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_4496_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_4497_cast_fp16 = softmax(axis = var_4195, x = var_4496_cast_fp16)[name = string("op_4497_cast_fp16")];
            tensor<int32, [4]> tile_74 = const()[name = string("tile_74"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_4498_axis_0 = const()[name = string("op_4498_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 1]> var_4498_cast_fp16_0, tensor<fp16, [1, 512, 1, 1]> var_4498_cast_fp16_1, tensor<fp16, [1, 512, 1, 1]> var_4498_cast_fp16_2, tensor<fp16, [1, 512, 1, 1]> var_4498_cast_fp16_3 = split(axis = var_4498_axis_0, split_sizes = tile_74, x = var_4497_cast_fp16)[name = string("op_4498_cast_fp16")];
            tensor<int32, [4]> tile_75 = const()[name = string("tile_75"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_4503_axis_0 = const()[name = string("op_4503_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_4503_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_4503_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_4503_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_4503_cast_fp16_3 = split(axis = var_4503_axis_0, split_sizes = tile_75, x = var_4230_cast_fp16_1)[name = string("op_4503_cast_fp16")];
            fp16 var_4508_to_fp16 = const()[name = string("op_4508_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_4509_cast_fp16 = mul(x = var_4498_cast_fp16_0, y = var_4508_to_fp16)[name = string("op_4509_cast_fp16")];
            string var_4511_equation_0 = const()[name = string("op_4511_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_4511_cast_fp16 = einsum(equation = var_4511_equation_0, values = (var_4503_cast_fp16_0, var_4509_cast_fp16))[name = string("op_4511_cast_fp16")];
            fp16 var_4512_to_fp16 = const()[name = string("op_4512_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_4513_cast_fp16 = mul(x = var_4498_cast_fp16_1, y = var_4512_to_fp16)[name = string("op_4513_cast_fp16")];
            string var_4515_equation_0 = const()[name = string("op_4515_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_4515_cast_fp16 = einsum(equation = var_4515_equation_0, values = (var_4503_cast_fp16_1, var_4513_cast_fp16))[name = string("op_4515_cast_fp16")];
            fp16 var_4516_to_fp16 = const()[name = string("op_4516_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_4517_cast_fp16 = mul(x = var_4498_cast_fp16_2, y = var_4516_to_fp16)[name = string("op_4517_cast_fp16")];
            string var_4519_equation_0 = const()[name = string("op_4519_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_4519_cast_fp16 = einsum(equation = var_4519_equation_0, values = (var_4503_cast_fp16_2, var_4517_cast_fp16))[name = string("op_4519_cast_fp16")];
            fp16 var_4520_to_fp16 = const()[name = string("op_4520_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_4521_cast_fp16 = mul(x = var_4498_cast_fp16_3, y = var_4520_to_fp16)[name = string("op_4521_cast_fp16")];
            string var_4523_equation_0 = const()[name = string("op_4523_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_4523_cast_fp16 = einsum(equation = var_4523_equation_0, values = (var_4503_cast_fp16_3, var_4521_cast_fp16))[name = string("op_4523_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_4524_cast_fp16 = add(x = var_4511_cast_fp16, y = var_4515_cast_fp16)[name = string("op_4524_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_4525_cast_fp16 = add(x = var_4519_cast_fp16, y = var_4523_cast_fp16)[name = string("op_4525_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_4526_cast_fp16 = add(x = var_4524_cast_fp16, y = var_4525_cast_fp16)[name = string("op_4526_cast_fp16")];
            fp16 _inversed_4528_y_0_to_fp16 = const()[name = string("_inversed_4528_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 1]> _inversed_4528_cast_fp16 = mul(x = var_4526_cast_fp16, y = _inversed_4528_y_0_to_fp16)[name = string("_inversed_4528_cast_fp16")];
            bool var_4530_interleave_0 = const()[name = string("op_4530_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 2048, 1, 1]> var_4530_cast_fp16 = concat(axis = var_4195, interleave = var_4530_interleave_0, values = (_inversed_4269_cast_fp16, _inversed_4306_cast_fp16, _inversed_4343_cast_fp16, _inversed_4380_cast_fp16, _inversed_4417_cast_fp16, _inversed_4454_cast_fp16, _inversed_4491_cast_fp16, _inversed_4528_cast_fp16))[name = string("op_4530_cast_fp16")];
            tensor<int32, [4]> var_4531 = const()[name = string("op_4531"), val = tensor<int32, [4]>([2, 4, 256, 1])];
            tensor<fp16, [2, 4, 256, 1]> var_4532_cast_fp16 = reshape(shape = var_4531, x = var_4530_cast_fp16)[name = string("op_4532_cast_fp16")];
            tensor<int32, [4]> transpose_7_perm_0 = const()[name = string("transpose_7_perm_0"), val = tensor<int32, [4]>([3, 0, 1, 2])];
            tensor<int32, [2]> var_4550 = const()[name = string("op_4550"), val = tensor<int32, [2]>([1, 2048])];
            tensor<fp16, [1, 2, 4, 256]> transpose_7_cast_fp16 = transpose(perm = transpose_7_perm_0, x = var_4532_cast_fp16)[name = string("transpose_25")];
            tensor<fp16, [1, 2048]> attention_output_cast_fp16 = reshape(shape = var_4550, x = transpose_7_cast_fp16)[name = string("attention_output_cast_fp16")];
            tensor<fp16, [1, 2048]> var_4552_cast_fp16 = sigmoid(x = gate_15_cast_fp16)[name = string("op_4552_cast_fp16")];
            tensor<fp16, [1, 2048]> input_201_cast_fp16 = mul(x = attention_output_cast_fp16, y = var_4552_cast_fp16)[name = string("input_201_cast_fp16")];
            tensor<fp16, [1024, 2048]> completions_3_o_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 2048]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(241976576))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(243549504))))[name = string("completions_3_o_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 1024]> linear_120_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = completions_3_o_proj_weight_promoted_to_fp16_palettized, x = input_201_cast_fp16)[name = string("linear_120_cast_fp16")];
            tensor<fp16, [1, 1024]> value_231_cast_fp16 = add(x = value_217_cast_fp16, y = linear_120_cast_fp16)[name = string("value_231_cast_fp16")];
            tensor<fp16, [1, 1024]> var_4557_cast_fp16 = mul(x = value_231_cast_fp16, y = value_231_cast_fp16)[name = string("op_4557_cast_fp16")];
            tensor<int32, [1]> variance_103_axes_0 = const()[name = string("variance_103_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_103_keep_dims_0 = const()[name = string("variance_103_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 1]> variance_103_cast_fp16 = reduce_mean(axes = variance_103_axes_0, keep_dims = variance_103_keep_dims_0, x = var_4557_cast_fp16)[name = string("variance_103_cast_fp16")];
            fp16 var_4560_to_fp16 = const()[name = string("op_4560_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 1]> var_4561_cast_fp16 = add(x = variance_103_cast_fp16, y = var_4560_to_fp16)[name = string("op_4561_cast_fp16")];
            fp32 var_4562_epsilon_0 = const()[name = string("op_4562_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 1]> var_4562_cast_fp16 = rsqrt(epsilon = var_4562_epsilon_0, x = var_4561_cast_fp16)[name = string("op_4562_cast_fp16")];
            tensor<fp16, [1, 1024]> var_4563_cast_fp16 = mul(x = value_231_cast_fp16, y = var_4562_cast_fp16)[name = string("op_4563_cast_fp16")];
            tensor<fp16, [1024]> var_4565_to_fp16 = const()[name = string("op_4565_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(243557760)))];
            tensor<fp16, [1, 1024]> input_203_cast_fp16 = mul(x = var_4563_cast_fp16, y = var_4565_to_fp16)[name = string("input_203_cast_fp16")];
            tensor<fp16, [3584, 1024]> completions_3_gate_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(243559872))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(246312448))))[name = string("completions_3_gate_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 3584]> linear_121_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = completions_3_gate_proj_weight_promoted_to_fp16_palettized, x = input_203_cast_fp16)[name = string("linear_121_cast_fp16")];
            tensor<fp16, [1, 3584]> var_4569_cast_fp16 = silu(x = linear_121_cast_fp16)[name = string("op_4569_cast_fp16")];
            tensor<fp16, [3584, 1024]> completions_3_up_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(246341184))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(249093760))))[name = string("completions_3_up_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 3584]> linear_122_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = completions_3_up_proj_weight_promoted_to_fp16_palettized, x = input_203_cast_fp16)[name = string("linear_122_cast_fp16")];
            tensor<fp16, [1, 3584]> input_207_cast_fp16 = mul(x = var_4569_cast_fp16, y = linear_122_cast_fp16)[name = string("input_207_cast_fp16")];
            tensor<fp16, [1024, 3584]> completions_3_down_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 3584]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(249122496))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(251875072))))[name = string("completions_3_down_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 1024]> linear_123_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = completions_3_down_proj_weight_promoted_to_fp16_palettized, x = input_207_cast_fp16)[name = string("linear_123_cast_fp16")];
            tensor<fp16, [1, 1024]> value_233_cast_fp16 = add(x = value_231_cast_fp16, y = linear_123_cast_fp16)[name = string("value_233_cast_fp16")];
            int32 var_4590 = const()[name = string("op_4590"), val = int32(-1)];
            tensor<fp16, [1, 1024]> var_4605_cast_fp16 = mul(x = value_233_cast_fp16, y = value_233_cast_fp16)[name = string("op_4605_cast_fp16")];
            tensor<int32, [1]> variance_105_axes_0 = const()[name = string("variance_105_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_105_keep_dims_0 = const()[name = string("variance_105_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 1]> variance_105_cast_fp16 = reduce_mean(axes = variance_105_axes_0, keep_dims = variance_105_keep_dims_0, x = var_4605_cast_fp16)[name = string("variance_105_cast_fp16")];
            fp16 var_4608_to_fp16 = const()[name = string("op_4608_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 1]> var_4609_cast_fp16 = add(x = variance_105_cast_fp16, y = var_4608_to_fp16)[name = string("op_4609_cast_fp16")];
            fp32 var_4610_epsilon_0 = const()[name = string("op_4610_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 1]> var_4610_cast_fp16 = rsqrt(epsilon = var_4610_epsilon_0, x = var_4609_cast_fp16)[name = string("op_4610_cast_fp16")];
            tensor<fp16, [1, 1024]> var_4611_cast_fp16 = mul(x = value_233_cast_fp16, y = var_4610_cast_fp16)[name = string("op_4611_cast_fp16")];
            tensor<fp16, [1024]> var_4613_to_fp16 = const()[name = string("op_4613_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(251883328)))];
            tensor<fp16, [1, 1024]> input_209_cast_fp16 = mul(x = var_4611_cast_fp16, y = var_4613_to_fp16)[name = string("input_209_cast_fp16")];
            tensor<fp16, [6144, 1024]> final_group_0_in_proj_qkv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(251885440))), lut = tensor<fp16, [384, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(256604096))))[name = string("final_group_0_in_proj_qkv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 6144]> linear_124_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = final_group_0_in_proj_qkv_weight_promoted_to_fp16_palettized, x = input_209_cast_fp16)[name = string("linear_124_cast_fp16")];
            tensor<int32, [2]> var_4617_perm_0 = const()[name = string("op_4617_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> mixed_25_axes_0 = const()[name = string("mixed_25_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [6144, 1]> var_4617_cast_fp16 = transpose(perm = var_4617_perm_0, x = linear_124_cast_fp16)[name = string("transpose_24")];
            tensor<fp16, [1, 6144, 1]> mixed_25_cast_fp16 = expand_dims(axes = mixed_25_axes_0, x = var_4617_cast_fp16)[name = string("mixed_25_cast_fp16")];
            bool convolution_input_25_interleave_0 = const()[name = string("convolution_input_25_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 6144, 4]> convolution_input_25_cast_fp16 = concat(axis = var_4590, interleave = convolution_input_25_interleave_0, values = (conv16, mixed_25_cast_fp16))[name = string("convolution_input_25_cast_fp16")];
            string input_211_pad_type_0 = const()[name = string("input_211_pad_type_0"), val = string("valid")];
            int32 input_211_groups_0 = const()[name = string("input_211_groups_0"), val = int32(6144)];
            tensor<int32, [1]> input_211_strides_0 = const()[name = string("input_211_strides_0"), val = tensor<int32, [1]>([1])];
            tensor<int32, [2]> input_211_pad_0 = const()[name = string("input_211_pad_0"), val = tensor<int32, [2]>([0, 0])];
            tensor<int32, [1]> input_211_dilations_0 = const()[name = string("input_211_dilations_0"), val = tensor<int32, [1]>([1])];
            tensor<fp16, [6144, 1, 4]> final_group_0_conv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1, 4]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(256653312))), lut = tensor<fp16, [384, 1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(256671808))))[name = string("final_group_0_conv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 6144, 1]> input_211_cast_fp16 = conv(dilations = input_211_dilations_0, groups = input_211_groups_0, pad = input_211_pad_0, pad_type = input_211_pad_type_0, strides = input_211_strides_0, weight = final_group_0_conv_weight_promoted_to_fp16_palettized, x = convolution_input_25_cast_fp16)[name = string("input_211_cast_fp16")];
            tensor<fp16, [1, 6144, 1]> var_4626_cast_fp16 = silu(x = input_211_cast_fp16)[name = string("op_4626_cast_fp16")];
            tensor<int32, [3]> var_4627_perm_0 = const()[name = string("op_4627_perm_0"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [3]> var_4630_begin_0 = const()[name = string("op_4630_begin_0"), val = tensor<int32, [3]>([0, 0, 1])];
            tensor<int32, [3]> var_4630_end_0 = const()[name = string("op_4630_end_0"), val = tensor<int32, [3]>([1, 6144, 4])];
            tensor<bool, [3]> var_4630_end_mask_0 = const()[name = string("op_4630_end_mask_0"), val = tensor<bool, [3]>([true, true, true])];
            tensor<fp16, [1, 6144, 3]> conv16_out = slice_by_index(begin = var_4630_begin_0, end = var_4630_end_0, end_mask = var_4630_end_mask_0, x = convolution_input_25_cast_fp16)[name = string("op_4630_cast_fp16")];
            tensor<int32, [3]> var_4631 = const()[name = string("op_4631"), val = tensor<int32, [3]>([2048, 2048, 2048])];
            int32 var_4632_axis_0 = const()[name = string("op_4632_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 1, 6144]> var_4627_cast_fp16 = transpose(perm = var_4627_perm_0, x = var_4626_cast_fp16)[name = string("transpose_23")];
            tensor<fp16, [1, 1, 2048]> var_4632_cast_fp16_0, tensor<fp16, [1, 1, 2048]> var_4632_cast_fp16_1, tensor<fp16, [1, 1, 2048]> var_4632_cast_fp16_2 = split(axis = var_4632_axis_0, split_sizes = var_4631, x = var_4627_cast_fp16)[name = string("op_4632_cast_fp16")];
            tensor<int32, [4]> var_4636 = const()[name = string("op_4636"), val = tensor<int32, [4]>([1, 1, 16, 128])];
            tensor<fp16, [1, 1, 16, 128]> value_237_cast_fp16 = reshape(shape = var_4636, x = var_4632_cast_fp16_0)[name = string("value_237_cast_fp16")];
            tensor<int32, [4]> var_4638 = const()[name = string("op_4638"), val = tensor<int32, [4]>([1, 1, 16, 128])];
            tensor<fp16, [1, 1, 16, 128]> value_239_cast_fp16 = reshape(shape = var_4638, x = var_4632_cast_fp16_1)[name = string("value_239_cast_fp16")];
            tensor<int32, [4]> var_4640 = const()[name = string("op_4640"), val = tensor<int32, [4]>([1, 1, 16, 128])];
            tensor<fp16, [1, 1, 16, 128]> value_241_cast_fp16 = reshape(shape = var_4640, x = var_4632_cast_fp16_2)[name = string("value_241_cast_fp16")];
            tensor<fp16, [1, 1, 16, 128]> var_4642_cast_fp16 = mul(x = value_237_cast_fp16, y = value_237_cast_fp16)[name = string("op_4642_cast_fp16")];
            tensor<int32, [1]> var_4644_axes_0 = const()[name = string("op_4644_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_4644_keep_dims_0 = const()[name = string("op_4644_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 1, 16, 1]> var_4644_cast_fp16 = reduce_sum(axes = var_4644_axes_0, keep_dims = var_4644_keep_dims_0, x = var_4642_cast_fp16)[name = string("op_4644_cast_fp16")];
            fp16 var_4645_to_fp16 = const()[name = string("op_4645_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 1, 16, 1]> var_4646_cast_fp16 = add(x = var_4644_cast_fp16, y = var_4645_to_fp16)[name = string("op_4646_cast_fp16")];
            fp32 var_4647_epsilon_0 = const()[name = string("op_4647_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 1, 16, 1]> var_4647_cast_fp16 = rsqrt(epsilon = var_4647_epsilon_0, x = var_4646_cast_fp16)[name = string("op_4647_cast_fp16")];
            tensor<fp16, [1, 1, 16, 128]> var_4648_cast_fp16 = mul(x = value_237_cast_fp16, y = var_4647_cast_fp16)[name = string("op_4648_cast_fp16")];
            tensor<int32, [4]> var_4649_perm_0 = const()[name = string("op_4649_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            fp16 _inversed_query_67_y_0_to_fp16 = const()[name = string("_inversed_query_67_y_0_to_fp16"), val = fp16(0x1.6ap-4)];
            tensor<fp16, [1, 16, 1, 128]> var_4649_cast_fp16 = transpose(perm = var_4649_perm_0, x = var_4648_cast_fp16)[name = string("transpose_22")];
            tensor<fp16, [1, 16, 1, 128]> _inversed_query_67_cast_fp16 = mul(x = var_4649_cast_fp16, y = _inversed_query_67_y_0_to_fp16)[name = string("_inversed_query_67_cast_fp16")];
            tensor<fp16, [1, 1, 16, 128]> var_4652_cast_fp16 = mul(x = value_239_cast_fp16, y = value_239_cast_fp16)[name = string("op_4652_cast_fp16")];
            tensor<int32, [1]> var_4654_axes_0 = const()[name = string("op_4654_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_4654_keep_dims_0 = const()[name = string("op_4654_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 1, 16, 1]> var_4654_cast_fp16 = reduce_sum(axes = var_4654_axes_0, keep_dims = var_4654_keep_dims_0, x = var_4652_cast_fp16)[name = string("op_4654_cast_fp16")];
            fp16 var_4655_to_fp16 = const()[name = string("op_4655_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 1, 16, 1]> var_4656_cast_fp16 = add(x = var_4654_cast_fp16, y = var_4655_to_fp16)[name = string("op_4656_cast_fp16")];
            fp32 var_4657_epsilon_0 = const()[name = string("op_4657_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 1, 16, 1]> var_4657_cast_fp16 = rsqrt(epsilon = var_4657_epsilon_0, x = var_4656_cast_fp16)[name = string("op_4657_cast_fp16")];
            tensor<fp16, [1, 1, 16, 128]> var_4658_cast_fp16 = mul(x = value_239_cast_fp16, y = var_4657_cast_fp16)[name = string("op_4658_cast_fp16")];
            tensor<int32, [4]> key_67_perm_0 = const()[name = string("key_67_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<int32, [4]> value_243_perm_0 = const()[name = string("value_243_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<fp16, [16, 1024]> final_group_0_in_proj_b_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(256721024))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(256733376))))[name = string("final_group_0_in_proj_b_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16]> linear_125_bias_0_to_fp16 = const()[name = string("linear_125_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(4862656)))];
            tensor<fp16, [1, 16]> linear_125_cast_fp16 = linear(bias = linear_125_bias_0_to_fp16, weight = final_group_0_in_proj_b_weight_promoted_to_fp16_palettized, x = input_209_cast_fp16)[name = string("linear_125_cast_fp16")];
            tensor<fp16, [1, 16]> var_4663_cast_fp16 = sigmoid(x = linear_125_cast_fp16)[name = string("op_4663_cast_fp16")];
            tensor<int32, [2]> var_4664_perm_0 = const()[name = string("op_4664_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> beta_25_axes_0 = const()[name = string("beta_25_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 1]> var_4664_cast_fp16 = transpose(perm = var_4664_perm_0, x = var_4663_cast_fp16)[name = string("transpose_21")];
            tensor<fp16, [1, 16, 1]> beta_25_cast_fp16 = expand_dims(axes = beta_25_axes_0, x = var_4664_cast_fp16)[name = string("beta_25_cast_fp16")];
            tensor<fp16, [16, 1024]> op_4670_weight_0_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(256733568))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(256745920))))[name = string("op_4670_weight_0_to_fp16_palettized")];
            tensor<fp16, [16]> var_4670_bias_0_to_fp16 = const()[name = string("op_4670_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(256746112)))];
            tensor<fp16, [1, 16]> var_4670_cast_fp16 = linear(bias = var_4670_bias_0_to_fp16, weight = op_4670_weight_0_to_fp16_palettized, x = input_209_cast_fp16)[name = string("op_4670_cast_fp16")];
            tensor<fp16, [1, 16]> var_4671_cast_fp16 = softplus(x = var_4670_cast_fp16)[name = string("op_4671_cast_fp16")];
            tensor<fp16, [16]> var_4667_promoted_to_fp16 = const()[name = string("op_4667_promoted_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(256746240)))];
            tensor<fp16, [1, 16]> var_4672_cast_fp16 = mul(x = var_4667_promoted_to_fp16, y = var_4671_cast_fp16)[name = string("op_4672_cast_fp16")];
            tensor<int32, [2]> var_4673_perm_0 = const()[name = string("op_4673_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> decay_25_axes_0 = const()[name = string("decay_25_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 1]> var_4673_cast_fp16 = transpose(perm = var_4673_perm_0, x = var_4672_cast_fp16)[name = string("transpose_20")];
            tensor<fp16, [1, 16, 1]> decay_25_cast_fp16 = expand_dims(axes = decay_25_axes_0, x = var_4673_cast_fp16)[name = string("decay_25_cast_fp16")];
            tensor<fp16, [2048, 1024]> final_group_0_in_proj_z_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [2048, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(256746368))), lut = tensor<fp16, [128, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(258319296))))[name = string("final_group_0_in_proj_z_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 2048]> linear_127_cast_fp16 = linear(bias = linear_3_bias_0_to_fp16, weight = final_group_0_in_proj_z_weight_promoted_to_fp16_palettized, x = input_209_cast_fp16)[name = string("linear_127_cast_fp16")];
            tensor<int32, [3]> var_4681_begin_0 = const()[name = string("op_4681_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_4681_end_0 = const()[name = string("op_4681_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_4681_end_mask_0 = const()[name = string("op_4681_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_4681_squeeze_mask_0 = const()[name = string("op_4681_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_4681_cast_fp16 = slice_by_index(begin = var_4681_begin_0, end = var_4681_end_0, end_mask = var_4681_end_mask_0, squeeze_mask = var_4681_squeeze_mask_0, x = decay_25_cast_fp16)[name = string("op_4681_cast_fp16")];
            tensor<fp16, [1, 16]> var_4682_cast_fp16 = exp(x = var_4681_cast_fp16)[name = string("op_4682_cast_fp16")];
            tensor<int32, [1]> var_4683_axes_0 = const()[name = string("op_4683_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_4683_cast_fp16 = expand_dims(axes = var_4683_axes_0, x = var_4682_cast_fp16)[name = string("op_4683_cast_fp16")];
            tensor<int32, [1]> var_4684_axes_0 = const()[name = string("op_4684_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_4684_cast_fp16 = expand_dims(axes = var_4684_axes_0, x = var_4683_cast_fp16)[name = string("op_4684_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_49_cast_fp16 = mul(x = rec16, y = var_4684_cast_fp16)[name = string("state_49_cast_fp16")];
            tensor<int32, [4]> key_token_25_begin_0 = const()[name = string("key_token_25_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> key_token_25_end_0 = const()[name = string("key_token_25_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> key_token_25_end_mask_0 = const()[name = string("key_token_25_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_25_squeeze_mask_0 = const()[name = string("key_token_25_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 1, 128]> key_67_cast_fp16 = transpose(perm = key_67_perm_0, x = var_4658_cast_fp16)[name = string("transpose_19")];
            tensor<fp16, [1, 16, 128]> key_token_25_cast_fp16 = slice_by_index(begin = key_token_25_begin_0, end = key_token_25_end_0, end_mask = key_token_25_end_mask_0, squeeze_mask = key_token_25_squeeze_mask_0, x = key_67_cast_fp16)[name = string("key_token_25_cast_fp16")];
            tensor<int32, [4]> value_token_25_begin_0 = const()[name = string("value_token_25_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> value_token_25_end_0 = const()[name = string("value_token_25_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> value_token_25_end_mask_0 = const()[name = string("value_token_25_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_25_squeeze_mask_0 = const()[name = string("value_token_25_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 1, 128]> value_243_cast_fp16 = transpose(perm = value_243_perm_0, x = value_241_cast_fp16)[name = string("transpose_18")];
            tensor<fp16, [1, 16, 128]> value_token_25_cast_fp16 = slice_by_index(begin = value_token_25_begin_0, end = value_token_25_end_0, end_mask = value_token_25_end_mask_0, squeeze_mask = value_token_25_squeeze_mask_0, x = value_243_cast_fp16)[name = string("value_token_25_cast_fp16")];
            tensor<int32, [1]> var_4692_axes_0 = const()[name = string("op_4692_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_4692_cast_fp16 = expand_dims(axes = var_4692_axes_0, x = key_token_25_cast_fp16)[name = string("op_4692_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4693_cast_fp16 = mul(x = state_49_cast_fp16, y = var_4692_cast_fp16)[name = string("op_4693_cast_fp16")];
            tensor<int32, [1]> memory_25_axes_0 = const()[name = string("memory_25_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_25_keep_dims_0 = const()[name = string("memory_25_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_25_cast_fp16 = reduce_sum(axes = memory_25_axes_0, keep_dims = memory_25_keep_dims_0, x = var_4693_cast_fp16)[name = string("memory_25_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_4696_cast_fp16 = sub(x = value_token_25_cast_fp16, y = memory_25_cast_fp16)[name = string("op_4696_cast_fp16")];
            tensor<int32, [3]> var_4699_begin_0 = const()[name = string("op_4699_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_4699_end_0 = const()[name = string("op_4699_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_4699_end_mask_0 = const()[name = string("op_4699_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_4699_squeeze_mask_0 = const()[name = string("op_4699_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_4699_cast_fp16 = slice_by_index(begin = var_4699_begin_0, end = var_4699_end_0, end_mask = var_4699_end_mask_0, squeeze_mask = var_4699_squeeze_mask_0, x = beta_25_cast_fp16)[name = string("op_4699_cast_fp16")];
            tensor<int32, [1]> var_4700_axes_0 = const()[name = string("op_4700_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_4700_cast_fp16 = expand_dims(axes = var_4700_axes_0, x = var_4699_cast_fp16)[name = string("op_4700_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_25_cast_fp16 = mul(x = var_4696_cast_fp16, y = var_4700_cast_fp16)[name = string("delta_25_cast_fp16")];
            tensor<int32, [1]> var_4703_axes_0 = const()[name = string("op_4703_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_4703_cast_fp16 = expand_dims(axes = var_4703_axes_0, x = delta_25_cast_fp16)[name = string("op_4703_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4704_cast_fp16 = mul(x = var_4692_cast_fp16, y = var_4703_cast_fp16)[name = string("op_4704_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> rec16_out = add(x = state_49_cast_fp16, y = var_4704_cast_fp16)[name = string("state_51_cast_fp16")];
            tensor<int32, [4]> var_4708_begin_0 = const()[name = string("op_4708_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> var_4708_end_0 = const()[name = string("op_4708_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> var_4708_end_mask_0 = const()[name = string("op_4708_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_4708_squeeze_mask_0 = const()[name = string("op_4708_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_4708_cast_fp16 = slice_by_index(begin = var_4708_begin_0, end = var_4708_end_0, end_mask = var_4708_end_mask_0, squeeze_mask = var_4708_squeeze_mask_0, x = _inversed_query_67_cast_fp16)[name = string("op_4708_cast_fp16")];
            tensor<int32, [1]> var_4709_axes_0 = const()[name = string("op_4709_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_4709_cast_fp16 = expand_dims(axes = var_4709_axes_0, x = var_4708_cast_fp16)[name = string("op_4709_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4710_cast_fp16 = mul(x = rec16_out, y = var_4709_cast_fp16)[name = string("op_4710_cast_fp16")];
            tensor<int32, [1]> var_4712_axes_0 = const()[name = string("op_4712_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_4712_keep_dims_0 = const()[name = string("op_4712_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_4712_cast_fp16 = reduce_sum(axes = var_4712_axes_0, keep_dims = var_4712_keep_dims_0, x = var_4710_cast_fp16)[name = string("op_4712_cast_fp16")];
            tensor<int32, [1]> attention_121_axes_0 = const()[name = string("attention_121_axes_0"), val = tensor<int32, [1]>([1])];
            tensor<fp16, [1, 1, 16, 128]> attention_121_cast_fp16 = expand_dims(axes = attention_121_axes_0, x = var_4712_cast_fp16)[name = string("attention_121_cast_fp16")];
            tensor<int32, [2]> var_4715 = const()[name = string("op_4715"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [16, 128]> attention_123_cast_fp16 = reshape(shape = var_4715, x = attention_121_cast_fp16)[name = string("attention_123_cast_fp16")];
            tensor<int32, [2]> var_4717 = const()[name = string("op_4717"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [16, 128]> input_213_cast_fp16 = reshape(shape = var_4717, x = linear_127_cast_fp16)[name = string("input_213_cast_fp16")];
            tensor<fp16, [16, 128]> var_4719_cast_fp16 = mul(x = attention_123_cast_fp16, y = attention_123_cast_fp16)[name = string("op_4719_cast_fp16")];
            tensor<int32, [1]> variance_107_axes_0 = const()[name = string("variance_107_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_107_keep_dims_0 = const()[name = string("variance_107_keep_dims_0"), val = bool(true)];
            tensor<fp16, [16, 1]> variance_107_cast_fp16 = reduce_mean(axes = variance_107_axes_0, keep_dims = variance_107_keep_dims_0, x = var_4719_cast_fp16)[name = string("variance_107_cast_fp16")];
            fp16 var_4722_to_fp16 = const()[name = string("op_4722_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [16, 1]> var_4723_cast_fp16 = add(x = variance_107_cast_fp16, y = var_4722_to_fp16)[name = string("op_4723_cast_fp16")];
            fp32 var_4724_epsilon_0 = const()[name = string("op_4724_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [16, 1]> var_4724_cast_fp16 = rsqrt(epsilon = var_4724_epsilon_0, x = var_4723_cast_fp16)[name = string("op_4724_cast_fp16")];
            tensor<fp16, [16, 128]> attention_125_cast_fp16 = mul(x = attention_123_cast_fp16, y = var_4724_cast_fp16)[name = string("attention_125_cast_fp16")];
            tensor<fp16, [128]> final_group_0_gated_norm_promoted_to_fp16 = const()[name = string("final_group_0_gated_norm_promoted_to_fp16"), val = tensor<fp16, [128]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(258335744)))];
            tensor<fp16, [16, 128]> attention_127_cast_fp16 = mul(x = attention_125_cast_fp16, y = final_group_0_gated_norm_promoted_to_fp16)[name = string("attention_127_cast_fp16")];
            tensor<fp16, [16, 128]> var_4727_cast_fp16 = silu(x = input_213_cast_fp16)[name = string("op_4727_cast_fp16")];
            tensor<fp16, [16, 128]> attention_129_cast_fp16 = mul(x = attention_127_cast_fp16, y = var_4727_cast_fp16)[name = string("attention_129_cast_fp16")];
            tensor<int32, [2]> var_4729 = const()[name = string("op_4729"), val = tensor<int32, [2]>([1, 2048])];
            tensor<fp16, [1, 2048]> input_215_cast_fp16 = reshape(shape = var_4729, x = attention_129_cast_fp16)[name = string("input_215_cast_fp16")];
            tensor<fp16, [1024, 2048]> final_group_0_out_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 2048]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(258336064))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(259908992))))[name = string("final_group_0_out_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 1024]> linear_128_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = final_group_0_out_proj_weight_promoted_to_fp16_palettized, x = input_215_cast_fp16)[name = string("linear_128_cast_fp16")];
            tensor<fp16, [1, 1024]> value_245_cast_fp16 = add(x = value_233_cast_fp16, y = linear_128_cast_fp16)[name = string("value_245_cast_fp16")];
            tensor<fp16, [1, 1024]> var_4734_cast_fp16 = mul(x = value_245_cast_fp16, y = value_245_cast_fp16)[name = string("op_4734_cast_fp16")];
            tensor<int32, [1]> variance_109_axes_0 = const()[name = string("variance_109_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_109_keep_dims_0 = const()[name = string("variance_109_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 1]> variance_109_cast_fp16 = reduce_mean(axes = variance_109_axes_0, keep_dims = variance_109_keep_dims_0, x = var_4734_cast_fp16)[name = string("variance_109_cast_fp16")];
            fp16 var_4737_to_fp16 = const()[name = string("op_4737_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 1]> var_4738_cast_fp16 = add(x = variance_109_cast_fp16, y = var_4737_to_fp16)[name = string("op_4738_cast_fp16")];
            fp32 var_4739_epsilon_0 = const()[name = string("op_4739_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 1]> var_4739_cast_fp16 = rsqrt(epsilon = var_4739_epsilon_0, x = var_4738_cast_fp16)[name = string("op_4739_cast_fp16")];
            tensor<fp16, [1, 1024]> var_4740_cast_fp16 = mul(x = value_245_cast_fp16, y = var_4739_cast_fp16)[name = string("op_4740_cast_fp16")];
            tensor<fp16, [1024]> var_4742_to_fp16 = const()[name = string("op_4742_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(259917248)))];
            tensor<fp16, [1, 1024]> input_217_cast_fp16 = mul(x = var_4740_cast_fp16, y = var_4742_to_fp16)[name = string("input_217_cast_fp16")];
            tensor<fp16, [3584, 1024]> final_group_0_gate_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(259919360))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(262671936))))[name = string("final_group_0_gate_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 3584]> linear_129_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = final_group_0_gate_proj_weight_promoted_to_fp16_palettized, x = input_217_cast_fp16)[name = string("linear_129_cast_fp16")];
            tensor<fp16, [1, 3584]> var_4746_cast_fp16 = silu(x = linear_129_cast_fp16)[name = string("op_4746_cast_fp16")];
            tensor<fp16, [3584, 1024]> final_group_0_up_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(262700672))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(265453248))))[name = string("final_group_0_up_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 3584]> linear_130_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = final_group_0_up_proj_weight_promoted_to_fp16_palettized, x = input_217_cast_fp16)[name = string("linear_130_cast_fp16")];
            tensor<fp16, [1, 3584]> input_221_cast_fp16 = mul(x = var_4746_cast_fp16, y = linear_130_cast_fp16)[name = string("input_221_cast_fp16")];
            tensor<fp16, [1024, 3584]> final_group_0_down_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 3584]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(265481984))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(268234560))))[name = string("final_group_0_down_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 1024]> linear_131_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = final_group_0_down_proj_weight_promoted_to_fp16_palettized, x = input_221_cast_fp16)[name = string("linear_131_cast_fp16")];
            tensor<fp16, [1, 1024]> value_247_cast_fp16 = add(x = value_245_cast_fp16, y = linear_131_cast_fp16)[name = string("value_247_cast_fp16")];
            int32 var_4771 = const()[name = string("op_4771"), val = int32(-1)];
            tensor<fp16, [1, 1024]> var_4786_cast_fp16 = mul(x = value_247_cast_fp16, y = value_247_cast_fp16)[name = string("op_4786_cast_fp16")];
            tensor<int32, [1]> variance_111_axes_0 = const()[name = string("variance_111_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_111_keep_dims_0 = const()[name = string("variance_111_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 1]> variance_111_cast_fp16 = reduce_mean(axes = variance_111_axes_0, keep_dims = variance_111_keep_dims_0, x = var_4786_cast_fp16)[name = string("variance_111_cast_fp16")];
            fp16 var_4789_to_fp16 = const()[name = string("op_4789_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 1]> var_4790_cast_fp16 = add(x = variance_111_cast_fp16, y = var_4789_to_fp16)[name = string("op_4790_cast_fp16")];
            fp32 var_4791_epsilon_0 = const()[name = string("op_4791_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 1]> var_4791_cast_fp16 = rsqrt(epsilon = var_4791_epsilon_0, x = var_4790_cast_fp16)[name = string("op_4791_cast_fp16")];
            tensor<fp16, [1, 1024]> var_4792_cast_fp16 = mul(x = value_247_cast_fp16, y = var_4791_cast_fp16)[name = string("op_4792_cast_fp16")];
            tensor<fp16, [1024]> var_4794_to_fp16 = const()[name = string("op_4794_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(268242816)))];
            tensor<fp16, [1, 1024]> input_223_cast_fp16 = mul(x = var_4792_cast_fp16, y = var_4794_to_fp16)[name = string("input_223_cast_fp16")];
            tensor<fp16, [6144, 1024]> final_group_1_in_proj_qkv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(268244928))), lut = tensor<fp16, [384, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(272963584))))[name = string("final_group_1_in_proj_qkv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 6144]> linear_132_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = final_group_1_in_proj_qkv_weight_promoted_to_fp16_palettized, x = input_223_cast_fp16)[name = string("linear_132_cast_fp16")];
            tensor<int32, [2]> var_4798_perm_0 = const()[name = string("op_4798_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> mixed_27_axes_0 = const()[name = string("mixed_27_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [6144, 1]> var_4798_cast_fp16 = transpose(perm = var_4798_perm_0, x = linear_132_cast_fp16)[name = string("transpose_17")];
            tensor<fp16, [1, 6144, 1]> mixed_27_cast_fp16 = expand_dims(axes = mixed_27_axes_0, x = var_4798_cast_fp16)[name = string("mixed_27_cast_fp16")];
            bool convolution_input_27_interleave_0 = const()[name = string("convolution_input_27_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 6144, 4]> convolution_input_27_cast_fp16 = concat(axis = var_4771, interleave = convolution_input_27_interleave_0, values = (conv17, mixed_27_cast_fp16))[name = string("convolution_input_27_cast_fp16")];
            string input_225_pad_type_0 = const()[name = string("input_225_pad_type_0"), val = string("valid")];
            int32 input_225_groups_0 = const()[name = string("input_225_groups_0"), val = int32(6144)];
            tensor<int32, [1]> input_225_strides_0 = const()[name = string("input_225_strides_0"), val = tensor<int32, [1]>([1])];
            tensor<int32, [2]> input_225_pad_0 = const()[name = string("input_225_pad_0"), val = tensor<int32, [2]>([0, 0])];
            tensor<int32, [1]> input_225_dilations_0 = const()[name = string("input_225_dilations_0"), val = tensor<int32, [1]>([1])];
            tensor<fp16, [6144, 1, 4]> final_group_1_conv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1, 4]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(273012800))), lut = tensor<fp16, [384, 1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(273031296))))[name = string("final_group_1_conv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 6144, 1]> input_225_cast_fp16 = conv(dilations = input_225_dilations_0, groups = input_225_groups_0, pad = input_225_pad_0, pad_type = input_225_pad_type_0, strides = input_225_strides_0, weight = final_group_1_conv_weight_promoted_to_fp16_palettized, x = convolution_input_27_cast_fp16)[name = string("input_225_cast_fp16")];
            tensor<fp16, [1, 6144, 1]> var_4807_cast_fp16 = silu(x = input_225_cast_fp16)[name = string("op_4807_cast_fp16")];
            tensor<int32, [3]> var_4808_perm_0 = const()[name = string("op_4808_perm_0"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [3]> var_4811_begin_0 = const()[name = string("op_4811_begin_0"), val = tensor<int32, [3]>([0, 0, 1])];
            tensor<int32, [3]> var_4811_end_0 = const()[name = string("op_4811_end_0"), val = tensor<int32, [3]>([1, 6144, 4])];
            tensor<bool, [3]> var_4811_end_mask_0 = const()[name = string("op_4811_end_mask_0"), val = tensor<bool, [3]>([true, true, true])];
            tensor<fp16, [1, 6144, 3]> conv17_out = slice_by_index(begin = var_4811_begin_0, end = var_4811_end_0, end_mask = var_4811_end_mask_0, x = convolution_input_27_cast_fp16)[name = string("op_4811_cast_fp16")];
            tensor<int32, [3]> var_4812 = const()[name = string("op_4812"), val = tensor<int32, [3]>([2048, 2048, 2048])];
            int32 var_4813_axis_0 = const()[name = string("op_4813_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 1, 6144]> var_4808_cast_fp16 = transpose(perm = var_4808_perm_0, x = var_4807_cast_fp16)[name = string("transpose_16")];
            tensor<fp16, [1, 1, 2048]> var_4813_cast_fp16_0, tensor<fp16, [1, 1, 2048]> var_4813_cast_fp16_1, tensor<fp16, [1, 1, 2048]> var_4813_cast_fp16_2 = split(axis = var_4813_axis_0, split_sizes = var_4812, x = var_4808_cast_fp16)[name = string("op_4813_cast_fp16")];
            tensor<int32, [4]> var_4817 = const()[name = string("op_4817"), val = tensor<int32, [4]>([1, 1, 16, 128])];
            tensor<fp16, [1, 1, 16, 128]> value_251_cast_fp16 = reshape(shape = var_4817, x = var_4813_cast_fp16_0)[name = string("value_251_cast_fp16")];
            tensor<int32, [4]> var_4819 = const()[name = string("op_4819"), val = tensor<int32, [4]>([1, 1, 16, 128])];
            tensor<fp16, [1, 1, 16, 128]> value_253_cast_fp16 = reshape(shape = var_4819, x = var_4813_cast_fp16_1)[name = string("value_253_cast_fp16")];
            tensor<int32, [4]> var_4821 = const()[name = string("op_4821"), val = tensor<int32, [4]>([1, 1, 16, 128])];
            tensor<fp16, [1, 1, 16, 128]> value_255_cast_fp16 = reshape(shape = var_4821, x = var_4813_cast_fp16_2)[name = string("value_255_cast_fp16")];
            tensor<fp16, [1, 1, 16, 128]> var_4823_cast_fp16 = mul(x = value_251_cast_fp16, y = value_251_cast_fp16)[name = string("op_4823_cast_fp16")];
            tensor<int32, [1]> var_4825_axes_0 = const()[name = string("op_4825_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_4825_keep_dims_0 = const()[name = string("op_4825_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 1, 16, 1]> var_4825_cast_fp16 = reduce_sum(axes = var_4825_axes_0, keep_dims = var_4825_keep_dims_0, x = var_4823_cast_fp16)[name = string("op_4825_cast_fp16")];
            fp16 var_4826_to_fp16 = const()[name = string("op_4826_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 1, 16, 1]> var_4827_cast_fp16 = add(x = var_4825_cast_fp16, y = var_4826_to_fp16)[name = string("op_4827_cast_fp16")];
            fp32 var_4828_epsilon_0 = const()[name = string("op_4828_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 1, 16, 1]> var_4828_cast_fp16 = rsqrt(epsilon = var_4828_epsilon_0, x = var_4827_cast_fp16)[name = string("op_4828_cast_fp16")];
            tensor<fp16, [1, 1, 16, 128]> var_4829_cast_fp16 = mul(x = value_251_cast_fp16, y = var_4828_cast_fp16)[name = string("op_4829_cast_fp16")];
            tensor<int32, [4]> var_4830_perm_0 = const()[name = string("op_4830_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            fp16 _inversed_query_71_y_0_to_fp16 = const()[name = string("_inversed_query_71_y_0_to_fp16"), val = fp16(0x1.6ap-4)];
            tensor<fp16, [1, 16, 1, 128]> var_4830_cast_fp16 = transpose(perm = var_4830_perm_0, x = var_4829_cast_fp16)[name = string("transpose_15")];
            tensor<fp16, [1, 16, 1, 128]> _inversed_query_71_cast_fp16 = mul(x = var_4830_cast_fp16, y = _inversed_query_71_y_0_to_fp16)[name = string("_inversed_query_71_cast_fp16")];
            tensor<fp16, [1, 1, 16, 128]> var_4833_cast_fp16 = mul(x = value_253_cast_fp16, y = value_253_cast_fp16)[name = string("op_4833_cast_fp16")];
            tensor<int32, [1]> var_4835_axes_0 = const()[name = string("op_4835_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_4835_keep_dims_0 = const()[name = string("op_4835_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 1, 16, 1]> var_4835_cast_fp16 = reduce_sum(axes = var_4835_axes_0, keep_dims = var_4835_keep_dims_0, x = var_4833_cast_fp16)[name = string("op_4835_cast_fp16")];
            fp16 var_4836_to_fp16 = const()[name = string("op_4836_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 1, 16, 1]> var_4837_cast_fp16 = add(x = var_4835_cast_fp16, y = var_4836_to_fp16)[name = string("op_4837_cast_fp16")];
            fp32 var_4838_epsilon_0 = const()[name = string("op_4838_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 1, 16, 1]> var_4838_cast_fp16 = rsqrt(epsilon = var_4838_epsilon_0, x = var_4837_cast_fp16)[name = string("op_4838_cast_fp16")];
            tensor<fp16, [1, 1, 16, 128]> var_4839_cast_fp16 = mul(x = value_253_cast_fp16, y = var_4838_cast_fp16)[name = string("op_4839_cast_fp16")];
            tensor<int32, [4]> key_71_perm_0 = const()[name = string("key_71_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<int32, [4]> value_257_perm_0 = const()[name = string("value_257_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<fp16, [16, 1024]> final_group_1_in_proj_b_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(273080512))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(273092864))))[name = string("final_group_1_in_proj_b_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16]> linear_133_bias_0_to_fp16 = const()[name = string("linear_133_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(4862656)))];
            tensor<fp16, [1, 16]> linear_133_cast_fp16 = linear(bias = linear_133_bias_0_to_fp16, weight = final_group_1_in_proj_b_weight_promoted_to_fp16_palettized, x = input_223_cast_fp16)[name = string("linear_133_cast_fp16")];
            tensor<fp16, [1, 16]> var_4844_cast_fp16 = sigmoid(x = linear_133_cast_fp16)[name = string("op_4844_cast_fp16")];
            tensor<int32, [2]> var_4845_perm_0 = const()[name = string("op_4845_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> beta_27_axes_0 = const()[name = string("beta_27_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 1]> var_4845_cast_fp16 = transpose(perm = var_4845_perm_0, x = var_4844_cast_fp16)[name = string("transpose_14")];
            tensor<fp16, [1, 16, 1]> beta_27_cast_fp16 = expand_dims(axes = beta_27_axes_0, x = var_4845_cast_fp16)[name = string("beta_27_cast_fp16")];
            tensor<fp16, [16, 1024]> op_4851_weight_0_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(273093056))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(273105408))))[name = string("op_4851_weight_0_to_fp16_palettized")];
            tensor<fp16, [16]> var_4851_bias_0_to_fp16 = const()[name = string("op_4851_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(273105600)))];
            tensor<fp16, [1, 16]> var_4851_cast_fp16 = linear(bias = var_4851_bias_0_to_fp16, weight = op_4851_weight_0_to_fp16_palettized, x = input_223_cast_fp16)[name = string("op_4851_cast_fp16")];
            tensor<fp16, [1, 16]> var_4852_cast_fp16 = softplus(x = var_4851_cast_fp16)[name = string("op_4852_cast_fp16")];
            tensor<fp16, [16]> var_4848_promoted_to_fp16 = const()[name = string("op_4848_promoted_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(273105728)))];
            tensor<fp16, [1, 16]> var_4853_cast_fp16 = mul(x = var_4848_promoted_to_fp16, y = var_4852_cast_fp16)[name = string("op_4853_cast_fp16")];
            tensor<int32, [2]> var_4854_perm_0 = const()[name = string("op_4854_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> decay_27_axes_0 = const()[name = string("decay_27_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 1]> var_4854_cast_fp16 = transpose(perm = var_4854_perm_0, x = var_4853_cast_fp16)[name = string("transpose_13")];
            tensor<fp16, [1, 16, 1]> decay_27_cast_fp16 = expand_dims(axes = decay_27_axes_0, x = var_4854_cast_fp16)[name = string("decay_27_cast_fp16")];
            tensor<fp16, [2048, 1024]> final_group_1_in_proj_z_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [2048, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(273105856))), lut = tensor<fp16, [128, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(274678784))))[name = string("final_group_1_in_proj_z_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 2048]> linear_135_cast_fp16 = linear(bias = linear_3_bias_0_to_fp16, weight = final_group_1_in_proj_z_weight_promoted_to_fp16_palettized, x = input_223_cast_fp16)[name = string("linear_135_cast_fp16")];
            tensor<int32, [3]> var_4862_begin_0 = const()[name = string("op_4862_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_4862_end_0 = const()[name = string("op_4862_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_4862_end_mask_0 = const()[name = string("op_4862_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_4862_squeeze_mask_0 = const()[name = string("op_4862_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_4862_cast_fp16 = slice_by_index(begin = var_4862_begin_0, end = var_4862_end_0, end_mask = var_4862_end_mask_0, squeeze_mask = var_4862_squeeze_mask_0, x = decay_27_cast_fp16)[name = string("op_4862_cast_fp16")];
            tensor<fp16, [1, 16]> var_4863_cast_fp16 = exp(x = var_4862_cast_fp16)[name = string("op_4863_cast_fp16")];
            tensor<int32, [1]> var_4864_axes_0 = const()[name = string("op_4864_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_4864_cast_fp16 = expand_dims(axes = var_4864_axes_0, x = var_4863_cast_fp16)[name = string("op_4864_cast_fp16")];
            tensor<int32, [1]> var_4865_axes_0 = const()[name = string("op_4865_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_4865_cast_fp16 = expand_dims(axes = var_4865_axes_0, x = var_4864_cast_fp16)[name = string("op_4865_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_53_cast_fp16 = mul(x = rec17, y = var_4865_cast_fp16)[name = string("state_53_cast_fp16")];
            tensor<int32, [4]> key_token_27_begin_0 = const()[name = string("key_token_27_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> key_token_27_end_0 = const()[name = string("key_token_27_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> key_token_27_end_mask_0 = const()[name = string("key_token_27_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_27_squeeze_mask_0 = const()[name = string("key_token_27_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 1, 128]> key_71_cast_fp16 = transpose(perm = key_71_perm_0, x = var_4839_cast_fp16)[name = string("transpose_12")];
            tensor<fp16, [1, 16, 128]> key_token_27_cast_fp16 = slice_by_index(begin = key_token_27_begin_0, end = key_token_27_end_0, end_mask = key_token_27_end_mask_0, squeeze_mask = key_token_27_squeeze_mask_0, x = key_71_cast_fp16)[name = string("key_token_27_cast_fp16")];
            tensor<int32, [4]> value_token_27_begin_0 = const()[name = string("value_token_27_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> value_token_27_end_0 = const()[name = string("value_token_27_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> value_token_27_end_mask_0 = const()[name = string("value_token_27_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_27_squeeze_mask_0 = const()[name = string("value_token_27_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 1, 128]> value_257_cast_fp16 = transpose(perm = value_257_perm_0, x = value_255_cast_fp16)[name = string("transpose_11")];
            tensor<fp16, [1, 16, 128]> value_token_27_cast_fp16 = slice_by_index(begin = value_token_27_begin_0, end = value_token_27_end_0, end_mask = value_token_27_end_mask_0, squeeze_mask = value_token_27_squeeze_mask_0, x = value_257_cast_fp16)[name = string("value_token_27_cast_fp16")];
            tensor<int32, [1]> var_4873_axes_0 = const()[name = string("op_4873_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_4873_cast_fp16 = expand_dims(axes = var_4873_axes_0, x = key_token_27_cast_fp16)[name = string("op_4873_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4874_cast_fp16 = mul(x = state_53_cast_fp16, y = var_4873_cast_fp16)[name = string("op_4874_cast_fp16")];
            tensor<int32, [1]> memory_27_axes_0 = const()[name = string("memory_27_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_27_keep_dims_0 = const()[name = string("memory_27_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_27_cast_fp16 = reduce_sum(axes = memory_27_axes_0, keep_dims = memory_27_keep_dims_0, x = var_4874_cast_fp16)[name = string("memory_27_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_4877_cast_fp16 = sub(x = value_token_27_cast_fp16, y = memory_27_cast_fp16)[name = string("op_4877_cast_fp16")];
            tensor<int32, [3]> var_4880_begin_0 = const()[name = string("op_4880_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_4880_end_0 = const()[name = string("op_4880_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_4880_end_mask_0 = const()[name = string("op_4880_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_4880_squeeze_mask_0 = const()[name = string("op_4880_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_4880_cast_fp16 = slice_by_index(begin = var_4880_begin_0, end = var_4880_end_0, end_mask = var_4880_end_mask_0, squeeze_mask = var_4880_squeeze_mask_0, x = beta_27_cast_fp16)[name = string("op_4880_cast_fp16")];
            tensor<int32, [1]> var_4881_axes_0 = const()[name = string("op_4881_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_4881_cast_fp16 = expand_dims(axes = var_4881_axes_0, x = var_4880_cast_fp16)[name = string("op_4881_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_27_cast_fp16 = mul(x = var_4877_cast_fp16, y = var_4881_cast_fp16)[name = string("delta_27_cast_fp16")];
            tensor<int32, [1]> var_4884_axes_0 = const()[name = string("op_4884_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_4884_cast_fp16 = expand_dims(axes = var_4884_axes_0, x = delta_27_cast_fp16)[name = string("op_4884_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4885_cast_fp16 = mul(x = var_4873_cast_fp16, y = var_4884_cast_fp16)[name = string("op_4885_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> rec17_out = add(x = state_53_cast_fp16, y = var_4885_cast_fp16)[name = string("state_55_cast_fp16")];
            tensor<int32, [4]> var_4889_begin_0 = const()[name = string("op_4889_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> var_4889_end_0 = const()[name = string("op_4889_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> var_4889_end_mask_0 = const()[name = string("op_4889_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_4889_squeeze_mask_0 = const()[name = string("op_4889_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_4889_cast_fp16 = slice_by_index(begin = var_4889_begin_0, end = var_4889_end_0, end_mask = var_4889_end_mask_0, squeeze_mask = var_4889_squeeze_mask_0, x = _inversed_query_71_cast_fp16)[name = string("op_4889_cast_fp16")];
            tensor<int32, [1]> var_4890_axes_0 = const()[name = string("op_4890_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_4890_cast_fp16 = expand_dims(axes = var_4890_axes_0, x = var_4889_cast_fp16)[name = string("op_4890_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4891_cast_fp16 = mul(x = rec17_out, y = var_4890_cast_fp16)[name = string("op_4891_cast_fp16")];
            tensor<int32, [1]> var_4893_axes_0 = const()[name = string("op_4893_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_4893_keep_dims_0 = const()[name = string("op_4893_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_4893_cast_fp16 = reduce_sum(axes = var_4893_axes_0, keep_dims = var_4893_keep_dims_0, x = var_4891_cast_fp16)[name = string("op_4893_cast_fp16")];
            tensor<int32, [1]> attention_131_axes_0 = const()[name = string("attention_131_axes_0"), val = tensor<int32, [1]>([1])];
            tensor<fp16, [1, 1, 16, 128]> attention_131_cast_fp16 = expand_dims(axes = attention_131_axes_0, x = var_4893_cast_fp16)[name = string("attention_131_cast_fp16")];
            tensor<int32, [2]> var_4896 = const()[name = string("op_4896"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [16, 128]> attention_133_cast_fp16 = reshape(shape = var_4896, x = attention_131_cast_fp16)[name = string("attention_133_cast_fp16")];
            tensor<int32, [2]> var_4898 = const()[name = string("op_4898"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [16, 128]> input_227_cast_fp16 = reshape(shape = var_4898, x = linear_135_cast_fp16)[name = string("input_227_cast_fp16")];
            tensor<fp16, [16, 128]> var_4900_cast_fp16 = mul(x = attention_133_cast_fp16, y = attention_133_cast_fp16)[name = string("op_4900_cast_fp16")];
            tensor<int32, [1]> variance_113_axes_0 = const()[name = string("variance_113_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_113_keep_dims_0 = const()[name = string("variance_113_keep_dims_0"), val = bool(true)];
            tensor<fp16, [16, 1]> variance_113_cast_fp16 = reduce_mean(axes = variance_113_axes_0, keep_dims = variance_113_keep_dims_0, x = var_4900_cast_fp16)[name = string("variance_113_cast_fp16")];
            fp16 var_4903_to_fp16 = const()[name = string("op_4903_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [16, 1]> var_4904_cast_fp16 = add(x = variance_113_cast_fp16, y = var_4903_to_fp16)[name = string("op_4904_cast_fp16")];
            fp32 var_4905_epsilon_0 = const()[name = string("op_4905_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [16, 1]> var_4905_cast_fp16 = rsqrt(epsilon = var_4905_epsilon_0, x = var_4904_cast_fp16)[name = string("op_4905_cast_fp16")];
            tensor<fp16, [16, 128]> attention_135_cast_fp16 = mul(x = attention_133_cast_fp16, y = var_4905_cast_fp16)[name = string("attention_135_cast_fp16")];
            tensor<fp16, [128]> final_group_1_gated_norm_promoted_to_fp16 = const()[name = string("final_group_1_gated_norm_promoted_to_fp16"), val = tensor<fp16, [128]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(274695232)))];
            tensor<fp16, [16, 128]> attention_137_cast_fp16 = mul(x = attention_135_cast_fp16, y = final_group_1_gated_norm_promoted_to_fp16)[name = string("attention_137_cast_fp16")];
            tensor<fp16, [16, 128]> var_4908_cast_fp16 = silu(x = input_227_cast_fp16)[name = string("op_4908_cast_fp16")];
            tensor<fp16, [16, 128]> attention_139_cast_fp16 = mul(x = attention_137_cast_fp16, y = var_4908_cast_fp16)[name = string("attention_139_cast_fp16")];
            tensor<int32, [2]> var_4910 = const()[name = string("op_4910"), val = tensor<int32, [2]>([1, 2048])];
            tensor<fp16, [1, 2048]> input_229_cast_fp16 = reshape(shape = var_4910, x = attention_139_cast_fp16)[name = string("input_229_cast_fp16")];
            tensor<fp16, [1024, 2048]> final_group_1_out_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 2048]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(274695552))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(276268480))))[name = string("final_group_1_out_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 1024]> linear_136_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = final_group_1_out_proj_weight_promoted_to_fp16_palettized, x = input_229_cast_fp16)[name = string("linear_136_cast_fp16")];
            tensor<fp16, [1, 1024]> value_259_cast_fp16 = add(x = value_247_cast_fp16, y = linear_136_cast_fp16)[name = string("value_259_cast_fp16")];
            tensor<fp16, [1, 1024]> var_4915_cast_fp16 = mul(x = value_259_cast_fp16, y = value_259_cast_fp16)[name = string("op_4915_cast_fp16")];
            tensor<int32, [1]> variance_115_axes_0 = const()[name = string("variance_115_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_115_keep_dims_0 = const()[name = string("variance_115_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 1]> variance_115_cast_fp16 = reduce_mean(axes = variance_115_axes_0, keep_dims = variance_115_keep_dims_0, x = var_4915_cast_fp16)[name = string("variance_115_cast_fp16")];
            fp16 var_4918_to_fp16 = const()[name = string("op_4918_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 1]> var_4919_cast_fp16 = add(x = variance_115_cast_fp16, y = var_4918_to_fp16)[name = string("op_4919_cast_fp16")];
            fp32 var_4920_epsilon_0 = const()[name = string("op_4920_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 1]> var_4920_cast_fp16 = rsqrt(epsilon = var_4920_epsilon_0, x = var_4919_cast_fp16)[name = string("op_4920_cast_fp16")];
            tensor<fp16, [1, 1024]> var_4921_cast_fp16 = mul(x = value_259_cast_fp16, y = var_4920_cast_fp16)[name = string("op_4921_cast_fp16")];
            tensor<fp16, [1024]> var_4923_to_fp16 = const()[name = string("op_4923_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(276276736)))];
            tensor<fp16, [1, 1024]> input_231_cast_fp16 = mul(x = var_4921_cast_fp16, y = var_4923_to_fp16)[name = string("input_231_cast_fp16")];
            tensor<fp16, [3584, 1024]> final_group_1_gate_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(276278848))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(279031424))))[name = string("final_group_1_gate_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 3584]> linear_137_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = final_group_1_gate_proj_weight_promoted_to_fp16_palettized, x = input_231_cast_fp16)[name = string("linear_137_cast_fp16")];
            tensor<fp16, [1, 3584]> var_4927_cast_fp16 = silu(x = linear_137_cast_fp16)[name = string("op_4927_cast_fp16")];
            tensor<fp16, [3584, 1024]> final_group_1_up_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(279060160))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(281812736))))[name = string("final_group_1_up_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 3584]> linear_138_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = final_group_1_up_proj_weight_promoted_to_fp16_palettized, x = input_231_cast_fp16)[name = string("linear_138_cast_fp16")];
            tensor<fp16, [1, 3584]> input_235_cast_fp16 = mul(x = var_4927_cast_fp16, y = linear_138_cast_fp16)[name = string("input_235_cast_fp16")];
            tensor<fp16, [1024, 3584]> final_group_1_down_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 3584]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(281841472))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(284594048))))[name = string("final_group_1_down_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 1024]> linear_139_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = final_group_1_down_proj_weight_promoted_to_fp16_palettized, x = input_235_cast_fp16)[name = string("linear_139_cast_fp16")];
            tensor<fp16, [1, 1024]> value_261_cast_fp16 = add(x = value_259_cast_fp16, y = linear_139_cast_fp16)[name = string("value_261_cast_fp16")];
            int32 var_4952 = const()[name = string("op_4952"), val = int32(-1)];
            tensor<fp16, [1, 1024]> var_4967_cast_fp16 = mul(x = value_261_cast_fp16, y = value_261_cast_fp16)[name = string("op_4967_cast_fp16")];
            tensor<int32, [1]> variance_117_axes_0 = const()[name = string("variance_117_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_117_keep_dims_0 = const()[name = string("variance_117_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 1]> variance_117_cast_fp16 = reduce_mean(axes = variance_117_axes_0, keep_dims = variance_117_keep_dims_0, x = var_4967_cast_fp16)[name = string("variance_117_cast_fp16")];
            fp16 var_4970_to_fp16 = const()[name = string("op_4970_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 1]> var_4971_cast_fp16 = add(x = variance_117_cast_fp16, y = var_4970_to_fp16)[name = string("op_4971_cast_fp16")];
            fp32 var_4972_epsilon_0 = const()[name = string("op_4972_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 1]> var_4972_cast_fp16 = rsqrt(epsilon = var_4972_epsilon_0, x = var_4971_cast_fp16)[name = string("op_4972_cast_fp16")];
            tensor<fp16, [1, 1024]> var_4973_cast_fp16 = mul(x = value_261_cast_fp16, y = var_4972_cast_fp16)[name = string("op_4973_cast_fp16")];
            tensor<fp16, [1024]> var_4975_to_fp16 = const()[name = string("op_4975_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(284602304)))];
            tensor<fp16, [1, 1024]> input_237_cast_fp16 = mul(x = var_4973_cast_fp16, y = var_4975_to_fp16)[name = string("input_237_cast_fp16")];
            tensor<fp16, [6144, 1024]> final_group_2_in_proj_qkv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(284604416))), lut = tensor<fp16, [384, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(289323072))))[name = string("final_group_2_in_proj_qkv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 6144]> linear_140_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = final_group_2_in_proj_qkv_weight_promoted_to_fp16_palettized, x = input_237_cast_fp16)[name = string("linear_140_cast_fp16")];
            tensor<int32, [2]> var_4979_perm_0 = const()[name = string("op_4979_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> mixed_axes_0 = const()[name = string("mixed_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [6144, 1]> var_4979_cast_fp16 = transpose(perm = var_4979_perm_0, x = linear_140_cast_fp16)[name = string("transpose_10")];
            tensor<fp16, [1, 6144, 1]> mixed_cast_fp16 = expand_dims(axes = mixed_axes_0, x = var_4979_cast_fp16)[name = string("mixed_cast_fp16")];
            bool convolution_input_interleave_0 = const()[name = string("convolution_input_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 6144, 4]> convolution_input_cast_fp16 = concat(axis = var_4952, interleave = convolution_input_interleave_0, values = (conv18, mixed_cast_fp16))[name = string("convolution_input_cast_fp16")];
            string input_239_pad_type_0 = const()[name = string("input_239_pad_type_0"), val = string("valid")];
            int32 input_239_groups_0 = const()[name = string("input_239_groups_0"), val = int32(6144)];
            tensor<int32, [1]> input_239_strides_0 = const()[name = string("input_239_strides_0"), val = tensor<int32, [1]>([1])];
            tensor<int32, [2]> input_239_pad_0 = const()[name = string("input_239_pad_0"), val = tensor<int32, [2]>([0, 0])];
            tensor<int32, [1]> input_239_dilations_0 = const()[name = string("input_239_dilations_0"), val = tensor<int32, [1]>([1])];
            tensor<fp16, [6144, 1, 4]> final_group_2_conv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1, 4]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(289372288))), lut = tensor<fp16, [384, 1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(289390784))))[name = string("final_group_2_conv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 6144, 1]> input_239_cast_fp16 = conv(dilations = input_239_dilations_0, groups = input_239_groups_0, pad = input_239_pad_0, pad_type = input_239_pad_type_0, strides = input_239_strides_0, weight = final_group_2_conv_weight_promoted_to_fp16_palettized, x = convolution_input_cast_fp16)[name = string("input_239_cast_fp16")];
            tensor<fp16, [1, 6144, 1]> var_4988_cast_fp16 = silu(x = input_239_cast_fp16)[name = string("op_4988_cast_fp16")];
            tensor<int32, [3]> var_4989_perm_0 = const()[name = string("op_4989_perm_0"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [3]> var_4992_begin_0 = const()[name = string("op_4992_begin_0"), val = tensor<int32, [3]>([0, 0, 1])];
            tensor<int32, [3]> var_4992_end_0 = const()[name = string("op_4992_end_0"), val = tensor<int32, [3]>([1, 6144, 4])];
            tensor<bool, [3]> var_4992_end_mask_0 = const()[name = string("op_4992_end_mask_0"), val = tensor<bool, [3]>([true, true, true])];
            tensor<fp16, [1, 6144, 3]> conv18_out = slice_by_index(begin = var_4992_begin_0, end = var_4992_end_0, end_mask = var_4992_end_mask_0, x = convolution_input_cast_fp16)[name = string("op_4992_cast_fp16")];
            tensor<int32, [3]> var_4993 = const()[name = string("op_4993"), val = tensor<int32, [3]>([2048, 2048, 2048])];
            int32 var_4994_axis_0 = const()[name = string("op_4994_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 1, 6144]> var_4989_cast_fp16 = transpose(perm = var_4989_perm_0, x = var_4988_cast_fp16)[name = string("transpose_9")];
            tensor<fp16, [1, 1, 2048]> var_4994_cast_fp16_0, tensor<fp16, [1, 1, 2048]> var_4994_cast_fp16_1, tensor<fp16, [1, 1, 2048]> var_4994_cast_fp16_2 = split(axis = var_4994_axis_0, split_sizes = var_4993, x = var_4989_cast_fp16)[name = string("op_4994_cast_fp16")];
            tensor<int32, [4]> var_4998 = const()[name = string("op_4998"), val = tensor<int32, [4]>([1, 1, 16, 128])];
            tensor<fp16, [1, 1, 16, 128]> value_265_cast_fp16 = reshape(shape = var_4998, x = var_4994_cast_fp16_0)[name = string("value_265_cast_fp16")];
            tensor<int32, [4]> var_5000 = const()[name = string("op_5000"), val = tensor<int32, [4]>([1, 1, 16, 128])];
            tensor<fp16, [1, 1, 16, 128]> value_267_cast_fp16 = reshape(shape = var_5000, x = var_4994_cast_fp16_1)[name = string("value_267_cast_fp16")];
            tensor<int32, [4]> var_5002 = const()[name = string("op_5002"), val = tensor<int32, [4]>([1, 1, 16, 128])];
            tensor<fp16, [1, 1, 16, 128]> value_269_cast_fp16 = reshape(shape = var_5002, x = var_4994_cast_fp16_2)[name = string("value_269_cast_fp16")];
            tensor<fp16, [1, 1, 16, 128]> var_5004_cast_fp16 = mul(x = value_265_cast_fp16, y = value_265_cast_fp16)[name = string("op_5004_cast_fp16")];
            tensor<int32, [1]> var_5006_axes_0 = const()[name = string("op_5006_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_5006_keep_dims_0 = const()[name = string("op_5006_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 1, 16, 1]> var_5006_cast_fp16 = reduce_sum(axes = var_5006_axes_0, keep_dims = var_5006_keep_dims_0, x = var_5004_cast_fp16)[name = string("op_5006_cast_fp16")];
            fp16 var_5007_to_fp16 = const()[name = string("op_5007_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 1, 16, 1]> var_5008_cast_fp16 = add(x = var_5006_cast_fp16, y = var_5007_to_fp16)[name = string("op_5008_cast_fp16")];
            fp32 var_5009_epsilon_0 = const()[name = string("op_5009_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 1, 16, 1]> var_5009_cast_fp16 = rsqrt(epsilon = var_5009_epsilon_0, x = var_5008_cast_fp16)[name = string("op_5009_cast_fp16")];
            tensor<fp16, [1, 1, 16, 128]> var_5010_cast_fp16 = mul(x = value_265_cast_fp16, y = var_5009_cast_fp16)[name = string("op_5010_cast_fp16")];
            tensor<int32, [4]> var_5011_perm_0 = const()[name = string("op_5011_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            fp16 _inversed_query_75_y_0_to_fp16 = const()[name = string("_inversed_query_75_y_0_to_fp16"), val = fp16(0x1.6ap-4)];
            tensor<fp16, [1, 16, 1, 128]> var_5011_cast_fp16 = transpose(perm = var_5011_perm_0, x = var_5010_cast_fp16)[name = string("transpose_8")];
            tensor<fp16, [1, 16, 1, 128]> _inversed_query_75_cast_fp16 = mul(x = var_5011_cast_fp16, y = _inversed_query_75_y_0_to_fp16)[name = string("_inversed_query_75_cast_fp16")];
            tensor<fp16, [1, 1, 16, 128]> var_5014_cast_fp16 = mul(x = value_267_cast_fp16, y = value_267_cast_fp16)[name = string("op_5014_cast_fp16")];
            tensor<int32, [1]> var_5016_axes_0 = const()[name = string("op_5016_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_5016_keep_dims_0 = const()[name = string("op_5016_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 1, 16, 1]> var_5016_cast_fp16 = reduce_sum(axes = var_5016_axes_0, keep_dims = var_5016_keep_dims_0, x = var_5014_cast_fp16)[name = string("op_5016_cast_fp16")];
            fp16 var_5017_to_fp16 = const()[name = string("op_5017_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 1, 16, 1]> var_5018_cast_fp16 = add(x = var_5016_cast_fp16, y = var_5017_to_fp16)[name = string("op_5018_cast_fp16")];
            fp32 var_5019_epsilon_0 = const()[name = string("op_5019_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 1, 16, 1]> var_5019_cast_fp16 = rsqrt(epsilon = var_5019_epsilon_0, x = var_5018_cast_fp16)[name = string("op_5019_cast_fp16")];
            tensor<fp16, [1, 1, 16, 128]> var_5020_cast_fp16 = mul(x = value_267_cast_fp16, y = var_5019_cast_fp16)[name = string("op_5020_cast_fp16")];
            tensor<int32, [4]> key_75_perm_0 = const()[name = string("key_75_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<int32, [4]> value_271_perm_0 = const()[name = string("value_271_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<fp16, [16, 1024]> final_group_2_in_proj_b_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(289440000))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(289452352))))[name = string("final_group_2_in_proj_b_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16]> linear_141_bias_0_to_fp16 = const()[name = string("linear_141_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(4862656)))];
            tensor<fp16, [1, 16]> linear_141_cast_fp16 = linear(bias = linear_141_bias_0_to_fp16, weight = final_group_2_in_proj_b_weight_promoted_to_fp16_palettized, x = input_237_cast_fp16)[name = string("linear_141_cast_fp16")];
            tensor<fp16, [1, 16]> var_5025_cast_fp16 = sigmoid(x = linear_141_cast_fp16)[name = string("op_5025_cast_fp16")];
            tensor<int32, [2]> var_5026_perm_0 = const()[name = string("op_5026_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> beta_axes_0 = const()[name = string("beta_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 1]> var_5026_cast_fp16 = transpose(perm = var_5026_perm_0, x = var_5025_cast_fp16)[name = string("transpose_7")];
            tensor<fp16, [1, 16, 1]> beta_cast_fp16 = expand_dims(axes = beta_axes_0, x = var_5026_cast_fp16)[name = string("beta_cast_fp16")];
            tensor<fp16, [16, 1024]> op_5032_weight_0_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(289452544))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(289464896))))[name = string("op_5032_weight_0_to_fp16_palettized")];
            tensor<fp16, [16]> var_5032_bias_0_to_fp16 = const()[name = string("op_5032_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(289465088)))];
            tensor<fp16, [1, 16]> var_5032_cast_fp16 = linear(bias = var_5032_bias_0_to_fp16, weight = op_5032_weight_0_to_fp16_palettized, x = input_237_cast_fp16)[name = string("op_5032_cast_fp16")];
            tensor<fp16, [1, 16]> var_5033_cast_fp16 = softplus(x = var_5032_cast_fp16)[name = string("op_5033_cast_fp16")];
            tensor<fp16, [16]> var_5029_promoted_to_fp16 = const()[name = string("op_5029_promoted_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(289465216)))];
            tensor<fp16, [1, 16]> var_5034_cast_fp16 = mul(x = var_5029_promoted_to_fp16, y = var_5033_cast_fp16)[name = string("op_5034_cast_fp16")];
            tensor<int32, [2]> var_5035_perm_0 = const()[name = string("op_5035_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> decay_axes_0 = const()[name = string("decay_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 1]> var_5035_cast_fp16 = transpose(perm = var_5035_perm_0, x = var_5034_cast_fp16)[name = string("transpose_6")];
            tensor<fp16, [1, 16, 1]> decay_cast_fp16 = expand_dims(axes = decay_axes_0, x = var_5035_cast_fp16)[name = string("decay_cast_fp16")];
            tensor<fp16, [2048, 1024]> final_group_2_in_proj_z_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [2048, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(289465344))), lut = tensor<fp16, [128, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(291038272))))[name = string("final_group_2_in_proj_z_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 2048]> linear_143_cast_fp16 = linear(bias = linear_3_bias_0_to_fp16, weight = final_group_2_in_proj_z_weight_promoted_to_fp16_palettized, x = input_237_cast_fp16)[name = string("linear_143_cast_fp16")];
            tensor<int32, [3]> var_5043_begin_0 = const()[name = string("op_5043_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_5043_end_0 = const()[name = string("op_5043_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_5043_end_mask_0 = const()[name = string("op_5043_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_5043_squeeze_mask_0 = const()[name = string("op_5043_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_5043_cast_fp16 = slice_by_index(begin = var_5043_begin_0, end = var_5043_end_0, end_mask = var_5043_end_mask_0, squeeze_mask = var_5043_squeeze_mask_0, x = decay_cast_fp16)[name = string("op_5043_cast_fp16")];
            tensor<fp16, [1, 16]> var_5044_cast_fp16 = exp(x = var_5043_cast_fp16)[name = string("op_5044_cast_fp16")];
            tensor<int32, [1]> var_5045_axes_0 = const()[name = string("op_5045_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_5045_cast_fp16 = expand_dims(axes = var_5045_axes_0, x = var_5044_cast_fp16)[name = string("op_5045_cast_fp16")];
            tensor<int32, [1]> var_5046_axes_0 = const()[name = string("op_5046_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_5046_cast_fp16 = expand_dims(axes = var_5046_axes_0, x = var_5045_cast_fp16)[name = string("op_5046_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_57_cast_fp16 = mul(x = rec18, y = var_5046_cast_fp16)[name = string("state_57_cast_fp16")];
            tensor<int32, [4]> key_token_begin_0 = const()[name = string("key_token_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> key_token_end_0 = const()[name = string("key_token_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> key_token_end_mask_0 = const()[name = string("key_token_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_squeeze_mask_0 = const()[name = string("key_token_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 1, 128]> key_75_cast_fp16 = transpose(perm = key_75_perm_0, x = var_5020_cast_fp16)[name = string("transpose_5")];
            tensor<fp16, [1, 16, 128]> key_token_cast_fp16 = slice_by_index(begin = key_token_begin_0, end = key_token_end_0, end_mask = key_token_end_mask_0, squeeze_mask = key_token_squeeze_mask_0, x = key_75_cast_fp16)[name = string("key_token_cast_fp16")];
            tensor<int32, [4]> value_token_begin_0 = const()[name = string("value_token_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> value_token_end_0 = const()[name = string("value_token_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> value_token_end_mask_0 = const()[name = string("value_token_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_squeeze_mask_0 = const()[name = string("value_token_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 1, 128]> value_271_cast_fp16 = transpose(perm = value_271_perm_0, x = value_269_cast_fp16)[name = string("transpose_4")];
            tensor<fp16, [1, 16, 128]> value_token_cast_fp16 = slice_by_index(begin = value_token_begin_0, end = value_token_end_0, end_mask = value_token_end_mask_0, squeeze_mask = value_token_squeeze_mask_0, x = value_271_cast_fp16)[name = string("value_token_cast_fp16")];
            tensor<int32, [1]> var_5054_axes_0 = const()[name = string("op_5054_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_5054_cast_fp16 = expand_dims(axes = var_5054_axes_0, x = key_token_cast_fp16)[name = string("op_5054_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_5055_cast_fp16 = mul(x = state_57_cast_fp16, y = var_5054_cast_fp16)[name = string("op_5055_cast_fp16")];
            tensor<int32, [1]> memory_axes_0 = const()[name = string("memory_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_keep_dims_0 = const()[name = string("memory_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_cast_fp16 = reduce_sum(axes = memory_axes_0, keep_dims = memory_keep_dims_0, x = var_5055_cast_fp16)[name = string("memory_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_5058_cast_fp16 = sub(x = value_token_cast_fp16, y = memory_cast_fp16)[name = string("op_5058_cast_fp16")];
            tensor<int32, [3]> var_5061_begin_0 = const()[name = string("op_5061_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_5061_end_0 = const()[name = string("op_5061_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_5061_end_mask_0 = const()[name = string("op_5061_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_5061_squeeze_mask_0 = const()[name = string("op_5061_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_5061_cast_fp16 = slice_by_index(begin = var_5061_begin_0, end = var_5061_end_0, end_mask = var_5061_end_mask_0, squeeze_mask = var_5061_squeeze_mask_0, x = beta_cast_fp16)[name = string("op_5061_cast_fp16")];
            tensor<int32, [1]> var_5062_axes_0 = const()[name = string("op_5062_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_5062_cast_fp16 = expand_dims(axes = var_5062_axes_0, x = var_5061_cast_fp16)[name = string("op_5062_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_cast_fp16 = mul(x = var_5058_cast_fp16, y = var_5062_cast_fp16)[name = string("delta_cast_fp16")];
            tensor<int32, [1]> var_5065_axes_0 = const()[name = string("op_5065_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_5065_cast_fp16 = expand_dims(axes = var_5065_axes_0, x = delta_cast_fp16)[name = string("op_5065_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_5066_cast_fp16 = mul(x = var_5054_cast_fp16, y = var_5065_cast_fp16)[name = string("op_5066_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> rec18_out = add(x = state_57_cast_fp16, y = var_5066_cast_fp16)[name = string("state_cast_fp16")];
            tensor<int32, [4]> var_5070_begin_0 = const()[name = string("op_5070_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> var_5070_end_0 = const()[name = string("op_5070_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> var_5070_end_mask_0 = const()[name = string("op_5070_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_5070_squeeze_mask_0 = const()[name = string("op_5070_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_5070_cast_fp16 = slice_by_index(begin = var_5070_begin_0, end = var_5070_end_0, end_mask = var_5070_end_mask_0, squeeze_mask = var_5070_squeeze_mask_0, x = _inversed_query_75_cast_fp16)[name = string("op_5070_cast_fp16")];
            tensor<int32, [1]> var_5071_axes_0 = const()[name = string("op_5071_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_5071_cast_fp16 = expand_dims(axes = var_5071_axes_0, x = var_5070_cast_fp16)[name = string("op_5071_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_5072_cast_fp16 = mul(x = rec18_out, y = var_5071_cast_fp16)[name = string("op_5072_cast_fp16")];
            tensor<int32, [1]> var_5074_axes_0 = const()[name = string("op_5074_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_5074_keep_dims_0 = const()[name = string("op_5074_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_5074_cast_fp16 = reduce_sum(axes = var_5074_axes_0, keep_dims = var_5074_keep_dims_0, x = var_5072_cast_fp16)[name = string("op_5074_cast_fp16")];
            tensor<int32, [1]> attention_141_axes_0 = const()[name = string("attention_141_axes_0"), val = tensor<int32, [1]>([1])];
            tensor<fp16, [1, 1, 16, 128]> attention_141_cast_fp16 = expand_dims(axes = attention_141_axes_0, x = var_5074_cast_fp16)[name = string("attention_141_cast_fp16")];
            tensor<int32, [2]> var_5077 = const()[name = string("op_5077"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [16, 128]> attention_143_cast_fp16 = reshape(shape = var_5077, x = attention_141_cast_fp16)[name = string("attention_143_cast_fp16")];
            tensor<int32, [2]> var_5079 = const()[name = string("op_5079"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [16, 128]> input_241_cast_fp16 = reshape(shape = var_5079, x = linear_143_cast_fp16)[name = string("input_241_cast_fp16")];
            tensor<fp16, [16, 128]> var_5081_cast_fp16 = mul(x = attention_143_cast_fp16, y = attention_143_cast_fp16)[name = string("op_5081_cast_fp16")];
            tensor<int32, [1]> variance_119_axes_0 = const()[name = string("variance_119_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_119_keep_dims_0 = const()[name = string("variance_119_keep_dims_0"), val = bool(true)];
            tensor<fp16, [16, 1]> variance_119_cast_fp16 = reduce_mean(axes = variance_119_axes_0, keep_dims = variance_119_keep_dims_0, x = var_5081_cast_fp16)[name = string("variance_119_cast_fp16")];
            fp16 var_5084_to_fp16 = const()[name = string("op_5084_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [16, 1]> var_5085_cast_fp16 = add(x = variance_119_cast_fp16, y = var_5084_to_fp16)[name = string("op_5085_cast_fp16")];
            fp32 var_5086_epsilon_0 = const()[name = string("op_5086_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [16, 1]> var_5086_cast_fp16 = rsqrt(epsilon = var_5086_epsilon_0, x = var_5085_cast_fp16)[name = string("op_5086_cast_fp16")];
            tensor<fp16, [16, 128]> attention_145_cast_fp16 = mul(x = attention_143_cast_fp16, y = var_5086_cast_fp16)[name = string("attention_145_cast_fp16")];
            tensor<fp16, [128]> final_group_2_gated_norm_promoted_to_fp16 = const()[name = string("final_group_2_gated_norm_promoted_to_fp16"), val = tensor<fp16, [128]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(291054720)))];
            tensor<fp16, [16, 128]> attention_147_cast_fp16 = mul(x = attention_145_cast_fp16, y = final_group_2_gated_norm_promoted_to_fp16)[name = string("attention_147_cast_fp16")];
            tensor<fp16, [16, 128]> var_5089_cast_fp16 = silu(x = input_241_cast_fp16)[name = string("op_5089_cast_fp16")];
            tensor<fp16, [16, 128]> attention_149_cast_fp16 = mul(x = attention_147_cast_fp16, y = var_5089_cast_fp16)[name = string("attention_149_cast_fp16")];
            tensor<int32, [2]> var_5091 = const()[name = string("op_5091"), val = tensor<int32, [2]>([1, 2048])];
            tensor<fp16, [1, 2048]> input_243_cast_fp16 = reshape(shape = var_5091, x = attention_149_cast_fp16)[name = string("input_243_cast_fp16")];
            tensor<fp16, [1024, 2048]> final_group_2_out_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 2048]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(291055040))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(292627968))))[name = string("final_group_2_out_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 1024]> linear_144_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = final_group_2_out_proj_weight_promoted_to_fp16_palettized, x = input_243_cast_fp16)[name = string("linear_144_cast_fp16")];
            tensor<fp16, [1, 1024]> value_273_cast_fp16 = add(x = value_261_cast_fp16, y = linear_144_cast_fp16)[name = string("value_273_cast_fp16")];
            tensor<fp16, [1, 1024]> var_5096_cast_fp16 = mul(x = value_273_cast_fp16, y = value_273_cast_fp16)[name = string("op_5096_cast_fp16")];
            tensor<int32, [1]> variance_121_axes_0 = const()[name = string("variance_121_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_121_keep_dims_0 = const()[name = string("variance_121_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 1]> variance_121_cast_fp16 = reduce_mean(axes = variance_121_axes_0, keep_dims = variance_121_keep_dims_0, x = var_5096_cast_fp16)[name = string("variance_121_cast_fp16")];
            fp16 var_5099_to_fp16 = const()[name = string("op_5099_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 1]> var_5100_cast_fp16 = add(x = variance_121_cast_fp16, y = var_5099_to_fp16)[name = string("op_5100_cast_fp16")];
            fp32 var_5101_epsilon_0 = const()[name = string("op_5101_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 1]> var_5101_cast_fp16 = rsqrt(epsilon = var_5101_epsilon_0, x = var_5100_cast_fp16)[name = string("op_5101_cast_fp16")];
            tensor<fp16, [1, 1024]> var_5102_cast_fp16 = mul(x = value_273_cast_fp16, y = var_5101_cast_fp16)[name = string("op_5102_cast_fp16")];
            tensor<fp16, [1024]> var_5104_to_fp16 = const()[name = string("op_5104_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(292636224)))];
            tensor<fp16, [1, 1024]> input_245_cast_fp16 = mul(x = var_5102_cast_fp16, y = var_5104_to_fp16)[name = string("input_245_cast_fp16")];
            tensor<fp16, [3584, 1024]> final_group_2_gate_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(292638336))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(295390912))))[name = string("final_group_2_gate_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 3584]> linear_145_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = final_group_2_gate_proj_weight_promoted_to_fp16_palettized, x = input_245_cast_fp16)[name = string("linear_145_cast_fp16")];
            tensor<fp16, [1, 3584]> var_5108_cast_fp16 = silu(x = linear_145_cast_fp16)[name = string("op_5108_cast_fp16")];
            tensor<fp16, [3584, 1024]> final_group_2_up_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(295419648))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(298172224))))[name = string("final_group_2_up_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 3584]> linear_146_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = final_group_2_up_proj_weight_promoted_to_fp16_palettized, x = input_245_cast_fp16)[name = string("linear_146_cast_fp16")];
            tensor<fp16, [1, 3584]> input_249_cast_fp16 = mul(x = var_5108_cast_fp16, y = linear_146_cast_fp16)[name = string("input_249_cast_fp16")];
            tensor<fp16, [1024, 3584]> final_group_2_down_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 3584]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(298200960))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(300953536))))[name = string("final_group_2_down_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 1024]> linear_147_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = final_group_2_down_proj_weight_promoted_to_fp16_palettized, x = input_249_cast_fp16)[name = string("linear_147_cast_fp16")];
            tensor<fp16, [1, 1024]> hidden_out = add(x = value_273_cast_fp16, y = linear_147_cast_fp16)[name = string("value_275_cast_fp16")];
            int32 var_5132 = const()[name = string("op_5132"), val = int32(-1)];
            tensor<fp16, [1, 1024]> var_5139_cast_fp16 = mul(x = hidden_out, y = hidden_out)[name = string("op_5139_cast_fp16")];
            tensor<int32, [1]> variance_123_axes_0 = const()[name = string("variance_123_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_123_keep_dims_0 = const()[name = string("variance_123_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 1]> variance_123_cast_fp16 = reduce_mean(axes = variance_123_axes_0, keep_dims = variance_123_keep_dims_0, x = var_5139_cast_fp16)[name = string("variance_123_cast_fp16")];
            fp16 var_5142_to_fp16 = const()[name = string("op_5142_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 1]> var_5143_cast_fp16 = add(x = variance_123_cast_fp16, y = var_5142_to_fp16)[name = string("op_5143_cast_fp16")];
            fp32 var_5144_epsilon_0 = const()[name = string("op_5144_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 1]> var_5144_cast_fp16 = rsqrt(epsilon = var_5144_epsilon_0, x = var_5143_cast_fp16)[name = string("op_5144_cast_fp16")];
            tensor<fp16, [1, 1024]> var_5145_cast_fp16 = mul(x = hidden_out, y = var_5144_cast_fp16)[name = string("op_5145_cast_fp16")];
            tensor<fp16, [1024]> var_5147_to_fp16 = const()[name = string("op_5147_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(300961792)))];
            tensor<fp16, [1, 1024]> input_cast_fp16 = mul(x = var_5145_cast_fp16, y = var_5147_to_fp16)[name = string("input_cast_fp16")];
            tensor<fp16, [4096, 1024]> projection19_q_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [4096, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(300963904))), lut = tensor<fp16, [256, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(304109696))))[name = string("projection19_q_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 4096]> linear_148_cast_fp16 = linear(bias = linear_24_bias_0_to_fp16, weight = projection19_q_proj_weight_promoted_to_fp16_palettized, x = input_cast_fp16)[name = string("linear_148_cast_fp16")];
            tensor<int32, [3]> var_5151 = const()[name = string("op_5151"), val = tensor<int32, [3]>([1, 8, 512])];
            tensor<fp16, [1, 8, 512]> query_and_gate_cast_fp16 = reshape(shape = var_5151, x = linear_148_cast_fp16)[name = string("query_and_gate_cast_fp16")];
            tensor<int32, [2]> var_5153_split_sizes_0 = const()[name = string("op_5153_split_sizes_0"), val = tensor<int32, [2]>([256, 256])];
            int32 var_5153_axis_0 = const()[name = string("op_5153_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 8, 256]> var_5153_cast_fp16_0, tensor<fp16, [1, 8, 256]> var_5153_cast_fp16_1 = split(axis = var_5153_axis_0, split_sizes = var_5153_split_sizes_0, x = query_and_gate_cast_fp16)[name = string("op_5153_cast_fp16")];
            tensor<fp16, [512, 1024]> projection19_k_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [512, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(304142528))), lut = tensor<fp16, [32, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(304535808))))[name = string("projection19_k_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 512]> linear_149_cast_fp16 = linear(bias = linear_25_bias_0_to_fp16, weight = projection19_k_proj_weight_promoted_to_fp16_palettized, x = input_cast_fp16)[name = string("linear_149_cast_fp16")];
            tensor<int32, [3]> var_5157 = const()[name = string("op_5157"), val = tensor<int32, [3]>([1, 2, 256])];
            tensor<fp16, [1, 2, 256]> value_279_cast_fp16 = reshape(shape = var_5157, x = linear_149_cast_fp16)[name = string("value_279_cast_fp16")];
            tensor<fp16, [512, 1024]> projection19_v_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [512, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(304539968))), lut = tensor<fp16, [32, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(304933248))))[name = string("projection19_v_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 512]> linear_150_cast_fp16 = linear(bias = linear_25_bias_0_to_fp16, weight = projection19_v_proj_weight_promoted_to_fp16_palettized, x = input_cast_fp16)[name = string("linear_150_cast_fp16")];
            tensor<int32, [3]> var_5161 = const()[name = string("op_5161"), val = tensor<int32, [3]>([1, 2, 256])];
            tensor<fp16, [1, 2, 256]> value_cast_fp16 = reshape(shape = var_5161, x = linear_150_cast_fp16)[name = string("value_cast_fp16")];
            tensor<fp16, [1, 8, 256]> var_5163_cast_fp16 = mul(x = var_5153_cast_fp16_0, y = var_5153_cast_fp16_0)[name = string("op_5163_cast_fp16")];
            tensor<int32, [1]> variance_125_axes_0 = const()[name = string("variance_125_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_125_keep_dims_0 = const()[name = string("variance_125_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 8, 1]> variance_125_cast_fp16 = reduce_mean(axes = variance_125_axes_0, keep_dims = variance_125_keep_dims_0, x = var_5163_cast_fp16)[name = string("variance_125_cast_fp16")];
            fp16 var_5166_to_fp16 = const()[name = string("op_5166_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 8, 1]> var_5167_cast_fp16 = add(x = variance_125_cast_fp16, y = var_5166_to_fp16)[name = string("op_5167_cast_fp16")];
            fp32 var_5168_epsilon_0 = const()[name = string("op_5168_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 8, 1]> var_5168_cast_fp16 = rsqrt(epsilon = var_5168_epsilon_0, x = var_5167_cast_fp16)[name = string("op_5168_cast_fp16")];
            tensor<fp16, [1, 8, 256]> var_5169_cast_fp16 = mul(x = var_5153_cast_fp16_0, y = var_5168_cast_fp16)[name = string("op_5169_cast_fp16")];
            tensor<fp16, [256]> var_5171_to_fp16 = const()[name = string("op_5171_to_fp16"), val = tensor<fp16, [256]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(304937408)))];
            tensor<fp16, [1, 8, 256]> var_5172_cast_fp16 = mul(x = var_5169_cast_fp16, y = var_5171_to_fp16)[name = string("op_5172_cast_fp16")];
            tensor<int32, [3]> var_5173 = const()[name = string("op_5173"), val = tensor<int32, [3]>([1, 0, 2])];
            tensor<int32, [1]> value_281_axes_0 = const()[name = string("value_281_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [8, 1, 256]> var_5174_cast_fp16 = transpose(perm = var_5173, x = var_5172_cast_fp16)[name = string("transpose_3")];
            tensor<fp16, [1, 8, 1, 256]> value_281_cast_fp16 = expand_dims(axes = value_281_axes_0, x = var_5174_cast_fp16)[name = string("value_281_cast_fp16")];
            tensor<fp16, [1, 2, 256]> var_5176_cast_fp16 = mul(x = value_279_cast_fp16, y = value_279_cast_fp16)[name = string("op_5176_cast_fp16")];
            tensor<int32, [1]> variance_axes_0 = const()[name = string("variance_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_keep_dims_0 = const()[name = string("variance_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 2, 1]> variance_cast_fp16 = reduce_mean(axes = variance_axes_0, keep_dims = variance_keep_dims_0, x = var_5176_cast_fp16)[name = string("variance_cast_fp16")];
            fp16 var_5179_to_fp16 = const()[name = string("op_5179_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 2, 1]> var_5180_cast_fp16 = add(x = variance_cast_fp16, y = var_5179_to_fp16)[name = string("op_5180_cast_fp16")];
            fp32 var_5181_epsilon_0 = const()[name = string("op_5181_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 2, 1]> var_5181_cast_fp16 = rsqrt(epsilon = var_5181_epsilon_0, x = var_5180_cast_fp16)[name = string("op_5181_cast_fp16")];
            tensor<fp16, [1, 2, 256]> var_5182_cast_fp16 = mul(x = value_279_cast_fp16, y = var_5181_cast_fp16)[name = string("op_5182_cast_fp16")];
            tensor<fp16, [256]> var_5184_to_fp16 = const()[name = string("op_5184_to_fp16"), val = tensor<fp16, [256]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(304937984)))];
            tensor<fp16, [1, 2, 256]> var_5185_cast_fp16 = mul(x = var_5182_cast_fp16, y = var_5184_to_fp16)[name = string("op_5185_cast_fp16")];
            tensor<int32, [3]> var_5186 = const()[name = string("op_5186"), val = tensor<int32, [3]>([1, 0, 2])];
            tensor<int32, [1]> value_283_axes_0 = const()[name = string("value_283_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [2, 1, 256]> var_5187_cast_fp16 = transpose(perm = var_5186, x = var_5185_cast_fp16)[name = string("transpose_2")];
            tensor<fp16, [1, 2, 1, 256]> value_283_cast_fp16 = expand_dims(axes = value_283_axes_0, x = var_5187_cast_fp16)[name = string("value_283_cast_fp16")];
            tensor<int32, [4]> rotated_17_begin_0 = const()[name = string("rotated_17_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> rotated_17_end_0 = const()[name = string("rotated_17_end_0"), val = tensor<int32, [4]>([1, 8, 1, 64])];
            tensor<bool, [4]> rotated_17_end_mask_0 = const()[name = string("rotated_17_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
            tensor<fp16, [1, 8, 1, 64]> rotated_17_cast_fp16 = slice_by_index(begin = rotated_17_begin_0, end = rotated_17_end_0, end_mask = rotated_17_end_mask_0, x = value_281_cast_fp16)[name = string("rotated_17_cast_fp16")];
            tensor<int32, [4]> passthrough_17_begin_0 = const()[name = string("passthrough_17_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 64])];
            tensor<int32, [4]> passthrough_17_end_0 = const()[name = string("passthrough_17_end_0"), val = tensor<int32, [4]>([1, 8, 1, 256])];
            tensor<bool, [4]> passthrough_17_end_mask_0 = const()[name = string("passthrough_17_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
            tensor<fp16, [1, 8, 1, 192]> passthrough_17_cast_fp16 = slice_by_index(begin = passthrough_17_begin_0, end = passthrough_17_end_0, end_mask = passthrough_17_end_mask_0, x = value_281_cast_fp16)[name = string("passthrough_17_cast_fp16")];
            tensor<int32, [2]> var_5191_split_sizes_0 = const()[name = string("op_5191_split_sizes_0"), val = tensor<int32, [2]>([32, 32])];
            int32 var_5191_axis_0 = const()[name = string("op_5191_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 8, 1, 32]> var_5191_cast_fp16_0, tensor<fp16, [1, 8, 1, 32]> var_5191_cast_fp16_1 = split(axis = var_5191_axis_0, split_sizes = var_5191_split_sizes_0, x = rotated_17_cast_fp16)[name = string("op_5191_cast_fp16")];
            fp16 const_175_promoted_to_fp16 = const()[name = string("const_175_promoted_to_fp16"), val = fp16(-0x1p+0)];
            tensor<fp16, [1, 8, 1, 32]> var_5193_cast_fp16 = mul(x = var_5191_cast_fp16_1, y = const_175_promoted_to_fp16)[name = string("op_5193_cast_fp16")];
            bool rotated_half_17_interleave_0 = const()[name = string("rotated_half_17_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 8, 1, 64]> rotated_half_17_cast_fp16 = concat(axis = var_5132, interleave = rotated_half_17_interleave_0, values = (var_5193_cast_fp16, var_5191_cast_fp16_0))[name = string("rotated_half_17_cast_fp16")];
            tensor<fp16, [1, 8, 1, 64]> var_5196_cast_fp16 = mul(x = rotated_17_cast_fp16, y = cos)[name = string("op_5196_cast_fp16")];
            tensor<fp16, [1, 8, 1, 64]> var_5197_cast_fp16 = mul(x = rotated_half_17_cast_fp16, y = sin)[name = string("op_5197_cast_fp16")];
            tensor<fp16, [1, 8, 1, 64]> var_5198_cast_fp16 = add(x = var_5196_cast_fp16, y = var_5197_cast_fp16)[name = string("op_5198_cast_fp16")];
            bool query_interleave_0 = const()[name = string("query_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 8, 1, 256]> query_cast_fp16 = concat(axis = var_5132, interleave = query_interleave_0, values = (var_5198_cast_fp16, passthrough_17_cast_fp16))[name = string("query_cast_fp16")];
            tensor<int32, [4]> rotated_begin_0 = const()[name = string("rotated_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> rotated_end_0 = const()[name = string("rotated_end_0"), val = tensor<int32, [4]>([1, 2, 1, 64])];
            tensor<bool, [4]> rotated_end_mask_0 = const()[name = string("rotated_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
            tensor<fp16, [1, 2, 1, 64]> rotated_cast_fp16 = slice_by_index(begin = rotated_begin_0, end = rotated_end_0, end_mask = rotated_end_mask_0, x = value_283_cast_fp16)[name = string("rotated_cast_fp16")];
            tensor<int32, [4]> passthrough_begin_0 = const()[name = string("passthrough_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 64])];
            tensor<int32, [4]> passthrough_end_0 = const()[name = string("passthrough_end_0"), val = tensor<int32, [4]>([1, 2, 1, 256])];
            tensor<bool, [4]> passthrough_end_mask_0 = const()[name = string("passthrough_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
            tensor<fp16, [1, 2, 1, 192]> passthrough_cast_fp16 = slice_by_index(begin = passthrough_begin_0, end = passthrough_end_0, end_mask = passthrough_end_mask_0, x = value_283_cast_fp16)[name = string("passthrough_cast_fp16")];
            tensor<int32, [2]> var_5203_split_sizes_0 = const()[name = string("op_5203_split_sizes_0"), val = tensor<int32, [2]>([32, 32])];
            int32 var_5203_axis_0 = const()[name = string("op_5203_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 2, 1, 32]> var_5203_cast_fp16_0, tensor<fp16, [1, 2, 1, 32]> var_5203_cast_fp16_1 = split(axis = var_5203_axis_0, split_sizes = var_5203_split_sizes_0, x = rotated_cast_fp16)[name = string("op_5203_cast_fp16")];
            fp16 const_176_promoted_to_fp16 = const()[name = string("const_176_promoted_to_fp16"), val = fp16(-0x1p+0)];
            tensor<fp16, [1, 2, 1, 32]> var_5205_cast_fp16 = mul(x = var_5203_cast_fp16_1, y = const_176_promoted_to_fp16)[name = string("op_5205_cast_fp16")];
            bool rotated_half_interleave_0 = const()[name = string("rotated_half_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 2, 1, 64]> rotated_half_cast_fp16 = concat(axis = var_5132, interleave = rotated_half_interleave_0, values = (var_5205_cast_fp16, var_5203_cast_fp16_0))[name = string("rotated_half_cast_fp16")];
            tensor<fp16, [1, 2, 1, 64]> var_5208_cast_fp16 = mul(x = rotated_cast_fp16, y = cos)[name = string("op_5208_cast_fp16")];
            tensor<fp16, [1, 2, 1, 64]> var_5209_cast_fp16 = mul(x = rotated_half_cast_fp16, y = sin)[name = string("op_5209_cast_fp16")];
            tensor<fp16, [1, 2, 1, 64]> var_5210_cast_fp16 = add(x = var_5208_cast_fp16, y = var_5209_cast_fp16)[name = string("op_5210_cast_fp16")];
            bool key_interleave_0 = const()[name = string("key_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 2, 1, 256]> key_cast_fp16 = concat(axis = var_5132, interleave = key_interleave_0, values = (var_5210_cast_fp16, passthrough_cast_fp16))[name = string("key_cast_fp16")];
            tensor<int32, [4]> var_5213 = const()[name = string("op_5213"), val = tensor<int32, [4]>([2, 4, 1, 256])];
            tensor<fp16, [2, 4, 1, 256]> var_5214_cast_fp16 = reshape(shape = var_5213, x = query_cast_fp16)[name = string("op_5214_cast_fp16")];
            tensor<int32, [4]> var_5215 = const()[name = string("op_5215"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<int32, [3]> var_5217 = const()[name = string("op_5217"), val = tensor<int32, [3]>([2, 1, 256])];
            tensor<fp16, [2, 1, 256]> k19 = reshape(shape = var_5217, x = key_cast_fp16)[name = string("op_5218_cast_fp16")];
            tensor<int32, [3]> var_5219 = const()[name = string("op_5219"), val = tensor<int32, [3]>([1, 0, 2])];
            tensor<int32, [2]> var_5221 = const()[name = string("op_5221"), val = tensor<int32, [2]>([1, 2048])];
            tensor<fp16, [1, 2048]> gate19 = reshape(shape = var_5221, x = var_5153_cast_fp16_1)[name = string("op_5222_cast_fp16")];
            tensor<fp16, [2, 1, 4, 256]> q19 = transpose(perm = var_5215, x = var_5214_cast_fp16)[name = string("transpose_0")];
            tensor<fp16, [2, 1, 256]> v19 = transpose(perm = var_5219, x = value_cast_fp16)[name = string("transpose_1")];
        } -> (hidden_out, conv0_out, rec0_out, conv1_out, rec1_out, conv2_out, rec2_out, conv4_out, rec4_out, conv5_out, rec5_out, conv6_out, rec6_out, conv8_out, rec8_out, conv9_out, rec9_out, conv10_out, rec10_out, conv12_out, rec12_out, conv13_out, rec13_out, conv14_out, rec14_out, conv16_out, rec16_out, conv17_out, rec17_out, conv18_out, rec18_out, q19, k19, v19, gate19);
    func prefill16<ios18>(tensor<fp16, [1, 6144, 3]> conv0, tensor<fp16, [1, 6144, 3]> conv1, tensor<fp16, [1, 6144, 3]> conv10, tensor<fp16, [1, 6144, 3]> conv12, tensor<fp16, [1, 6144, 3]> conv13, tensor<fp16, [1, 6144, 3]> conv14, tensor<fp16, [1, 6144, 3]> conv16, tensor<fp16, [1, 6144, 3]> conv17, tensor<fp16, [1, 6144, 3]> conv18, tensor<fp16, [1, 6144, 3]> conv2, tensor<fp16, [1, 6144, 3]> conv4, tensor<fp16, [1, 6144, 3]> conv5, tensor<fp16, [1, 6144, 3]> conv6, tensor<fp16, [1, 6144, 3]> conv8, tensor<fp16, [1, 6144, 3]> conv9, tensor<fp16, [1, 1, 16, 64]> cos, tensor<int32, [1]> current_pos, tensor<fp16, [16, 1024]> hidden, state<tensor<fp16, [4, 2, 2, 2048, 256]>> kv_cache, tensor<fp16, [1, 16, 2048]> mask3, tensor<fp16, [1, 16, 128, 128]> rec0, tensor<fp16, [1, 16, 128, 128]> rec1, tensor<fp16, [1, 16, 128, 128]> rec10, tensor<fp16, [1, 16, 128, 128]> rec12, tensor<fp16, [1, 16, 128, 128]> rec13, tensor<fp16, [1, 16, 128, 128]> rec14, tensor<fp16, [1, 16, 128, 128]> rec16, tensor<fp16, [1, 16, 128, 128]> rec17, tensor<fp16, [1, 16, 128, 128]> rec18, tensor<fp16, [1, 16, 128, 128]> rec2, tensor<fp16, [1, 16, 128, 128]> rec4, tensor<fp16, [1, 16, 128, 128]> rec5, tensor<fp16, [1, 16, 128, 128]> rec6, tensor<fp16, [1, 16, 128, 128]> rec8, tensor<fp16, [1, 16, 128, 128]> rec9, tensor<fp16, [1, 1, 16, 64]> sin) {
            int32 var_131 = const()[name = string("op_131"), val = int32(-1)];
            tensor<fp16, [16, 1024]> var_146_cast_fp16 = mul(x = hidden, y = hidden)[name = string("op_146_cast_fp16")];
            tensor<int32, [1]> variance_1_axes_0 = const()[name = string("variance_1_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_1_keep_dims_0 = const()[name = string("variance_1_keep_dims_0"), val = bool(true)];
            tensor<fp16, [16, 1]> variance_1_cast_fp16 = reduce_mean(axes = variance_1_axes_0, keep_dims = variance_1_keep_dims_0, x = var_146_cast_fp16)[name = string("variance_1_cast_fp16")];
            fp16 var_149_to_fp16 = const()[name = string("op_149_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [16, 1]> var_150_cast_fp16 = add(x = variance_1_cast_fp16, y = var_149_to_fp16)[name = string("op_150_cast_fp16")];
            fp32 var_151_epsilon_0 = const()[name = string("op_151_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [16, 1]> var_151_cast_fp16 = rsqrt(epsilon = var_151_epsilon_0, x = var_150_cast_fp16)[name = string("op_151_cast_fp16")];
            tensor<fp16, [16, 1024]> var_152_cast_fp16 = mul(x = hidden, y = var_151_cast_fp16)[name = string("op_152_cast_fp16")];
            tensor<fp16, [1024]> var_154_to_fp16 = const()[name = string("op_154_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(64)))];
            tensor<fp16, [16, 1024]> input_1_cast_fp16 = mul(x = var_152_cast_fp16, y = var_154_to_fp16)[name = string("input_1_cast_fp16")];
            tensor<fp16, [6144, 1024]> groups_0_0_in_proj_qkv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(2176))), lut = tensor<fp16, [384, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(4720832))))[name = string("groups_0_0_in_proj_qkv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [6144]> linear_0_bias_0_to_fp16 = const()[name = string("linear_0_bias_0_to_fp16"), val = tensor<fp16, [6144]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(4770048)))];
            tensor<fp16, [16, 6144]> linear_0_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = groups_0_0_in_proj_qkv_weight_promoted_to_fp16_palettized, x = input_1_cast_fp16)[name = string("linear_0_cast_fp16")];
            tensor<int32, [2]> var_158_perm_0 = const()[name = string("op_158_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> mixed_1_axes_0 = const()[name = string("mixed_1_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [6144, 16]> var_158_cast_fp16 = transpose(perm = var_158_perm_0, x = linear_0_cast_fp16)[name = string("transpose_141")];
            tensor<fp16, [1, 6144, 16]> mixed_1_cast_fp16 = expand_dims(axes = mixed_1_axes_0, x = var_158_cast_fp16)[name = string("mixed_1_cast_fp16")];
            bool convolution_input_1_interleave_0 = const()[name = string("convolution_input_1_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 6144, 19]> convolution_input_1_cast_fp16 = concat(axis = var_131, interleave = convolution_input_1_interleave_0, values = (conv0, mixed_1_cast_fp16))[name = string("convolution_input_1_cast_fp16")];
            string input_3_pad_type_0 = const()[name = string("input_3_pad_type_0"), val = string("valid")];
            int32 input_3_groups_0 = const()[name = string("input_3_groups_0"), val = int32(6144)];
            tensor<int32, [1]> input_3_strides_0 = const()[name = string("input_3_strides_0"), val = tensor<int32, [1]>([1])];
            tensor<int32, [2]> input_3_pad_0 = const()[name = string("input_3_pad_0"), val = tensor<int32, [2]>([0, 0])];
            tensor<int32, [1]> input_3_dilations_0 = const()[name = string("input_3_dilations_0"), val = tensor<int32, [1]>([1])];
            tensor<fp16, [6144, 1, 4]> groups_0_0_conv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1, 4]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(4782400))), lut = tensor<fp16, [384, 1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(4800896))))[name = string("groups_0_0_conv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 6144, 16]> input_3_cast_fp16 = conv(dilations = input_3_dilations_0, groups = input_3_groups_0, pad = input_3_pad_0, pad_type = input_3_pad_type_0, strides = input_3_strides_0, weight = groups_0_0_conv_weight_promoted_to_fp16_palettized, x = convolution_input_1_cast_fp16)[name = string("input_3_cast_fp16")];
            tensor<fp16, [1, 6144, 16]> var_167_cast_fp16 = silu(x = input_3_cast_fp16)[name = string("op_167_cast_fp16")];
            tensor<int32, [3]> var_168_perm_0 = const()[name = string("op_168_perm_0"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [3]> var_171_begin_0 = const()[name = string("op_171_begin_0"), val = tensor<int32, [3]>([0, 0, 16])];
            tensor<int32, [3]> var_171_end_0 = const()[name = string("op_171_end_0"), val = tensor<int32, [3]>([1, 6144, 19])];
            tensor<bool, [3]> var_171_end_mask_0 = const()[name = string("op_171_end_mask_0"), val = tensor<bool, [3]>([true, true, true])];
            tensor<fp16, [1, 6144, 3]> conv0_out = slice_by_index(begin = var_171_begin_0, end = var_171_end_0, end_mask = var_171_end_mask_0, x = convolution_input_1_cast_fp16)[name = string("op_171_cast_fp16")];
            tensor<int32, [3]> var_172 = const()[name = string("op_172"), val = tensor<int32, [3]>([2048, 2048, 2048])];
            int32 var_173_axis_0 = const()[name = string("op_173_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 16, 6144]> var_168_cast_fp16 = transpose(perm = var_168_perm_0, x = var_167_cast_fp16)[name = string("transpose_140")];
            tensor<fp16, [1, 16, 2048]> var_173_cast_fp16_0, tensor<fp16, [1, 16, 2048]> var_173_cast_fp16_1, tensor<fp16, [1, 16, 2048]> var_173_cast_fp16_2 = split(axis = var_173_axis_0, split_sizes = var_172, x = var_168_cast_fp16)[name = string("op_173_cast_fp16")];
            tensor<int32, [4]> var_177 = const()[name = string("op_177"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<fp16, [1, 16, 16, 128]> value_5_cast_fp16 = reshape(shape = var_177, x = var_173_cast_fp16_0)[name = string("value_5_cast_fp16")];
            tensor<int32, [4]> var_179 = const()[name = string("op_179"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<fp16, [1, 16, 16, 128]> value_7_cast_fp16 = reshape(shape = var_179, x = var_173_cast_fp16_1)[name = string("value_7_cast_fp16")];
            tensor<int32, [4]> var_181 = const()[name = string("op_181"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<fp16, [1, 16, 16, 128]> value_9_cast_fp16 = reshape(shape = var_181, x = var_173_cast_fp16_2)[name = string("value_9_cast_fp16")];
            tensor<fp16, [1, 16, 16, 128]> var_183_cast_fp16 = mul(x = value_5_cast_fp16, y = value_5_cast_fp16)[name = string("op_183_cast_fp16")];
            tensor<int32, [1]> var_185_axes_0 = const()[name = string("op_185_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_185_keep_dims_0 = const()[name = string("op_185_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 16, 16, 1]> var_185_cast_fp16 = reduce_sum(axes = var_185_axes_0, keep_dims = var_185_keep_dims_0, x = var_183_cast_fp16)[name = string("op_185_cast_fp16")];
            fp16 var_186_to_fp16 = const()[name = string("op_186_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 16, 16, 1]> var_187_cast_fp16 = add(x = var_185_cast_fp16, y = var_186_to_fp16)[name = string("op_187_cast_fp16")];
            fp32 var_188_epsilon_0 = const()[name = string("op_188_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 16, 16, 1]> var_188_cast_fp16 = rsqrt(epsilon = var_188_epsilon_0, x = var_187_cast_fp16)[name = string("op_188_cast_fp16")];
            tensor<fp16, [1, 16, 16, 128]> var_189_cast_fp16 = mul(x = value_5_cast_fp16, y = var_188_cast_fp16)[name = string("op_189_cast_fp16")];
            tensor<int32, [4]> var_190_perm_0 = const()[name = string("op_190_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            fp16 _inversed_query_3_y_0_to_fp16 = const()[name = string("_inversed_query_3_y_0_to_fp16"), val = fp16(0x1.6ap-4)];
            tensor<fp16, [1, 16, 16, 128]> var_190_cast_fp16 = transpose(perm = var_190_perm_0, x = var_189_cast_fp16)[name = string("transpose_139")];
            tensor<fp16, [1, 16, 16, 128]> _inversed_query_3_cast_fp16 = mul(x = var_190_cast_fp16, y = _inversed_query_3_y_0_to_fp16)[name = string("_inversed_query_3_cast_fp16")];
            tensor<fp16, [1, 16, 16, 128]> var_193_cast_fp16 = mul(x = value_7_cast_fp16, y = value_7_cast_fp16)[name = string("op_193_cast_fp16")];
            tensor<int32, [1]> var_195_axes_0 = const()[name = string("op_195_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_195_keep_dims_0 = const()[name = string("op_195_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 16, 16, 1]> var_195_cast_fp16 = reduce_sum(axes = var_195_axes_0, keep_dims = var_195_keep_dims_0, x = var_193_cast_fp16)[name = string("op_195_cast_fp16")];
            fp16 var_196_to_fp16 = const()[name = string("op_196_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 16, 16, 1]> var_197_cast_fp16 = add(x = var_195_cast_fp16, y = var_196_to_fp16)[name = string("op_197_cast_fp16")];
            fp32 var_198_epsilon_0 = const()[name = string("op_198_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 16, 16, 1]> var_198_cast_fp16 = rsqrt(epsilon = var_198_epsilon_0, x = var_197_cast_fp16)[name = string("op_198_cast_fp16")];
            tensor<fp16, [1, 16, 16, 128]> var_199_cast_fp16 = mul(x = value_7_cast_fp16, y = var_198_cast_fp16)[name = string("op_199_cast_fp16")];
            tensor<int32, [4]> key_3_perm_0 = const()[name = string("key_3_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<int32, [4]> value_11_perm_0 = const()[name = string("value_11_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<fp16, [16, 1024]> groups_0_0_in_proj_b_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(4850112))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(4862464))))[name = string("groups_0_0_in_proj_b_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16]> linear_1_bias_0_to_fp16 = const()[name = string("linear_1_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(4862656)))];
            tensor<fp16, [16, 16]> linear_1_cast_fp16 = linear(bias = linear_1_bias_0_to_fp16, weight = groups_0_0_in_proj_b_weight_promoted_to_fp16_palettized, x = input_1_cast_fp16)[name = string("linear_1_cast_fp16")];
            tensor<fp16, [16, 16]> var_204_cast_fp16 = sigmoid(x = linear_1_cast_fp16)[name = string("op_204_cast_fp16")];
            tensor<int32, [2]> var_205_perm_0 = const()[name = string("op_205_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> beta_1_axes_0 = const()[name = string("beta_1_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 16]> var_205_cast_fp16 = transpose(perm = var_205_perm_0, x = var_204_cast_fp16)[name = string("transpose_138")];
            tensor<fp16, [1, 16, 16]> beta_1_cast_fp16 = expand_dims(axes = beta_1_axes_0, x = var_205_cast_fp16)[name = string("beta_1_cast_fp16")];
            tensor<fp16, [16, 1024]> op_211_weight_0_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(4862784))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(4875136))))[name = string("op_211_weight_0_to_fp16_palettized")];
            tensor<fp16, [16]> var_211_bias_0_to_fp16 = const()[name = string("op_211_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(4875328)))];
            tensor<fp16, [16, 16]> var_211_cast_fp16 = linear(bias = var_211_bias_0_to_fp16, weight = op_211_weight_0_to_fp16_palettized, x = input_1_cast_fp16)[name = string("op_211_cast_fp16")];
            tensor<fp16, [16, 16]> var_212_cast_fp16 = softplus(x = var_211_cast_fp16)[name = string("op_212_cast_fp16")];
            tensor<fp16, [16]> var_208_promoted_to_fp16 = const()[name = string("op_208_promoted_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(4875456)))];
            tensor<fp16, [16, 16]> var_213_cast_fp16 = mul(x = var_208_promoted_to_fp16, y = var_212_cast_fp16)[name = string("op_213_cast_fp16")];
            tensor<int32, [2]> var_214_perm_0 = const()[name = string("op_214_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> decay_1_axes_0 = const()[name = string("decay_1_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 16]> var_214_cast_fp16 = transpose(perm = var_214_perm_0, x = var_213_cast_fp16)[name = string("transpose_137")];
            tensor<fp16, [1, 16, 16]> decay_1_cast_fp16 = expand_dims(axes = decay_1_axes_0, x = var_214_cast_fp16)[name = string("decay_1_cast_fp16")];
            tensor<fp16, [2048, 1024]> groups_0_0_in_proj_z_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [2048, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(4875584))), lut = tensor<fp16, [128, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(6448512))))[name = string("groups_0_0_in_proj_z_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [2048]> linear_3_bias_0_to_fp16 = const()[name = string("linear_3_bias_0_to_fp16"), val = tensor<fp16, [2048]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(6464960)))];
            tensor<fp16, [16, 2048]> linear_3_cast_fp16 = linear(bias = linear_3_bias_0_to_fp16, weight = groups_0_0_in_proj_z_weight_promoted_to_fp16_palettized, x = input_1_cast_fp16)[name = string("linear_3_cast_fp16")];
            tensor<int32, [3]> var_222_begin_0 = const()[name = string("op_222_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_222_end_0 = const()[name = string("op_222_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_222_end_mask_0 = const()[name = string("op_222_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_222_squeeze_mask_0 = const()[name = string("op_222_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_222_cast_fp16 = slice_by_index(begin = var_222_begin_0, end = var_222_end_0, end_mask = var_222_end_mask_0, squeeze_mask = var_222_squeeze_mask_0, x = decay_1_cast_fp16)[name = string("op_222_cast_fp16")];
            tensor<fp16, [1, 16]> var_223_cast_fp16 = exp(x = var_222_cast_fp16)[name = string("op_223_cast_fp16")];
            tensor<int32, [1]> var_224_axes_0 = const()[name = string("op_224_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_224_cast_fp16 = expand_dims(axes = var_224_axes_0, x = var_223_cast_fp16)[name = string("op_224_cast_fp16")];
            tensor<int32, [1]> var_225_axes_0 = const()[name = string("op_225_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_225_cast_fp16 = expand_dims(axes = var_225_axes_0, x = var_224_cast_fp16)[name = string("op_225_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_1_cast_fp16 = mul(x = rec0, y = var_225_cast_fp16)[name = string("state_1_cast_fp16")];
            tensor<int32, [4]> key_token_1_begin_0 = const()[name = string("key_token_1_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> key_token_1_end_0 = const()[name = string("key_token_1_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> key_token_1_end_mask_0 = const()[name = string("key_token_1_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_1_squeeze_mask_0 = const()[name = string("key_token_1_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 16, 128]> key_3_cast_fp16 = transpose(perm = key_3_perm_0, x = var_199_cast_fp16)[name = string("transpose_136")];
            tensor<fp16, [1, 16, 128]> key_token_1_cast_fp16 = slice_by_index(begin = key_token_1_begin_0, end = key_token_1_end_0, end_mask = key_token_1_end_mask_0, squeeze_mask = key_token_1_squeeze_mask_0, x = key_3_cast_fp16)[name = string("key_token_1_cast_fp16")];
            tensor<int32, [4]> value_token_1_begin_0 = const()[name = string("value_token_1_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> value_token_1_end_0 = const()[name = string("value_token_1_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> value_token_1_end_mask_0 = const()[name = string("value_token_1_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_1_squeeze_mask_0 = const()[name = string("value_token_1_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 16, 128]> value_11_cast_fp16 = transpose(perm = value_11_perm_0, x = value_9_cast_fp16)[name = string("transpose_135")];
            tensor<fp16, [1, 16, 128]> value_token_1_cast_fp16 = slice_by_index(begin = value_token_1_begin_0, end = value_token_1_end_0, end_mask = value_token_1_end_mask_0, squeeze_mask = value_token_1_squeeze_mask_0, x = value_11_cast_fp16)[name = string("value_token_1_cast_fp16")];
            tensor<int32, [1]> var_233_axes_0 = const()[name = string("op_233_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_233_cast_fp16 = expand_dims(axes = var_233_axes_0, x = key_token_1_cast_fp16)[name = string("op_233_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_234_cast_fp16 = mul(x = state_1_cast_fp16, y = var_233_cast_fp16)[name = string("op_234_cast_fp16")];
            tensor<int32, [1]> memory_1_axes_0 = const()[name = string("memory_1_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_1_keep_dims_0 = const()[name = string("memory_1_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_1_cast_fp16 = reduce_sum(axes = memory_1_axes_0, keep_dims = memory_1_keep_dims_0, x = var_234_cast_fp16)[name = string("memory_1_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_237_cast_fp16 = sub(x = value_token_1_cast_fp16, y = memory_1_cast_fp16)[name = string("op_237_cast_fp16")];
            tensor<int32, [3]> var_240_begin_0 = const()[name = string("op_240_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_240_end_0 = const()[name = string("op_240_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_240_end_mask_0 = const()[name = string("op_240_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_240_squeeze_mask_0 = const()[name = string("op_240_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_240_cast_fp16 = slice_by_index(begin = var_240_begin_0, end = var_240_end_0, end_mask = var_240_end_mask_0, squeeze_mask = var_240_squeeze_mask_0, x = beta_1_cast_fp16)[name = string("op_240_cast_fp16")];
            tensor<int32, [1]> var_241_axes_0 = const()[name = string("op_241_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_241_cast_fp16 = expand_dims(axes = var_241_axes_0, x = var_240_cast_fp16)[name = string("op_241_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_1_cast_fp16 = mul(x = var_237_cast_fp16, y = var_241_cast_fp16)[name = string("delta_1_cast_fp16")];
            tensor<int32, [1]> var_244_axes_0 = const()[name = string("op_244_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_244_cast_fp16 = expand_dims(axes = var_244_axes_0, x = delta_1_cast_fp16)[name = string("op_244_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_245_cast_fp16 = mul(x = var_233_cast_fp16, y = var_244_cast_fp16)[name = string("op_245_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_3_cast_fp16 = add(x = state_1_cast_fp16, y = var_245_cast_fp16)[name = string("state_3_cast_fp16")];
            tensor<int32, [4]> var_249_begin_0 = const()[name = string("op_249_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> var_249_end_0 = const()[name = string("op_249_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> var_249_end_mask_0 = const()[name = string("op_249_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_249_squeeze_mask_0 = const()[name = string("op_249_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_249_cast_fp16 = slice_by_index(begin = var_249_begin_0, end = var_249_end_0, end_mask = var_249_end_mask_0, squeeze_mask = var_249_squeeze_mask_0, x = _inversed_query_3_cast_fp16)[name = string("op_249_cast_fp16")];
            tensor<int32, [1]> var_250_axes_0 = const()[name = string("op_250_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_250_cast_fp16 = expand_dims(axes = var_250_axes_0, x = var_249_cast_fp16)[name = string("op_250_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_251_cast_fp16 = mul(x = state_3_cast_fp16, y = var_250_cast_fp16)[name = string("op_251_cast_fp16")];
            tensor<int32, [1]> var_253_axes_0 = const()[name = string("op_253_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_253_keep_dims_0 = const()[name = string("op_253_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_253_cast_fp16 = reduce_sum(axes = var_253_axes_0, keep_dims = var_253_keep_dims_0, x = var_251_cast_fp16)[name = string("op_253_cast_fp16")];
            tensor<int32, [3]> var_256_begin_0 = const()[name = string("op_256_begin_0"), val = tensor<int32, [3]>([0, 0, 1])];
            tensor<int32, [3]> var_256_end_0 = const()[name = string("op_256_end_0"), val = tensor<int32, [3]>([1, 16, 2])];
            tensor<bool, [3]> var_256_end_mask_0 = const()[name = string("op_256_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_256_squeeze_mask_0 = const()[name = string("op_256_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_256_cast_fp16 = slice_by_index(begin = var_256_begin_0, end = var_256_end_0, end_mask = var_256_end_mask_0, squeeze_mask = var_256_squeeze_mask_0, x = decay_1_cast_fp16)[name = string("op_256_cast_fp16")];
            tensor<fp16, [1, 16]> var_257_cast_fp16 = exp(x = var_256_cast_fp16)[name = string("op_257_cast_fp16")];
            tensor<int32, [1]> var_258_axes_0 = const()[name = string("op_258_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_258_cast_fp16 = expand_dims(axes = var_258_axes_0, x = var_257_cast_fp16)[name = string("op_258_cast_fp16")];
            tensor<int32, [1]> var_259_axes_0 = const()[name = string("op_259_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_259_cast_fp16 = expand_dims(axes = var_259_axes_0, x = var_258_cast_fp16)[name = string("op_259_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_5_cast_fp16 = mul(x = state_3_cast_fp16, y = var_259_cast_fp16)[name = string("state_5_cast_fp16")];
            tensor<int32, [4]> key_token_3_begin_0 = const()[name = string("key_token_3_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> key_token_3_end_0 = const()[name = string("key_token_3_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> key_token_3_end_mask_0 = const()[name = string("key_token_3_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_3_squeeze_mask_0 = const()[name = string("key_token_3_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_3_cast_fp16 = slice_by_index(begin = key_token_3_begin_0, end = key_token_3_end_0, end_mask = key_token_3_end_mask_0, squeeze_mask = key_token_3_squeeze_mask_0, x = key_3_cast_fp16)[name = string("key_token_3_cast_fp16")];
            tensor<int32, [4]> value_token_3_begin_0 = const()[name = string("value_token_3_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> value_token_3_end_0 = const()[name = string("value_token_3_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> value_token_3_end_mask_0 = const()[name = string("value_token_3_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_3_squeeze_mask_0 = const()[name = string("value_token_3_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_3_cast_fp16 = slice_by_index(begin = value_token_3_begin_0, end = value_token_3_end_0, end_mask = value_token_3_end_mask_0, squeeze_mask = value_token_3_squeeze_mask_0, x = value_11_cast_fp16)[name = string("value_token_3_cast_fp16")];
            tensor<int32, [1]> var_267_axes_0 = const()[name = string("op_267_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_267_cast_fp16 = expand_dims(axes = var_267_axes_0, x = key_token_3_cast_fp16)[name = string("op_267_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_268_cast_fp16 = mul(x = state_5_cast_fp16, y = var_267_cast_fp16)[name = string("op_268_cast_fp16")];
            tensor<int32, [1]> memory_3_axes_0 = const()[name = string("memory_3_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_3_keep_dims_0 = const()[name = string("memory_3_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_3_cast_fp16 = reduce_sum(axes = memory_3_axes_0, keep_dims = memory_3_keep_dims_0, x = var_268_cast_fp16)[name = string("memory_3_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_271_cast_fp16 = sub(x = value_token_3_cast_fp16, y = memory_3_cast_fp16)[name = string("op_271_cast_fp16")];
            tensor<int32, [3]> var_274_begin_0 = const()[name = string("op_274_begin_0"), val = tensor<int32, [3]>([0, 0, 1])];
            tensor<int32, [3]> var_274_end_0 = const()[name = string("op_274_end_0"), val = tensor<int32, [3]>([1, 16, 2])];
            tensor<bool, [3]> var_274_end_mask_0 = const()[name = string("op_274_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_274_squeeze_mask_0 = const()[name = string("op_274_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_274_cast_fp16 = slice_by_index(begin = var_274_begin_0, end = var_274_end_0, end_mask = var_274_end_mask_0, squeeze_mask = var_274_squeeze_mask_0, x = beta_1_cast_fp16)[name = string("op_274_cast_fp16")];
            tensor<int32, [1]> var_275_axes_0 = const()[name = string("op_275_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_275_cast_fp16 = expand_dims(axes = var_275_axes_0, x = var_274_cast_fp16)[name = string("op_275_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_3_cast_fp16 = mul(x = var_271_cast_fp16, y = var_275_cast_fp16)[name = string("delta_3_cast_fp16")];
            tensor<int32, [1]> var_278_axes_0 = const()[name = string("op_278_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_278_cast_fp16 = expand_dims(axes = var_278_axes_0, x = delta_3_cast_fp16)[name = string("op_278_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_279_cast_fp16 = mul(x = var_267_cast_fp16, y = var_278_cast_fp16)[name = string("op_279_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_7_cast_fp16 = add(x = state_5_cast_fp16, y = var_279_cast_fp16)[name = string("state_7_cast_fp16")];
            tensor<int32, [4]> var_283_begin_0 = const()[name = string("op_283_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> var_283_end_0 = const()[name = string("op_283_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> var_283_end_mask_0 = const()[name = string("op_283_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_283_squeeze_mask_0 = const()[name = string("op_283_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_283_cast_fp16 = slice_by_index(begin = var_283_begin_0, end = var_283_end_0, end_mask = var_283_end_mask_0, squeeze_mask = var_283_squeeze_mask_0, x = _inversed_query_3_cast_fp16)[name = string("op_283_cast_fp16")];
            tensor<int32, [1]> var_284_axes_0 = const()[name = string("op_284_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_284_cast_fp16 = expand_dims(axes = var_284_axes_0, x = var_283_cast_fp16)[name = string("op_284_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_285_cast_fp16 = mul(x = state_7_cast_fp16, y = var_284_cast_fp16)[name = string("op_285_cast_fp16")];
            tensor<int32, [1]> var_287_axes_0 = const()[name = string("op_287_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_287_keep_dims_0 = const()[name = string("op_287_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_287_cast_fp16 = reduce_sum(axes = var_287_axes_0, keep_dims = var_287_keep_dims_0, x = var_285_cast_fp16)[name = string("op_287_cast_fp16")];
            tensor<int32, [3]> var_290_begin_0 = const()[name = string("op_290_begin_0"), val = tensor<int32, [3]>([0, 0, 2])];
            tensor<int32, [3]> var_290_end_0 = const()[name = string("op_290_end_0"), val = tensor<int32, [3]>([1, 16, 3])];
            tensor<bool, [3]> var_290_end_mask_0 = const()[name = string("op_290_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_290_squeeze_mask_0 = const()[name = string("op_290_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_290_cast_fp16 = slice_by_index(begin = var_290_begin_0, end = var_290_end_0, end_mask = var_290_end_mask_0, squeeze_mask = var_290_squeeze_mask_0, x = decay_1_cast_fp16)[name = string("op_290_cast_fp16")];
            tensor<fp16, [1, 16]> var_291_cast_fp16 = exp(x = var_290_cast_fp16)[name = string("op_291_cast_fp16")];
            tensor<int32, [1]> var_292_axes_0 = const()[name = string("op_292_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_292_cast_fp16 = expand_dims(axes = var_292_axes_0, x = var_291_cast_fp16)[name = string("op_292_cast_fp16")];
            tensor<int32, [1]> var_293_axes_0 = const()[name = string("op_293_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_293_cast_fp16 = expand_dims(axes = var_293_axes_0, x = var_292_cast_fp16)[name = string("op_293_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_9_cast_fp16 = mul(x = state_7_cast_fp16, y = var_293_cast_fp16)[name = string("state_9_cast_fp16")];
            tensor<int32, [4]> key_token_5_begin_0 = const()[name = string("key_token_5_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> key_token_5_end_0 = const()[name = string("key_token_5_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> key_token_5_end_mask_0 = const()[name = string("key_token_5_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_5_squeeze_mask_0 = const()[name = string("key_token_5_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_5_cast_fp16 = slice_by_index(begin = key_token_5_begin_0, end = key_token_5_end_0, end_mask = key_token_5_end_mask_0, squeeze_mask = key_token_5_squeeze_mask_0, x = key_3_cast_fp16)[name = string("key_token_5_cast_fp16")];
            tensor<int32, [4]> value_token_5_begin_0 = const()[name = string("value_token_5_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> value_token_5_end_0 = const()[name = string("value_token_5_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> value_token_5_end_mask_0 = const()[name = string("value_token_5_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_5_squeeze_mask_0 = const()[name = string("value_token_5_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_5_cast_fp16 = slice_by_index(begin = value_token_5_begin_0, end = value_token_5_end_0, end_mask = value_token_5_end_mask_0, squeeze_mask = value_token_5_squeeze_mask_0, x = value_11_cast_fp16)[name = string("value_token_5_cast_fp16")];
            tensor<int32, [1]> var_301_axes_0 = const()[name = string("op_301_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_301_cast_fp16 = expand_dims(axes = var_301_axes_0, x = key_token_5_cast_fp16)[name = string("op_301_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_302_cast_fp16 = mul(x = state_9_cast_fp16, y = var_301_cast_fp16)[name = string("op_302_cast_fp16")];
            tensor<int32, [1]> memory_5_axes_0 = const()[name = string("memory_5_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_5_keep_dims_0 = const()[name = string("memory_5_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_5_cast_fp16 = reduce_sum(axes = memory_5_axes_0, keep_dims = memory_5_keep_dims_0, x = var_302_cast_fp16)[name = string("memory_5_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_305_cast_fp16 = sub(x = value_token_5_cast_fp16, y = memory_5_cast_fp16)[name = string("op_305_cast_fp16")];
            tensor<int32, [3]> var_308_begin_0 = const()[name = string("op_308_begin_0"), val = tensor<int32, [3]>([0, 0, 2])];
            tensor<int32, [3]> var_308_end_0 = const()[name = string("op_308_end_0"), val = tensor<int32, [3]>([1, 16, 3])];
            tensor<bool, [3]> var_308_end_mask_0 = const()[name = string("op_308_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_308_squeeze_mask_0 = const()[name = string("op_308_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_308_cast_fp16 = slice_by_index(begin = var_308_begin_0, end = var_308_end_0, end_mask = var_308_end_mask_0, squeeze_mask = var_308_squeeze_mask_0, x = beta_1_cast_fp16)[name = string("op_308_cast_fp16")];
            tensor<int32, [1]> var_309_axes_0 = const()[name = string("op_309_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_309_cast_fp16 = expand_dims(axes = var_309_axes_0, x = var_308_cast_fp16)[name = string("op_309_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_5_cast_fp16 = mul(x = var_305_cast_fp16, y = var_309_cast_fp16)[name = string("delta_5_cast_fp16")];
            tensor<int32, [1]> var_312_axes_0 = const()[name = string("op_312_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_312_cast_fp16 = expand_dims(axes = var_312_axes_0, x = delta_5_cast_fp16)[name = string("op_312_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_313_cast_fp16 = mul(x = var_301_cast_fp16, y = var_312_cast_fp16)[name = string("op_313_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_11_cast_fp16 = add(x = state_9_cast_fp16, y = var_313_cast_fp16)[name = string("state_11_cast_fp16")];
            tensor<int32, [4]> var_317_begin_0 = const()[name = string("op_317_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> var_317_end_0 = const()[name = string("op_317_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> var_317_end_mask_0 = const()[name = string("op_317_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_317_squeeze_mask_0 = const()[name = string("op_317_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_317_cast_fp16 = slice_by_index(begin = var_317_begin_0, end = var_317_end_0, end_mask = var_317_end_mask_0, squeeze_mask = var_317_squeeze_mask_0, x = _inversed_query_3_cast_fp16)[name = string("op_317_cast_fp16")];
            tensor<int32, [1]> var_318_axes_0 = const()[name = string("op_318_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_318_cast_fp16 = expand_dims(axes = var_318_axes_0, x = var_317_cast_fp16)[name = string("op_318_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_319_cast_fp16 = mul(x = state_11_cast_fp16, y = var_318_cast_fp16)[name = string("op_319_cast_fp16")];
            tensor<int32, [1]> var_321_axes_0 = const()[name = string("op_321_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_321_keep_dims_0 = const()[name = string("op_321_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_321_cast_fp16 = reduce_sum(axes = var_321_axes_0, keep_dims = var_321_keep_dims_0, x = var_319_cast_fp16)[name = string("op_321_cast_fp16")];
            tensor<int32, [3]> var_324_begin_0 = const()[name = string("op_324_begin_0"), val = tensor<int32, [3]>([0, 0, 3])];
            tensor<int32, [3]> var_324_end_0 = const()[name = string("op_324_end_0"), val = tensor<int32, [3]>([1, 16, 4])];
            tensor<bool, [3]> var_324_end_mask_0 = const()[name = string("op_324_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_324_squeeze_mask_0 = const()[name = string("op_324_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_324_cast_fp16 = slice_by_index(begin = var_324_begin_0, end = var_324_end_0, end_mask = var_324_end_mask_0, squeeze_mask = var_324_squeeze_mask_0, x = decay_1_cast_fp16)[name = string("op_324_cast_fp16")];
            tensor<fp16, [1, 16]> var_325_cast_fp16 = exp(x = var_324_cast_fp16)[name = string("op_325_cast_fp16")];
            tensor<int32, [1]> var_326_axes_0 = const()[name = string("op_326_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_326_cast_fp16 = expand_dims(axes = var_326_axes_0, x = var_325_cast_fp16)[name = string("op_326_cast_fp16")];
            tensor<int32, [1]> var_327_axes_0 = const()[name = string("op_327_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_327_cast_fp16 = expand_dims(axes = var_327_axes_0, x = var_326_cast_fp16)[name = string("op_327_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_13_cast_fp16 = mul(x = state_11_cast_fp16, y = var_327_cast_fp16)[name = string("state_13_cast_fp16")];
            tensor<int32, [4]> key_token_7_begin_0 = const()[name = string("key_token_7_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> key_token_7_end_0 = const()[name = string("key_token_7_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> key_token_7_end_mask_0 = const()[name = string("key_token_7_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_7_squeeze_mask_0 = const()[name = string("key_token_7_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_7_cast_fp16 = slice_by_index(begin = key_token_7_begin_0, end = key_token_7_end_0, end_mask = key_token_7_end_mask_0, squeeze_mask = key_token_7_squeeze_mask_0, x = key_3_cast_fp16)[name = string("key_token_7_cast_fp16")];
            tensor<int32, [4]> value_token_7_begin_0 = const()[name = string("value_token_7_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> value_token_7_end_0 = const()[name = string("value_token_7_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> value_token_7_end_mask_0 = const()[name = string("value_token_7_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_7_squeeze_mask_0 = const()[name = string("value_token_7_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_7_cast_fp16 = slice_by_index(begin = value_token_7_begin_0, end = value_token_7_end_0, end_mask = value_token_7_end_mask_0, squeeze_mask = value_token_7_squeeze_mask_0, x = value_11_cast_fp16)[name = string("value_token_7_cast_fp16")];
            tensor<int32, [1]> var_335_axes_0 = const()[name = string("op_335_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_335_cast_fp16 = expand_dims(axes = var_335_axes_0, x = key_token_7_cast_fp16)[name = string("op_335_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_336_cast_fp16 = mul(x = state_13_cast_fp16, y = var_335_cast_fp16)[name = string("op_336_cast_fp16")];
            tensor<int32, [1]> memory_7_axes_0 = const()[name = string("memory_7_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_7_keep_dims_0 = const()[name = string("memory_7_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_7_cast_fp16 = reduce_sum(axes = memory_7_axes_0, keep_dims = memory_7_keep_dims_0, x = var_336_cast_fp16)[name = string("memory_7_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_339_cast_fp16 = sub(x = value_token_7_cast_fp16, y = memory_7_cast_fp16)[name = string("op_339_cast_fp16")];
            tensor<int32, [3]> var_342_begin_0 = const()[name = string("op_342_begin_0"), val = tensor<int32, [3]>([0, 0, 3])];
            tensor<int32, [3]> var_342_end_0 = const()[name = string("op_342_end_0"), val = tensor<int32, [3]>([1, 16, 4])];
            tensor<bool, [3]> var_342_end_mask_0 = const()[name = string("op_342_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_342_squeeze_mask_0 = const()[name = string("op_342_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_342_cast_fp16 = slice_by_index(begin = var_342_begin_0, end = var_342_end_0, end_mask = var_342_end_mask_0, squeeze_mask = var_342_squeeze_mask_0, x = beta_1_cast_fp16)[name = string("op_342_cast_fp16")];
            tensor<int32, [1]> var_343_axes_0 = const()[name = string("op_343_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_343_cast_fp16 = expand_dims(axes = var_343_axes_0, x = var_342_cast_fp16)[name = string("op_343_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_7_cast_fp16 = mul(x = var_339_cast_fp16, y = var_343_cast_fp16)[name = string("delta_7_cast_fp16")];
            tensor<int32, [1]> var_346_axes_0 = const()[name = string("op_346_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_346_cast_fp16 = expand_dims(axes = var_346_axes_0, x = delta_7_cast_fp16)[name = string("op_346_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_347_cast_fp16 = mul(x = var_335_cast_fp16, y = var_346_cast_fp16)[name = string("op_347_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_15_cast_fp16 = add(x = state_13_cast_fp16, y = var_347_cast_fp16)[name = string("state_15_cast_fp16")];
            tensor<int32, [4]> var_351_begin_0 = const()[name = string("op_351_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> var_351_end_0 = const()[name = string("op_351_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> var_351_end_mask_0 = const()[name = string("op_351_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_351_squeeze_mask_0 = const()[name = string("op_351_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_351_cast_fp16 = slice_by_index(begin = var_351_begin_0, end = var_351_end_0, end_mask = var_351_end_mask_0, squeeze_mask = var_351_squeeze_mask_0, x = _inversed_query_3_cast_fp16)[name = string("op_351_cast_fp16")];
            tensor<int32, [1]> var_352_axes_0 = const()[name = string("op_352_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_352_cast_fp16 = expand_dims(axes = var_352_axes_0, x = var_351_cast_fp16)[name = string("op_352_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_353_cast_fp16 = mul(x = state_15_cast_fp16, y = var_352_cast_fp16)[name = string("op_353_cast_fp16")];
            tensor<int32, [1]> var_355_axes_0 = const()[name = string("op_355_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_355_keep_dims_0 = const()[name = string("op_355_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_355_cast_fp16 = reduce_sum(axes = var_355_axes_0, keep_dims = var_355_keep_dims_0, x = var_353_cast_fp16)[name = string("op_355_cast_fp16")];
            tensor<int32, [3]> var_358_begin_0 = const()[name = string("op_358_begin_0"), val = tensor<int32, [3]>([0, 0, 4])];
            tensor<int32, [3]> var_358_end_0 = const()[name = string("op_358_end_0"), val = tensor<int32, [3]>([1, 16, 5])];
            tensor<bool, [3]> var_358_end_mask_0 = const()[name = string("op_358_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_358_squeeze_mask_0 = const()[name = string("op_358_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_358_cast_fp16 = slice_by_index(begin = var_358_begin_0, end = var_358_end_0, end_mask = var_358_end_mask_0, squeeze_mask = var_358_squeeze_mask_0, x = decay_1_cast_fp16)[name = string("op_358_cast_fp16")];
            tensor<fp16, [1, 16]> var_359_cast_fp16 = exp(x = var_358_cast_fp16)[name = string("op_359_cast_fp16")];
            tensor<int32, [1]> var_360_axes_0 = const()[name = string("op_360_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_360_cast_fp16 = expand_dims(axes = var_360_axes_0, x = var_359_cast_fp16)[name = string("op_360_cast_fp16")];
            tensor<int32, [1]> var_361_axes_0 = const()[name = string("op_361_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_361_cast_fp16 = expand_dims(axes = var_361_axes_0, x = var_360_cast_fp16)[name = string("op_361_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_17_cast_fp16 = mul(x = state_15_cast_fp16, y = var_361_cast_fp16)[name = string("state_17_cast_fp16")];
            tensor<int32, [4]> key_token_9_begin_0 = const()[name = string("key_token_9_begin_0"), val = tensor<int32, [4]>([0, 0, 4, 0])];
            tensor<int32, [4]> key_token_9_end_0 = const()[name = string("key_token_9_end_0"), val = tensor<int32, [4]>([1, 16, 5, 128])];
            tensor<bool, [4]> key_token_9_end_mask_0 = const()[name = string("key_token_9_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_9_squeeze_mask_0 = const()[name = string("key_token_9_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_9_cast_fp16 = slice_by_index(begin = key_token_9_begin_0, end = key_token_9_end_0, end_mask = key_token_9_end_mask_0, squeeze_mask = key_token_9_squeeze_mask_0, x = key_3_cast_fp16)[name = string("key_token_9_cast_fp16")];
            tensor<int32, [4]> value_token_9_begin_0 = const()[name = string("value_token_9_begin_0"), val = tensor<int32, [4]>([0, 0, 4, 0])];
            tensor<int32, [4]> value_token_9_end_0 = const()[name = string("value_token_9_end_0"), val = tensor<int32, [4]>([1, 16, 5, 128])];
            tensor<bool, [4]> value_token_9_end_mask_0 = const()[name = string("value_token_9_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_9_squeeze_mask_0 = const()[name = string("value_token_9_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_9_cast_fp16 = slice_by_index(begin = value_token_9_begin_0, end = value_token_9_end_0, end_mask = value_token_9_end_mask_0, squeeze_mask = value_token_9_squeeze_mask_0, x = value_11_cast_fp16)[name = string("value_token_9_cast_fp16")];
            tensor<int32, [1]> var_369_axes_0 = const()[name = string("op_369_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_369_cast_fp16 = expand_dims(axes = var_369_axes_0, x = key_token_9_cast_fp16)[name = string("op_369_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_370_cast_fp16 = mul(x = state_17_cast_fp16, y = var_369_cast_fp16)[name = string("op_370_cast_fp16")];
            tensor<int32, [1]> memory_9_axes_0 = const()[name = string("memory_9_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_9_keep_dims_0 = const()[name = string("memory_9_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_9_cast_fp16 = reduce_sum(axes = memory_9_axes_0, keep_dims = memory_9_keep_dims_0, x = var_370_cast_fp16)[name = string("memory_9_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_373_cast_fp16 = sub(x = value_token_9_cast_fp16, y = memory_9_cast_fp16)[name = string("op_373_cast_fp16")];
            tensor<int32, [3]> var_376_begin_0 = const()[name = string("op_376_begin_0"), val = tensor<int32, [3]>([0, 0, 4])];
            tensor<int32, [3]> var_376_end_0 = const()[name = string("op_376_end_0"), val = tensor<int32, [3]>([1, 16, 5])];
            tensor<bool, [3]> var_376_end_mask_0 = const()[name = string("op_376_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_376_squeeze_mask_0 = const()[name = string("op_376_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_376_cast_fp16 = slice_by_index(begin = var_376_begin_0, end = var_376_end_0, end_mask = var_376_end_mask_0, squeeze_mask = var_376_squeeze_mask_0, x = beta_1_cast_fp16)[name = string("op_376_cast_fp16")];
            tensor<int32, [1]> var_377_axes_0 = const()[name = string("op_377_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_377_cast_fp16 = expand_dims(axes = var_377_axes_0, x = var_376_cast_fp16)[name = string("op_377_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_9_cast_fp16 = mul(x = var_373_cast_fp16, y = var_377_cast_fp16)[name = string("delta_9_cast_fp16")];
            tensor<int32, [1]> var_380_axes_0 = const()[name = string("op_380_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_380_cast_fp16 = expand_dims(axes = var_380_axes_0, x = delta_9_cast_fp16)[name = string("op_380_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_381_cast_fp16 = mul(x = var_369_cast_fp16, y = var_380_cast_fp16)[name = string("op_381_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_19_cast_fp16 = add(x = state_17_cast_fp16, y = var_381_cast_fp16)[name = string("state_19_cast_fp16")];
            tensor<int32, [4]> var_385_begin_0 = const()[name = string("op_385_begin_0"), val = tensor<int32, [4]>([0, 0, 4, 0])];
            tensor<int32, [4]> var_385_end_0 = const()[name = string("op_385_end_0"), val = tensor<int32, [4]>([1, 16, 5, 128])];
            tensor<bool, [4]> var_385_end_mask_0 = const()[name = string("op_385_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_385_squeeze_mask_0 = const()[name = string("op_385_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_385_cast_fp16 = slice_by_index(begin = var_385_begin_0, end = var_385_end_0, end_mask = var_385_end_mask_0, squeeze_mask = var_385_squeeze_mask_0, x = _inversed_query_3_cast_fp16)[name = string("op_385_cast_fp16")];
            tensor<int32, [1]> var_386_axes_0 = const()[name = string("op_386_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_386_cast_fp16 = expand_dims(axes = var_386_axes_0, x = var_385_cast_fp16)[name = string("op_386_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_387_cast_fp16 = mul(x = state_19_cast_fp16, y = var_386_cast_fp16)[name = string("op_387_cast_fp16")];
            tensor<int32, [1]> var_389_axes_0 = const()[name = string("op_389_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_389_keep_dims_0 = const()[name = string("op_389_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_389_cast_fp16 = reduce_sum(axes = var_389_axes_0, keep_dims = var_389_keep_dims_0, x = var_387_cast_fp16)[name = string("op_389_cast_fp16")];
            tensor<int32, [3]> var_392_begin_0 = const()[name = string("op_392_begin_0"), val = tensor<int32, [3]>([0, 0, 5])];
            tensor<int32, [3]> var_392_end_0 = const()[name = string("op_392_end_0"), val = tensor<int32, [3]>([1, 16, 6])];
            tensor<bool, [3]> var_392_end_mask_0 = const()[name = string("op_392_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_392_squeeze_mask_0 = const()[name = string("op_392_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_392_cast_fp16 = slice_by_index(begin = var_392_begin_0, end = var_392_end_0, end_mask = var_392_end_mask_0, squeeze_mask = var_392_squeeze_mask_0, x = decay_1_cast_fp16)[name = string("op_392_cast_fp16")];
            tensor<fp16, [1, 16]> var_393_cast_fp16 = exp(x = var_392_cast_fp16)[name = string("op_393_cast_fp16")];
            tensor<int32, [1]> var_394_axes_0 = const()[name = string("op_394_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_394_cast_fp16 = expand_dims(axes = var_394_axes_0, x = var_393_cast_fp16)[name = string("op_394_cast_fp16")];
            tensor<int32, [1]> var_395_axes_0 = const()[name = string("op_395_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_395_cast_fp16 = expand_dims(axes = var_395_axes_0, x = var_394_cast_fp16)[name = string("op_395_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_21_cast_fp16 = mul(x = state_19_cast_fp16, y = var_395_cast_fp16)[name = string("state_21_cast_fp16")];
            tensor<int32, [4]> key_token_11_begin_0 = const()[name = string("key_token_11_begin_0"), val = tensor<int32, [4]>([0, 0, 5, 0])];
            tensor<int32, [4]> key_token_11_end_0 = const()[name = string("key_token_11_end_0"), val = tensor<int32, [4]>([1, 16, 6, 128])];
            tensor<bool, [4]> key_token_11_end_mask_0 = const()[name = string("key_token_11_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_11_squeeze_mask_0 = const()[name = string("key_token_11_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_11_cast_fp16 = slice_by_index(begin = key_token_11_begin_0, end = key_token_11_end_0, end_mask = key_token_11_end_mask_0, squeeze_mask = key_token_11_squeeze_mask_0, x = key_3_cast_fp16)[name = string("key_token_11_cast_fp16")];
            tensor<int32, [4]> value_token_11_begin_0 = const()[name = string("value_token_11_begin_0"), val = tensor<int32, [4]>([0, 0, 5, 0])];
            tensor<int32, [4]> value_token_11_end_0 = const()[name = string("value_token_11_end_0"), val = tensor<int32, [4]>([1, 16, 6, 128])];
            tensor<bool, [4]> value_token_11_end_mask_0 = const()[name = string("value_token_11_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_11_squeeze_mask_0 = const()[name = string("value_token_11_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_11_cast_fp16 = slice_by_index(begin = value_token_11_begin_0, end = value_token_11_end_0, end_mask = value_token_11_end_mask_0, squeeze_mask = value_token_11_squeeze_mask_0, x = value_11_cast_fp16)[name = string("value_token_11_cast_fp16")];
            tensor<int32, [1]> var_403_axes_0 = const()[name = string("op_403_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_403_cast_fp16 = expand_dims(axes = var_403_axes_0, x = key_token_11_cast_fp16)[name = string("op_403_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_404_cast_fp16 = mul(x = state_21_cast_fp16, y = var_403_cast_fp16)[name = string("op_404_cast_fp16")];
            tensor<int32, [1]> memory_11_axes_0 = const()[name = string("memory_11_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_11_keep_dims_0 = const()[name = string("memory_11_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_11_cast_fp16 = reduce_sum(axes = memory_11_axes_0, keep_dims = memory_11_keep_dims_0, x = var_404_cast_fp16)[name = string("memory_11_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_407_cast_fp16 = sub(x = value_token_11_cast_fp16, y = memory_11_cast_fp16)[name = string("op_407_cast_fp16")];
            tensor<int32, [3]> var_410_begin_0 = const()[name = string("op_410_begin_0"), val = tensor<int32, [3]>([0, 0, 5])];
            tensor<int32, [3]> var_410_end_0 = const()[name = string("op_410_end_0"), val = tensor<int32, [3]>([1, 16, 6])];
            tensor<bool, [3]> var_410_end_mask_0 = const()[name = string("op_410_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_410_squeeze_mask_0 = const()[name = string("op_410_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_410_cast_fp16 = slice_by_index(begin = var_410_begin_0, end = var_410_end_0, end_mask = var_410_end_mask_0, squeeze_mask = var_410_squeeze_mask_0, x = beta_1_cast_fp16)[name = string("op_410_cast_fp16")];
            tensor<int32, [1]> var_411_axes_0 = const()[name = string("op_411_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_411_cast_fp16 = expand_dims(axes = var_411_axes_0, x = var_410_cast_fp16)[name = string("op_411_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_11_cast_fp16 = mul(x = var_407_cast_fp16, y = var_411_cast_fp16)[name = string("delta_11_cast_fp16")];
            tensor<int32, [1]> var_414_axes_0 = const()[name = string("op_414_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_414_cast_fp16 = expand_dims(axes = var_414_axes_0, x = delta_11_cast_fp16)[name = string("op_414_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_415_cast_fp16 = mul(x = var_403_cast_fp16, y = var_414_cast_fp16)[name = string("op_415_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_23_cast_fp16 = add(x = state_21_cast_fp16, y = var_415_cast_fp16)[name = string("state_23_cast_fp16")];
            tensor<int32, [4]> var_419_begin_0 = const()[name = string("op_419_begin_0"), val = tensor<int32, [4]>([0, 0, 5, 0])];
            tensor<int32, [4]> var_419_end_0 = const()[name = string("op_419_end_0"), val = tensor<int32, [4]>([1, 16, 6, 128])];
            tensor<bool, [4]> var_419_end_mask_0 = const()[name = string("op_419_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_419_squeeze_mask_0 = const()[name = string("op_419_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_419_cast_fp16 = slice_by_index(begin = var_419_begin_0, end = var_419_end_0, end_mask = var_419_end_mask_0, squeeze_mask = var_419_squeeze_mask_0, x = _inversed_query_3_cast_fp16)[name = string("op_419_cast_fp16")];
            tensor<int32, [1]> var_420_axes_0 = const()[name = string("op_420_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_420_cast_fp16 = expand_dims(axes = var_420_axes_0, x = var_419_cast_fp16)[name = string("op_420_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_421_cast_fp16 = mul(x = state_23_cast_fp16, y = var_420_cast_fp16)[name = string("op_421_cast_fp16")];
            tensor<int32, [1]> var_423_axes_0 = const()[name = string("op_423_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_423_keep_dims_0 = const()[name = string("op_423_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_423_cast_fp16 = reduce_sum(axes = var_423_axes_0, keep_dims = var_423_keep_dims_0, x = var_421_cast_fp16)[name = string("op_423_cast_fp16")];
            tensor<int32, [3]> var_426_begin_0 = const()[name = string("op_426_begin_0"), val = tensor<int32, [3]>([0, 0, 6])];
            tensor<int32, [3]> var_426_end_0 = const()[name = string("op_426_end_0"), val = tensor<int32, [3]>([1, 16, 7])];
            tensor<bool, [3]> var_426_end_mask_0 = const()[name = string("op_426_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_426_squeeze_mask_0 = const()[name = string("op_426_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_426_cast_fp16 = slice_by_index(begin = var_426_begin_0, end = var_426_end_0, end_mask = var_426_end_mask_0, squeeze_mask = var_426_squeeze_mask_0, x = decay_1_cast_fp16)[name = string("op_426_cast_fp16")];
            tensor<fp16, [1, 16]> var_427_cast_fp16 = exp(x = var_426_cast_fp16)[name = string("op_427_cast_fp16")];
            tensor<int32, [1]> var_428_axes_0 = const()[name = string("op_428_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_428_cast_fp16 = expand_dims(axes = var_428_axes_0, x = var_427_cast_fp16)[name = string("op_428_cast_fp16")];
            tensor<int32, [1]> var_429_axes_0 = const()[name = string("op_429_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_429_cast_fp16 = expand_dims(axes = var_429_axes_0, x = var_428_cast_fp16)[name = string("op_429_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_25_cast_fp16 = mul(x = state_23_cast_fp16, y = var_429_cast_fp16)[name = string("state_25_cast_fp16")];
            tensor<int32, [4]> key_token_13_begin_0 = const()[name = string("key_token_13_begin_0"), val = tensor<int32, [4]>([0, 0, 6, 0])];
            tensor<int32, [4]> key_token_13_end_0 = const()[name = string("key_token_13_end_0"), val = tensor<int32, [4]>([1, 16, 7, 128])];
            tensor<bool, [4]> key_token_13_end_mask_0 = const()[name = string("key_token_13_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_13_squeeze_mask_0 = const()[name = string("key_token_13_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_13_cast_fp16 = slice_by_index(begin = key_token_13_begin_0, end = key_token_13_end_0, end_mask = key_token_13_end_mask_0, squeeze_mask = key_token_13_squeeze_mask_0, x = key_3_cast_fp16)[name = string("key_token_13_cast_fp16")];
            tensor<int32, [4]> value_token_13_begin_0 = const()[name = string("value_token_13_begin_0"), val = tensor<int32, [4]>([0, 0, 6, 0])];
            tensor<int32, [4]> value_token_13_end_0 = const()[name = string("value_token_13_end_0"), val = tensor<int32, [4]>([1, 16, 7, 128])];
            tensor<bool, [4]> value_token_13_end_mask_0 = const()[name = string("value_token_13_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_13_squeeze_mask_0 = const()[name = string("value_token_13_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_13_cast_fp16 = slice_by_index(begin = value_token_13_begin_0, end = value_token_13_end_0, end_mask = value_token_13_end_mask_0, squeeze_mask = value_token_13_squeeze_mask_0, x = value_11_cast_fp16)[name = string("value_token_13_cast_fp16")];
            tensor<int32, [1]> var_437_axes_0 = const()[name = string("op_437_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_437_cast_fp16 = expand_dims(axes = var_437_axes_0, x = key_token_13_cast_fp16)[name = string("op_437_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_438_cast_fp16 = mul(x = state_25_cast_fp16, y = var_437_cast_fp16)[name = string("op_438_cast_fp16")];
            tensor<int32, [1]> memory_13_axes_0 = const()[name = string("memory_13_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_13_keep_dims_0 = const()[name = string("memory_13_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_13_cast_fp16 = reduce_sum(axes = memory_13_axes_0, keep_dims = memory_13_keep_dims_0, x = var_438_cast_fp16)[name = string("memory_13_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_441_cast_fp16 = sub(x = value_token_13_cast_fp16, y = memory_13_cast_fp16)[name = string("op_441_cast_fp16")];
            tensor<int32, [3]> var_444_begin_0 = const()[name = string("op_444_begin_0"), val = tensor<int32, [3]>([0, 0, 6])];
            tensor<int32, [3]> var_444_end_0 = const()[name = string("op_444_end_0"), val = tensor<int32, [3]>([1, 16, 7])];
            tensor<bool, [3]> var_444_end_mask_0 = const()[name = string("op_444_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_444_squeeze_mask_0 = const()[name = string("op_444_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_444_cast_fp16 = slice_by_index(begin = var_444_begin_0, end = var_444_end_0, end_mask = var_444_end_mask_0, squeeze_mask = var_444_squeeze_mask_0, x = beta_1_cast_fp16)[name = string("op_444_cast_fp16")];
            tensor<int32, [1]> var_445_axes_0 = const()[name = string("op_445_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_445_cast_fp16 = expand_dims(axes = var_445_axes_0, x = var_444_cast_fp16)[name = string("op_445_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_13_cast_fp16 = mul(x = var_441_cast_fp16, y = var_445_cast_fp16)[name = string("delta_13_cast_fp16")];
            tensor<int32, [1]> var_448_axes_0 = const()[name = string("op_448_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_448_cast_fp16 = expand_dims(axes = var_448_axes_0, x = delta_13_cast_fp16)[name = string("op_448_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_449_cast_fp16 = mul(x = var_437_cast_fp16, y = var_448_cast_fp16)[name = string("op_449_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_27_cast_fp16 = add(x = state_25_cast_fp16, y = var_449_cast_fp16)[name = string("state_27_cast_fp16")];
            tensor<int32, [4]> var_453_begin_0 = const()[name = string("op_453_begin_0"), val = tensor<int32, [4]>([0, 0, 6, 0])];
            tensor<int32, [4]> var_453_end_0 = const()[name = string("op_453_end_0"), val = tensor<int32, [4]>([1, 16, 7, 128])];
            tensor<bool, [4]> var_453_end_mask_0 = const()[name = string("op_453_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_453_squeeze_mask_0 = const()[name = string("op_453_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_453_cast_fp16 = slice_by_index(begin = var_453_begin_0, end = var_453_end_0, end_mask = var_453_end_mask_0, squeeze_mask = var_453_squeeze_mask_0, x = _inversed_query_3_cast_fp16)[name = string("op_453_cast_fp16")];
            tensor<int32, [1]> var_454_axes_0 = const()[name = string("op_454_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_454_cast_fp16 = expand_dims(axes = var_454_axes_0, x = var_453_cast_fp16)[name = string("op_454_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_455_cast_fp16 = mul(x = state_27_cast_fp16, y = var_454_cast_fp16)[name = string("op_455_cast_fp16")];
            tensor<int32, [1]> var_457_axes_0 = const()[name = string("op_457_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_457_keep_dims_0 = const()[name = string("op_457_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_457_cast_fp16 = reduce_sum(axes = var_457_axes_0, keep_dims = var_457_keep_dims_0, x = var_455_cast_fp16)[name = string("op_457_cast_fp16")];
            tensor<int32, [3]> var_460_begin_0 = const()[name = string("op_460_begin_0"), val = tensor<int32, [3]>([0, 0, 7])];
            tensor<int32, [3]> var_460_end_0 = const()[name = string("op_460_end_0"), val = tensor<int32, [3]>([1, 16, 8])];
            tensor<bool, [3]> var_460_end_mask_0 = const()[name = string("op_460_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_460_squeeze_mask_0 = const()[name = string("op_460_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_460_cast_fp16 = slice_by_index(begin = var_460_begin_0, end = var_460_end_0, end_mask = var_460_end_mask_0, squeeze_mask = var_460_squeeze_mask_0, x = decay_1_cast_fp16)[name = string("op_460_cast_fp16")];
            tensor<fp16, [1, 16]> var_461_cast_fp16 = exp(x = var_460_cast_fp16)[name = string("op_461_cast_fp16")];
            tensor<int32, [1]> var_462_axes_0 = const()[name = string("op_462_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_462_cast_fp16 = expand_dims(axes = var_462_axes_0, x = var_461_cast_fp16)[name = string("op_462_cast_fp16")];
            tensor<int32, [1]> var_463_axes_0 = const()[name = string("op_463_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_463_cast_fp16 = expand_dims(axes = var_463_axes_0, x = var_462_cast_fp16)[name = string("op_463_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_29_cast_fp16 = mul(x = state_27_cast_fp16, y = var_463_cast_fp16)[name = string("state_29_cast_fp16")];
            tensor<int32, [4]> key_token_15_begin_0 = const()[name = string("key_token_15_begin_0"), val = tensor<int32, [4]>([0, 0, 7, 0])];
            tensor<int32, [4]> key_token_15_end_0 = const()[name = string("key_token_15_end_0"), val = tensor<int32, [4]>([1, 16, 8, 128])];
            tensor<bool, [4]> key_token_15_end_mask_0 = const()[name = string("key_token_15_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_15_squeeze_mask_0 = const()[name = string("key_token_15_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_15_cast_fp16 = slice_by_index(begin = key_token_15_begin_0, end = key_token_15_end_0, end_mask = key_token_15_end_mask_0, squeeze_mask = key_token_15_squeeze_mask_0, x = key_3_cast_fp16)[name = string("key_token_15_cast_fp16")];
            tensor<int32, [4]> value_token_15_begin_0 = const()[name = string("value_token_15_begin_0"), val = tensor<int32, [4]>([0, 0, 7, 0])];
            tensor<int32, [4]> value_token_15_end_0 = const()[name = string("value_token_15_end_0"), val = tensor<int32, [4]>([1, 16, 8, 128])];
            tensor<bool, [4]> value_token_15_end_mask_0 = const()[name = string("value_token_15_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_15_squeeze_mask_0 = const()[name = string("value_token_15_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_15_cast_fp16 = slice_by_index(begin = value_token_15_begin_0, end = value_token_15_end_0, end_mask = value_token_15_end_mask_0, squeeze_mask = value_token_15_squeeze_mask_0, x = value_11_cast_fp16)[name = string("value_token_15_cast_fp16")];
            tensor<int32, [1]> var_471_axes_0 = const()[name = string("op_471_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_471_cast_fp16 = expand_dims(axes = var_471_axes_0, x = key_token_15_cast_fp16)[name = string("op_471_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_472_cast_fp16 = mul(x = state_29_cast_fp16, y = var_471_cast_fp16)[name = string("op_472_cast_fp16")];
            tensor<int32, [1]> memory_15_axes_0 = const()[name = string("memory_15_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_15_keep_dims_0 = const()[name = string("memory_15_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_15_cast_fp16 = reduce_sum(axes = memory_15_axes_0, keep_dims = memory_15_keep_dims_0, x = var_472_cast_fp16)[name = string("memory_15_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_475_cast_fp16 = sub(x = value_token_15_cast_fp16, y = memory_15_cast_fp16)[name = string("op_475_cast_fp16")];
            tensor<int32, [3]> var_478_begin_0 = const()[name = string("op_478_begin_0"), val = tensor<int32, [3]>([0, 0, 7])];
            tensor<int32, [3]> var_478_end_0 = const()[name = string("op_478_end_0"), val = tensor<int32, [3]>([1, 16, 8])];
            tensor<bool, [3]> var_478_end_mask_0 = const()[name = string("op_478_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_478_squeeze_mask_0 = const()[name = string("op_478_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_478_cast_fp16 = slice_by_index(begin = var_478_begin_0, end = var_478_end_0, end_mask = var_478_end_mask_0, squeeze_mask = var_478_squeeze_mask_0, x = beta_1_cast_fp16)[name = string("op_478_cast_fp16")];
            tensor<int32, [1]> var_479_axes_0 = const()[name = string("op_479_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_479_cast_fp16 = expand_dims(axes = var_479_axes_0, x = var_478_cast_fp16)[name = string("op_479_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_15_cast_fp16 = mul(x = var_475_cast_fp16, y = var_479_cast_fp16)[name = string("delta_15_cast_fp16")];
            tensor<int32, [1]> var_482_axes_0 = const()[name = string("op_482_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_482_cast_fp16 = expand_dims(axes = var_482_axes_0, x = delta_15_cast_fp16)[name = string("op_482_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_483_cast_fp16 = mul(x = var_471_cast_fp16, y = var_482_cast_fp16)[name = string("op_483_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_31_cast_fp16 = add(x = state_29_cast_fp16, y = var_483_cast_fp16)[name = string("state_31_cast_fp16")];
            tensor<int32, [4]> var_487_begin_0 = const()[name = string("op_487_begin_0"), val = tensor<int32, [4]>([0, 0, 7, 0])];
            tensor<int32, [4]> var_487_end_0 = const()[name = string("op_487_end_0"), val = tensor<int32, [4]>([1, 16, 8, 128])];
            tensor<bool, [4]> var_487_end_mask_0 = const()[name = string("op_487_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_487_squeeze_mask_0 = const()[name = string("op_487_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_487_cast_fp16 = slice_by_index(begin = var_487_begin_0, end = var_487_end_0, end_mask = var_487_end_mask_0, squeeze_mask = var_487_squeeze_mask_0, x = _inversed_query_3_cast_fp16)[name = string("op_487_cast_fp16")];
            tensor<int32, [1]> var_488_axes_0 = const()[name = string("op_488_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_488_cast_fp16 = expand_dims(axes = var_488_axes_0, x = var_487_cast_fp16)[name = string("op_488_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_489_cast_fp16 = mul(x = state_31_cast_fp16, y = var_488_cast_fp16)[name = string("op_489_cast_fp16")];
            tensor<int32, [1]> var_491_axes_0 = const()[name = string("op_491_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_491_keep_dims_0 = const()[name = string("op_491_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_491_cast_fp16 = reduce_sum(axes = var_491_axes_0, keep_dims = var_491_keep_dims_0, x = var_489_cast_fp16)[name = string("op_491_cast_fp16")];
            tensor<int32, [3]> var_494_begin_0 = const()[name = string("op_494_begin_0"), val = tensor<int32, [3]>([0, 0, 8])];
            tensor<int32, [3]> var_494_end_0 = const()[name = string("op_494_end_0"), val = tensor<int32, [3]>([1, 16, 9])];
            tensor<bool, [3]> var_494_end_mask_0 = const()[name = string("op_494_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_494_squeeze_mask_0 = const()[name = string("op_494_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_494_cast_fp16 = slice_by_index(begin = var_494_begin_0, end = var_494_end_0, end_mask = var_494_end_mask_0, squeeze_mask = var_494_squeeze_mask_0, x = decay_1_cast_fp16)[name = string("op_494_cast_fp16")];
            tensor<fp16, [1, 16]> var_495_cast_fp16 = exp(x = var_494_cast_fp16)[name = string("op_495_cast_fp16")];
            tensor<int32, [1]> var_496_axes_0 = const()[name = string("op_496_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_496_cast_fp16 = expand_dims(axes = var_496_axes_0, x = var_495_cast_fp16)[name = string("op_496_cast_fp16")];
            tensor<int32, [1]> var_497_axes_0 = const()[name = string("op_497_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_497_cast_fp16 = expand_dims(axes = var_497_axes_0, x = var_496_cast_fp16)[name = string("op_497_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_33_cast_fp16 = mul(x = state_31_cast_fp16, y = var_497_cast_fp16)[name = string("state_33_cast_fp16")];
            tensor<int32, [4]> key_token_17_begin_0 = const()[name = string("key_token_17_begin_0"), val = tensor<int32, [4]>([0, 0, 8, 0])];
            tensor<int32, [4]> key_token_17_end_0 = const()[name = string("key_token_17_end_0"), val = tensor<int32, [4]>([1, 16, 9, 128])];
            tensor<bool, [4]> key_token_17_end_mask_0 = const()[name = string("key_token_17_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_17_squeeze_mask_0 = const()[name = string("key_token_17_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_17_cast_fp16 = slice_by_index(begin = key_token_17_begin_0, end = key_token_17_end_0, end_mask = key_token_17_end_mask_0, squeeze_mask = key_token_17_squeeze_mask_0, x = key_3_cast_fp16)[name = string("key_token_17_cast_fp16")];
            tensor<int32, [4]> value_token_17_begin_0 = const()[name = string("value_token_17_begin_0"), val = tensor<int32, [4]>([0, 0, 8, 0])];
            tensor<int32, [4]> value_token_17_end_0 = const()[name = string("value_token_17_end_0"), val = tensor<int32, [4]>([1, 16, 9, 128])];
            tensor<bool, [4]> value_token_17_end_mask_0 = const()[name = string("value_token_17_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_17_squeeze_mask_0 = const()[name = string("value_token_17_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_17_cast_fp16 = slice_by_index(begin = value_token_17_begin_0, end = value_token_17_end_0, end_mask = value_token_17_end_mask_0, squeeze_mask = value_token_17_squeeze_mask_0, x = value_11_cast_fp16)[name = string("value_token_17_cast_fp16")];
            tensor<int32, [1]> var_505_axes_0 = const()[name = string("op_505_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_505_cast_fp16 = expand_dims(axes = var_505_axes_0, x = key_token_17_cast_fp16)[name = string("op_505_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_506_cast_fp16 = mul(x = state_33_cast_fp16, y = var_505_cast_fp16)[name = string("op_506_cast_fp16")];
            tensor<int32, [1]> memory_17_axes_0 = const()[name = string("memory_17_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_17_keep_dims_0 = const()[name = string("memory_17_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_17_cast_fp16 = reduce_sum(axes = memory_17_axes_0, keep_dims = memory_17_keep_dims_0, x = var_506_cast_fp16)[name = string("memory_17_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_509_cast_fp16 = sub(x = value_token_17_cast_fp16, y = memory_17_cast_fp16)[name = string("op_509_cast_fp16")];
            tensor<int32, [3]> var_512_begin_0 = const()[name = string("op_512_begin_0"), val = tensor<int32, [3]>([0, 0, 8])];
            tensor<int32, [3]> var_512_end_0 = const()[name = string("op_512_end_0"), val = tensor<int32, [3]>([1, 16, 9])];
            tensor<bool, [3]> var_512_end_mask_0 = const()[name = string("op_512_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_512_squeeze_mask_0 = const()[name = string("op_512_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_512_cast_fp16 = slice_by_index(begin = var_512_begin_0, end = var_512_end_0, end_mask = var_512_end_mask_0, squeeze_mask = var_512_squeeze_mask_0, x = beta_1_cast_fp16)[name = string("op_512_cast_fp16")];
            tensor<int32, [1]> var_513_axes_0 = const()[name = string("op_513_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_513_cast_fp16 = expand_dims(axes = var_513_axes_0, x = var_512_cast_fp16)[name = string("op_513_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_17_cast_fp16 = mul(x = var_509_cast_fp16, y = var_513_cast_fp16)[name = string("delta_17_cast_fp16")];
            tensor<int32, [1]> var_516_axes_0 = const()[name = string("op_516_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_516_cast_fp16 = expand_dims(axes = var_516_axes_0, x = delta_17_cast_fp16)[name = string("op_516_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_517_cast_fp16 = mul(x = var_505_cast_fp16, y = var_516_cast_fp16)[name = string("op_517_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_35_cast_fp16 = add(x = state_33_cast_fp16, y = var_517_cast_fp16)[name = string("state_35_cast_fp16")];
            tensor<int32, [4]> var_521_begin_0 = const()[name = string("op_521_begin_0"), val = tensor<int32, [4]>([0, 0, 8, 0])];
            tensor<int32, [4]> var_521_end_0 = const()[name = string("op_521_end_0"), val = tensor<int32, [4]>([1, 16, 9, 128])];
            tensor<bool, [4]> var_521_end_mask_0 = const()[name = string("op_521_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_521_squeeze_mask_0 = const()[name = string("op_521_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_521_cast_fp16 = slice_by_index(begin = var_521_begin_0, end = var_521_end_0, end_mask = var_521_end_mask_0, squeeze_mask = var_521_squeeze_mask_0, x = _inversed_query_3_cast_fp16)[name = string("op_521_cast_fp16")];
            tensor<int32, [1]> var_522_axes_0 = const()[name = string("op_522_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_522_cast_fp16 = expand_dims(axes = var_522_axes_0, x = var_521_cast_fp16)[name = string("op_522_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_523_cast_fp16 = mul(x = state_35_cast_fp16, y = var_522_cast_fp16)[name = string("op_523_cast_fp16")];
            tensor<int32, [1]> var_525_axes_0 = const()[name = string("op_525_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_525_keep_dims_0 = const()[name = string("op_525_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_525_cast_fp16 = reduce_sum(axes = var_525_axes_0, keep_dims = var_525_keep_dims_0, x = var_523_cast_fp16)[name = string("op_525_cast_fp16")];
            tensor<int32, [3]> var_528_begin_0 = const()[name = string("op_528_begin_0"), val = tensor<int32, [3]>([0, 0, 9])];
            tensor<int32, [3]> var_528_end_0 = const()[name = string("op_528_end_0"), val = tensor<int32, [3]>([1, 16, 10])];
            tensor<bool, [3]> var_528_end_mask_0 = const()[name = string("op_528_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_528_squeeze_mask_0 = const()[name = string("op_528_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_528_cast_fp16 = slice_by_index(begin = var_528_begin_0, end = var_528_end_0, end_mask = var_528_end_mask_0, squeeze_mask = var_528_squeeze_mask_0, x = decay_1_cast_fp16)[name = string("op_528_cast_fp16")];
            tensor<fp16, [1, 16]> var_529_cast_fp16 = exp(x = var_528_cast_fp16)[name = string("op_529_cast_fp16")];
            tensor<int32, [1]> var_530_axes_0 = const()[name = string("op_530_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_530_cast_fp16 = expand_dims(axes = var_530_axes_0, x = var_529_cast_fp16)[name = string("op_530_cast_fp16")];
            tensor<int32, [1]> var_531_axes_0 = const()[name = string("op_531_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_531_cast_fp16 = expand_dims(axes = var_531_axes_0, x = var_530_cast_fp16)[name = string("op_531_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_37_cast_fp16 = mul(x = state_35_cast_fp16, y = var_531_cast_fp16)[name = string("state_37_cast_fp16")];
            tensor<int32, [4]> key_token_19_begin_0 = const()[name = string("key_token_19_begin_0"), val = tensor<int32, [4]>([0, 0, 9, 0])];
            tensor<int32, [4]> key_token_19_end_0 = const()[name = string("key_token_19_end_0"), val = tensor<int32, [4]>([1, 16, 10, 128])];
            tensor<bool, [4]> key_token_19_end_mask_0 = const()[name = string("key_token_19_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_19_squeeze_mask_0 = const()[name = string("key_token_19_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_19_cast_fp16 = slice_by_index(begin = key_token_19_begin_0, end = key_token_19_end_0, end_mask = key_token_19_end_mask_0, squeeze_mask = key_token_19_squeeze_mask_0, x = key_3_cast_fp16)[name = string("key_token_19_cast_fp16")];
            tensor<int32, [4]> value_token_19_begin_0 = const()[name = string("value_token_19_begin_0"), val = tensor<int32, [4]>([0, 0, 9, 0])];
            tensor<int32, [4]> value_token_19_end_0 = const()[name = string("value_token_19_end_0"), val = tensor<int32, [4]>([1, 16, 10, 128])];
            tensor<bool, [4]> value_token_19_end_mask_0 = const()[name = string("value_token_19_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_19_squeeze_mask_0 = const()[name = string("value_token_19_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_19_cast_fp16 = slice_by_index(begin = value_token_19_begin_0, end = value_token_19_end_0, end_mask = value_token_19_end_mask_0, squeeze_mask = value_token_19_squeeze_mask_0, x = value_11_cast_fp16)[name = string("value_token_19_cast_fp16")];
            tensor<int32, [1]> var_539_axes_0 = const()[name = string("op_539_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_539_cast_fp16 = expand_dims(axes = var_539_axes_0, x = key_token_19_cast_fp16)[name = string("op_539_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_540_cast_fp16 = mul(x = state_37_cast_fp16, y = var_539_cast_fp16)[name = string("op_540_cast_fp16")];
            tensor<int32, [1]> memory_19_axes_0 = const()[name = string("memory_19_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_19_keep_dims_0 = const()[name = string("memory_19_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_19_cast_fp16 = reduce_sum(axes = memory_19_axes_0, keep_dims = memory_19_keep_dims_0, x = var_540_cast_fp16)[name = string("memory_19_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_543_cast_fp16 = sub(x = value_token_19_cast_fp16, y = memory_19_cast_fp16)[name = string("op_543_cast_fp16")];
            tensor<int32, [3]> var_546_begin_0 = const()[name = string("op_546_begin_0"), val = tensor<int32, [3]>([0, 0, 9])];
            tensor<int32, [3]> var_546_end_0 = const()[name = string("op_546_end_0"), val = tensor<int32, [3]>([1, 16, 10])];
            tensor<bool, [3]> var_546_end_mask_0 = const()[name = string("op_546_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_546_squeeze_mask_0 = const()[name = string("op_546_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_546_cast_fp16 = slice_by_index(begin = var_546_begin_0, end = var_546_end_0, end_mask = var_546_end_mask_0, squeeze_mask = var_546_squeeze_mask_0, x = beta_1_cast_fp16)[name = string("op_546_cast_fp16")];
            tensor<int32, [1]> var_547_axes_0 = const()[name = string("op_547_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_547_cast_fp16 = expand_dims(axes = var_547_axes_0, x = var_546_cast_fp16)[name = string("op_547_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_19_cast_fp16 = mul(x = var_543_cast_fp16, y = var_547_cast_fp16)[name = string("delta_19_cast_fp16")];
            tensor<int32, [1]> var_550_axes_0 = const()[name = string("op_550_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_550_cast_fp16 = expand_dims(axes = var_550_axes_0, x = delta_19_cast_fp16)[name = string("op_550_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_551_cast_fp16 = mul(x = var_539_cast_fp16, y = var_550_cast_fp16)[name = string("op_551_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_39_cast_fp16 = add(x = state_37_cast_fp16, y = var_551_cast_fp16)[name = string("state_39_cast_fp16")];
            tensor<int32, [4]> var_555_begin_0 = const()[name = string("op_555_begin_0"), val = tensor<int32, [4]>([0, 0, 9, 0])];
            tensor<int32, [4]> var_555_end_0 = const()[name = string("op_555_end_0"), val = tensor<int32, [4]>([1, 16, 10, 128])];
            tensor<bool, [4]> var_555_end_mask_0 = const()[name = string("op_555_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_555_squeeze_mask_0 = const()[name = string("op_555_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_555_cast_fp16 = slice_by_index(begin = var_555_begin_0, end = var_555_end_0, end_mask = var_555_end_mask_0, squeeze_mask = var_555_squeeze_mask_0, x = _inversed_query_3_cast_fp16)[name = string("op_555_cast_fp16")];
            tensor<int32, [1]> var_556_axes_0 = const()[name = string("op_556_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_556_cast_fp16 = expand_dims(axes = var_556_axes_0, x = var_555_cast_fp16)[name = string("op_556_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_557_cast_fp16 = mul(x = state_39_cast_fp16, y = var_556_cast_fp16)[name = string("op_557_cast_fp16")];
            tensor<int32, [1]> var_559_axes_0 = const()[name = string("op_559_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_559_keep_dims_0 = const()[name = string("op_559_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_559_cast_fp16 = reduce_sum(axes = var_559_axes_0, keep_dims = var_559_keep_dims_0, x = var_557_cast_fp16)[name = string("op_559_cast_fp16")];
            tensor<int32, [3]> var_562_begin_0 = const()[name = string("op_562_begin_0"), val = tensor<int32, [3]>([0, 0, 10])];
            tensor<int32, [3]> var_562_end_0 = const()[name = string("op_562_end_0"), val = tensor<int32, [3]>([1, 16, 11])];
            tensor<bool, [3]> var_562_end_mask_0 = const()[name = string("op_562_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_562_squeeze_mask_0 = const()[name = string("op_562_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_562_cast_fp16 = slice_by_index(begin = var_562_begin_0, end = var_562_end_0, end_mask = var_562_end_mask_0, squeeze_mask = var_562_squeeze_mask_0, x = decay_1_cast_fp16)[name = string("op_562_cast_fp16")];
            tensor<fp16, [1, 16]> var_563_cast_fp16 = exp(x = var_562_cast_fp16)[name = string("op_563_cast_fp16")];
            tensor<int32, [1]> var_564_axes_0 = const()[name = string("op_564_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_564_cast_fp16 = expand_dims(axes = var_564_axes_0, x = var_563_cast_fp16)[name = string("op_564_cast_fp16")];
            tensor<int32, [1]> var_565_axes_0 = const()[name = string("op_565_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_565_cast_fp16 = expand_dims(axes = var_565_axes_0, x = var_564_cast_fp16)[name = string("op_565_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_41_cast_fp16 = mul(x = state_39_cast_fp16, y = var_565_cast_fp16)[name = string("state_41_cast_fp16")];
            tensor<int32, [4]> key_token_21_begin_0 = const()[name = string("key_token_21_begin_0"), val = tensor<int32, [4]>([0, 0, 10, 0])];
            tensor<int32, [4]> key_token_21_end_0 = const()[name = string("key_token_21_end_0"), val = tensor<int32, [4]>([1, 16, 11, 128])];
            tensor<bool, [4]> key_token_21_end_mask_0 = const()[name = string("key_token_21_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_21_squeeze_mask_0 = const()[name = string("key_token_21_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_21_cast_fp16 = slice_by_index(begin = key_token_21_begin_0, end = key_token_21_end_0, end_mask = key_token_21_end_mask_0, squeeze_mask = key_token_21_squeeze_mask_0, x = key_3_cast_fp16)[name = string("key_token_21_cast_fp16")];
            tensor<int32, [4]> value_token_21_begin_0 = const()[name = string("value_token_21_begin_0"), val = tensor<int32, [4]>([0, 0, 10, 0])];
            tensor<int32, [4]> value_token_21_end_0 = const()[name = string("value_token_21_end_0"), val = tensor<int32, [4]>([1, 16, 11, 128])];
            tensor<bool, [4]> value_token_21_end_mask_0 = const()[name = string("value_token_21_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_21_squeeze_mask_0 = const()[name = string("value_token_21_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_21_cast_fp16 = slice_by_index(begin = value_token_21_begin_0, end = value_token_21_end_0, end_mask = value_token_21_end_mask_0, squeeze_mask = value_token_21_squeeze_mask_0, x = value_11_cast_fp16)[name = string("value_token_21_cast_fp16")];
            tensor<int32, [1]> var_573_axes_0 = const()[name = string("op_573_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_573_cast_fp16 = expand_dims(axes = var_573_axes_0, x = key_token_21_cast_fp16)[name = string("op_573_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_574_cast_fp16 = mul(x = state_41_cast_fp16, y = var_573_cast_fp16)[name = string("op_574_cast_fp16")];
            tensor<int32, [1]> memory_21_axes_0 = const()[name = string("memory_21_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_21_keep_dims_0 = const()[name = string("memory_21_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_21_cast_fp16 = reduce_sum(axes = memory_21_axes_0, keep_dims = memory_21_keep_dims_0, x = var_574_cast_fp16)[name = string("memory_21_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_577_cast_fp16 = sub(x = value_token_21_cast_fp16, y = memory_21_cast_fp16)[name = string("op_577_cast_fp16")];
            tensor<int32, [3]> var_580_begin_0 = const()[name = string("op_580_begin_0"), val = tensor<int32, [3]>([0, 0, 10])];
            tensor<int32, [3]> var_580_end_0 = const()[name = string("op_580_end_0"), val = tensor<int32, [3]>([1, 16, 11])];
            tensor<bool, [3]> var_580_end_mask_0 = const()[name = string("op_580_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_580_squeeze_mask_0 = const()[name = string("op_580_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_580_cast_fp16 = slice_by_index(begin = var_580_begin_0, end = var_580_end_0, end_mask = var_580_end_mask_0, squeeze_mask = var_580_squeeze_mask_0, x = beta_1_cast_fp16)[name = string("op_580_cast_fp16")];
            tensor<int32, [1]> var_581_axes_0 = const()[name = string("op_581_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_581_cast_fp16 = expand_dims(axes = var_581_axes_0, x = var_580_cast_fp16)[name = string("op_581_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_21_cast_fp16 = mul(x = var_577_cast_fp16, y = var_581_cast_fp16)[name = string("delta_21_cast_fp16")];
            tensor<int32, [1]> var_584_axes_0 = const()[name = string("op_584_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_584_cast_fp16 = expand_dims(axes = var_584_axes_0, x = delta_21_cast_fp16)[name = string("op_584_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_585_cast_fp16 = mul(x = var_573_cast_fp16, y = var_584_cast_fp16)[name = string("op_585_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_43_cast_fp16 = add(x = state_41_cast_fp16, y = var_585_cast_fp16)[name = string("state_43_cast_fp16")];
            tensor<int32, [4]> var_589_begin_0 = const()[name = string("op_589_begin_0"), val = tensor<int32, [4]>([0, 0, 10, 0])];
            tensor<int32, [4]> var_589_end_0 = const()[name = string("op_589_end_0"), val = tensor<int32, [4]>([1, 16, 11, 128])];
            tensor<bool, [4]> var_589_end_mask_0 = const()[name = string("op_589_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_589_squeeze_mask_0 = const()[name = string("op_589_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_589_cast_fp16 = slice_by_index(begin = var_589_begin_0, end = var_589_end_0, end_mask = var_589_end_mask_0, squeeze_mask = var_589_squeeze_mask_0, x = _inversed_query_3_cast_fp16)[name = string("op_589_cast_fp16")];
            tensor<int32, [1]> var_590_axes_0 = const()[name = string("op_590_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_590_cast_fp16 = expand_dims(axes = var_590_axes_0, x = var_589_cast_fp16)[name = string("op_590_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_591_cast_fp16 = mul(x = state_43_cast_fp16, y = var_590_cast_fp16)[name = string("op_591_cast_fp16")];
            tensor<int32, [1]> var_593_axes_0 = const()[name = string("op_593_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_593_keep_dims_0 = const()[name = string("op_593_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_593_cast_fp16 = reduce_sum(axes = var_593_axes_0, keep_dims = var_593_keep_dims_0, x = var_591_cast_fp16)[name = string("op_593_cast_fp16")];
            tensor<int32, [3]> var_596_begin_0 = const()[name = string("op_596_begin_0"), val = tensor<int32, [3]>([0, 0, 11])];
            tensor<int32, [3]> var_596_end_0 = const()[name = string("op_596_end_0"), val = tensor<int32, [3]>([1, 16, 12])];
            tensor<bool, [3]> var_596_end_mask_0 = const()[name = string("op_596_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_596_squeeze_mask_0 = const()[name = string("op_596_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_596_cast_fp16 = slice_by_index(begin = var_596_begin_0, end = var_596_end_0, end_mask = var_596_end_mask_0, squeeze_mask = var_596_squeeze_mask_0, x = decay_1_cast_fp16)[name = string("op_596_cast_fp16")];
            tensor<fp16, [1, 16]> var_597_cast_fp16 = exp(x = var_596_cast_fp16)[name = string("op_597_cast_fp16")];
            tensor<int32, [1]> var_598_axes_0 = const()[name = string("op_598_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_598_cast_fp16 = expand_dims(axes = var_598_axes_0, x = var_597_cast_fp16)[name = string("op_598_cast_fp16")];
            tensor<int32, [1]> var_599_axes_0 = const()[name = string("op_599_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_599_cast_fp16 = expand_dims(axes = var_599_axes_0, x = var_598_cast_fp16)[name = string("op_599_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_45_cast_fp16 = mul(x = state_43_cast_fp16, y = var_599_cast_fp16)[name = string("state_45_cast_fp16")];
            tensor<int32, [4]> key_token_23_begin_0 = const()[name = string("key_token_23_begin_0"), val = tensor<int32, [4]>([0, 0, 11, 0])];
            tensor<int32, [4]> key_token_23_end_0 = const()[name = string("key_token_23_end_0"), val = tensor<int32, [4]>([1, 16, 12, 128])];
            tensor<bool, [4]> key_token_23_end_mask_0 = const()[name = string("key_token_23_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_23_squeeze_mask_0 = const()[name = string("key_token_23_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_23_cast_fp16 = slice_by_index(begin = key_token_23_begin_0, end = key_token_23_end_0, end_mask = key_token_23_end_mask_0, squeeze_mask = key_token_23_squeeze_mask_0, x = key_3_cast_fp16)[name = string("key_token_23_cast_fp16")];
            tensor<int32, [4]> value_token_23_begin_0 = const()[name = string("value_token_23_begin_0"), val = tensor<int32, [4]>([0, 0, 11, 0])];
            tensor<int32, [4]> value_token_23_end_0 = const()[name = string("value_token_23_end_0"), val = tensor<int32, [4]>([1, 16, 12, 128])];
            tensor<bool, [4]> value_token_23_end_mask_0 = const()[name = string("value_token_23_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_23_squeeze_mask_0 = const()[name = string("value_token_23_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_23_cast_fp16 = slice_by_index(begin = value_token_23_begin_0, end = value_token_23_end_0, end_mask = value_token_23_end_mask_0, squeeze_mask = value_token_23_squeeze_mask_0, x = value_11_cast_fp16)[name = string("value_token_23_cast_fp16")];
            tensor<int32, [1]> var_607_axes_0 = const()[name = string("op_607_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_607_cast_fp16 = expand_dims(axes = var_607_axes_0, x = key_token_23_cast_fp16)[name = string("op_607_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_608_cast_fp16 = mul(x = state_45_cast_fp16, y = var_607_cast_fp16)[name = string("op_608_cast_fp16")];
            tensor<int32, [1]> memory_23_axes_0 = const()[name = string("memory_23_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_23_keep_dims_0 = const()[name = string("memory_23_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_23_cast_fp16 = reduce_sum(axes = memory_23_axes_0, keep_dims = memory_23_keep_dims_0, x = var_608_cast_fp16)[name = string("memory_23_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_611_cast_fp16 = sub(x = value_token_23_cast_fp16, y = memory_23_cast_fp16)[name = string("op_611_cast_fp16")];
            tensor<int32, [3]> var_614_begin_0 = const()[name = string("op_614_begin_0"), val = tensor<int32, [3]>([0, 0, 11])];
            tensor<int32, [3]> var_614_end_0 = const()[name = string("op_614_end_0"), val = tensor<int32, [3]>([1, 16, 12])];
            tensor<bool, [3]> var_614_end_mask_0 = const()[name = string("op_614_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_614_squeeze_mask_0 = const()[name = string("op_614_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_614_cast_fp16 = slice_by_index(begin = var_614_begin_0, end = var_614_end_0, end_mask = var_614_end_mask_0, squeeze_mask = var_614_squeeze_mask_0, x = beta_1_cast_fp16)[name = string("op_614_cast_fp16")];
            tensor<int32, [1]> var_615_axes_0 = const()[name = string("op_615_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_615_cast_fp16 = expand_dims(axes = var_615_axes_0, x = var_614_cast_fp16)[name = string("op_615_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_23_cast_fp16 = mul(x = var_611_cast_fp16, y = var_615_cast_fp16)[name = string("delta_23_cast_fp16")];
            tensor<int32, [1]> var_618_axes_0 = const()[name = string("op_618_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_618_cast_fp16 = expand_dims(axes = var_618_axes_0, x = delta_23_cast_fp16)[name = string("op_618_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_619_cast_fp16 = mul(x = var_607_cast_fp16, y = var_618_cast_fp16)[name = string("op_619_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_47_cast_fp16 = add(x = state_45_cast_fp16, y = var_619_cast_fp16)[name = string("state_47_cast_fp16")];
            tensor<int32, [4]> var_623_begin_0 = const()[name = string("op_623_begin_0"), val = tensor<int32, [4]>([0, 0, 11, 0])];
            tensor<int32, [4]> var_623_end_0 = const()[name = string("op_623_end_0"), val = tensor<int32, [4]>([1, 16, 12, 128])];
            tensor<bool, [4]> var_623_end_mask_0 = const()[name = string("op_623_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_623_squeeze_mask_0 = const()[name = string("op_623_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_623_cast_fp16 = slice_by_index(begin = var_623_begin_0, end = var_623_end_0, end_mask = var_623_end_mask_0, squeeze_mask = var_623_squeeze_mask_0, x = _inversed_query_3_cast_fp16)[name = string("op_623_cast_fp16")];
            tensor<int32, [1]> var_624_axes_0 = const()[name = string("op_624_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_624_cast_fp16 = expand_dims(axes = var_624_axes_0, x = var_623_cast_fp16)[name = string("op_624_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_625_cast_fp16 = mul(x = state_47_cast_fp16, y = var_624_cast_fp16)[name = string("op_625_cast_fp16")];
            tensor<int32, [1]> var_627_axes_0 = const()[name = string("op_627_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_627_keep_dims_0 = const()[name = string("op_627_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_627_cast_fp16 = reduce_sum(axes = var_627_axes_0, keep_dims = var_627_keep_dims_0, x = var_625_cast_fp16)[name = string("op_627_cast_fp16")];
            tensor<int32, [3]> var_630_begin_0 = const()[name = string("op_630_begin_0"), val = tensor<int32, [3]>([0, 0, 12])];
            tensor<int32, [3]> var_630_end_0 = const()[name = string("op_630_end_0"), val = tensor<int32, [3]>([1, 16, 13])];
            tensor<bool, [3]> var_630_end_mask_0 = const()[name = string("op_630_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_630_squeeze_mask_0 = const()[name = string("op_630_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_630_cast_fp16 = slice_by_index(begin = var_630_begin_0, end = var_630_end_0, end_mask = var_630_end_mask_0, squeeze_mask = var_630_squeeze_mask_0, x = decay_1_cast_fp16)[name = string("op_630_cast_fp16")];
            tensor<fp16, [1, 16]> var_631_cast_fp16 = exp(x = var_630_cast_fp16)[name = string("op_631_cast_fp16")];
            tensor<int32, [1]> var_632_axes_0 = const()[name = string("op_632_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_632_cast_fp16 = expand_dims(axes = var_632_axes_0, x = var_631_cast_fp16)[name = string("op_632_cast_fp16")];
            tensor<int32, [1]> var_633_axes_0 = const()[name = string("op_633_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_633_cast_fp16 = expand_dims(axes = var_633_axes_0, x = var_632_cast_fp16)[name = string("op_633_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_49_cast_fp16 = mul(x = state_47_cast_fp16, y = var_633_cast_fp16)[name = string("state_49_cast_fp16")];
            tensor<int32, [4]> key_token_25_begin_0 = const()[name = string("key_token_25_begin_0"), val = tensor<int32, [4]>([0, 0, 12, 0])];
            tensor<int32, [4]> key_token_25_end_0 = const()[name = string("key_token_25_end_0"), val = tensor<int32, [4]>([1, 16, 13, 128])];
            tensor<bool, [4]> key_token_25_end_mask_0 = const()[name = string("key_token_25_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_25_squeeze_mask_0 = const()[name = string("key_token_25_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_25_cast_fp16 = slice_by_index(begin = key_token_25_begin_0, end = key_token_25_end_0, end_mask = key_token_25_end_mask_0, squeeze_mask = key_token_25_squeeze_mask_0, x = key_3_cast_fp16)[name = string("key_token_25_cast_fp16")];
            tensor<int32, [4]> value_token_25_begin_0 = const()[name = string("value_token_25_begin_0"), val = tensor<int32, [4]>([0, 0, 12, 0])];
            tensor<int32, [4]> value_token_25_end_0 = const()[name = string("value_token_25_end_0"), val = tensor<int32, [4]>([1, 16, 13, 128])];
            tensor<bool, [4]> value_token_25_end_mask_0 = const()[name = string("value_token_25_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_25_squeeze_mask_0 = const()[name = string("value_token_25_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_25_cast_fp16 = slice_by_index(begin = value_token_25_begin_0, end = value_token_25_end_0, end_mask = value_token_25_end_mask_0, squeeze_mask = value_token_25_squeeze_mask_0, x = value_11_cast_fp16)[name = string("value_token_25_cast_fp16")];
            tensor<int32, [1]> var_641_axes_0 = const()[name = string("op_641_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_641_cast_fp16 = expand_dims(axes = var_641_axes_0, x = key_token_25_cast_fp16)[name = string("op_641_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_642_cast_fp16 = mul(x = state_49_cast_fp16, y = var_641_cast_fp16)[name = string("op_642_cast_fp16")];
            tensor<int32, [1]> memory_25_axes_0 = const()[name = string("memory_25_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_25_keep_dims_0 = const()[name = string("memory_25_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_25_cast_fp16 = reduce_sum(axes = memory_25_axes_0, keep_dims = memory_25_keep_dims_0, x = var_642_cast_fp16)[name = string("memory_25_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_645_cast_fp16 = sub(x = value_token_25_cast_fp16, y = memory_25_cast_fp16)[name = string("op_645_cast_fp16")];
            tensor<int32, [3]> var_648_begin_0 = const()[name = string("op_648_begin_0"), val = tensor<int32, [3]>([0, 0, 12])];
            tensor<int32, [3]> var_648_end_0 = const()[name = string("op_648_end_0"), val = tensor<int32, [3]>([1, 16, 13])];
            tensor<bool, [3]> var_648_end_mask_0 = const()[name = string("op_648_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_648_squeeze_mask_0 = const()[name = string("op_648_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_648_cast_fp16 = slice_by_index(begin = var_648_begin_0, end = var_648_end_0, end_mask = var_648_end_mask_0, squeeze_mask = var_648_squeeze_mask_0, x = beta_1_cast_fp16)[name = string("op_648_cast_fp16")];
            tensor<int32, [1]> var_649_axes_0 = const()[name = string("op_649_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_649_cast_fp16 = expand_dims(axes = var_649_axes_0, x = var_648_cast_fp16)[name = string("op_649_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_25_cast_fp16 = mul(x = var_645_cast_fp16, y = var_649_cast_fp16)[name = string("delta_25_cast_fp16")];
            tensor<int32, [1]> var_652_axes_0 = const()[name = string("op_652_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_652_cast_fp16 = expand_dims(axes = var_652_axes_0, x = delta_25_cast_fp16)[name = string("op_652_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_653_cast_fp16 = mul(x = var_641_cast_fp16, y = var_652_cast_fp16)[name = string("op_653_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_51_cast_fp16 = add(x = state_49_cast_fp16, y = var_653_cast_fp16)[name = string("state_51_cast_fp16")];
            tensor<int32, [4]> var_657_begin_0 = const()[name = string("op_657_begin_0"), val = tensor<int32, [4]>([0, 0, 12, 0])];
            tensor<int32, [4]> var_657_end_0 = const()[name = string("op_657_end_0"), val = tensor<int32, [4]>([1, 16, 13, 128])];
            tensor<bool, [4]> var_657_end_mask_0 = const()[name = string("op_657_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_657_squeeze_mask_0 = const()[name = string("op_657_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_657_cast_fp16 = slice_by_index(begin = var_657_begin_0, end = var_657_end_0, end_mask = var_657_end_mask_0, squeeze_mask = var_657_squeeze_mask_0, x = _inversed_query_3_cast_fp16)[name = string("op_657_cast_fp16")];
            tensor<int32, [1]> var_658_axes_0 = const()[name = string("op_658_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_658_cast_fp16 = expand_dims(axes = var_658_axes_0, x = var_657_cast_fp16)[name = string("op_658_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_659_cast_fp16 = mul(x = state_51_cast_fp16, y = var_658_cast_fp16)[name = string("op_659_cast_fp16")];
            tensor<int32, [1]> var_661_axes_0 = const()[name = string("op_661_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_661_keep_dims_0 = const()[name = string("op_661_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_661_cast_fp16 = reduce_sum(axes = var_661_axes_0, keep_dims = var_661_keep_dims_0, x = var_659_cast_fp16)[name = string("op_661_cast_fp16")];
            tensor<int32, [3]> var_664_begin_0 = const()[name = string("op_664_begin_0"), val = tensor<int32, [3]>([0, 0, 13])];
            tensor<int32, [3]> var_664_end_0 = const()[name = string("op_664_end_0"), val = tensor<int32, [3]>([1, 16, 14])];
            tensor<bool, [3]> var_664_end_mask_0 = const()[name = string("op_664_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_664_squeeze_mask_0 = const()[name = string("op_664_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_664_cast_fp16 = slice_by_index(begin = var_664_begin_0, end = var_664_end_0, end_mask = var_664_end_mask_0, squeeze_mask = var_664_squeeze_mask_0, x = decay_1_cast_fp16)[name = string("op_664_cast_fp16")];
            tensor<fp16, [1, 16]> var_665_cast_fp16 = exp(x = var_664_cast_fp16)[name = string("op_665_cast_fp16")];
            tensor<int32, [1]> var_666_axes_0 = const()[name = string("op_666_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_666_cast_fp16 = expand_dims(axes = var_666_axes_0, x = var_665_cast_fp16)[name = string("op_666_cast_fp16")];
            tensor<int32, [1]> var_667_axes_0 = const()[name = string("op_667_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_667_cast_fp16 = expand_dims(axes = var_667_axes_0, x = var_666_cast_fp16)[name = string("op_667_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_53_cast_fp16 = mul(x = state_51_cast_fp16, y = var_667_cast_fp16)[name = string("state_53_cast_fp16")];
            tensor<int32, [4]> key_token_27_begin_0 = const()[name = string("key_token_27_begin_0"), val = tensor<int32, [4]>([0, 0, 13, 0])];
            tensor<int32, [4]> key_token_27_end_0 = const()[name = string("key_token_27_end_0"), val = tensor<int32, [4]>([1, 16, 14, 128])];
            tensor<bool, [4]> key_token_27_end_mask_0 = const()[name = string("key_token_27_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_27_squeeze_mask_0 = const()[name = string("key_token_27_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_27_cast_fp16 = slice_by_index(begin = key_token_27_begin_0, end = key_token_27_end_0, end_mask = key_token_27_end_mask_0, squeeze_mask = key_token_27_squeeze_mask_0, x = key_3_cast_fp16)[name = string("key_token_27_cast_fp16")];
            tensor<int32, [4]> value_token_27_begin_0 = const()[name = string("value_token_27_begin_0"), val = tensor<int32, [4]>([0, 0, 13, 0])];
            tensor<int32, [4]> value_token_27_end_0 = const()[name = string("value_token_27_end_0"), val = tensor<int32, [4]>([1, 16, 14, 128])];
            tensor<bool, [4]> value_token_27_end_mask_0 = const()[name = string("value_token_27_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_27_squeeze_mask_0 = const()[name = string("value_token_27_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_27_cast_fp16 = slice_by_index(begin = value_token_27_begin_0, end = value_token_27_end_0, end_mask = value_token_27_end_mask_0, squeeze_mask = value_token_27_squeeze_mask_0, x = value_11_cast_fp16)[name = string("value_token_27_cast_fp16")];
            tensor<int32, [1]> var_675_axes_0 = const()[name = string("op_675_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_675_cast_fp16 = expand_dims(axes = var_675_axes_0, x = key_token_27_cast_fp16)[name = string("op_675_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_676_cast_fp16 = mul(x = state_53_cast_fp16, y = var_675_cast_fp16)[name = string("op_676_cast_fp16")];
            tensor<int32, [1]> memory_27_axes_0 = const()[name = string("memory_27_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_27_keep_dims_0 = const()[name = string("memory_27_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_27_cast_fp16 = reduce_sum(axes = memory_27_axes_0, keep_dims = memory_27_keep_dims_0, x = var_676_cast_fp16)[name = string("memory_27_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_679_cast_fp16 = sub(x = value_token_27_cast_fp16, y = memory_27_cast_fp16)[name = string("op_679_cast_fp16")];
            tensor<int32, [3]> var_682_begin_0 = const()[name = string("op_682_begin_0"), val = tensor<int32, [3]>([0, 0, 13])];
            tensor<int32, [3]> var_682_end_0 = const()[name = string("op_682_end_0"), val = tensor<int32, [3]>([1, 16, 14])];
            tensor<bool, [3]> var_682_end_mask_0 = const()[name = string("op_682_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_682_squeeze_mask_0 = const()[name = string("op_682_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_682_cast_fp16 = slice_by_index(begin = var_682_begin_0, end = var_682_end_0, end_mask = var_682_end_mask_0, squeeze_mask = var_682_squeeze_mask_0, x = beta_1_cast_fp16)[name = string("op_682_cast_fp16")];
            tensor<int32, [1]> var_683_axes_0 = const()[name = string("op_683_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_683_cast_fp16 = expand_dims(axes = var_683_axes_0, x = var_682_cast_fp16)[name = string("op_683_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_27_cast_fp16 = mul(x = var_679_cast_fp16, y = var_683_cast_fp16)[name = string("delta_27_cast_fp16")];
            tensor<int32, [1]> var_686_axes_0 = const()[name = string("op_686_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_686_cast_fp16 = expand_dims(axes = var_686_axes_0, x = delta_27_cast_fp16)[name = string("op_686_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_687_cast_fp16 = mul(x = var_675_cast_fp16, y = var_686_cast_fp16)[name = string("op_687_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_55_cast_fp16 = add(x = state_53_cast_fp16, y = var_687_cast_fp16)[name = string("state_55_cast_fp16")];
            tensor<int32, [4]> var_691_begin_0 = const()[name = string("op_691_begin_0"), val = tensor<int32, [4]>([0, 0, 13, 0])];
            tensor<int32, [4]> var_691_end_0 = const()[name = string("op_691_end_0"), val = tensor<int32, [4]>([1, 16, 14, 128])];
            tensor<bool, [4]> var_691_end_mask_0 = const()[name = string("op_691_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_691_squeeze_mask_0 = const()[name = string("op_691_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_691_cast_fp16 = slice_by_index(begin = var_691_begin_0, end = var_691_end_0, end_mask = var_691_end_mask_0, squeeze_mask = var_691_squeeze_mask_0, x = _inversed_query_3_cast_fp16)[name = string("op_691_cast_fp16")];
            tensor<int32, [1]> var_692_axes_0 = const()[name = string("op_692_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_692_cast_fp16 = expand_dims(axes = var_692_axes_0, x = var_691_cast_fp16)[name = string("op_692_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_693_cast_fp16 = mul(x = state_55_cast_fp16, y = var_692_cast_fp16)[name = string("op_693_cast_fp16")];
            tensor<int32, [1]> var_695_axes_0 = const()[name = string("op_695_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_695_keep_dims_0 = const()[name = string("op_695_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_695_cast_fp16 = reduce_sum(axes = var_695_axes_0, keep_dims = var_695_keep_dims_0, x = var_693_cast_fp16)[name = string("op_695_cast_fp16")];
            tensor<int32, [3]> var_698_begin_0 = const()[name = string("op_698_begin_0"), val = tensor<int32, [3]>([0, 0, 14])];
            tensor<int32, [3]> var_698_end_0 = const()[name = string("op_698_end_0"), val = tensor<int32, [3]>([1, 16, 15])];
            tensor<bool, [3]> var_698_end_mask_0 = const()[name = string("op_698_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_698_squeeze_mask_0 = const()[name = string("op_698_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_698_cast_fp16 = slice_by_index(begin = var_698_begin_0, end = var_698_end_0, end_mask = var_698_end_mask_0, squeeze_mask = var_698_squeeze_mask_0, x = decay_1_cast_fp16)[name = string("op_698_cast_fp16")];
            tensor<fp16, [1, 16]> var_699_cast_fp16 = exp(x = var_698_cast_fp16)[name = string("op_699_cast_fp16")];
            tensor<int32, [1]> var_700_axes_0 = const()[name = string("op_700_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_700_cast_fp16 = expand_dims(axes = var_700_axes_0, x = var_699_cast_fp16)[name = string("op_700_cast_fp16")];
            tensor<int32, [1]> var_701_axes_0 = const()[name = string("op_701_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_701_cast_fp16 = expand_dims(axes = var_701_axes_0, x = var_700_cast_fp16)[name = string("op_701_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_57_cast_fp16 = mul(x = state_55_cast_fp16, y = var_701_cast_fp16)[name = string("state_57_cast_fp16")];
            tensor<int32, [4]> key_token_29_begin_0 = const()[name = string("key_token_29_begin_0"), val = tensor<int32, [4]>([0, 0, 14, 0])];
            tensor<int32, [4]> key_token_29_end_0 = const()[name = string("key_token_29_end_0"), val = tensor<int32, [4]>([1, 16, 15, 128])];
            tensor<bool, [4]> key_token_29_end_mask_0 = const()[name = string("key_token_29_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_29_squeeze_mask_0 = const()[name = string("key_token_29_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_29_cast_fp16 = slice_by_index(begin = key_token_29_begin_0, end = key_token_29_end_0, end_mask = key_token_29_end_mask_0, squeeze_mask = key_token_29_squeeze_mask_0, x = key_3_cast_fp16)[name = string("key_token_29_cast_fp16")];
            tensor<int32, [4]> value_token_29_begin_0 = const()[name = string("value_token_29_begin_0"), val = tensor<int32, [4]>([0, 0, 14, 0])];
            tensor<int32, [4]> value_token_29_end_0 = const()[name = string("value_token_29_end_0"), val = tensor<int32, [4]>([1, 16, 15, 128])];
            tensor<bool, [4]> value_token_29_end_mask_0 = const()[name = string("value_token_29_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_29_squeeze_mask_0 = const()[name = string("value_token_29_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_29_cast_fp16 = slice_by_index(begin = value_token_29_begin_0, end = value_token_29_end_0, end_mask = value_token_29_end_mask_0, squeeze_mask = value_token_29_squeeze_mask_0, x = value_11_cast_fp16)[name = string("value_token_29_cast_fp16")];
            tensor<int32, [1]> var_709_axes_0 = const()[name = string("op_709_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_709_cast_fp16 = expand_dims(axes = var_709_axes_0, x = key_token_29_cast_fp16)[name = string("op_709_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_710_cast_fp16 = mul(x = state_57_cast_fp16, y = var_709_cast_fp16)[name = string("op_710_cast_fp16")];
            tensor<int32, [1]> memory_29_axes_0 = const()[name = string("memory_29_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_29_keep_dims_0 = const()[name = string("memory_29_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_29_cast_fp16 = reduce_sum(axes = memory_29_axes_0, keep_dims = memory_29_keep_dims_0, x = var_710_cast_fp16)[name = string("memory_29_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_713_cast_fp16 = sub(x = value_token_29_cast_fp16, y = memory_29_cast_fp16)[name = string("op_713_cast_fp16")];
            tensor<int32, [3]> var_716_begin_0 = const()[name = string("op_716_begin_0"), val = tensor<int32, [3]>([0, 0, 14])];
            tensor<int32, [3]> var_716_end_0 = const()[name = string("op_716_end_0"), val = tensor<int32, [3]>([1, 16, 15])];
            tensor<bool, [3]> var_716_end_mask_0 = const()[name = string("op_716_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_716_squeeze_mask_0 = const()[name = string("op_716_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_716_cast_fp16 = slice_by_index(begin = var_716_begin_0, end = var_716_end_0, end_mask = var_716_end_mask_0, squeeze_mask = var_716_squeeze_mask_0, x = beta_1_cast_fp16)[name = string("op_716_cast_fp16")];
            tensor<int32, [1]> var_717_axes_0 = const()[name = string("op_717_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_717_cast_fp16 = expand_dims(axes = var_717_axes_0, x = var_716_cast_fp16)[name = string("op_717_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_29_cast_fp16 = mul(x = var_713_cast_fp16, y = var_717_cast_fp16)[name = string("delta_29_cast_fp16")];
            tensor<int32, [1]> var_720_axes_0 = const()[name = string("op_720_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_720_cast_fp16 = expand_dims(axes = var_720_axes_0, x = delta_29_cast_fp16)[name = string("op_720_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_721_cast_fp16 = mul(x = var_709_cast_fp16, y = var_720_cast_fp16)[name = string("op_721_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_59_cast_fp16 = add(x = state_57_cast_fp16, y = var_721_cast_fp16)[name = string("state_59_cast_fp16")];
            tensor<int32, [4]> var_725_begin_0 = const()[name = string("op_725_begin_0"), val = tensor<int32, [4]>([0, 0, 14, 0])];
            tensor<int32, [4]> var_725_end_0 = const()[name = string("op_725_end_0"), val = tensor<int32, [4]>([1, 16, 15, 128])];
            tensor<bool, [4]> var_725_end_mask_0 = const()[name = string("op_725_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_725_squeeze_mask_0 = const()[name = string("op_725_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_725_cast_fp16 = slice_by_index(begin = var_725_begin_0, end = var_725_end_0, end_mask = var_725_end_mask_0, squeeze_mask = var_725_squeeze_mask_0, x = _inversed_query_3_cast_fp16)[name = string("op_725_cast_fp16")];
            tensor<int32, [1]> var_726_axes_0 = const()[name = string("op_726_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_726_cast_fp16 = expand_dims(axes = var_726_axes_0, x = var_725_cast_fp16)[name = string("op_726_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_727_cast_fp16 = mul(x = state_59_cast_fp16, y = var_726_cast_fp16)[name = string("op_727_cast_fp16")];
            tensor<int32, [1]> var_729_axes_0 = const()[name = string("op_729_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_729_keep_dims_0 = const()[name = string("op_729_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_729_cast_fp16 = reduce_sum(axes = var_729_axes_0, keep_dims = var_729_keep_dims_0, x = var_727_cast_fp16)[name = string("op_729_cast_fp16")];
            tensor<int32, [3]> var_732_begin_0 = const()[name = string("op_732_begin_0"), val = tensor<int32, [3]>([0, 0, 15])];
            tensor<int32, [3]> var_732_end_0 = const()[name = string("op_732_end_0"), val = tensor<int32, [3]>([1, 16, 16])];
            tensor<bool, [3]> var_732_end_mask_0 = const()[name = string("op_732_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_732_squeeze_mask_0 = const()[name = string("op_732_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_732_cast_fp16 = slice_by_index(begin = var_732_begin_0, end = var_732_end_0, end_mask = var_732_end_mask_0, squeeze_mask = var_732_squeeze_mask_0, x = decay_1_cast_fp16)[name = string("op_732_cast_fp16")];
            tensor<fp16, [1, 16]> var_733_cast_fp16 = exp(x = var_732_cast_fp16)[name = string("op_733_cast_fp16")];
            tensor<int32, [1]> var_734_axes_0 = const()[name = string("op_734_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_734_cast_fp16 = expand_dims(axes = var_734_axes_0, x = var_733_cast_fp16)[name = string("op_734_cast_fp16")];
            tensor<int32, [1]> var_735_axes_0 = const()[name = string("op_735_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_735_cast_fp16 = expand_dims(axes = var_735_axes_0, x = var_734_cast_fp16)[name = string("op_735_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_61_cast_fp16 = mul(x = state_59_cast_fp16, y = var_735_cast_fp16)[name = string("state_61_cast_fp16")];
            tensor<int32, [4]> key_token_31_begin_0 = const()[name = string("key_token_31_begin_0"), val = tensor<int32, [4]>([0, 0, 15, 0])];
            tensor<int32, [4]> key_token_31_end_0 = const()[name = string("key_token_31_end_0"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<bool, [4]> key_token_31_end_mask_0 = const()[name = string("key_token_31_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_31_squeeze_mask_0 = const()[name = string("key_token_31_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_31_cast_fp16 = slice_by_index(begin = key_token_31_begin_0, end = key_token_31_end_0, end_mask = key_token_31_end_mask_0, squeeze_mask = key_token_31_squeeze_mask_0, x = key_3_cast_fp16)[name = string("key_token_31_cast_fp16")];
            tensor<int32, [4]> value_token_31_begin_0 = const()[name = string("value_token_31_begin_0"), val = tensor<int32, [4]>([0, 0, 15, 0])];
            tensor<int32, [4]> value_token_31_end_0 = const()[name = string("value_token_31_end_0"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<bool, [4]> value_token_31_end_mask_0 = const()[name = string("value_token_31_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_31_squeeze_mask_0 = const()[name = string("value_token_31_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_31_cast_fp16 = slice_by_index(begin = value_token_31_begin_0, end = value_token_31_end_0, end_mask = value_token_31_end_mask_0, squeeze_mask = value_token_31_squeeze_mask_0, x = value_11_cast_fp16)[name = string("value_token_31_cast_fp16")];
            tensor<int32, [1]> var_743_axes_0 = const()[name = string("op_743_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_743_cast_fp16 = expand_dims(axes = var_743_axes_0, x = key_token_31_cast_fp16)[name = string("op_743_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_744_cast_fp16 = mul(x = state_61_cast_fp16, y = var_743_cast_fp16)[name = string("op_744_cast_fp16")];
            tensor<int32, [1]> memory_31_axes_0 = const()[name = string("memory_31_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_31_keep_dims_0 = const()[name = string("memory_31_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_31_cast_fp16 = reduce_sum(axes = memory_31_axes_0, keep_dims = memory_31_keep_dims_0, x = var_744_cast_fp16)[name = string("memory_31_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_747_cast_fp16 = sub(x = value_token_31_cast_fp16, y = memory_31_cast_fp16)[name = string("op_747_cast_fp16")];
            tensor<int32, [3]> var_750_begin_0 = const()[name = string("op_750_begin_0"), val = tensor<int32, [3]>([0, 0, 15])];
            tensor<int32, [3]> var_750_end_0 = const()[name = string("op_750_end_0"), val = tensor<int32, [3]>([1, 16, 16])];
            tensor<bool, [3]> var_750_end_mask_0 = const()[name = string("op_750_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_750_squeeze_mask_0 = const()[name = string("op_750_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_750_cast_fp16 = slice_by_index(begin = var_750_begin_0, end = var_750_end_0, end_mask = var_750_end_mask_0, squeeze_mask = var_750_squeeze_mask_0, x = beta_1_cast_fp16)[name = string("op_750_cast_fp16")];
            tensor<int32, [1]> var_751_axes_0 = const()[name = string("op_751_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_751_cast_fp16 = expand_dims(axes = var_751_axes_0, x = var_750_cast_fp16)[name = string("op_751_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_31_cast_fp16 = mul(x = var_747_cast_fp16, y = var_751_cast_fp16)[name = string("delta_31_cast_fp16")];
            tensor<int32, [1]> var_754_axes_0 = const()[name = string("op_754_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_754_cast_fp16 = expand_dims(axes = var_754_axes_0, x = delta_31_cast_fp16)[name = string("op_754_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_755_cast_fp16 = mul(x = var_743_cast_fp16, y = var_754_cast_fp16)[name = string("op_755_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> rec0_out = add(x = state_61_cast_fp16, y = var_755_cast_fp16)[name = string("state_63_cast_fp16")];
            tensor<int32, [4]> var_759_begin_0 = const()[name = string("op_759_begin_0"), val = tensor<int32, [4]>([0, 0, 15, 0])];
            tensor<int32, [4]> var_759_end_0 = const()[name = string("op_759_end_0"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<bool, [4]> var_759_end_mask_0 = const()[name = string("op_759_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_759_squeeze_mask_0 = const()[name = string("op_759_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_759_cast_fp16 = slice_by_index(begin = var_759_begin_0, end = var_759_end_0, end_mask = var_759_end_mask_0, squeeze_mask = var_759_squeeze_mask_0, x = _inversed_query_3_cast_fp16)[name = string("op_759_cast_fp16")];
            tensor<int32, [1]> var_760_axes_0 = const()[name = string("op_760_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_760_cast_fp16 = expand_dims(axes = var_760_axes_0, x = var_759_cast_fp16)[name = string("op_760_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_761_cast_fp16 = mul(x = rec0_out, y = var_760_cast_fp16)[name = string("op_761_cast_fp16")];
            tensor<int32, [1]> var_763_axes_0 = const()[name = string("op_763_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_763_keep_dims_0 = const()[name = string("op_763_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_763_cast_fp16 = reduce_sum(axes = var_763_axes_0, keep_dims = var_763_keep_dims_0, x = var_761_cast_fp16)[name = string("op_763_cast_fp16")];
            int32 attention_1_axis_0 = const()[name = string("attention_1_axis_0"), val = int32(1)];
            tensor<fp16, [1, 16, 16, 128]> attention_1_cast_fp16 = stack(axis = attention_1_axis_0, values = (var_253_cast_fp16, var_287_cast_fp16, var_321_cast_fp16, var_355_cast_fp16, var_389_cast_fp16, var_423_cast_fp16, var_457_cast_fp16, var_491_cast_fp16, var_525_cast_fp16, var_559_cast_fp16, var_593_cast_fp16, var_627_cast_fp16, var_661_cast_fp16, var_695_cast_fp16, var_729_cast_fp16, var_763_cast_fp16))[name = string("attention_1_cast_fp16")];
            tensor<int32, [2]> var_766 = const()[name = string("op_766"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [256, 128]> attention_3_cast_fp16 = reshape(shape = var_766, x = attention_1_cast_fp16)[name = string("attention_3_cast_fp16")];
            tensor<int32, [2]> var_768 = const()[name = string("op_768"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [256, 128]> input_5_cast_fp16 = reshape(shape = var_768, x = linear_3_cast_fp16)[name = string("input_5_cast_fp16")];
            tensor<fp16, [256, 128]> var_770_cast_fp16 = mul(x = attention_3_cast_fp16, y = attention_3_cast_fp16)[name = string("op_770_cast_fp16")];
            tensor<int32, [1]> variance_3_axes_0 = const()[name = string("variance_3_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_3_keep_dims_0 = const()[name = string("variance_3_keep_dims_0"), val = bool(true)];
            tensor<fp16, [256, 1]> variance_3_cast_fp16 = reduce_mean(axes = variance_3_axes_0, keep_dims = variance_3_keep_dims_0, x = var_770_cast_fp16)[name = string("variance_3_cast_fp16")];
            fp16 var_773_to_fp16 = const()[name = string("op_773_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [256, 1]> var_774_cast_fp16 = add(x = variance_3_cast_fp16, y = var_773_to_fp16)[name = string("op_774_cast_fp16")];
            fp32 var_775_epsilon_0 = const()[name = string("op_775_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [256, 1]> var_775_cast_fp16 = rsqrt(epsilon = var_775_epsilon_0, x = var_774_cast_fp16)[name = string("op_775_cast_fp16")];
            tensor<fp16, [256, 128]> attention_5_cast_fp16 = mul(x = attention_3_cast_fp16, y = var_775_cast_fp16)[name = string("attention_5_cast_fp16")];
            tensor<fp16, [128]> groups_0_0_gated_norm_promoted_to_fp16 = const()[name = string("groups_0_0_gated_norm_promoted_to_fp16"), val = tensor<fp16, [128]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(6469120)))];
            tensor<fp16, [256, 128]> attention_7_cast_fp16 = mul(x = attention_5_cast_fp16, y = groups_0_0_gated_norm_promoted_to_fp16)[name = string("attention_7_cast_fp16")];
            tensor<fp16, [256, 128]> var_778_cast_fp16 = silu(x = input_5_cast_fp16)[name = string("op_778_cast_fp16")];
            tensor<fp16, [256, 128]> attention_9_cast_fp16 = mul(x = attention_7_cast_fp16, y = var_778_cast_fp16)[name = string("attention_9_cast_fp16")];
            tensor<int32, [2]> var_780 = const()[name = string("op_780"), val = tensor<int32, [2]>([16, 2048])];
            tensor<fp16, [16, 2048]> input_7_cast_fp16 = reshape(shape = var_780, x = attention_9_cast_fp16)[name = string("input_7_cast_fp16")];
            tensor<fp16, [1024, 2048]> groups_0_0_out_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 2048]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(6469440))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(8042368))))[name = string("groups_0_0_out_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1024]> linear_4_bias_0_to_fp16 = const()[name = string("linear_4_bias_0_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(8050624)))];
            tensor<fp16, [16, 1024]> linear_4_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = groups_0_0_out_proj_weight_promoted_to_fp16_palettized, x = input_7_cast_fp16)[name = string("linear_4_cast_fp16")];
            tensor<fp16, [16, 1024]> value_13_cast_fp16 = add(x = hidden, y = linear_4_cast_fp16)[name = string("value_13_cast_fp16")];
            tensor<fp16, [16, 1024]> var_785_cast_fp16 = mul(x = value_13_cast_fp16, y = value_13_cast_fp16)[name = string("op_785_cast_fp16")];
            tensor<int32, [1]> variance_5_axes_0 = const()[name = string("variance_5_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_5_keep_dims_0 = const()[name = string("variance_5_keep_dims_0"), val = bool(true)];
            tensor<fp16, [16, 1]> variance_5_cast_fp16 = reduce_mean(axes = variance_5_axes_0, keep_dims = variance_5_keep_dims_0, x = var_785_cast_fp16)[name = string("variance_5_cast_fp16")];
            fp16 var_788_to_fp16 = const()[name = string("op_788_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [16, 1]> var_789_cast_fp16 = add(x = variance_5_cast_fp16, y = var_788_to_fp16)[name = string("op_789_cast_fp16")];
            fp32 var_790_epsilon_0 = const()[name = string("op_790_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [16, 1]> var_790_cast_fp16 = rsqrt(epsilon = var_790_epsilon_0, x = var_789_cast_fp16)[name = string("op_790_cast_fp16")];
            tensor<fp16, [16, 1024]> var_791_cast_fp16 = mul(x = value_13_cast_fp16, y = var_790_cast_fp16)[name = string("op_791_cast_fp16")];
            tensor<fp16, [1024]> var_793_to_fp16 = const()[name = string("op_793_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(8052736)))];
            tensor<fp16, [16, 1024]> input_9_cast_fp16 = mul(x = var_791_cast_fp16, y = var_793_to_fp16)[name = string("input_9_cast_fp16")];
            tensor<fp16, [3584, 1024]> groups_0_0_gate_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(8054848))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(10807424))))[name = string("groups_0_0_gate_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [3584]> linear_5_bias_0_to_fp16 = const()[name = string("linear_5_bias_0_to_fp16"), val = tensor<fp16, [3584]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(10836160)))];
            tensor<fp16, [16, 3584]> linear_5_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = groups_0_0_gate_proj_weight_promoted_to_fp16_palettized, x = input_9_cast_fp16)[name = string("linear_5_cast_fp16")];
            tensor<fp16, [16, 3584]> var_797_cast_fp16 = silu(x = linear_5_cast_fp16)[name = string("op_797_cast_fp16")];
            tensor<fp16, [3584, 1024]> groups_0_0_up_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(10843392))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(13595968))))[name = string("groups_0_0_up_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 3584]> linear_6_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = groups_0_0_up_proj_weight_promoted_to_fp16_palettized, x = input_9_cast_fp16)[name = string("linear_6_cast_fp16")];
            tensor<fp16, [16, 3584]> input_13_cast_fp16 = mul(x = var_797_cast_fp16, y = linear_6_cast_fp16)[name = string("input_13_cast_fp16")];
            tensor<fp16, [1024, 3584]> groups_0_0_down_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 3584]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(13624704))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(16377280))))[name = string("groups_0_0_down_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 1024]> linear_7_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = groups_0_0_down_proj_weight_promoted_to_fp16_palettized, x = input_13_cast_fp16)[name = string("linear_7_cast_fp16")];
            tensor<fp16, [16, 1024]> value_15_cast_fp16 = add(x = value_13_cast_fp16, y = linear_7_cast_fp16)[name = string("value_15_cast_fp16")];
            int32 var_835 = const()[name = string("op_835"), val = int32(-1)];
            tensor<fp16, [16, 1024]> var_850_cast_fp16 = mul(x = value_15_cast_fp16, y = value_15_cast_fp16)[name = string("op_850_cast_fp16")];
            tensor<int32, [1]> variance_7_axes_0 = const()[name = string("variance_7_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_7_keep_dims_0 = const()[name = string("variance_7_keep_dims_0"), val = bool(true)];
            tensor<fp16, [16, 1]> variance_7_cast_fp16 = reduce_mean(axes = variance_7_axes_0, keep_dims = variance_7_keep_dims_0, x = var_850_cast_fp16)[name = string("variance_7_cast_fp16")];
            fp16 var_853_to_fp16 = const()[name = string("op_853_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [16, 1]> var_854_cast_fp16 = add(x = variance_7_cast_fp16, y = var_853_to_fp16)[name = string("op_854_cast_fp16")];
            fp32 var_855_epsilon_0 = const()[name = string("op_855_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [16, 1]> var_855_cast_fp16 = rsqrt(epsilon = var_855_epsilon_0, x = var_854_cast_fp16)[name = string("op_855_cast_fp16")];
            tensor<fp16, [16, 1024]> var_856_cast_fp16 = mul(x = value_15_cast_fp16, y = var_855_cast_fp16)[name = string("op_856_cast_fp16")];
            tensor<fp16, [1024]> var_858_to_fp16 = const()[name = string("op_858_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(16385536)))];
            tensor<fp16, [16, 1024]> input_15_cast_fp16 = mul(x = var_856_cast_fp16, y = var_858_to_fp16)[name = string("input_15_cast_fp16")];
            tensor<fp16, [6144, 1024]> groups_0_1_in_proj_qkv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(16387648))), lut = tensor<fp16, [384, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(21106304))))[name = string("groups_0_1_in_proj_qkv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 6144]> linear_8_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = groups_0_1_in_proj_qkv_weight_promoted_to_fp16_palettized, x = input_15_cast_fp16)[name = string("linear_8_cast_fp16")];
            tensor<int32, [2]> var_862_perm_0 = const()[name = string("op_862_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> mixed_3_axes_0 = const()[name = string("mixed_3_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [6144, 16]> var_862_cast_fp16 = transpose(perm = var_862_perm_0, x = linear_8_cast_fp16)[name = string("transpose_134")];
            tensor<fp16, [1, 6144, 16]> mixed_3_cast_fp16 = expand_dims(axes = mixed_3_axes_0, x = var_862_cast_fp16)[name = string("mixed_3_cast_fp16")];
            bool convolution_input_3_interleave_0 = const()[name = string("convolution_input_3_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 6144, 19]> convolution_input_3_cast_fp16 = concat(axis = var_835, interleave = convolution_input_3_interleave_0, values = (conv1, mixed_3_cast_fp16))[name = string("convolution_input_3_cast_fp16")];
            string input_17_pad_type_0 = const()[name = string("input_17_pad_type_0"), val = string("valid")];
            int32 input_17_groups_0 = const()[name = string("input_17_groups_0"), val = int32(6144)];
            tensor<int32, [1]> input_17_strides_0 = const()[name = string("input_17_strides_0"), val = tensor<int32, [1]>([1])];
            tensor<int32, [2]> input_17_pad_0 = const()[name = string("input_17_pad_0"), val = tensor<int32, [2]>([0, 0])];
            tensor<int32, [1]> input_17_dilations_0 = const()[name = string("input_17_dilations_0"), val = tensor<int32, [1]>([1])];
            tensor<fp16, [6144, 1, 4]> groups_0_1_conv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1, 4]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(21155520))), lut = tensor<fp16, [384, 1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(21174016))))[name = string("groups_0_1_conv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 6144, 16]> input_17_cast_fp16 = conv(dilations = input_17_dilations_0, groups = input_17_groups_0, pad = input_17_pad_0, pad_type = input_17_pad_type_0, strides = input_17_strides_0, weight = groups_0_1_conv_weight_promoted_to_fp16_palettized, x = convolution_input_3_cast_fp16)[name = string("input_17_cast_fp16")];
            tensor<fp16, [1, 6144, 16]> var_871_cast_fp16 = silu(x = input_17_cast_fp16)[name = string("op_871_cast_fp16")];
            tensor<int32, [3]> var_872_perm_0 = const()[name = string("op_872_perm_0"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [3]> var_875_begin_0 = const()[name = string("op_875_begin_0"), val = tensor<int32, [3]>([0, 0, 16])];
            tensor<int32, [3]> var_875_end_0 = const()[name = string("op_875_end_0"), val = tensor<int32, [3]>([1, 6144, 19])];
            tensor<bool, [3]> var_875_end_mask_0 = const()[name = string("op_875_end_mask_0"), val = tensor<bool, [3]>([true, true, true])];
            tensor<fp16, [1, 6144, 3]> conv1_out = slice_by_index(begin = var_875_begin_0, end = var_875_end_0, end_mask = var_875_end_mask_0, x = convolution_input_3_cast_fp16)[name = string("op_875_cast_fp16")];
            tensor<int32, [3]> var_876 = const()[name = string("op_876"), val = tensor<int32, [3]>([2048, 2048, 2048])];
            int32 var_877_axis_0 = const()[name = string("op_877_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 16, 6144]> var_872_cast_fp16 = transpose(perm = var_872_perm_0, x = var_871_cast_fp16)[name = string("transpose_133")];
            tensor<fp16, [1, 16, 2048]> var_877_cast_fp16_0, tensor<fp16, [1, 16, 2048]> var_877_cast_fp16_1, tensor<fp16, [1, 16, 2048]> var_877_cast_fp16_2 = split(axis = var_877_axis_0, split_sizes = var_876, x = var_872_cast_fp16)[name = string("op_877_cast_fp16")];
            tensor<int32, [4]> var_881 = const()[name = string("op_881"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<fp16, [1, 16, 16, 128]> value_19_cast_fp16 = reshape(shape = var_881, x = var_877_cast_fp16_0)[name = string("value_19_cast_fp16")];
            tensor<int32, [4]> var_883 = const()[name = string("op_883"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<fp16, [1, 16, 16, 128]> value_21_cast_fp16 = reshape(shape = var_883, x = var_877_cast_fp16_1)[name = string("value_21_cast_fp16")];
            tensor<int32, [4]> var_885 = const()[name = string("op_885"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<fp16, [1, 16, 16, 128]> value_23_cast_fp16 = reshape(shape = var_885, x = var_877_cast_fp16_2)[name = string("value_23_cast_fp16")];
            tensor<fp16, [1, 16, 16, 128]> var_887_cast_fp16 = mul(x = value_19_cast_fp16, y = value_19_cast_fp16)[name = string("op_887_cast_fp16")];
            tensor<int32, [1]> var_889_axes_0 = const()[name = string("op_889_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_889_keep_dims_0 = const()[name = string("op_889_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 16, 16, 1]> var_889_cast_fp16 = reduce_sum(axes = var_889_axes_0, keep_dims = var_889_keep_dims_0, x = var_887_cast_fp16)[name = string("op_889_cast_fp16")];
            fp16 var_890_to_fp16 = const()[name = string("op_890_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 16, 16, 1]> var_891_cast_fp16 = add(x = var_889_cast_fp16, y = var_890_to_fp16)[name = string("op_891_cast_fp16")];
            fp32 var_892_epsilon_0 = const()[name = string("op_892_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 16, 16, 1]> var_892_cast_fp16 = rsqrt(epsilon = var_892_epsilon_0, x = var_891_cast_fp16)[name = string("op_892_cast_fp16")];
            tensor<fp16, [1, 16, 16, 128]> var_893_cast_fp16 = mul(x = value_19_cast_fp16, y = var_892_cast_fp16)[name = string("op_893_cast_fp16")];
            tensor<int32, [4]> var_894_perm_0 = const()[name = string("op_894_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            fp16 _inversed_query_7_y_0_to_fp16 = const()[name = string("_inversed_query_7_y_0_to_fp16"), val = fp16(0x1.6ap-4)];
            tensor<fp16, [1, 16, 16, 128]> var_894_cast_fp16 = transpose(perm = var_894_perm_0, x = var_893_cast_fp16)[name = string("transpose_132")];
            tensor<fp16, [1, 16, 16, 128]> _inversed_query_7_cast_fp16 = mul(x = var_894_cast_fp16, y = _inversed_query_7_y_0_to_fp16)[name = string("_inversed_query_7_cast_fp16")];
            tensor<fp16, [1, 16, 16, 128]> var_897_cast_fp16 = mul(x = value_21_cast_fp16, y = value_21_cast_fp16)[name = string("op_897_cast_fp16")];
            tensor<int32, [1]> var_899_axes_0 = const()[name = string("op_899_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_899_keep_dims_0 = const()[name = string("op_899_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 16, 16, 1]> var_899_cast_fp16 = reduce_sum(axes = var_899_axes_0, keep_dims = var_899_keep_dims_0, x = var_897_cast_fp16)[name = string("op_899_cast_fp16")];
            fp16 var_900_to_fp16 = const()[name = string("op_900_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 16, 16, 1]> var_901_cast_fp16 = add(x = var_899_cast_fp16, y = var_900_to_fp16)[name = string("op_901_cast_fp16")];
            fp32 var_902_epsilon_0 = const()[name = string("op_902_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 16, 16, 1]> var_902_cast_fp16 = rsqrt(epsilon = var_902_epsilon_0, x = var_901_cast_fp16)[name = string("op_902_cast_fp16")];
            tensor<fp16, [1, 16, 16, 128]> var_903_cast_fp16 = mul(x = value_21_cast_fp16, y = var_902_cast_fp16)[name = string("op_903_cast_fp16")];
            tensor<int32, [4]> key_7_perm_0 = const()[name = string("key_7_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<int32, [4]> value_25_perm_0 = const()[name = string("value_25_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<fp16, [16, 1024]> groups_0_1_in_proj_b_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(21223232))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(21235584))))[name = string("groups_0_1_in_proj_b_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16]> linear_9_bias_0_to_fp16 = const()[name = string("linear_9_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(4862656)))];
            tensor<fp16, [16, 16]> linear_9_cast_fp16 = linear(bias = linear_9_bias_0_to_fp16, weight = groups_0_1_in_proj_b_weight_promoted_to_fp16_palettized, x = input_15_cast_fp16)[name = string("linear_9_cast_fp16")];
            tensor<fp16, [16, 16]> var_908_cast_fp16 = sigmoid(x = linear_9_cast_fp16)[name = string("op_908_cast_fp16")];
            tensor<int32, [2]> var_909_perm_0 = const()[name = string("op_909_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> beta_3_axes_0 = const()[name = string("beta_3_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 16]> var_909_cast_fp16 = transpose(perm = var_909_perm_0, x = var_908_cast_fp16)[name = string("transpose_131")];
            tensor<fp16, [1, 16, 16]> beta_3_cast_fp16 = expand_dims(axes = beta_3_axes_0, x = var_909_cast_fp16)[name = string("beta_3_cast_fp16")];
            tensor<fp16, [16, 1024]> op_915_weight_0_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(21235776))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(21248128))))[name = string("op_915_weight_0_to_fp16_palettized")];
            tensor<fp16, [16]> var_915_bias_0_to_fp16 = const()[name = string("op_915_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(21248320)))];
            tensor<fp16, [16, 16]> var_915_cast_fp16 = linear(bias = var_915_bias_0_to_fp16, weight = op_915_weight_0_to_fp16_palettized, x = input_15_cast_fp16)[name = string("op_915_cast_fp16")];
            tensor<fp16, [16, 16]> var_916_cast_fp16 = softplus(x = var_915_cast_fp16)[name = string("op_916_cast_fp16")];
            tensor<fp16, [16]> var_912_promoted_to_fp16 = const()[name = string("op_912_promoted_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(21248448)))];
            tensor<fp16, [16, 16]> var_917_cast_fp16 = mul(x = var_912_promoted_to_fp16, y = var_916_cast_fp16)[name = string("op_917_cast_fp16")];
            tensor<int32, [2]> var_918_perm_0 = const()[name = string("op_918_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> decay_3_axes_0 = const()[name = string("decay_3_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 16]> var_918_cast_fp16 = transpose(perm = var_918_perm_0, x = var_917_cast_fp16)[name = string("transpose_130")];
            tensor<fp16, [1, 16, 16]> decay_3_cast_fp16 = expand_dims(axes = decay_3_axes_0, x = var_918_cast_fp16)[name = string("decay_3_cast_fp16")];
            tensor<fp16, [2048, 1024]> groups_0_1_in_proj_z_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [2048, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(21248576))), lut = tensor<fp16, [128, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(22821504))))[name = string("groups_0_1_in_proj_z_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 2048]> linear_11_cast_fp16 = linear(bias = linear_3_bias_0_to_fp16, weight = groups_0_1_in_proj_z_weight_promoted_to_fp16_palettized, x = input_15_cast_fp16)[name = string("linear_11_cast_fp16")];
            tensor<int32, [3]> var_926_begin_0 = const()[name = string("op_926_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_926_end_0 = const()[name = string("op_926_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_926_end_mask_0 = const()[name = string("op_926_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_926_squeeze_mask_0 = const()[name = string("op_926_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_926_cast_fp16 = slice_by_index(begin = var_926_begin_0, end = var_926_end_0, end_mask = var_926_end_mask_0, squeeze_mask = var_926_squeeze_mask_0, x = decay_3_cast_fp16)[name = string("op_926_cast_fp16")];
            tensor<fp16, [1, 16]> var_927_cast_fp16 = exp(x = var_926_cast_fp16)[name = string("op_927_cast_fp16")];
            tensor<int32, [1]> var_928_axes_0 = const()[name = string("op_928_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_928_cast_fp16 = expand_dims(axes = var_928_axes_0, x = var_927_cast_fp16)[name = string("op_928_cast_fp16")];
            tensor<int32, [1]> var_929_axes_0 = const()[name = string("op_929_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_929_cast_fp16 = expand_dims(axes = var_929_axes_0, x = var_928_cast_fp16)[name = string("op_929_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_65_cast_fp16 = mul(x = rec1, y = var_929_cast_fp16)[name = string("state_65_cast_fp16")];
            tensor<int32, [4]> key_token_33_begin_0 = const()[name = string("key_token_33_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> key_token_33_end_0 = const()[name = string("key_token_33_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> key_token_33_end_mask_0 = const()[name = string("key_token_33_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_33_squeeze_mask_0 = const()[name = string("key_token_33_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 16, 128]> key_7_cast_fp16 = transpose(perm = key_7_perm_0, x = var_903_cast_fp16)[name = string("transpose_129")];
            tensor<fp16, [1, 16, 128]> key_token_33_cast_fp16 = slice_by_index(begin = key_token_33_begin_0, end = key_token_33_end_0, end_mask = key_token_33_end_mask_0, squeeze_mask = key_token_33_squeeze_mask_0, x = key_7_cast_fp16)[name = string("key_token_33_cast_fp16")];
            tensor<int32, [4]> value_token_33_begin_0 = const()[name = string("value_token_33_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> value_token_33_end_0 = const()[name = string("value_token_33_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> value_token_33_end_mask_0 = const()[name = string("value_token_33_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_33_squeeze_mask_0 = const()[name = string("value_token_33_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 16, 128]> value_25_cast_fp16 = transpose(perm = value_25_perm_0, x = value_23_cast_fp16)[name = string("transpose_128")];
            tensor<fp16, [1, 16, 128]> value_token_33_cast_fp16 = slice_by_index(begin = value_token_33_begin_0, end = value_token_33_end_0, end_mask = value_token_33_end_mask_0, squeeze_mask = value_token_33_squeeze_mask_0, x = value_25_cast_fp16)[name = string("value_token_33_cast_fp16")];
            tensor<int32, [1]> var_937_axes_0 = const()[name = string("op_937_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_937_cast_fp16 = expand_dims(axes = var_937_axes_0, x = key_token_33_cast_fp16)[name = string("op_937_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_938_cast_fp16 = mul(x = state_65_cast_fp16, y = var_937_cast_fp16)[name = string("op_938_cast_fp16")];
            tensor<int32, [1]> memory_33_axes_0 = const()[name = string("memory_33_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_33_keep_dims_0 = const()[name = string("memory_33_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_33_cast_fp16 = reduce_sum(axes = memory_33_axes_0, keep_dims = memory_33_keep_dims_0, x = var_938_cast_fp16)[name = string("memory_33_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_941_cast_fp16 = sub(x = value_token_33_cast_fp16, y = memory_33_cast_fp16)[name = string("op_941_cast_fp16")];
            tensor<int32, [3]> var_944_begin_0 = const()[name = string("op_944_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_944_end_0 = const()[name = string("op_944_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_944_end_mask_0 = const()[name = string("op_944_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_944_squeeze_mask_0 = const()[name = string("op_944_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_944_cast_fp16 = slice_by_index(begin = var_944_begin_0, end = var_944_end_0, end_mask = var_944_end_mask_0, squeeze_mask = var_944_squeeze_mask_0, x = beta_3_cast_fp16)[name = string("op_944_cast_fp16")];
            tensor<int32, [1]> var_945_axes_0 = const()[name = string("op_945_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_945_cast_fp16 = expand_dims(axes = var_945_axes_0, x = var_944_cast_fp16)[name = string("op_945_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_33_cast_fp16 = mul(x = var_941_cast_fp16, y = var_945_cast_fp16)[name = string("delta_33_cast_fp16")];
            tensor<int32, [1]> var_948_axes_0 = const()[name = string("op_948_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_948_cast_fp16 = expand_dims(axes = var_948_axes_0, x = delta_33_cast_fp16)[name = string("op_948_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_949_cast_fp16 = mul(x = var_937_cast_fp16, y = var_948_cast_fp16)[name = string("op_949_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_67_cast_fp16 = add(x = state_65_cast_fp16, y = var_949_cast_fp16)[name = string("state_67_cast_fp16")];
            tensor<int32, [4]> var_953_begin_0 = const()[name = string("op_953_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> var_953_end_0 = const()[name = string("op_953_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> var_953_end_mask_0 = const()[name = string("op_953_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_953_squeeze_mask_0 = const()[name = string("op_953_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_953_cast_fp16 = slice_by_index(begin = var_953_begin_0, end = var_953_end_0, end_mask = var_953_end_mask_0, squeeze_mask = var_953_squeeze_mask_0, x = _inversed_query_7_cast_fp16)[name = string("op_953_cast_fp16")];
            tensor<int32, [1]> var_954_axes_0 = const()[name = string("op_954_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_954_cast_fp16 = expand_dims(axes = var_954_axes_0, x = var_953_cast_fp16)[name = string("op_954_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_955_cast_fp16 = mul(x = state_67_cast_fp16, y = var_954_cast_fp16)[name = string("op_955_cast_fp16")];
            tensor<int32, [1]> var_957_axes_0 = const()[name = string("op_957_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_957_keep_dims_0 = const()[name = string("op_957_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_957_cast_fp16 = reduce_sum(axes = var_957_axes_0, keep_dims = var_957_keep_dims_0, x = var_955_cast_fp16)[name = string("op_957_cast_fp16")];
            tensor<int32, [3]> var_960_begin_0 = const()[name = string("op_960_begin_0"), val = tensor<int32, [3]>([0, 0, 1])];
            tensor<int32, [3]> var_960_end_0 = const()[name = string("op_960_end_0"), val = tensor<int32, [3]>([1, 16, 2])];
            tensor<bool, [3]> var_960_end_mask_0 = const()[name = string("op_960_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_960_squeeze_mask_0 = const()[name = string("op_960_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_960_cast_fp16 = slice_by_index(begin = var_960_begin_0, end = var_960_end_0, end_mask = var_960_end_mask_0, squeeze_mask = var_960_squeeze_mask_0, x = decay_3_cast_fp16)[name = string("op_960_cast_fp16")];
            tensor<fp16, [1, 16]> var_961_cast_fp16 = exp(x = var_960_cast_fp16)[name = string("op_961_cast_fp16")];
            tensor<int32, [1]> var_962_axes_0 = const()[name = string("op_962_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_962_cast_fp16 = expand_dims(axes = var_962_axes_0, x = var_961_cast_fp16)[name = string("op_962_cast_fp16")];
            tensor<int32, [1]> var_963_axes_0 = const()[name = string("op_963_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_963_cast_fp16 = expand_dims(axes = var_963_axes_0, x = var_962_cast_fp16)[name = string("op_963_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_69_cast_fp16 = mul(x = state_67_cast_fp16, y = var_963_cast_fp16)[name = string("state_69_cast_fp16")];
            tensor<int32, [4]> key_token_35_begin_0 = const()[name = string("key_token_35_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> key_token_35_end_0 = const()[name = string("key_token_35_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> key_token_35_end_mask_0 = const()[name = string("key_token_35_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_35_squeeze_mask_0 = const()[name = string("key_token_35_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_35_cast_fp16 = slice_by_index(begin = key_token_35_begin_0, end = key_token_35_end_0, end_mask = key_token_35_end_mask_0, squeeze_mask = key_token_35_squeeze_mask_0, x = key_7_cast_fp16)[name = string("key_token_35_cast_fp16")];
            tensor<int32, [4]> value_token_35_begin_0 = const()[name = string("value_token_35_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> value_token_35_end_0 = const()[name = string("value_token_35_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> value_token_35_end_mask_0 = const()[name = string("value_token_35_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_35_squeeze_mask_0 = const()[name = string("value_token_35_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_35_cast_fp16 = slice_by_index(begin = value_token_35_begin_0, end = value_token_35_end_0, end_mask = value_token_35_end_mask_0, squeeze_mask = value_token_35_squeeze_mask_0, x = value_25_cast_fp16)[name = string("value_token_35_cast_fp16")];
            tensor<int32, [1]> var_971_axes_0 = const()[name = string("op_971_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_971_cast_fp16 = expand_dims(axes = var_971_axes_0, x = key_token_35_cast_fp16)[name = string("op_971_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_972_cast_fp16 = mul(x = state_69_cast_fp16, y = var_971_cast_fp16)[name = string("op_972_cast_fp16")];
            tensor<int32, [1]> memory_35_axes_0 = const()[name = string("memory_35_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_35_keep_dims_0 = const()[name = string("memory_35_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_35_cast_fp16 = reduce_sum(axes = memory_35_axes_0, keep_dims = memory_35_keep_dims_0, x = var_972_cast_fp16)[name = string("memory_35_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_975_cast_fp16 = sub(x = value_token_35_cast_fp16, y = memory_35_cast_fp16)[name = string("op_975_cast_fp16")];
            tensor<int32, [3]> var_978_begin_0 = const()[name = string("op_978_begin_0"), val = tensor<int32, [3]>([0, 0, 1])];
            tensor<int32, [3]> var_978_end_0 = const()[name = string("op_978_end_0"), val = tensor<int32, [3]>([1, 16, 2])];
            tensor<bool, [3]> var_978_end_mask_0 = const()[name = string("op_978_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_978_squeeze_mask_0 = const()[name = string("op_978_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_978_cast_fp16 = slice_by_index(begin = var_978_begin_0, end = var_978_end_0, end_mask = var_978_end_mask_0, squeeze_mask = var_978_squeeze_mask_0, x = beta_3_cast_fp16)[name = string("op_978_cast_fp16")];
            tensor<int32, [1]> var_979_axes_0 = const()[name = string("op_979_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_979_cast_fp16 = expand_dims(axes = var_979_axes_0, x = var_978_cast_fp16)[name = string("op_979_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_35_cast_fp16 = mul(x = var_975_cast_fp16, y = var_979_cast_fp16)[name = string("delta_35_cast_fp16")];
            tensor<int32, [1]> var_982_axes_0 = const()[name = string("op_982_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_982_cast_fp16 = expand_dims(axes = var_982_axes_0, x = delta_35_cast_fp16)[name = string("op_982_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_983_cast_fp16 = mul(x = var_971_cast_fp16, y = var_982_cast_fp16)[name = string("op_983_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_71_cast_fp16 = add(x = state_69_cast_fp16, y = var_983_cast_fp16)[name = string("state_71_cast_fp16")];
            tensor<int32, [4]> var_987_begin_0 = const()[name = string("op_987_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> var_987_end_0 = const()[name = string("op_987_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> var_987_end_mask_0 = const()[name = string("op_987_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_987_squeeze_mask_0 = const()[name = string("op_987_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_987_cast_fp16 = slice_by_index(begin = var_987_begin_0, end = var_987_end_0, end_mask = var_987_end_mask_0, squeeze_mask = var_987_squeeze_mask_0, x = _inversed_query_7_cast_fp16)[name = string("op_987_cast_fp16")];
            tensor<int32, [1]> var_988_axes_0 = const()[name = string("op_988_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_988_cast_fp16 = expand_dims(axes = var_988_axes_0, x = var_987_cast_fp16)[name = string("op_988_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_989_cast_fp16 = mul(x = state_71_cast_fp16, y = var_988_cast_fp16)[name = string("op_989_cast_fp16")];
            tensor<int32, [1]> var_991_axes_0 = const()[name = string("op_991_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_991_keep_dims_0 = const()[name = string("op_991_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_991_cast_fp16 = reduce_sum(axes = var_991_axes_0, keep_dims = var_991_keep_dims_0, x = var_989_cast_fp16)[name = string("op_991_cast_fp16")];
            tensor<int32, [3]> var_994_begin_0 = const()[name = string("op_994_begin_0"), val = tensor<int32, [3]>([0, 0, 2])];
            tensor<int32, [3]> var_994_end_0 = const()[name = string("op_994_end_0"), val = tensor<int32, [3]>([1, 16, 3])];
            tensor<bool, [3]> var_994_end_mask_0 = const()[name = string("op_994_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_994_squeeze_mask_0 = const()[name = string("op_994_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_994_cast_fp16 = slice_by_index(begin = var_994_begin_0, end = var_994_end_0, end_mask = var_994_end_mask_0, squeeze_mask = var_994_squeeze_mask_0, x = decay_3_cast_fp16)[name = string("op_994_cast_fp16")];
            tensor<fp16, [1, 16]> var_995_cast_fp16 = exp(x = var_994_cast_fp16)[name = string("op_995_cast_fp16")];
            tensor<int32, [1]> var_996_axes_0 = const()[name = string("op_996_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_996_cast_fp16 = expand_dims(axes = var_996_axes_0, x = var_995_cast_fp16)[name = string("op_996_cast_fp16")];
            tensor<int32, [1]> var_997_axes_0 = const()[name = string("op_997_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_997_cast_fp16 = expand_dims(axes = var_997_axes_0, x = var_996_cast_fp16)[name = string("op_997_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_73_cast_fp16 = mul(x = state_71_cast_fp16, y = var_997_cast_fp16)[name = string("state_73_cast_fp16")];
            tensor<int32, [4]> key_token_37_begin_0 = const()[name = string("key_token_37_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> key_token_37_end_0 = const()[name = string("key_token_37_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> key_token_37_end_mask_0 = const()[name = string("key_token_37_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_37_squeeze_mask_0 = const()[name = string("key_token_37_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_37_cast_fp16 = slice_by_index(begin = key_token_37_begin_0, end = key_token_37_end_0, end_mask = key_token_37_end_mask_0, squeeze_mask = key_token_37_squeeze_mask_0, x = key_7_cast_fp16)[name = string("key_token_37_cast_fp16")];
            tensor<int32, [4]> value_token_37_begin_0 = const()[name = string("value_token_37_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> value_token_37_end_0 = const()[name = string("value_token_37_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> value_token_37_end_mask_0 = const()[name = string("value_token_37_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_37_squeeze_mask_0 = const()[name = string("value_token_37_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_37_cast_fp16 = slice_by_index(begin = value_token_37_begin_0, end = value_token_37_end_0, end_mask = value_token_37_end_mask_0, squeeze_mask = value_token_37_squeeze_mask_0, x = value_25_cast_fp16)[name = string("value_token_37_cast_fp16")];
            tensor<int32, [1]> var_1005_axes_0 = const()[name = string("op_1005_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1005_cast_fp16 = expand_dims(axes = var_1005_axes_0, x = key_token_37_cast_fp16)[name = string("op_1005_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1006_cast_fp16 = mul(x = state_73_cast_fp16, y = var_1005_cast_fp16)[name = string("op_1006_cast_fp16")];
            tensor<int32, [1]> memory_37_axes_0 = const()[name = string("memory_37_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_37_keep_dims_0 = const()[name = string("memory_37_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_37_cast_fp16 = reduce_sum(axes = memory_37_axes_0, keep_dims = memory_37_keep_dims_0, x = var_1006_cast_fp16)[name = string("memory_37_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_1009_cast_fp16 = sub(x = value_token_37_cast_fp16, y = memory_37_cast_fp16)[name = string("op_1009_cast_fp16")];
            tensor<int32, [3]> var_1012_begin_0 = const()[name = string("op_1012_begin_0"), val = tensor<int32, [3]>([0, 0, 2])];
            tensor<int32, [3]> var_1012_end_0 = const()[name = string("op_1012_end_0"), val = tensor<int32, [3]>([1, 16, 3])];
            tensor<bool, [3]> var_1012_end_mask_0 = const()[name = string("op_1012_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1012_squeeze_mask_0 = const()[name = string("op_1012_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1012_cast_fp16 = slice_by_index(begin = var_1012_begin_0, end = var_1012_end_0, end_mask = var_1012_end_mask_0, squeeze_mask = var_1012_squeeze_mask_0, x = beta_3_cast_fp16)[name = string("op_1012_cast_fp16")];
            tensor<int32, [1]> var_1013_axes_0 = const()[name = string("op_1013_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1013_cast_fp16 = expand_dims(axes = var_1013_axes_0, x = var_1012_cast_fp16)[name = string("op_1013_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_37_cast_fp16 = mul(x = var_1009_cast_fp16, y = var_1013_cast_fp16)[name = string("delta_37_cast_fp16")];
            tensor<int32, [1]> var_1016_axes_0 = const()[name = string("op_1016_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_1016_cast_fp16 = expand_dims(axes = var_1016_axes_0, x = delta_37_cast_fp16)[name = string("op_1016_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1017_cast_fp16 = mul(x = var_1005_cast_fp16, y = var_1016_cast_fp16)[name = string("op_1017_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_75_cast_fp16 = add(x = state_73_cast_fp16, y = var_1017_cast_fp16)[name = string("state_75_cast_fp16")];
            tensor<int32, [4]> var_1021_begin_0 = const()[name = string("op_1021_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> var_1021_end_0 = const()[name = string("op_1021_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> var_1021_end_mask_0 = const()[name = string("op_1021_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_1021_squeeze_mask_0 = const()[name = string("op_1021_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_1021_cast_fp16 = slice_by_index(begin = var_1021_begin_0, end = var_1021_end_0, end_mask = var_1021_end_mask_0, squeeze_mask = var_1021_squeeze_mask_0, x = _inversed_query_7_cast_fp16)[name = string("op_1021_cast_fp16")];
            tensor<int32, [1]> var_1022_axes_0 = const()[name = string("op_1022_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1022_cast_fp16 = expand_dims(axes = var_1022_axes_0, x = var_1021_cast_fp16)[name = string("op_1022_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1023_cast_fp16 = mul(x = state_75_cast_fp16, y = var_1022_cast_fp16)[name = string("op_1023_cast_fp16")];
            tensor<int32, [1]> var_1025_axes_0 = const()[name = string("op_1025_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_1025_keep_dims_0 = const()[name = string("op_1025_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_1025_cast_fp16 = reduce_sum(axes = var_1025_axes_0, keep_dims = var_1025_keep_dims_0, x = var_1023_cast_fp16)[name = string("op_1025_cast_fp16")];
            tensor<int32, [3]> var_1028_begin_0 = const()[name = string("op_1028_begin_0"), val = tensor<int32, [3]>([0, 0, 3])];
            tensor<int32, [3]> var_1028_end_0 = const()[name = string("op_1028_end_0"), val = tensor<int32, [3]>([1, 16, 4])];
            tensor<bool, [3]> var_1028_end_mask_0 = const()[name = string("op_1028_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1028_squeeze_mask_0 = const()[name = string("op_1028_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1028_cast_fp16 = slice_by_index(begin = var_1028_begin_0, end = var_1028_end_0, end_mask = var_1028_end_mask_0, squeeze_mask = var_1028_squeeze_mask_0, x = decay_3_cast_fp16)[name = string("op_1028_cast_fp16")];
            tensor<fp16, [1, 16]> var_1029_cast_fp16 = exp(x = var_1028_cast_fp16)[name = string("op_1029_cast_fp16")];
            tensor<int32, [1]> var_1030_axes_0 = const()[name = string("op_1030_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1030_cast_fp16 = expand_dims(axes = var_1030_axes_0, x = var_1029_cast_fp16)[name = string("op_1030_cast_fp16")];
            tensor<int32, [1]> var_1031_axes_0 = const()[name = string("op_1031_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_1031_cast_fp16 = expand_dims(axes = var_1031_axes_0, x = var_1030_cast_fp16)[name = string("op_1031_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_77_cast_fp16 = mul(x = state_75_cast_fp16, y = var_1031_cast_fp16)[name = string("state_77_cast_fp16")];
            tensor<int32, [4]> key_token_39_begin_0 = const()[name = string("key_token_39_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> key_token_39_end_0 = const()[name = string("key_token_39_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> key_token_39_end_mask_0 = const()[name = string("key_token_39_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_39_squeeze_mask_0 = const()[name = string("key_token_39_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_39_cast_fp16 = slice_by_index(begin = key_token_39_begin_0, end = key_token_39_end_0, end_mask = key_token_39_end_mask_0, squeeze_mask = key_token_39_squeeze_mask_0, x = key_7_cast_fp16)[name = string("key_token_39_cast_fp16")];
            tensor<int32, [4]> value_token_39_begin_0 = const()[name = string("value_token_39_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> value_token_39_end_0 = const()[name = string("value_token_39_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> value_token_39_end_mask_0 = const()[name = string("value_token_39_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_39_squeeze_mask_0 = const()[name = string("value_token_39_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_39_cast_fp16 = slice_by_index(begin = value_token_39_begin_0, end = value_token_39_end_0, end_mask = value_token_39_end_mask_0, squeeze_mask = value_token_39_squeeze_mask_0, x = value_25_cast_fp16)[name = string("value_token_39_cast_fp16")];
            tensor<int32, [1]> var_1039_axes_0 = const()[name = string("op_1039_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1039_cast_fp16 = expand_dims(axes = var_1039_axes_0, x = key_token_39_cast_fp16)[name = string("op_1039_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1040_cast_fp16 = mul(x = state_77_cast_fp16, y = var_1039_cast_fp16)[name = string("op_1040_cast_fp16")];
            tensor<int32, [1]> memory_39_axes_0 = const()[name = string("memory_39_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_39_keep_dims_0 = const()[name = string("memory_39_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_39_cast_fp16 = reduce_sum(axes = memory_39_axes_0, keep_dims = memory_39_keep_dims_0, x = var_1040_cast_fp16)[name = string("memory_39_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_1043_cast_fp16 = sub(x = value_token_39_cast_fp16, y = memory_39_cast_fp16)[name = string("op_1043_cast_fp16")];
            tensor<int32, [3]> var_1046_begin_0 = const()[name = string("op_1046_begin_0"), val = tensor<int32, [3]>([0, 0, 3])];
            tensor<int32, [3]> var_1046_end_0 = const()[name = string("op_1046_end_0"), val = tensor<int32, [3]>([1, 16, 4])];
            tensor<bool, [3]> var_1046_end_mask_0 = const()[name = string("op_1046_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1046_squeeze_mask_0 = const()[name = string("op_1046_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1046_cast_fp16 = slice_by_index(begin = var_1046_begin_0, end = var_1046_end_0, end_mask = var_1046_end_mask_0, squeeze_mask = var_1046_squeeze_mask_0, x = beta_3_cast_fp16)[name = string("op_1046_cast_fp16")];
            tensor<int32, [1]> var_1047_axes_0 = const()[name = string("op_1047_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1047_cast_fp16 = expand_dims(axes = var_1047_axes_0, x = var_1046_cast_fp16)[name = string("op_1047_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_39_cast_fp16 = mul(x = var_1043_cast_fp16, y = var_1047_cast_fp16)[name = string("delta_39_cast_fp16")];
            tensor<int32, [1]> var_1050_axes_0 = const()[name = string("op_1050_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_1050_cast_fp16 = expand_dims(axes = var_1050_axes_0, x = delta_39_cast_fp16)[name = string("op_1050_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1051_cast_fp16 = mul(x = var_1039_cast_fp16, y = var_1050_cast_fp16)[name = string("op_1051_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_79_cast_fp16 = add(x = state_77_cast_fp16, y = var_1051_cast_fp16)[name = string("state_79_cast_fp16")];
            tensor<int32, [4]> var_1055_begin_0 = const()[name = string("op_1055_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> var_1055_end_0 = const()[name = string("op_1055_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> var_1055_end_mask_0 = const()[name = string("op_1055_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_1055_squeeze_mask_0 = const()[name = string("op_1055_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_1055_cast_fp16 = slice_by_index(begin = var_1055_begin_0, end = var_1055_end_0, end_mask = var_1055_end_mask_0, squeeze_mask = var_1055_squeeze_mask_0, x = _inversed_query_7_cast_fp16)[name = string("op_1055_cast_fp16")];
            tensor<int32, [1]> var_1056_axes_0 = const()[name = string("op_1056_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1056_cast_fp16 = expand_dims(axes = var_1056_axes_0, x = var_1055_cast_fp16)[name = string("op_1056_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1057_cast_fp16 = mul(x = state_79_cast_fp16, y = var_1056_cast_fp16)[name = string("op_1057_cast_fp16")];
            tensor<int32, [1]> var_1059_axes_0 = const()[name = string("op_1059_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_1059_keep_dims_0 = const()[name = string("op_1059_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_1059_cast_fp16 = reduce_sum(axes = var_1059_axes_0, keep_dims = var_1059_keep_dims_0, x = var_1057_cast_fp16)[name = string("op_1059_cast_fp16")];
            tensor<int32, [3]> var_1062_begin_0 = const()[name = string("op_1062_begin_0"), val = tensor<int32, [3]>([0, 0, 4])];
            tensor<int32, [3]> var_1062_end_0 = const()[name = string("op_1062_end_0"), val = tensor<int32, [3]>([1, 16, 5])];
            tensor<bool, [3]> var_1062_end_mask_0 = const()[name = string("op_1062_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1062_squeeze_mask_0 = const()[name = string("op_1062_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1062_cast_fp16 = slice_by_index(begin = var_1062_begin_0, end = var_1062_end_0, end_mask = var_1062_end_mask_0, squeeze_mask = var_1062_squeeze_mask_0, x = decay_3_cast_fp16)[name = string("op_1062_cast_fp16")];
            tensor<fp16, [1, 16]> var_1063_cast_fp16 = exp(x = var_1062_cast_fp16)[name = string("op_1063_cast_fp16")];
            tensor<int32, [1]> var_1064_axes_0 = const()[name = string("op_1064_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1064_cast_fp16 = expand_dims(axes = var_1064_axes_0, x = var_1063_cast_fp16)[name = string("op_1064_cast_fp16")];
            tensor<int32, [1]> var_1065_axes_0 = const()[name = string("op_1065_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_1065_cast_fp16 = expand_dims(axes = var_1065_axes_0, x = var_1064_cast_fp16)[name = string("op_1065_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_81_cast_fp16 = mul(x = state_79_cast_fp16, y = var_1065_cast_fp16)[name = string("state_81_cast_fp16")];
            tensor<int32, [4]> key_token_41_begin_0 = const()[name = string("key_token_41_begin_0"), val = tensor<int32, [4]>([0, 0, 4, 0])];
            tensor<int32, [4]> key_token_41_end_0 = const()[name = string("key_token_41_end_0"), val = tensor<int32, [4]>([1, 16, 5, 128])];
            tensor<bool, [4]> key_token_41_end_mask_0 = const()[name = string("key_token_41_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_41_squeeze_mask_0 = const()[name = string("key_token_41_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_41_cast_fp16 = slice_by_index(begin = key_token_41_begin_0, end = key_token_41_end_0, end_mask = key_token_41_end_mask_0, squeeze_mask = key_token_41_squeeze_mask_0, x = key_7_cast_fp16)[name = string("key_token_41_cast_fp16")];
            tensor<int32, [4]> value_token_41_begin_0 = const()[name = string("value_token_41_begin_0"), val = tensor<int32, [4]>([0, 0, 4, 0])];
            tensor<int32, [4]> value_token_41_end_0 = const()[name = string("value_token_41_end_0"), val = tensor<int32, [4]>([1, 16, 5, 128])];
            tensor<bool, [4]> value_token_41_end_mask_0 = const()[name = string("value_token_41_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_41_squeeze_mask_0 = const()[name = string("value_token_41_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_41_cast_fp16 = slice_by_index(begin = value_token_41_begin_0, end = value_token_41_end_0, end_mask = value_token_41_end_mask_0, squeeze_mask = value_token_41_squeeze_mask_0, x = value_25_cast_fp16)[name = string("value_token_41_cast_fp16")];
            tensor<int32, [1]> var_1073_axes_0 = const()[name = string("op_1073_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1073_cast_fp16 = expand_dims(axes = var_1073_axes_0, x = key_token_41_cast_fp16)[name = string("op_1073_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1074_cast_fp16 = mul(x = state_81_cast_fp16, y = var_1073_cast_fp16)[name = string("op_1074_cast_fp16")];
            tensor<int32, [1]> memory_41_axes_0 = const()[name = string("memory_41_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_41_keep_dims_0 = const()[name = string("memory_41_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_41_cast_fp16 = reduce_sum(axes = memory_41_axes_0, keep_dims = memory_41_keep_dims_0, x = var_1074_cast_fp16)[name = string("memory_41_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_1077_cast_fp16 = sub(x = value_token_41_cast_fp16, y = memory_41_cast_fp16)[name = string("op_1077_cast_fp16")];
            tensor<int32, [3]> var_1080_begin_0 = const()[name = string("op_1080_begin_0"), val = tensor<int32, [3]>([0, 0, 4])];
            tensor<int32, [3]> var_1080_end_0 = const()[name = string("op_1080_end_0"), val = tensor<int32, [3]>([1, 16, 5])];
            tensor<bool, [3]> var_1080_end_mask_0 = const()[name = string("op_1080_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1080_squeeze_mask_0 = const()[name = string("op_1080_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1080_cast_fp16 = slice_by_index(begin = var_1080_begin_0, end = var_1080_end_0, end_mask = var_1080_end_mask_0, squeeze_mask = var_1080_squeeze_mask_0, x = beta_3_cast_fp16)[name = string("op_1080_cast_fp16")];
            tensor<int32, [1]> var_1081_axes_0 = const()[name = string("op_1081_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1081_cast_fp16 = expand_dims(axes = var_1081_axes_0, x = var_1080_cast_fp16)[name = string("op_1081_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_41_cast_fp16 = mul(x = var_1077_cast_fp16, y = var_1081_cast_fp16)[name = string("delta_41_cast_fp16")];
            tensor<int32, [1]> var_1084_axes_0 = const()[name = string("op_1084_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_1084_cast_fp16 = expand_dims(axes = var_1084_axes_0, x = delta_41_cast_fp16)[name = string("op_1084_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1085_cast_fp16 = mul(x = var_1073_cast_fp16, y = var_1084_cast_fp16)[name = string("op_1085_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_83_cast_fp16 = add(x = state_81_cast_fp16, y = var_1085_cast_fp16)[name = string("state_83_cast_fp16")];
            tensor<int32, [4]> var_1089_begin_0 = const()[name = string("op_1089_begin_0"), val = tensor<int32, [4]>([0, 0, 4, 0])];
            tensor<int32, [4]> var_1089_end_0 = const()[name = string("op_1089_end_0"), val = tensor<int32, [4]>([1, 16, 5, 128])];
            tensor<bool, [4]> var_1089_end_mask_0 = const()[name = string("op_1089_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_1089_squeeze_mask_0 = const()[name = string("op_1089_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_1089_cast_fp16 = slice_by_index(begin = var_1089_begin_0, end = var_1089_end_0, end_mask = var_1089_end_mask_0, squeeze_mask = var_1089_squeeze_mask_0, x = _inversed_query_7_cast_fp16)[name = string("op_1089_cast_fp16")];
            tensor<int32, [1]> var_1090_axes_0 = const()[name = string("op_1090_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1090_cast_fp16 = expand_dims(axes = var_1090_axes_0, x = var_1089_cast_fp16)[name = string("op_1090_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1091_cast_fp16 = mul(x = state_83_cast_fp16, y = var_1090_cast_fp16)[name = string("op_1091_cast_fp16")];
            tensor<int32, [1]> var_1093_axes_0 = const()[name = string("op_1093_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_1093_keep_dims_0 = const()[name = string("op_1093_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_1093_cast_fp16 = reduce_sum(axes = var_1093_axes_0, keep_dims = var_1093_keep_dims_0, x = var_1091_cast_fp16)[name = string("op_1093_cast_fp16")];
            tensor<int32, [3]> var_1096_begin_0 = const()[name = string("op_1096_begin_0"), val = tensor<int32, [3]>([0, 0, 5])];
            tensor<int32, [3]> var_1096_end_0 = const()[name = string("op_1096_end_0"), val = tensor<int32, [3]>([1, 16, 6])];
            tensor<bool, [3]> var_1096_end_mask_0 = const()[name = string("op_1096_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1096_squeeze_mask_0 = const()[name = string("op_1096_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1096_cast_fp16 = slice_by_index(begin = var_1096_begin_0, end = var_1096_end_0, end_mask = var_1096_end_mask_0, squeeze_mask = var_1096_squeeze_mask_0, x = decay_3_cast_fp16)[name = string("op_1096_cast_fp16")];
            tensor<fp16, [1, 16]> var_1097_cast_fp16 = exp(x = var_1096_cast_fp16)[name = string("op_1097_cast_fp16")];
            tensor<int32, [1]> var_1098_axes_0 = const()[name = string("op_1098_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1098_cast_fp16 = expand_dims(axes = var_1098_axes_0, x = var_1097_cast_fp16)[name = string("op_1098_cast_fp16")];
            tensor<int32, [1]> var_1099_axes_0 = const()[name = string("op_1099_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_1099_cast_fp16 = expand_dims(axes = var_1099_axes_0, x = var_1098_cast_fp16)[name = string("op_1099_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_85_cast_fp16 = mul(x = state_83_cast_fp16, y = var_1099_cast_fp16)[name = string("state_85_cast_fp16")];
            tensor<int32, [4]> key_token_43_begin_0 = const()[name = string("key_token_43_begin_0"), val = tensor<int32, [4]>([0, 0, 5, 0])];
            tensor<int32, [4]> key_token_43_end_0 = const()[name = string("key_token_43_end_0"), val = tensor<int32, [4]>([1, 16, 6, 128])];
            tensor<bool, [4]> key_token_43_end_mask_0 = const()[name = string("key_token_43_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_43_squeeze_mask_0 = const()[name = string("key_token_43_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_43_cast_fp16 = slice_by_index(begin = key_token_43_begin_0, end = key_token_43_end_0, end_mask = key_token_43_end_mask_0, squeeze_mask = key_token_43_squeeze_mask_0, x = key_7_cast_fp16)[name = string("key_token_43_cast_fp16")];
            tensor<int32, [4]> value_token_43_begin_0 = const()[name = string("value_token_43_begin_0"), val = tensor<int32, [4]>([0, 0, 5, 0])];
            tensor<int32, [4]> value_token_43_end_0 = const()[name = string("value_token_43_end_0"), val = tensor<int32, [4]>([1, 16, 6, 128])];
            tensor<bool, [4]> value_token_43_end_mask_0 = const()[name = string("value_token_43_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_43_squeeze_mask_0 = const()[name = string("value_token_43_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_43_cast_fp16 = slice_by_index(begin = value_token_43_begin_0, end = value_token_43_end_0, end_mask = value_token_43_end_mask_0, squeeze_mask = value_token_43_squeeze_mask_0, x = value_25_cast_fp16)[name = string("value_token_43_cast_fp16")];
            tensor<int32, [1]> var_1107_axes_0 = const()[name = string("op_1107_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1107_cast_fp16 = expand_dims(axes = var_1107_axes_0, x = key_token_43_cast_fp16)[name = string("op_1107_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1108_cast_fp16 = mul(x = state_85_cast_fp16, y = var_1107_cast_fp16)[name = string("op_1108_cast_fp16")];
            tensor<int32, [1]> memory_43_axes_0 = const()[name = string("memory_43_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_43_keep_dims_0 = const()[name = string("memory_43_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_43_cast_fp16 = reduce_sum(axes = memory_43_axes_0, keep_dims = memory_43_keep_dims_0, x = var_1108_cast_fp16)[name = string("memory_43_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_1111_cast_fp16 = sub(x = value_token_43_cast_fp16, y = memory_43_cast_fp16)[name = string("op_1111_cast_fp16")];
            tensor<int32, [3]> var_1114_begin_0 = const()[name = string("op_1114_begin_0"), val = tensor<int32, [3]>([0, 0, 5])];
            tensor<int32, [3]> var_1114_end_0 = const()[name = string("op_1114_end_0"), val = tensor<int32, [3]>([1, 16, 6])];
            tensor<bool, [3]> var_1114_end_mask_0 = const()[name = string("op_1114_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1114_squeeze_mask_0 = const()[name = string("op_1114_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1114_cast_fp16 = slice_by_index(begin = var_1114_begin_0, end = var_1114_end_0, end_mask = var_1114_end_mask_0, squeeze_mask = var_1114_squeeze_mask_0, x = beta_3_cast_fp16)[name = string("op_1114_cast_fp16")];
            tensor<int32, [1]> var_1115_axes_0 = const()[name = string("op_1115_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1115_cast_fp16 = expand_dims(axes = var_1115_axes_0, x = var_1114_cast_fp16)[name = string("op_1115_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_43_cast_fp16 = mul(x = var_1111_cast_fp16, y = var_1115_cast_fp16)[name = string("delta_43_cast_fp16")];
            tensor<int32, [1]> var_1118_axes_0 = const()[name = string("op_1118_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_1118_cast_fp16 = expand_dims(axes = var_1118_axes_0, x = delta_43_cast_fp16)[name = string("op_1118_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1119_cast_fp16 = mul(x = var_1107_cast_fp16, y = var_1118_cast_fp16)[name = string("op_1119_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_87_cast_fp16 = add(x = state_85_cast_fp16, y = var_1119_cast_fp16)[name = string("state_87_cast_fp16")];
            tensor<int32, [4]> var_1123_begin_0 = const()[name = string("op_1123_begin_0"), val = tensor<int32, [4]>([0, 0, 5, 0])];
            tensor<int32, [4]> var_1123_end_0 = const()[name = string("op_1123_end_0"), val = tensor<int32, [4]>([1, 16, 6, 128])];
            tensor<bool, [4]> var_1123_end_mask_0 = const()[name = string("op_1123_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_1123_squeeze_mask_0 = const()[name = string("op_1123_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_1123_cast_fp16 = slice_by_index(begin = var_1123_begin_0, end = var_1123_end_0, end_mask = var_1123_end_mask_0, squeeze_mask = var_1123_squeeze_mask_0, x = _inversed_query_7_cast_fp16)[name = string("op_1123_cast_fp16")];
            tensor<int32, [1]> var_1124_axes_0 = const()[name = string("op_1124_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1124_cast_fp16 = expand_dims(axes = var_1124_axes_0, x = var_1123_cast_fp16)[name = string("op_1124_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1125_cast_fp16 = mul(x = state_87_cast_fp16, y = var_1124_cast_fp16)[name = string("op_1125_cast_fp16")];
            tensor<int32, [1]> var_1127_axes_0 = const()[name = string("op_1127_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_1127_keep_dims_0 = const()[name = string("op_1127_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_1127_cast_fp16 = reduce_sum(axes = var_1127_axes_0, keep_dims = var_1127_keep_dims_0, x = var_1125_cast_fp16)[name = string("op_1127_cast_fp16")];
            tensor<int32, [3]> var_1130_begin_0 = const()[name = string("op_1130_begin_0"), val = tensor<int32, [3]>([0, 0, 6])];
            tensor<int32, [3]> var_1130_end_0 = const()[name = string("op_1130_end_0"), val = tensor<int32, [3]>([1, 16, 7])];
            tensor<bool, [3]> var_1130_end_mask_0 = const()[name = string("op_1130_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1130_squeeze_mask_0 = const()[name = string("op_1130_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1130_cast_fp16 = slice_by_index(begin = var_1130_begin_0, end = var_1130_end_0, end_mask = var_1130_end_mask_0, squeeze_mask = var_1130_squeeze_mask_0, x = decay_3_cast_fp16)[name = string("op_1130_cast_fp16")];
            tensor<fp16, [1, 16]> var_1131_cast_fp16 = exp(x = var_1130_cast_fp16)[name = string("op_1131_cast_fp16")];
            tensor<int32, [1]> var_1132_axes_0 = const()[name = string("op_1132_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1132_cast_fp16 = expand_dims(axes = var_1132_axes_0, x = var_1131_cast_fp16)[name = string("op_1132_cast_fp16")];
            tensor<int32, [1]> var_1133_axes_0 = const()[name = string("op_1133_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_1133_cast_fp16 = expand_dims(axes = var_1133_axes_0, x = var_1132_cast_fp16)[name = string("op_1133_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_89_cast_fp16 = mul(x = state_87_cast_fp16, y = var_1133_cast_fp16)[name = string("state_89_cast_fp16")];
            tensor<int32, [4]> key_token_45_begin_0 = const()[name = string("key_token_45_begin_0"), val = tensor<int32, [4]>([0, 0, 6, 0])];
            tensor<int32, [4]> key_token_45_end_0 = const()[name = string("key_token_45_end_0"), val = tensor<int32, [4]>([1, 16, 7, 128])];
            tensor<bool, [4]> key_token_45_end_mask_0 = const()[name = string("key_token_45_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_45_squeeze_mask_0 = const()[name = string("key_token_45_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_45_cast_fp16 = slice_by_index(begin = key_token_45_begin_0, end = key_token_45_end_0, end_mask = key_token_45_end_mask_0, squeeze_mask = key_token_45_squeeze_mask_0, x = key_7_cast_fp16)[name = string("key_token_45_cast_fp16")];
            tensor<int32, [4]> value_token_45_begin_0 = const()[name = string("value_token_45_begin_0"), val = tensor<int32, [4]>([0, 0, 6, 0])];
            tensor<int32, [4]> value_token_45_end_0 = const()[name = string("value_token_45_end_0"), val = tensor<int32, [4]>([1, 16, 7, 128])];
            tensor<bool, [4]> value_token_45_end_mask_0 = const()[name = string("value_token_45_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_45_squeeze_mask_0 = const()[name = string("value_token_45_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_45_cast_fp16 = slice_by_index(begin = value_token_45_begin_0, end = value_token_45_end_0, end_mask = value_token_45_end_mask_0, squeeze_mask = value_token_45_squeeze_mask_0, x = value_25_cast_fp16)[name = string("value_token_45_cast_fp16")];
            tensor<int32, [1]> var_1141_axes_0 = const()[name = string("op_1141_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1141_cast_fp16 = expand_dims(axes = var_1141_axes_0, x = key_token_45_cast_fp16)[name = string("op_1141_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1142_cast_fp16 = mul(x = state_89_cast_fp16, y = var_1141_cast_fp16)[name = string("op_1142_cast_fp16")];
            tensor<int32, [1]> memory_45_axes_0 = const()[name = string("memory_45_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_45_keep_dims_0 = const()[name = string("memory_45_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_45_cast_fp16 = reduce_sum(axes = memory_45_axes_0, keep_dims = memory_45_keep_dims_0, x = var_1142_cast_fp16)[name = string("memory_45_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_1145_cast_fp16 = sub(x = value_token_45_cast_fp16, y = memory_45_cast_fp16)[name = string("op_1145_cast_fp16")];
            tensor<int32, [3]> var_1148_begin_0 = const()[name = string("op_1148_begin_0"), val = tensor<int32, [3]>([0, 0, 6])];
            tensor<int32, [3]> var_1148_end_0 = const()[name = string("op_1148_end_0"), val = tensor<int32, [3]>([1, 16, 7])];
            tensor<bool, [3]> var_1148_end_mask_0 = const()[name = string("op_1148_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1148_squeeze_mask_0 = const()[name = string("op_1148_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1148_cast_fp16 = slice_by_index(begin = var_1148_begin_0, end = var_1148_end_0, end_mask = var_1148_end_mask_0, squeeze_mask = var_1148_squeeze_mask_0, x = beta_3_cast_fp16)[name = string("op_1148_cast_fp16")];
            tensor<int32, [1]> var_1149_axes_0 = const()[name = string("op_1149_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1149_cast_fp16 = expand_dims(axes = var_1149_axes_0, x = var_1148_cast_fp16)[name = string("op_1149_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_45_cast_fp16 = mul(x = var_1145_cast_fp16, y = var_1149_cast_fp16)[name = string("delta_45_cast_fp16")];
            tensor<int32, [1]> var_1152_axes_0 = const()[name = string("op_1152_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_1152_cast_fp16 = expand_dims(axes = var_1152_axes_0, x = delta_45_cast_fp16)[name = string("op_1152_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1153_cast_fp16 = mul(x = var_1141_cast_fp16, y = var_1152_cast_fp16)[name = string("op_1153_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_91_cast_fp16 = add(x = state_89_cast_fp16, y = var_1153_cast_fp16)[name = string("state_91_cast_fp16")];
            tensor<int32, [4]> var_1157_begin_0 = const()[name = string("op_1157_begin_0"), val = tensor<int32, [4]>([0, 0, 6, 0])];
            tensor<int32, [4]> var_1157_end_0 = const()[name = string("op_1157_end_0"), val = tensor<int32, [4]>([1, 16, 7, 128])];
            tensor<bool, [4]> var_1157_end_mask_0 = const()[name = string("op_1157_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_1157_squeeze_mask_0 = const()[name = string("op_1157_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_1157_cast_fp16 = slice_by_index(begin = var_1157_begin_0, end = var_1157_end_0, end_mask = var_1157_end_mask_0, squeeze_mask = var_1157_squeeze_mask_0, x = _inversed_query_7_cast_fp16)[name = string("op_1157_cast_fp16")];
            tensor<int32, [1]> var_1158_axes_0 = const()[name = string("op_1158_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1158_cast_fp16 = expand_dims(axes = var_1158_axes_0, x = var_1157_cast_fp16)[name = string("op_1158_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1159_cast_fp16 = mul(x = state_91_cast_fp16, y = var_1158_cast_fp16)[name = string("op_1159_cast_fp16")];
            tensor<int32, [1]> var_1161_axes_0 = const()[name = string("op_1161_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_1161_keep_dims_0 = const()[name = string("op_1161_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_1161_cast_fp16 = reduce_sum(axes = var_1161_axes_0, keep_dims = var_1161_keep_dims_0, x = var_1159_cast_fp16)[name = string("op_1161_cast_fp16")];
            tensor<int32, [3]> var_1164_begin_0 = const()[name = string("op_1164_begin_0"), val = tensor<int32, [3]>([0, 0, 7])];
            tensor<int32, [3]> var_1164_end_0 = const()[name = string("op_1164_end_0"), val = tensor<int32, [3]>([1, 16, 8])];
            tensor<bool, [3]> var_1164_end_mask_0 = const()[name = string("op_1164_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1164_squeeze_mask_0 = const()[name = string("op_1164_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1164_cast_fp16 = slice_by_index(begin = var_1164_begin_0, end = var_1164_end_0, end_mask = var_1164_end_mask_0, squeeze_mask = var_1164_squeeze_mask_0, x = decay_3_cast_fp16)[name = string("op_1164_cast_fp16")];
            tensor<fp16, [1, 16]> var_1165_cast_fp16 = exp(x = var_1164_cast_fp16)[name = string("op_1165_cast_fp16")];
            tensor<int32, [1]> var_1166_axes_0 = const()[name = string("op_1166_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1166_cast_fp16 = expand_dims(axes = var_1166_axes_0, x = var_1165_cast_fp16)[name = string("op_1166_cast_fp16")];
            tensor<int32, [1]> var_1167_axes_0 = const()[name = string("op_1167_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_1167_cast_fp16 = expand_dims(axes = var_1167_axes_0, x = var_1166_cast_fp16)[name = string("op_1167_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_93_cast_fp16 = mul(x = state_91_cast_fp16, y = var_1167_cast_fp16)[name = string("state_93_cast_fp16")];
            tensor<int32, [4]> key_token_47_begin_0 = const()[name = string("key_token_47_begin_0"), val = tensor<int32, [4]>([0, 0, 7, 0])];
            tensor<int32, [4]> key_token_47_end_0 = const()[name = string("key_token_47_end_0"), val = tensor<int32, [4]>([1, 16, 8, 128])];
            tensor<bool, [4]> key_token_47_end_mask_0 = const()[name = string("key_token_47_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_47_squeeze_mask_0 = const()[name = string("key_token_47_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_47_cast_fp16 = slice_by_index(begin = key_token_47_begin_0, end = key_token_47_end_0, end_mask = key_token_47_end_mask_0, squeeze_mask = key_token_47_squeeze_mask_0, x = key_7_cast_fp16)[name = string("key_token_47_cast_fp16")];
            tensor<int32, [4]> value_token_47_begin_0 = const()[name = string("value_token_47_begin_0"), val = tensor<int32, [4]>([0, 0, 7, 0])];
            tensor<int32, [4]> value_token_47_end_0 = const()[name = string("value_token_47_end_0"), val = tensor<int32, [4]>([1, 16, 8, 128])];
            tensor<bool, [4]> value_token_47_end_mask_0 = const()[name = string("value_token_47_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_47_squeeze_mask_0 = const()[name = string("value_token_47_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_47_cast_fp16 = slice_by_index(begin = value_token_47_begin_0, end = value_token_47_end_0, end_mask = value_token_47_end_mask_0, squeeze_mask = value_token_47_squeeze_mask_0, x = value_25_cast_fp16)[name = string("value_token_47_cast_fp16")];
            tensor<int32, [1]> var_1175_axes_0 = const()[name = string("op_1175_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1175_cast_fp16 = expand_dims(axes = var_1175_axes_0, x = key_token_47_cast_fp16)[name = string("op_1175_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1176_cast_fp16 = mul(x = state_93_cast_fp16, y = var_1175_cast_fp16)[name = string("op_1176_cast_fp16")];
            tensor<int32, [1]> memory_47_axes_0 = const()[name = string("memory_47_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_47_keep_dims_0 = const()[name = string("memory_47_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_47_cast_fp16 = reduce_sum(axes = memory_47_axes_0, keep_dims = memory_47_keep_dims_0, x = var_1176_cast_fp16)[name = string("memory_47_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_1179_cast_fp16 = sub(x = value_token_47_cast_fp16, y = memory_47_cast_fp16)[name = string("op_1179_cast_fp16")];
            tensor<int32, [3]> var_1182_begin_0 = const()[name = string("op_1182_begin_0"), val = tensor<int32, [3]>([0, 0, 7])];
            tensor<int32, [3]> var_1182_end_0 = const()[name = string("op_1182_end_0"), val = tensor<int32, [3]>([1, 16, 8])];
            tensor<bool, [3]> var_1182_end_mask_0 = const()[name = string("op_1182_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1182_squeeze_mask_0 = const()[name = string("op_1182_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1182_cast_fp16 = slice_by_index(begin = var_1182_begin_0, end = var_1182_end_0, end_mask = var_1182_end_mask_0, squeeze_mask = var_1182_squeeze_mask_0, x = beta_3_cast_fp16)[name = string("op_1182_cast_fp16")];
            tensor<int32, [1]> var_1183_axes_0 = const()[name = string("op_1183_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1183_cast_fp16 = expand_dims(axes = var_1183_axes_0, x = var_1182_cast_fp16)[name = string("op_1183_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_47_cast_fp16 = mul(x = var_1179_cast_fp16, y = var_1183_cast_fp16)[name = string("delta_47_cast_fp16")];
            tensor<int32, [1]> var_1186_axes_0 = const()[name = string("op_1186_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_1186_cast_fp16 = expand_dims(axes = var_1186_axes_0, x = delta_47_cast_fp16)[name = string("op_1186_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1187_cast_fp16 = mul(x = var_1175_cast_fp16, y = var_1186_cast_fp16)[name = string("op_1187_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_95_cast_fp16 = add(x = state_93_cast_fp16, y = var_1187_cast_fp16)[name = string("state_95_cast_fp16")];
            tensor<int32, [4]> var_1191_begin_0 = const()[name = string("op_1191_begin_0"), val = tensor<int32, [4]>([0, 0, 7, 0])];
            tensor<int32, [4]> var_1191_end_0 = const()[name = string("op_1191_end_0"), val = tensor<int32, [4]>([1, 16, 8, 128])];
            tensor<bool, [4]> var_1191_end_mask_0 = const()[name = string("op_1191_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_1191_squeeze_mask_0 = const()[name = string("op_1191_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_1191_cast_fp16 = slice_by_index(begin = var_1191_begin_0, end = var_1191_end_0, end_mask = var_1191_end_mask_0, squeeze_mask = var_1191_squeeze_mask_0, x = _inversed_query_7_cast_fp16)[name = string("op_1191_cast_fp16")];
            tensor<int32, [1]> var_1192_axes_0 = const()[name = string("op_1192_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1192_cast_fp16 = expand_dims(axes = var_1192_axes_0, x = var_1191_cast_fp16)[name = string("op_1192_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1193_cast_fp16 = mul(x = state_95_cast_fp16, y = var_1192_cast_fp16)[name = string("op_1193_cast_fp16")];
            tensor<int32, [1]> var_1195_axes_0 = const()[name = string("op_1195_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_1195_keep_dims_0 = const()[name = string("op_1195_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_1195_cast_fp16 = reduce_sum(axes = var_1195_axes_0, keep_dims = var_1195_keep_dims_0, x = var_1193_cast_fp16)[name = string("op_1195_cast_fp16")];
            tensor<int32, [3]> var_1198_begin_0 = const()[name = string("op_1198_begin_0"), val = tensor<int32, [3]>([0, 0, 8])];
            tensor<int32, [3]> var_1198_end_0 = const()[name = string("op_1198_end_0"), val = tensor<int32, [3]>([1, 16, 9])];
            tensor<bool, [3]> var_1198_end_mask_0 = const()[name = string("op_1198_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1198_squeeze_mask_0 = const()[name = string("op_1198_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1198_cast_fp16 = slice_by_index(begin = var_1198_begin_0, end = var_1198_end_0, end_mask = var_1198_end_mask_0, squeeze_mask = var_1198_squeeze_mask_0, x = decay_3_cast_fp16)[name = string("op_1198_cast_fp16")];
            tensor<fp16, [1, 16]> var_1199_cast_fp16 = exp(x = var_1198_cast_fp16)[name = string("op_1199_cast_fp16")];
            tensor<int32, [1]> var_1200_axes_0 = const()[name = string("op_1200_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1200_cast_fp16 = expand_dims(axes = var_1200_axes_0, x = var_1199_cast_fp16)[name = string("op_1200_cast_fp16")];
            tensor<int32, [1]> var_1201_axes_0 = const()[name = string("op_1201_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_1201_cast_fp16 = expand_dims(axes = var_1201_axes_0, x = var_1200_cast_fp16)[name = string("op_1201_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_97_cast_fp16 = mul(x = state_95_cast_fp16, y = var_1201_cast_fp16)[name = string("state_97_cast_fp16")];
            tensor<int32, [4]> key_token_49_begin_0 = const()[name = string("key_token_49_begin_0"), val = tensor<int32, [4]>([0, 0, 8, 0])];
            tensor<int32, [4]> key_token_49_end_0 = const()[name = string("key_token_49_end_0"), val = tensor<int32, [4]>([1, 16, 9, 128])];
            tensor<bool, [4]> key_token_49_end_mask_0 = const()[name = string("key_token_49_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_49_squeeze_mask_0 = const()[name = string("key_token_49_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_49_cast_fp16 = slice_by_index(begin = key_token_49_begin_0, end = key_token_49_end_0, end_mask = key_token_49_end_mask_0, squeeze_mask = key_token_49_squeeze_mask_0, x = key_7_cast_fp16)[name = string("key_token_49_cast_fp16")];
            tensor<int32, [4]> value_token_49_begin_0 = const()[name = string("value_token_49_begin_0"), val = tensor<int32, [4]>([0, 0, 8, 0])];
            tensor<int32, [4]> value_token_49_end_0 = const()[name = string("value_token_49_end_0"), val = tensor<int32, [4]>([1, 16, 9, 128])];
            tensor<bool, [4]> value_token_49_end_mask_0 = const()[name = string("value_token_49_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_49_squeeze_mask_0 = const()[name = string("value_token_49_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_49_cast_fp16 = slice_by_index(begin = value_token_49_begin_0, end = value_token_49_end_0, end_mask = value_token_49_end_mask_0, squeeze_mask = value_token_49_squeeze_mask_0, x = value_25_cast_fp16)[name = string("value_token_49_cast_fp16")];
            tensor<int32, [1]> var_1209_axes_0 = const()[name = string("op_1209_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1209_cast_fp16 = expand_dims(axes = var_1209_axes_0, x = key_token_49_cast_fp16)[name = string("op_1209_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1210_cast_fp16 = mul(x = state_97_cast_fp16, y = var_1209_cast_fp16)[name = string("op_1210_cast_fp16")];
            tensor<int32, [1]> memory_49_axes_0 = const()[name = string("memory_49_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_49_keep_dims_0 = const()[name = string("memory_49_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_49_cast_fp16 = reduce_sum(axes = memory_49_axes_0, keep_dims = memory_49_keep_dims_0, x = var_1210_cast_fp16)[name = string("memory_49_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_1213_cast_fp16 = sub(x = value_token_49_cast_fp16, y = memory_49_cast_fp16)[name = string("op_1213_cast_fp16")];
            tensor<int32, [3]> var_1216_begin_0 = const()[name = string("op_1216_begin_0"), val = tensor<int32, [3]>([0, 0, 8])];
            tensor<int32, [3]> var_1216_end_0 = const()[name = string("op_1216_end_0"), val = tensor<int32, [3]>([1, 16, 9])];
            tensor<bool, [3]> var_1216_end_mask_0 = const()[name = string("op_1216_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1216_squeeze_mask_0 = const()[name = string("op_1216_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1216_cast_fp16 = slice_by_index(begin = var_1216_begin_0, end = var_1216_end_0, end_mask = var_1216_end_mask_0, squeeze_mask = var_1216_squeeze_mask_0, x = beta_3_cast_fp16)[name = string("op_1216_cast_fp16")];
            tensor<int32, [1]> var_1217_axes_0 = const()[name = string("op_1217_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1217_cast_fp16 = expand_dims(axes = var_1217_axes_0, x = var_1216_cast_fp16)[name = string("op_1217_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_49_cast_fp16 = mul(x = var_1213_cast_fp16, y = var_1217_cast_fp16)[name = string("delta_49_cast_fp16")];
            tensor<int32, [1]> var_1220_axes_0 = const()[name = string("op_1220_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_1220_cast_fp16 = expand_dims(axes = var_1220_axes_0, x = delta_49_cast_fp16)[name = string("op_1220_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1221_cast_fp16 = mul(x = var_1209_cast_fp16, y = var_1220_cast_fp16)[name = string("op_1221_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_99_cast_fp16 = add(x = state_97_cast_fp16, y = var_1221_cast_fp16)[name = string("state_99_cast_fp16")];
            tensor<int32, [4]> var_1225_begin_0 = const()[name = string("op_1225_begin_0"), val = tensor<int32, [4]>([0, 0, 8, 0])];
            tensor<int32, [4]> var_1225_end_0 = const()[name = string("op_1225_end_0"), val = tensor<int32, [4]>([1, 16, 9, 128])];
            tensor<bool, [4]> var_1225_end_mask_0 = const()[name = string("op_1225_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_1225_squeeze_mask_0 = const()[name = string("op_1225_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_1225_cast_fp16 = slice_by_index(begin = var_1225_begin_0, end = var_1225_end_0, end_mask = var_1225_end_mask_0, squeeze_mask = var_1225_squeeze_mask_0, x = _inversed_query_7_cast_fp16)[name = string("op_1225_cast_fp16")];
            tensor<int32, [1]> var_1226_axes_0 = const()[name = string("op_1226_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1226_cast_fp16 = expand_dims(axes = var_1226_axes_0, x = var_1225_cast_fp16)[name = string("op_1226_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1227_cast_fp16 = mul(x = state_99_cast_fp16, y = var_1226_cast_fp16)[name = string("op_1227_cast_fp16")];
            tensor<int32, [1]> var_1229_axes_0 = const()[name = string("op_1229_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_1229_keep_dims_0 = const()[name = string("op_1229_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_1229_cast_fp16 = reduce_sum(axes = var_1229_axes_0, keep_dims = var_1229_keep_dims_0, x = var_1227_cast_fp16)[name = string("op_1229_cast_fp16")];
            tensor<int32, [3]> var_1232_begin_0 = const()[name = string("op_1232_begin_0"), val = tensor<int32, [3]>([0, 0, 9])];
            tensor<int32, [3]> var_1232_end_0 = const()[name = string("op_1232_end_0"), val = tensor<int32, [3]>([1, 16, 10])];
            tensor<bool, [3]> var_1232_end_mask_0 = const()[name = string("op_1232_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1232_squeeze_mask_0 = const()[name = string("op_1232_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1232_cast_fp16 = slice_by_index(begin = var_1232_begin_0, end = var_1232_end_0, end_mask = var_1232_end_mask_0, squeeze_mask = var_1232_squeeze_mask_0, x = decay_3_cast_fp16)[name = string("op_1232_cast_fp16")];
            tensor<fp16, [1, 16]> var_1233_cast_fp16 = exp(x = var_1232_cast_fp16)[name = string("op_1233_cast_fp16")];
            tensor<int32, [1]> var_1234_axes_0 = const()[name = string("op_1234_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1234_cast_fp16 = expand_dims(axes = var_1234_axes_0, x = var_1233_cast_fp16)[name = string("op_1234_cast_fp16")];
            tensor<int32, [1]> var_1235_axes_0 = const()[name = string("op_1235_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_1235_cast_fp16 = expand_dims(axes = var_1235_axes_0, x = var_1234_cast_fp16)[name = string("op_1235_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_101_cast_fp16 = mul(x = state_99_cast_fp16, y = var_1235_cast_fp16)[name = string("state_101_cast_fp16")];
            tensor<int32, [4]> key_token_51_begin_0 = const()[name = string("key_token_51_begin_0"), val = tensor<int32, [4]>([0, 0, 9, 0])];
            tensor<int32, [4]> key_token_51_end_0 = const()[name = string("key_token_51_end_0"), val = tensor<int32, [4]>([1, 16, 10, 128])];
            tensor<bool, [4]> key_token_51_end_mask_0 = const()[name = string("key_token_51_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_51_squeeze_mask_0 = const()[name = string("key_token_51_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_51_cast_fp16 = slice_by_index(begin = key_token_51_begin_0, end = key_token_51_end_0, end_mask = key_token_51_end_mask_0, squeeze_mask = key_token_51_squeeze_mask_0, x = key_7_cast_fp16)[name = string("key_token_51_cast_fp16")];
            tensor<int32, [4]> value_token_51_begin_0 = const()[name = string("value_token_51_begin_0"), val = tensor<int32, [4]>([0, 0, 9, 0])];
            tensor<int32, [4]> value_token_51_end_0 = const()[name = string("value_token_51_end_0"), val = tensor<int32, [4]>([1, 16, 10, 128])];
            tensor<bool, [4]> value_token_51_end_mask_0 = const()[name = string("value_token_51_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_51_squeeze_mask_0 = const()[name = string("value_token_51_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_51_cast_fp16 = slice_by_index(begin = value_token_51_begin_0, end = value_token_51_end_0, end_mask = value_token_51_end_mask_0, squeeze_mask = value_token_51_squeeze_mask_0, x = value_25_cast_fp16)[name = string("value_token_51_cast_fp16")];
            tensor<int32, [1]> var_1243_axes_0 = const()[name = string("op_1243_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1243_cast_fp16 = expand_dims(axes = var_1243_axes_0, x = key_token_51_cast_fp16)[name = string("op_1243_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1244_cast_fp16 = mul(x = state_101_cast_fp16, y = var_1243_cast_fp16)[name = string("op_1244_cast_fp16")];
            tensor<int32, [1]> memory_51_axes_0 = const()[name = string("memory_51_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_51_keep_dims_0 = const()[name = string("memory_51_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_51_cast_fp16 = reduce_sum(axes = memory_51_axes_0, keep_dims = memory_51_keep_dims_0, x = var_1244_cast_fp16)[name = string("memory_51_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_1247_cast_fp16 = sub(x = value_token_51_cast_fp16, y = memory_51_cast_fp16)[name = string("op_1247_cast_fp16")];
            tensor<int32, [3]> var_1250_begin_0 = const()[name = string("op_1250_begin_0"), val = tensor<int32, [3]>([0, 0, 9])];
            tensor<int32, [3]> var_1250_end_0 = const()[name = string("op_1250_end_0"), val = tensor<int32, [3]>([1, 16, 10])];
            tensor<bool, [3]> var_1250_end_mask_0 = const()[name = string("op_1250_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1250_squeeze_mask_0 = const()[name = string("op_1250_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1250_cast_fp16 = slice_by_index(begin = var_1250_begin_0, end = var_1250_end_0, end_mask = var_1250_end_mask_0, squeeze_mask = var_1250_squeeze_mask_0, x = beta_3_cast_fp16)[name = string("op_1250_cast_fp16")];
            tensor<int32, [1]> var_1251_axes_0 = const()[name = string("op_1251_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1251_cast_fp16 = expand_dims(axes = var_1251_axes_0, x = var_1250_cast_fp16)[name = string("op_1251_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_51_cast_fp16 = mul(x = var_1247_cast_fp16, y = var_1251_cast_fp16)[name = string("delta_51_cast_fp16")];
            tensor<int32, [1]> var_1254_axes_0 = const()[name = string("op_1254_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_1254_cast_fp16 = expand_dims(axes = var_1254_axes_0, x = delta_51_cast_fp16)[name = string("op_1254_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1255_cast_fp16 = mul(x = var_1243_cast_fp16, y = var_1254_cast_fp16)[name = string("op_1255_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_103_cast_fp16 = add(x = state_101_cast_fp16, y = var_1255_cast_fp16)[name = string("state_103_cast_fp16")];
            tensor<int32, [4]> var_1259_begin_0 = const()[name = string("op_1259_begin_0"), val = tensor<int32, [4]>([0, 0, 9, 0])];
            tensor<int32, [4]> var_1259_end_0 = const()[name = string("op_1259_end_0"), val = tensor<int32, [4]>([1, 16, 10, 128])];
            tensor<bool, [4]> var_1259_end_mask_0 = const()[name = string("op_1259_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_1259_squeeze_mask_0 = const()[name = string("op_1259_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_1259_cast_fp16 = slice_by_index(begin = var_1259_begin_0, end = var_1259_end_0, end_mask = var_1259_end_mask_0, squeeze_mask = var_1259_squeeze_mask_0, x = _inversed_query_7_cast_fp16)[name = string("op_1259_cast_fp16")];
            tensor<int32, [1]> var_1260_axes_0 = const()[name = string("op_1260_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1260_cast_fp16 = expand_dims(axes = var_1260_axes_0, x = var_1259_cast_fp16)[name = string("op_1260_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1261_cast_fp16 = mul(x = state_103_cast_fp16, y = var_1260_cast_fp16)[name = string("op_1261_cast_fp16")];
            tensor<int32, [1]> var_1263_axes_0 = const()[name = string("op_1263_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_1263_keep_dims_0 = const()[name = string("op_1263_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_1263_cast_fp16 = reduce_sum(axes = var_1263_axes_0, keep_dims = var_1263_keep_dims_0, x = var_1261_cast_fp16)[name = string("op_1263_cast_fp16")];
            tensor<int32, [3]> var_1266_begin_0 = const()[name = string("op_1266_begin_0"), val = tensor<int32, [3]>([0, 0, 10])];
            tensor<int32, [3]> var_1266_end_0 = const()[name = string("op_1266_end_0"), val = tensor<int32, [3]>([1, 16, 11])];
            tensor<bool, [3]> var_1266_end_mask_0 = const()[name = string("op_1266_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1266_squeeze_mask_0 = const()[name = string("op_1266_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1266_cast_fp16 = slice_by_index(begin = var_1266_begin_0, end = var_1266_end_0, end_mask = var_1266_end_mask_0, squeeze_mask = var_1266_squeeze_mask_0, x = decay_3_cast_fp16)[name = string("op_1266_cast_fp16")];
            tensor<fp16, [1, 16]> var_1267_cast_fp16 = exp(x = var_1266_cast_fp16)[name = string("op_1267_cast_fp16")];
            tensor<int32, [1]> var_1268_axes_0 = const()[name = string("op_1268_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1268_cast_fp16 = expand_dims(axes = var_1268_axes_0, x = var_1267_cast_fp16)[name = string("op_1268_cast_fp16")];
            tensor<int32, [1]> var_1269_axes_0 = const()[name = string("op_1269_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_1269_cast_fp16 = expand_dims(axes = var_1269_axes_0, x = var_1268_cast_fp16)[name = string("op_1269_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_105_cast_fp16 = mul(x = state_103_cast_fp16, y = var_1269_cast_fp16)[name = string("state_105_cast_fp16")];
            tensor<int32, [4]> key_token_53_begin_0 = const()[name = string("key_token_53_begin_0"), val = tensor<int32, [4]>([0, 0, 10, 0])];
            tensor<int32, [4]> key_token_53_end_0 = const()[name = string("key_token_53_end_0"), val = tensor<int32, [4]>([1, 16, 11, 128])];
            tensor<bool, [4]> key_token_53_end_mask_0 = const()[name = string("key_token_53_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_53_squeeze_mask_0 = const()[name = string("key_token_53_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_53_cast_fp16 = slice_by_index(begin = key_token_53_begin_0, end = key_token_53_end_0, end_mask = key_token_53_end_mask_0, squeeze_mask = key_token_53_squeeze_mask_0, x = key_7_cast_fp16)[name = string("key_token_53_cast_fp16")];
            tensor<int32, [4]> value_token_53_begin_0 = const()[name = string("value_token_53_begin_0"), val = tensor<int32, [4]>([0, 0, 10, 0])];
            tensor<int32, [4]> value_token_53_end_0 = const()[name = string("value_token_53_end_0"), val = tensor<int32, [4]>([1, 16, 11, 128])];
            tensor<bool, [4]> value_token_53_end_mask_0 = const()[name = string("value_token_53_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_53_squeeze_mask_0 = const()[name = string("value_token_53_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_53_cast_fp16 = slice_by_index(begin = value_token_53_begin_0, end = value_token_53_end_0, end_mask = value_token_53_end_mask_0, squeeze_mask = value_token_53_squeeze_mask_0, x = value_25_cast_fp16)[name = string("value_token_53_cast_fp16")];
            tensor<int32, [1]> var_1277_axes_0 = const()[name = string("op_1277_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1277_cast_fp16 = expand_dims(axes = var_1277_axes_0, x = key_token_53_cast_fp16)[name = string("op_1277_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1278_cast_fp16 = mul(x = state_105_cast_fp16, y = var_1277_cast_fp16)[name = string("op_1278_cast_fp16")];
            tensor<int32, [1]> memory_53_axes_0 = const()[name = string("memory_53_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_53_keep_dims_0 = const()[name = string("memory_53_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_53_cast_fp16 = reduce_sum(axes = memory_53_axes_0, keep_dims = memory_53_keep_dims_0, x = var_1278_cast_fp16)[name = string("memory_53_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_1281_cast_fp16 = sub(x = value_token_53_cast_fp16, y = memory_53_cast_fp16)[name = string("op_1281_cast_fp16")];
            tensor<int32, [3]> var_1284_begin_0 = const()[name = string("op_1284_begin_0"), val = tensor<int32, [3]>([0, 0, 10])];
            tensor<int32, [3]> var_1284_end_0 = const()[name = string("op_1284_end_0"), val = tensor<int32, [3]>([1, 16, 11])];
            tensor<bool, [3]> var_1284_end_mask_0 = const()[name = string("op_1284_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1284_squeeze_mask_0 = const()[name = string("op_1284_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1284_cast_fp16 = slice_by_index(begin = var_1284_begin_0, end = var_1284_end_0, end_mask = var_1284_end_mask_0, squeeze_mask = var_1284_squeeze_mask_0, x = beta_3_cast_fp16)[name = string("op_1284_cast_fp16")];
            tensor<int32, [1]> var_1285_axes_0 = const()[name = string("op_1285_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1285_cast_fp16 = expand_dims(axes = var_1285_axes_0, x = var_1284_cast_fp16)[name = string("op_1285_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_53_cast_fp16 = mul(x = var_1281_cast_fp16, y = var_1285_cast_fp16)[name = string("delta_53_cast_fp16")];
            tensor<int32, [1]> var_1288_axes_0 = const()[name = string("op_1288_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_1288_cast_fp16 = expand_dims(axes = var_1288_axes_0, x = delta_53_cast_fp16)[name = string("op_1288_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1289_cast_fp16 = mul(x = var_1277_cast_fp16, y = var_1288_cast_fp16)[name = string("op_1289_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_107_cast_fp16 = add(x = state_105_cast_fp16, y = var_1289_cast_fp16)[name = string("state_107_cast_fp16")];
            tensor<int32, [4]> var_1293_begin_0 = const()[name = string("op_1293_begin_0"), val = tensor<int32, [4]>([0, 0, 10, 0])];
            tensor<int32, [4]> var_1293_end_0 = const()[name = string("op_1293_end_0"), val = tensor<int32, [4]>([1, 16, 11, 128])];
            tensor<bool, [4]> var_1293_end_mask_0 = const()[name = string("op_1293_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_1293_squeeze_mask_0 = const()[name = string("op_1293_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_1293_cast_fp16 = slice_by_index(begin = var_1293_begin_0, end = var_1293_end_0, end_mask = var_1293_end_mask_0, squeeze_mask = var_1293_squeeze_mask_0, x = _inversed_query_7_cast_fp16)[name = string("op_1293_cast_fp16")];
            tensor<int32, [1]> var_1294_axes_0 = const()[name = string("op_1294_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1294_cast_fp16 = expand_dims(axes = var_1294_axes_0, x = var_1293_cast_fp16)[name = string("op_1294_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1295_cast_fp16 = mul(x = state_107_cast_fp16, y = var_1294_cast_fp16)[name = string("op_1295_cast_fp16")];
            tensor<int32, [1]> var_1297_axes_0 = const()[name = string("op_1297_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_1297_keep_dims_0 = const()[name = string("op_1297_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_1297_cast_fp16 = reduce_sum(axes = var_1297_axes_0, keep_dims = var_1297_keep_dims_0, x = var_1295_cast_fp16)[name = string("op_1297_cast_fp16")];
            tensor<int32, [3]> var_1300_begin_0 = const()[name = string("op_1300_begin_0"), val = tensor<int32, [3]>([0, 0, 11])];
            tensor<int32, [3]> var_1300_end_0 = const()[name = string("op_1300_end_0"), val = tensor<int32, [3]>([1, 16, 12])];
            tensor<bool, [3]> var_1300_end_mask_0 = const()[name = string("op_1300_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1300_squeeze_mask_0 = const()[name = string("op_1300_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1300_cast_fp16 = slice_by_index(begin = var_1300_begin_0, end = var_1300_end_0, end_mask = var_1300_end_mask_0, squeeze_mask = var_1300_squeeze_mask_0, x = decay_3_cast_fp16)[name = string("op_1300_cast_fp16")];
            tensor<fp16, [1, 16]> var_1301_cast_fp16 = exp(x = var_1300_cast_fp16)[name = string("op_1301_cast_fp16")];
            tensor<int32, [1]> var_1302_axes_0 = const()[name = string("op_1302_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1302_cast_fp16 = expand_dims(axes = var_1302_axes_0, x = var_1301_cast_fp16)[name = string("op_1302_cast_fp16")];
            tensor<int32, [1]> var_1303_axes_0 = const()[name = string("op_1303_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_1303_cast_fp16 = expand_dims(axes = var_1303_axes_0, x = var_1302_cast_fp16)[name = string("op_1303_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_109_cast_fp16 = mul(x = state_107_cast_fp16, y = var_1303_cast_fp16)[name = string("state_109_cast_fp16")];
            tensor<int32, [4]> key_token_55_begin_0 = const()[name = string("key_token_55_begin_0"), val = tensor<int32, [4]>([0, 0, 11, 0])];
            tensor<int32, [4]> key_token_55_end_0 = const()[name = string("key_token_55_end_0"), val = tensor<int32, [4]>([1, 16, 12, 128])];
            tensor<bool, [4]> key_token_55_end_mask_0 = const()[name = string("key_token_55_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_55_squeeze_mask_0 = const()[name = string("key_token_55_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_55_cast_fp16 = slice_by_index(begin = key_token_55_begin_0, end = key_token_55_end_0, end_mask = key_token_55_end_mask_0, squeeze_mask = key_token_55_squeeze_mask_0, x = key_7_cast_fp16)[name = string("key_token_55_cast_fp16")];
            tensor<int32, [4]> value_token_55_begin_0 = const()[name = string("value_token_55_begin_0"), val = tensor<int32, [4]>([0, 0, 11, 0])];
            tensor<int32, [4]> value_token_55_end_0 = const()[name = string("value_token_55_end_0"), val = tensor<int32, [4]>([1, 16, 12, 128])];
            tensor<bool, [4]> value_token_55_end_mask_0 = const()[name = string("value_token_55_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_55_squeeze_mask_0 = const()[name = string("value_token_55_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_55_cast_fp16 = slice_by_index(begin = value_token_55_begin_0, end = value_token_55_end_0, end_mask = value_token_55_end_mask_0, squeeze_mask = value_token_55_squeeze_mask_0, x = value_25_cast_fp16)[name = string("value_token_55_cast_fp16")];
            tensor<int32, [1]> var_1311_axes_0 = const()[name = string("op_1311_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1311_cast_fp16 = expand_dims(axes = var_1311_axes_0, x = key_token_55_cast_fp16)[name = string("op_1311_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1312_cast_fp16 = mul(x = state_109_cast_fp16, y = var_1311_cast_fp16)[name = string("op_1312_cast_fp16")];
            tensor<int32, [1]> memory_55_axes_0 = const()[name = string("memory_55_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_55_keep_dims_0 = const()[name = string("memory_55_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_55_cast_fp16 = reduce_sum(axes = memory_55_axes_0, keep_dims = memory_55_keep_dims_0, x = var_1312_cast_fp16)[name = string("memory_55_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_1315_cast_fp16 = sub(x = value_token_55_cast_fp16, y = memory_55_cast_fp16)[name = string("op_1315_cast_fp16")];
            tensor<int32, [3]> var_1318_begin_0 = const()[name = string("op_1318_begin_0"), val = tensor<int32, [3]>([0, 0, 11])];
            tensor<int32, [3]> var_1318_end_0 = const()[name = string("op_1318_end_0"), val = tensor<int32, [3]>([1, 16, 12])];
            tensor<bool, [3]> var_1318_end_mask_0 = const()[name = string("op_1318_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1318_squeeze_mask_0 = const()[name = string("op_1318_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1318_cast_fp16 = slice_by_index(begin = var_1318_begin_0, end = var_1318_end_0, end_mask = var_1318_end_mask_0, squeeze_mask = var_1318_squeeze_mask_0, x = beta_3_cast_fp16)[name = string("op_1318_cast_fp16")];
            tensor<int32, [1]> var_1319_axes_0 = const()[name = string("op_1319_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1319_cast_fp16 = expand_dims(axes = var_1319_axes_0, x = var_1318_cast_fp16)[name = string("op_1319_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_55_cast_fp16 = mul(x = var_1315_cast_fp16, y = var_1319_cast_fp16)[name = string("delta_55_cast_fp16")];
            tensor<int32, [1]> var_1322_axes_0 = const()[name = string("op_1322_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_1322_cast_fp16 = expand_dims(axes = var_1322_axes_0, x = delta_55_cast_fp16)[name = string("op_1322_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1323_cast_fp16 = mul(x = var_1311_cast_fp16, y = var_1322_cast_fp16)[name = string("op_1323_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_111_cast_fp16 = add(x = state_109_cast_fp16, y = var_1323_cast_fp16)[name = string("state_111_cast_fp16")];
            tensor<int32, [4]> var_1327_begin_0 = const()[name = string("op_1327_begin_0"), val = tensor<int32, [4]>([0, 0, 11, 0])];
            tensor<int32, [4]> var_1327_end_0 = const()[name = string("op_1327_end_0"), val = tensor<int32, [4]>([1, 16, 12, 128])];
            tensor<bool, [4]> var_1327_end_mask_0 = const()[name = string("op_1327_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_1327_squeeze_mask_0 = const()[name = string("op_1327_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_1327_cast_fp16 = slice_by_index(begin = var_1327_begin_0, end = var_1327_end_0, end_mask = var_1327_end_mask_0, squeeze_mask = var_1327_squeeze_mask_0, x = _inversed_query_7_cast_fp16)[name = string("op_1327_cast_fp16")];
            tensor<int32, [1]> var_1328_axes_0 = const()[name = string("op_1328_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1328_cast_fp16 = expand_dims(axes = var_1328_axes_0, x = var_1327_cast_fp16)[name = string("op_1328_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1329_cast_fp16 = mul(x = state_111_cast_fp16, y = var_1328_cast_fp16)[name = string("op_1329_cast_fp16")];
            tensor<int32, [1]> var_1331_axes_0 = const()[name = string("op_1331_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_1331_keep_dims_0 = const()[name = string("op_1331_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_1331_cast_fp16 = reduce_sum(axes = var_1331_axes_0, keep_dims = var_1331_keep_dims_0, x = var_1329_cast_fp16)[name = string("op_1331_cast_fp16")];
            tensor<int32, [3]> var_1334_begin_0 = const()[name = string("op_1334_begin_0"), val = tensor<int32, [3]>([0, 0, 12])];
            tensor<int32, [3]> var_1334_end_0 = const()[name = string("op_1334_end_0"), val = tensor<int32, [3]>([1, 16, 13])];
            tensor<bool, [3]> var_1334_end_mask_0 = const()[name = string("op_1334_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1334_squeeze_mask_0 = const()[name = string("op_1334_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1334_cast_fp16 = slice_by_index(begin = var_1334_begin_0, end = var_1334_end_0, end_mask = var_1334_end_mask_0, squeeze_mask = var_1334_squeeze_mask_0, x = decay_3_cast_fp16)[name = string("op_1334_cast_fp16")];
            tensor<fp16, [1, 16]> var_1335_cast_fp16 = exp(x = var_1334_cast_fp16)[name = string("op_1335_cast_fp16")];
            tensor<int32, [1]> var_1336_axes_0 = const()[name = string("op_1336_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1336_cast_fp16 = expand_dims(axes = var_1336_axes_0, x = var_1335_cast_fp16)[name = string("op_1336_cast_fp16")];
            tensor<int32, [1]> var_1337_axes_0 = const()[name = string("op_1337_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_1337_cast_fp16 = expand_dims(axes = var_1337_axes_0, x = var_1336_cast_fp16)[name = string("op_1337_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_113_cast_fp16 = mul(x = state_111_cast_fp16, y = var_1337_cast_fp16)[name = string("state_113_cast_fp16")];
            tensor<int32, [4]> key_token_57_begin_0 = const()[name = string("key_token_57_begin_0"), val = tensor<int32, [4]>([0, 0, 12, 0])];
            tensor<int32, [4]> key_token_57_end_0 = const()[name = string("key_token_57_end_0"), val = tensor<int32, [4]>([1, 16, 13, 128])];
            tensor<bool, [4]> key_token_57_end_mask_0 = const()[name = string("key_token_57_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_57_squeeze_mask_0 = const()[name = string("key_token_57_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_57_cast_fp16 = slice_by_index(begin = key_token_57_begin_0, end = key_token_57_end_0, end_mask = key_token_57_end_mask_0, squeeze_mask = key_token_57_squeeze_mask_0, x = key_7_cast_fp16)[name = string("key_token_57_cast_fp16")];
            tensor<int32, [4]> value_token_57_begin_0 = const()[name = string("value_token_57_begin_0"), val = tensor<int32, [4]>([0, 0, 12, 0])];
            tensor<int32, [4]> value_token_57_end_0 = const()[name = string("value_token_57_end_0"), val = tensor<int32, [4]>([1, 16, 13, 128])];
            tensor<bool, [4]> value_token_57_end_mask_0 = const()[name = string("value_token_57_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_57_squeeze_mask_0 = const()[name = string("value_token_57_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_57_cast_fp16 = slice_by_index(begin = value_token_57_begin_0, end = value_token_57_end_0, end_mask = value_token_57_end_mask_0, squeeze_mask = value_token_57_squeeze_mask_0, x = value_25_cast_fp16)[name = string("value_token_57_cast_fp16")];
            tensor<int32, [1]> var_1345_axes_0 = const()[name = string("op_1345_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1345_cast_fp16 = expand_dims(axes = var_1345_axes_0, x = key_token_57_cast_fp16)[name = string("op_1345_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1346_cast_fp16 = mul(x = state_113_cast_fp16, y = var_1345_cast_fp16)[name = string("op_1346_cast_fp16")];
            tensor<int32, [1]> memory_57_axes_0 = const()[name = string("memory_57_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_57_keep_dims_0 = const()[name = string("memory_57_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_57_cast_fp16 = reduce_sum(axes = memory_57_axes_0, keep_dims = memory_57_keep_dims_0, x = var_1346_cast_fp16)[name = string("memory_57_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_1349_cast_fp16 = sub(x = value_token_57_cast_fp16, y = memory_57_cast_fp16)[name = string("op_1349_cast_fp16")];
            tensor<int32, [3]> var_1352_begin_0 = const()[name = string("op_1352_begin_0"), val = tensor<int32, [3]>([0, 0, 12])];
            tensor<int32, [3]> var_1352_end_0 = const()[name = string("op_1352_end_0"), val = tensor<int32, [3]>([1, 16, 13])];
            tensor<bool, [3]> var_1352_end_mask_0 = const()[name = string("op_1352_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1352_squeeze_mask_0 = const()[name = string("op_1352_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1352_cast_fp16 = slice_by_index(begin = var_1352_begin_0, end = var_1352_end_0, end_mask = var_1352_end_mask_0, squeeze_mask = var_1352_squeeze_mask_0, x = beta_3_cast_fp16)[name = string("op_1352_cast_fp16")];
            tensor<int32, [1]> var_1353_axes_0 = const()[name = string("op_1353_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1353_cast_fp16 = expand_dims(axes = var_1353_axes_0, x = var_1352_cast_fp16)[name = string("op_1353_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_57_cast_fp16 = mul(x = var_1349_cast_fp16, y = var_1353_cast_fp16)[name = string("delta_57_cast_fp16")];
            tensor<int32, [1]> var_1356_axes_0 = const()[name = string("op_1356_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_1356_cast_fp16 = expand_dims(axes = var_1356_axes_0, x = delta_57_cast_fp16)[name = string("op_1356_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1357_cast_fp16 = mul(x = var_1345_cast_fp16, y = var_1356_cast_fp16)[name = string("op_1357_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_115_cast_fp16 = add(x = state_113_cast_fp16, y = var_1357_cast_fp16)[name = string("state_115_cast_fp16")];
            tensor<int32, [4]> var_1361_begin_0 = const()[name = string("op_1361_begin_0"), val = tensor<int32, [4]>([0, 0, 12, 0])];
            tensor<int32, [4]> var_1361_end_0 = const()[name = string("op_1361_end_0"), val = tensor<int32, [4]>([1, 16, 13, 128])];
            tensor<bool, [4]> var_1361_end_mask_0 = const()[name = string("op_1361_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_1361_squeeze_mask_0 = const()[name = string("op_1361_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_1361_cast_fp16 = slice_by_index(begin = var_1361_begin_0, end = var_1361_end_0, end_mask = var_1361_end_mask_0, squeeze_mask = var_1361_squeeze_mask_0, x = _inversed_query_7_cast_fp16)[name = string("op_1361_cast_fp16")];
            tensor<int32, [1]> var_1362_axes_0 = const()[name = string("op_1362_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1362_cast_fp16 = expand_dims(axes = var_1362_axes_0, x = var_1361_cast_fp16)[name = string("op_1362_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1363_cast_fp16 = mul(x = state_115_cast_fp16, y = var_1362_cast_fp16)[name = string("op_1363_cast_fp16")];
            tensor<int32, [1]> var_1365_axes_0 = const()[name = string("op_1365_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_1365_keep_dims_0 = const()[name = string("op_1365_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_1365_cast_fp16 = reduce_sum(axes = var_1365_axes_0, keep_dims = var_1365_keep_dims_0, x = var_1363_cast_fp16)[name = string("op_1365_cast_fp16")];
            tensor<int32, [3]> var_1368_begin_0 = const()[name = string("op_1368_begin_0"), val = tensor<int32, [3]>([0, 0, 13])];
            tensor<int32, [3]> var_1368_end_0 = const()[name = string("op_1368_end_0"), val = tensor<int32, [3]>([1, 16, 14])];
            tensor<bool, [3]> var_1368_end_mask_0 = const()[name = string("op_1368_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1368_squeeze_mask_0 = const()[name = string("op_1368_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1368_cast_fp16 = slice_by_index(begin = var_1368_begin_0, end = var_1368_end_0, end_mask = var_1368_end_mask_0, squeeze_mask = var_1368_squeeze_mask_0, x = decay_3_cast_fp16)[name = string("op_1368_cast_fp16")];
            tensor<fp16, [1, 16]> var_1369_cast_fp16 = exp(x = var_1368_cast_fp16)[name = string("op_1369_cast_fp16")];
            tensor<int32, [1]> var_1370_axes_0 = const()[name = string("op_1370_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1370_cast_fp16 = expand_dims(axes = var_1370_axes_0, x = var_1369_cast_fp16)[name = string("op_1370_cast_fp16")];
            tensor<int32, [1]> var_1371_axes_0 = const()[name = string("op_1371_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_1371_cast_fp16 = expand_dims(axes = var_1371_axes_0, x = var_1370_cast_fp16)[name = string("op_1371_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_117_cast_fp16 = mul(x = state_115_cast_fp16, y = var_1371_cast_fp16)[name = string("state_117_cast_fp16")];
            tensor<int32, [4]> key_token_59_begin_0 = const()[name = string("key_token_59_begin_0"), val = tensor<int32, [4]>([0, 0, 13, 0])];
            tensor<int32, [4]> key_token_59_end_0 = const()[name = string("key_token_59_end_0"), val = tensor<int32, [4]>([1, 16, 14, 128])];
            tensor<bool, [4]> key_token_59_end_mask_0 = const()[name = string("key_token_59_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_59_squeeze_mask_0 = const()[name = string("key_token_59_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_59_cast_fp16 = slice_by_index(begin = key_token_59_begin_0, end = key_token_59_end_0, end_mask = key_token_59_end_mask_0, squeeze_mask = key_token_59_squeeze_mask_0, x = key_7_cast_fp16)[name = string("key_token_59_cast_fp16")];
            tensor<int32, [4]> value_token_59_begin_0 = const()[name = string("value_token_59_begin_0"), val = tensor<int32, [4]>([0, 0, 13, 0])];
            tensor<int32, [4]> value_token_59_end_0 = const()[name = string("value_token_59_end_0"), val = tensor<int32, [4]>([1, 16, 14, 128])];
            tensor<bool, [4]> value_token_59_end_mask_0 = const()[name = string("value_token_59_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_59_squeeze_mask_0 = const()[name = string("value_token_59_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_59_cast_fp16 = slice_by_index(begin = value_token_59_begin_0, end = value_token_59_end_0, end_mask = value_token_59_end_mask_0, squeeze_mask = value_token_59_squeeze_mask_0, x = value_25_cast_fp16)[name = string("value_token_59_cast_fp16")];
            tensor<int32, [1]> var_1379_axes_0 = const()[name = string("op_1379_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1379_cast_fp16 = expand_dims(axes = var_1379_axes_0, x = key_token_59_cast_fp16)[name = string("op_1379_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1380_cast_fp16 = mul(x = state_117_cast_fp16, y = var_1379_cast_fp16)[name = string("op_1380_cast_fp16")];
            tensor<int32, [1]> memory_59_axes_0 = const()[name = string("memory_59_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_59_keep_dims_0 = const()[name = string("memory_59_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_59_cast_fp16 = reduce_sum(axes = memory_59_axes_0, keep_dims = memory_59_keep_dims_0, x = var_1380_cast_fp16)[name = string("memory_59_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_1383_cast_fp16 = sub(x = value_token_59_cast_fp16, y = memory_59_cast_fp16)[name = string("op_1383_cast_fp16")];
            tensor<int32, [3]> var_1386_begin_0 = const()[name = string("op_1386_begin_0"), val = tensor<int32, [3]>([0, 0, 13])];
            tensor<int32, [3]> var_1386_end_0 = const()[name = string("op_1386_end_0"), val = tensor<int32, [3]>([1, 16, 14])];
            tensor<bool, [3]> var_1386_end_mask_0 = const()[name = string("op_1386_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1386_squeeze_mask_0 = const()[name = string("op_1386_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1386_cast_fp16 = slice_by_index(begin = var_1386_begin_0, end = var_1386_end_0, end_mask = var_1386_end_mask_0, squeeze_mask = var_1386_squeeze_mask_0, x = beta_3_cast_fp16)[name = string("op_1386_cast_fp16")];
            tensor<int32, [1]> var_1387_axes_0 = const()[name = string("op_1387_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1387_cast_fp16 = expand_dims(axes = var_1387_axes_0, x = var_1386_cast_fp16)[name = string("op_1387_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_59_cast_fp16 = mul(x = var_1383_cast_fp16, y = var_1387_cast_fp16)[name = string("delta_59_cast_fp16")];
            tensor<int32, [1]> var_1390_axes_0 = const()[name = string("op_1390_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_1390_cast_fp16 = expand_dims(axes = var_1390_axes_0, x = delta_59_cast_fp16)[name = string("op_1390_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1391_cast_fp16 = mul(x = var_1379_cast_fp16, y = var_1390_cast_fp16)[name = string("op_1391_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_119_cast_fp16 = add(x = state_117_cast_fp16, y = var_1391_cast_fp16)[name = string("state_119_cast_fp16")];
            tensor<int32, [4]> var_1395_begin_0 = const()[name = string("op_1395_begin_0"), val = tensor<int32, [4]>([0, 0, 13, 0])];
            tensor<int32, [4]> var_1395_end_0 = const()[name = string("op_1395_end_0"), val = tensor<int32, [4]>([1, 16, 14, 128])];
            tensor<bool, [4]> var_1395_end_mask_0 = const()[name = string("op_1395_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_1395_squeeze_mask_0 = const()[name = string("op_1395_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_1395_cast_fp16 = slice_by_index(begin = var_1395_begin_0, end = var_1395_end_0, end_mask = var_1395_end_mask_0, squeeze_mask = var_1395_squeeze_mask_0, x = _inversed_query_7_cast_fp16)[name = string("op_1395_cast_fp16")];
            tensor<int32, [1]> var_1396_axes_0 = const()[name = string("op_1396_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1396_cast_fp16 = expand_dims(axes = var_1396_axes_0, x = var_1395_cast_fp16)[name = string("op_1396_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1397_cast_fp16 = mul(x = state_119_cast_fp16, y = var_1396_cast_fp16)[name = string("op_1397_cast_fp16")];
            tensor<int32, [1]> var_1399_axes_0 = const()[name = string("op_1399_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_1399_keep_dims_0 = const()[name = string("op_1399_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_1399_cast_fp16 = reduce_sum(axes = var_1399_axes_0, keep_dims = var_1399_keep_dims_0, x = var_1397_cast_fp16)[name = string("op_1399_cast_fp16")];
            tensor<int32, [3]> var_1402_begin_0 = const()[name = string("op_1402_begin_0"), val = tensor<int32, [3]>([0, 0, 14])];
            tensor<int32, [3]> var_1402_end_0 = const()[name = string("op_1402_end_0"), val = tensor<int32, [3]>([1, 16, 15])];
            tensor<bool, [3]> var_1402_end_mask_0 = const()[name = string("op_1402_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1402_squeeze_mask_0 = const()[name = string("op_1402_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1402_cast_fp16 = slice_by_index(begin = var_1402_begin_0, end = var_1402_end_0, end_mask = var_1402_end_mask_0, squeeze_mask = var_1402_squeeze_mask_0, x = decay_3_cast_fp16)[name = string("op_1402_cast_fp16")];
            tensor<fp16, [1, 16]> var_1403_cast_fp16 = exp(x = var_1402_cast_fp16)[name = string("op_1403_cast_fp16")];
            tensor<int32, [1]> var_1404_axes_0 = const()[name = string("op_1404_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1404_cast_fp16 = expand_dims(axes = var_1404_axes_0, x = var_1403_cast_fp16)[name = string("op_1404_cast_fp16")];
            tensor<int32, [1]> var_1405_axes_0 = const()[name = string("op_1405_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_1405_cast_fp16 = expand_dims(axes = var_1405_axes_0, x = var_1404_cast_fp16)[name = string("op_1405_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_121_cast_fp16 = mul(x = state_119_cast_fp16, y = var_1405_cast_fp16)[name = string("state_121_cast_fp16")];
            tensor<int32, [4]> key_token_61_begin_0 = const()[name = string("key_token_61_begin_0"), val = tensor<int32, [4]>([0, 0, 14, 0])];
            tensor<int32, [4]> key_token_61_end_0 = const()[name = string("key_token_61_end_0"), val = tensor<int32, [4]>([1, 16, 15, 128])];
            tensor<bool, [4]> key_token_61_end_mask_0 = const()[name = string("key_token_61_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_61_squeeze_mask_0 = const()[name = string("key_token_61_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_61_cast_fp16 = slice_by_index(begin = key_token_61_begin_0, end = key_token_61_end_0, end_mask = key_token_61_end_mask_0, squeeze_mask = key_token_61_squeeze_mask_0, x = key_7_cast_fp16)[name = string("key_token_61_cast_fp16")];
            tensor<int32, [4]> value_token_61_begin_0 = const()[name = string("value_token_61_begin_0"), val = tensor<int32, [4]>([0, 0, 14, 0])];
            tensor<int32, [4]> value_token_61_end_0 = const()[name = string("value_token_61_end_0"), val = tensor<int32, [4]>([1, 16, 15, 128])];
            tensor<bool, [4]> value_token_61_end_mask_0 = const()[name = string("value_token_61_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_61_squeeze_mask_0 = const()[name = string("value_token_61_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_61_cast_fp16 = slice_by_index(begin = value_token_61_begin_0, end = value_token_61_end_0, end_mask = value_token_61_end_mask_0, squeeze_mask = value_token_61_squeeze_mask_0, x = value_25_cast_fp16)[name = string("value_token_61_cast_fp16")];
            tensor<int32, [1]> var_1413_axes_0 = const()[name = string("op_1413_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1413_cast_fp16 = expand_dims(axes = var_1413_axes_0, x = key_token_61_cast_fp16)[name = string("op_1413_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1414_cast_fp16 = mul(x = state_121_cast_fp16, y = var_1413_cast_fp16)[name = string("op_1414_cast_fp16")];
            tensor<int32, [1]> memory_61_axes_0 = const()[name = string("memory_61_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_61_keep_dims_0 = const()[name = string("memory_61_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_61_cast_fp16 = reduce_sum(axes = memory_61_axes_0, keep_dims = memory_61_keep_dims_0, x = var_1414_cast_fp16)[name = string("memory_61_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_1417_cast_fp16 = sub(x = value_token_61_cast_fp16, y = memory_61_cast_fp16)[name = string("op_1417_cast_fp16")];
            tensor<int32, [3]> var_1420_begin_0 = const()[name = string("op_1420_begin_0"), val = tensor<int32, [3]>([0, 0, 14])];
            tensor<int32, [3]> var_1420_end_0 = const()[name = string("op_1420_end_0"), val = tensor<int32, [3]>([1, 16, 15])];
            tensor<bool, [3]> var_1420_end_mask_0 = const()[name = string("op_1420_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1420_squeeze_mask_0 = const()[name = string("op_1420_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1420_cast_fp16 = slice_by_index(begin = var_1420_begin_0, end = var_1420_end_0, end_mask = var_1420_end_mask_0, squeeze_mask = var_1420_squeeze_mask_0, x = beta_3_cast_fp16)[name = string("op_1420_cast_fp16")];
            tensor<int32, [1]> var_1421_axes_0 = const()[name = string("op_1421_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1421_cast_fp16 = expand_dims(axes = var_1421_axes_0, x = var_1420_cast_fp16)[name = string("op_1421_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_61_cast_fp16 = mul(x = var_1417_cast_fp16, y = var_1421_cast_fp16)[name = string("delta_61_cast_fp16")];
            tensor<int32, [1]> var_1424_axes_0 = const()[name = string("op_1424_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_1424_cast_fp16 = expand_dims(axes = var_1424_axes_0, x = delta_61_cast_fp16)[name = string("op_1424_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1425_cast_fp16 = mul(x = var_1413_cast_fp16, y = var_1424_cast_fp16)[name = string("op_1425_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_123_cast_fp16 = add(x = state_121_cast_fp16, y = var_1425_cast_fp16)[name = string("state_123_cast_fp16")];
            tensor<int32, [4]> var_1429_begin_0 = const()[name = string("op_1429_begin_0"), val = tensor<int32, [4]>([0, 0, 14, 0])];
            tensor<int32, [4]> var_1429_end_0 = const()[name = string("op_1429_end_0"), val = tensor<int32, [4]>([1, 16, 15, 128])];
            tensor<bool, [4]> var_1429_end_mask_0 = const()[name = string("op_1429_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_1429_squeeze_mask_0 = const()[name = string("op_1429_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_1429_cast_fp16 = slice_by_index(begin = var_1429_begin_0, end = var_1429_end_0, end_mask = var_1429_end_mask_0, squeeze_mask = var_1429_squeeze_mask_0, x = _inversed_query_7_cast_fp16)[name = string("op_1429_cast_fp16")];
            tensor<int32, [1]> var_1430_axes_0 = const()[name = string("op_1430_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1430_cast_fp16 = expand_dims(axes = var_1430_axes_0, x = var_1429_cast_fp16)[name = string("op_1430_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1431_cast_fp16 = mul(x = state_123_cast_fp16, y = var_1430_cast_fp16)[name = string("op_1431_cast_fp16")];
            tensor<int32, [1]> var_1433_axes_0 = const()[name = string("op_1433_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_1433_keep_dims_0 = const()[name = string("op_1433_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_1433_cast_fp16 = reduce_sum(axes = var_1433_axes_0, keep_dims = var_1433_keep_dims_0, x = var_1431_cast_fp16)[name = string("op_1433_cast_fp16")];
            tensor<int32, [3]> var_1436_begin_0 = const()[name = string("op_1436_begin_0"), val = tensor<int32, [3]>([0, 0, 15])];
            tensor<int32, [3]> var_1436_end_0 = const()[name = string("op_1436_end_0"), val = tensor<int32, [3]>([1, 16, 16])];
            tensor<bool, [3]> var_1436_end_mask_0 = const()[name = string("op_1436_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1436_squeeze_mask_0 = const()[name = string("op_1436_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1436_cast_fp16 = slice_by_index(begin = var_1436_begin_0, end = var_1436_end_0, end_mask = var_1436_end_mask_0, squeeze_mask = var_1436_squeeze_mask_0, x = decay_3_cast_fp16)[name = string("op_1436_cast_fp16")];
            tensor<fp16, [1, 16]> var_1437_cast_fp16 = exp(x = var_1436_cast_fp16)[name = string("op_1437_cast_fp16")];
            tensor<int32, [1]> var_1438_axes_0 = const()[name = string("op_1438_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1438_cast_fp16 = expand_dims(axes = var_1438_axes_0, x = var_1437_cast_fp16)[name = string("op_1438_cast_fp16")];
            tensor<int32, [1]> var_1439_axes_0 = const()[name = string("op_1439_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_1439_cast_fp16 = expand_dims(axes = var_1439_axes_0, x = var_1438_cast_fp16)[name = string("op_1439_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_125_cast_fp16 = mul(x = state_123_cast_fp16, y = var_1439_cast_fp16)[name = string("state_125_cast_fp16")];
            tensor<int32, [4]> key_token_63_begin_0 = const()[name = string("key_token_63_begin_0"), val = tensor<int32, [4]>([0, 0, 15, 0])];
            tensor<int32, [4]> key_token_63_end_0 = const()[name = string("key_token_63_end_0"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<bool, [4]> key_token_63_end_mask_0 = const()[name = string("key_token_63_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_63_squeeze_mask_0 = const()[name = string("key_token_63_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_63_cast_fp16 = slice_by_index(begin = key_token_63_begin_0, end = key_token_63_end_0, end_mask = key_token_63_end_mask_0, squeeze_mask = key_token_63_squeeze_mask_0, x = key_7_cast_fp16)[name = string("key_token_63_cast_fp16")];
            tensor<int32, [4]> value_token_63_begin_0 = const()[name = string("value_token_63_begin_0"), val = tensor<int32, [4]>([0, 0, 15, 0])];
            tensor<int32, [4]> value_token_63_end_0 = const()[name = string("value_token_63_end_0"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<bool, [4]> value_token_63_end_mask_0 = const()[name = string("value_token_63_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_63_squeeze_mask_0 = const()[name = string("value_token_63_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_63_cast_fp16 = slice_by_index(begin = value_token_63_begin_0, end = value_token_63_end_0, end_mask = value_token_63_end_mask_0, squeeze_mask = value_token_63_squeeze_mask_0, x = value_25_cast_fp16)[name = string("value_token_63_cast_fp16")];
            tensor<int32, [1]> var_1447_axes_0 = const()[name = string("op_1447_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1447_cast_fp16 = expand_dims(axes = var_1447_axes_0, x = key_token_63_cast_fp16)[name = string("op_1447_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1448_cast_fp16 = mul(x = state_125_cast_fp16, y = var_1447_cast_fp16)[name = string("op_1448_cast_fp16")];
            tensor<int32, [1]> memory_63_axes_0 = const()[name = string("memory_63_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_63_keep_dims_0 = const()[name = string("memory_63_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_63_cast_fp16 = reduce_sum(axes = memory_63_axes_0, keep_dims = memory_63_keep_dims_0, x = var_1448_cast_fp16)[name = string("memory_63_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_1451_cast_fp16 = sub(x = value_token_63_cast_fp16, y = memory_63_cast_fp16)[name = string("op_1451_cast_fp16")];
            tensor<int32, [3]> var_1454_begin_0 = const()[name = string("op_1454_begin_0"), val = tensor<int32, [3]>([0, 0, 15])];
            tensor<int32, [3]> var_1454_end_0 = const()[name = string("op_1454_end_0"), val = tensor<int32, [3]>([1, 16, 16])];
            tensor<bool, [3]> var_1454_end_mask_0 = const()[name = string("op_1454_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1454_squeeze_mask_0 = const()[name = string("op_1454_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1454_cast_fp16 = slice_by_index(begin = var_1454_begin_0, end = var_1454_end_0, end_mask = var_1454_end_mask_0, squeeze_mask = var_1454_squeeze_mask_0, x = beta_3_cast_fp16)[name = string("op_1454_cast_fp16")];
            tensor<int32, [1]> var_1455_axes_0 = const()[name = string("op_1455_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1455_cast_fp16 = expand_dims(axes = var_1455_axes_0, x = var_1454_cast_fp16)[name = string("op_1455_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_63_cast_fp16 = mul(x = var_1451_cast_fp16, y = var_1455_cast_fp16)[name = string("delta_63_cast_fp16")];
            tensor<int32, [1]> var_1458_axes_0 = const()[name = string("op_1458_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_1458_cast_fp16 = expand_dims(axes = var_1458_axes_0, x = delta_63_cast_fp16)[name = string("op_1458_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1459_cast_fp16 = mul(x = var_1447_cast_fp16, y = var_1458_cast_fp16)[name = string("op_1459_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> rec1_out = add(x = state_125_cast_fp16, y = var_1459_cast_fp16)[name = string("state_127_cast_fp16")];
            tensor<int32, [4]> var_1463_begin_0 = const()[name = string("op_1463_begin_0"), val = tensor<int32, [4]>([0, 0, 15, 0])];
            tensor<int32, [4]> var_1463_end_0 = const()[name = string("op_1463_end_0"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<bool, [4]> var_1463_end_mask_0 = const()[name = string("op_1463_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_1463_squeeze_mask_0 = const()[name = string("op_1463_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_1463_cast_fp16 = slice_by_index(begin = var_1463_begin_0, end = var_1463_end_0, end_mask = var_1463_end_mask_0, squeeze_mask = var_1463_squeeze_mask_0, x = _inversed_query_7_cast_fp16)[name = string("op_1463_cast_fp16")];
            tensor<int32, [1]> var_1464_axes_0 = const()[name = string("op_1464_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1464_cast_fp16 = expand_dims(axes = var_1464_axes_0, x = var_1463_cast_fp16)[name = string("op_1464_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1465_cast_fp16 = mul(x = rec1_out, y = var_1464_cast_fp16)[name = string("op_1465_cast_fp16")];
            tensor<int32, [1]> var_1467_axes_0 = const()[name = string("op_1467_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_1467_keep_dims_0 = const()[name = string("op_1467_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_1467_cast_fp16 = reduce_sum(axes = var_1467_axes_0, keep_dims = var_1467_keep_dims_0, x = var_1465_cast_fp16)[name = string("op_1467_cast_fp16")];
            int32 attention_11_axis_0 = const()[name = string("attention_11_axis_0"), val = int32(1)];
            tensor<fp16, [1, 16, 16, 128]> attention_11_cast_fp16 = stack(axis = attention_11_axis_0, values = (var_957_cast_fp16, var_991_cast_fp16, var_1025_cast_fp16, var_1059_cast_fp16, var_1093_cast_fp16, var_1127_cast_fp16, var_1161_cast_fp16, var_1195_cast_fp16, var_1229_cast_fp16, var_1263_cast_fp16, var_1297_cast_fp16, var_1331_cast_fp16, var_1365_cast_fp16, var_1399_cast_fp16, var_1433_cast_fp16, var_1467_cast_fp16))[name = string("attention_11_cast_fp16")];
            tensor<int32, [2]> var_1470 = const()[name = string("op_1470"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [256, 128]> attention_13_cast_fp16 = reshape(shape = var_1470, x = attention_11_cast_fp16)[name = string("attention_13_cast_fp16")];
            tensor<int32, [2]> var_1472 = const()[name = string("op_1472"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [256, 128]> input_19_cast_fp16 = reshape(shape = var_1472, x = linear_11_cast_fp16)[name = string("input_19_cast_fp16")];
            tensor<fp16, [256, 128]> var_1474_cast_fp16 = mul(x = attention_13_cast_fp16, y = attention_13_cast_fp16)[name = string("op_1474_cast_fp16")];
            tensor<int32, [1]> variance_9_axes_0 = const()[name = string("variance_9_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_9_keep_dims_0 = const()[name = string("variance_9_keep_dims_0"), val = bool(true)];
            tensor<fp16, [256, 1]> variance_9_cast_fp16 = reduce_mean(axes = variance_9_axes_0, keep_dims = variance_9_keep_dims_0, x = var_1474_cast_fp16)[name = string("variance_9_cast_fp16")];
            fp16 var_1477_to_fp16 = const()[name = string("op_1477_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [256, 1]> var_1478_cast_fp16 = add(x = variance_9_cast_fp16, y = var_1477_to_fp16)[name = string("op_1478_cast_fp16")];
            fp32 var_1479_epsilon_0 = const()[name = string("op_1479_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [256, 1]> var_1479_cast_fp16 = rsqrt(epsilon = var_1479_epsilon_0, x = var_1478_cast_fp16)[name = string("op_1479_cast_fp16")];
            tensor<fp16, [256, 128]> attention_15_cast_fp16 = mul(x = attention_13_cast_fp16, y = var_1479_cast_fp16)[name = string("attention_15_cast_fp16")];
            tensor<fp16, [128]> groups_0_1_gated_norm_promoted_to_fp16 = const()[name = string("groups_0_1_gated_norm_promoted_to_fp16"), val = tensor<fp16, [128]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(22837952)))];
            tensor<fp16, [256, 128]> attention_17_cast_fp16 = mul(x = attention_15_cast_fp16, y = groups_0_1_gated_norm_promoted_to_fp16)[name = string("attention_17_cast_fp16")];
            tensor<fp16, [256, 128]> var_1482_cast_fp16 = silu(x = input_19_cast_fp16)[name = string("op_1482_cast_fp16")];
            tensor<fp16, [256, 128]> attention_19_cast_fp16 = mul(x = attention_17_cast_fp16, y = var_1482_cast_fp16)[name = string("attention_19_cast_fp16")];
            tensor<int32, [2]> var_1484 = const()[name = string("op_1484"), val = tensor<int32, [2]>([16, 2048])];
            tensor<fp16, [16, 2048]> input_21_cast_fp16 = reshape(shape = var_1484, x = attention_19_cast_fp16)[name = string("input_21_cast_fp16")];
            tensor<fp16, [1024, 2048]> groups_0_1_out_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 2048]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(22838272))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(24411200))))[name = string("groups_0_1_out_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 1024]> linear_12_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = groups_0_1_out_proj_weight_promoted_to_fp16_palettized, x = input_21_cast_fp16)[name = string("linear_12_cast_fp16")];
            tensor<fp16, [16, 1024]> value_27_cast_fp16 = add(x = value_15_cast_fp16, y = linear_12_cast_fp16)[name = string("value_27_cast_fp16")];
            tensor<fp16, [16, 1024]> var_1489_cast_fp16 = mul(x = value_27_cast_fp16, y = value_27_cast_fp16)[name = string("op_1489_cast_fp16")];
            tensor<int32, [1]> variance_11_axes_0 = const()[name = string("variance_11_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_11_keep_dims_0 = const()[name = string("variance_11_keep_dims_0"), val = bool(true)];
            tensor<fp16, [16, 1]> variance_11_cast_fp16 = reduce_mean(axes = variance_11_axes_0, keep_dims = variance_11_keep_dims_0, x = var_1489_cast_fp16)[name = string("variance_11_cast_fp16")];
            fp16 var_1492_to_fp16 = const()[name = string("op_1492_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [16, 1]> var_1493_cast_fp16 = add(x = variance_11_cast_fp16, y = var_1492_to_fp16)[name = string("op_1493_cast_fp16")];
            fp32 var_1494_epsilon_0 = const()[name = string("op_1494_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [16, 1]> var_1494_cast_fp16 = rsqrt(epsilon = var_1494_epsilon_0, x = var_1493_cast_fp16)[name = string("op_1494_cast_fp16")];
            tensor<fp16, [16, 1024]> var_1495_cast_fp16 = mul(x = value_27_cast_fp16, y = var_1494_cast_fp16)[name = string("op_1495_cast_fp16")];
            tensor<fp16, [1024]> var_1497_to_fp16 = const()[name = string("op_1497_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(24419456)))];
            tensor<fp16, [16, 1024]> input_23_cast_fp16 = mul(x = var_1495_cast_fp16, y = var_1497_to_fp16)[name = string("input_23_cast_fp16")];
            tensor<fp16, [3584, 1024]> groups_0_1_gate_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(24421568))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(27174144))))[name = string("groups_0_1_gate_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 3584]> linear_13_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = groups_0_1_gate_proj_weight_promoted_to_fp16_palettized, x = input_23_cast_fp16)[name = string("linear_13_cast_fp16")];
            tensor<fp16, [16, 3584]> var_1501_cast_fp16 = silu(x = linear_13_cast_fp16)[name = string("op_1501_cast_fp16")];
            tensor<fp16, [3584, 1024]> groups_0_1_up_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(27202880))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(29955456))))[name = string("groups_0_1_up_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 3584]> linear_14_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = groups_0_1_up_proj_weight_promoted_to_fp16_palettized, x = input_23_cast_fp16)[name = string("linear_14_cast_fp16")];
            tensor<fp16, [16, 3584]> input_27_cast_fp16 = mul(x = var_1501_cast_fp16, y = linear_14_cast_fp16)[name = string("input_27_cast_fp16")];
            tensor<fp16, [1024, 3584]> groups_0_1_down_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 3584]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(29984192))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(32736768))))[name = string("groups_0_1_down_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 1024]> linear_15_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = groups_0_1_down_proj_weight_promoted_to_fp16_palettized, x = input_27_cast_fp16)[name = string("linear_15_cast_fp16")];
            tensor<fp16, [16, 1024]> value_29_cast_fp16 = add(x = value_27_cast_fp16, y = linear_15_cast_fp16)[name = string("value_29_cast_fp16")];
            int32 var_1539 = const()[name = string("op_1539"), val = int32(-1)];
            tensor<fp16, [16, 1024]> var_1554_cast_fp16 = mul(x = value_29_cast_fp16, y = value_29_cast_fp16)[name = string("op_1554_cast_fp16")];
            tensor<int32, [1]> variance_13_axes_0 = const()[name = string("variance_13_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_13_keep_dims_0 = const()[name = string("variance_13_keep_dims_0"), val = bool(true)];
            tensor<fp16, [16, 1]> variance_13_cast_fp16 = reduce_mean(axes = variance_13_axes_0, keep_dims = variance_13_keep_dims_0, x = var_1554_cast_fp16)[name = string("variance_13_cast_fp16")];
            fp16 var_1557_to_fp16 = const()[name = string("op_1557_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [16, 1]> var_1558_cast_fp16 = add(x = variance_13_cast_fp16, y = var_1557_to_fp16)[name = string("op_1558_cast_fp16")];
            fp32 var_1559_epsilon_0 = const()[name = string("op_1559_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [16, 1]> var_1559_cast_fp16 = rsqrt(epsilon = var_1559_epsilon_0, x = var_1558_cast_fp16)[name = string("op_1559_cast_fp16")];
            tensor<fp16, [16, 1024]> var_1560_cast_fp16 = mul(x = value_29_cast_fp16, y = var_1559_cast_fp16)[name = string("op_1560_cast_fp16")];
            tensor<fp16, [1024]> var_1562_to_fp16 = const()[name = string("op_1562_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(32745024)))];
            tensor<fp16, [16, 1024]> input_29_cast_fp16 = mul(x = var_1560_cast_fp16, y = var_1562_to_fp16)[name = string("input_29_cast_fp16")];
            tensor<fp16, [6144, 1024]> groups_0_2_in_proj_qkv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(32747136))), lut = tensor<fp16, [384, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(37465792))))[name = string("groups_0_2_in_proj_qkv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 6144]> linear_16_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = groups_0_2_in_proj_qkv_weight_promoted_to_fp16_palettized, x = input_29_cast_fp16)[name = string("linear_16_cast_fp16")];
            tensor<int32, [2]> var_1566_perm_0 = const()[name = string("op_1566_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> mixed_5_axes_0 = const()[name = string("mixed_5_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [6144, 16]> var_1566_cast_fp16 = transpose(perm = var_1566_perm_0, x = linear_16_cast_fp16)[name = string("transpose_127")];
            tensor<fp16, [1, 6144, 16]> mixed_5_cast_fp16 = expand_dims(axes = mixed_5_axes_0, x = var_1566_cast_fp16)[name = string("mixed_5_cast_fp16")];
            bool convolution_input_5_interleave_0 = const()[name = string("convolution_input_5_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 6144, 19]> convolution_input_5_cast_fp16 = concat(axis = var_1539, interleave = convolution_input_5_interleave_0, values = (conv2, mixed_5_cast_fp16))[name = string("convolution_input_5_cast_fp16")];
            string input_31_pad_type_0 = const()[name = string("input_31_pad_type_0"), val = string("valid")];
            int32 input_31_groups_0 = const()[name = string("input_31_groups_0"), val = int32(6144)];
            tensor<int32, [1]> input_31_strides_0 = const()[name = string("input_31_strides_0"), val = tensor<int32, [1]>([1])];
            tensor<int32, [2]> input_31_pad_0 = const()[name = string("input_31_pad_0"), val = tensor<int32, [2]>([0, 0])];
            tensor<int32, [1]> input_31_dilations_0 = const()[name = string("input_31_dilations_0"), val = tensor<int32, [1]>([1])];
            tensor<fp16, [6144, 1, 4]> groups_0_2_conv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1, 4]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(37515008))), lut = tensor<fp16, [384, 1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(37533504))))[name = string("groups_0_2_conv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 6144, 16]> input_31_cast_fp16 = conv(dilations = input_31_dilations_0, groups = input_31_groups_0, pad = input_31_pad_0, pad_type = input_31_pad_type_0, strides = input_31_strides_0, weight = groups_0_2_conv_weight_promoted_to_fp16_palettized, x = convolution_input_5_cast_fp16)[name = string("input_31_cast_fp16")];
            tensor<fp16, [1, 6144, 16]> var_1575_cast_fp16 = silu(x = input_31_cast_fp16)[name = string("op_1575_cast_fp16")];
            tensor<int32, [3]> var_1576_perm_0 = const()[name = string("op_1576_perm_0"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [3]> var_1579_begin_0 = const()[name = string("op_1579_begin_0"), val = tensor<int32, [3]>([0, 0, 16])];
            tensor<int32, [3]> var_1579_end_0 = const()[name = string("op_1579_end_0"), val = tensor<int32, [3]>([1, 6144, 19])];
            tensor<bool, [3]> var_1579_end_mask_0 = const()[name = string("op_1579_end_mask_0"), val = tensor<bool, [3]>([true, true, true])];
            tensor<fp16, [1, 6144, 3]> conv2_out = slice_by_index(begin = var_1579_begin_0, end = var_1579_end_0, end_mask = var_1579_end_mask_0, x = convolution_input_5_cast_fp16)[name = string("op_1579_cast_fp16")];
            tensor<int32, [3]> var_1580 = const()[name = string("op_1580"), val = tensor<int32, [3]>([2048, 2048, 2048])];
            int32 var_1581_axis_0 = const()[name = string("op_1581_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 16, 6144]> var_1576_cast_fp16 = transpose(perm = var_1576_perm_0, x = var_1575_cast_fp16)[name = string("transpose_126")];
            tensor<fp16, [1, 16, 2048]> var_1581_cast_fp16_0, tensor<fp16, [1, 16, 2048]> var_1581_cast_fp16_1, tensor<fp16, [1, 16, 2048]> var_1581_cast_fp16_2 = split(axis = var_1581_axis_0, split_sizes = var_1580, x = var_1576_cast_fp16)[name = string("op_1581_cast_fp16")];
            tensor<int32, [4]> var_1585 = const()[name = string("op_1585"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<fp16, [1, 16, 16, 128]> value_33_cast_fp16 = reshape(shape = var_1585, x = var_1581_cast_fp16_0)[name = string("value_33_cast_fp16")];
            tensor<int32, [4]> var_1587 = const()[name = string("op_1587"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<fp16, [1, 16, 16, 128]> value_35_cast_fp16 = reshape(shape = var_1587, x = var_1581_cast_fp16_1)[name = string("value_35_cast_fp16")];
            tensor<int32, [4]> var_1589 = const()[name = string("op_1589"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<fp16, [1, 16, 16, 128]> value_37_cast_fp16 = reshape(shape = var_1589, x = var_1581_cast_fp16_2)[name = string("value_37_cast_fp16")];
            tensor<fp16, [1, 16, 16, 128]> var_1591_cast_fp16 = mul(x = value_33_cast_fp16, y = value_33_cast_fp16)[name = string("op_1591_cast_fp16")];
            tensor<int32, [1]> var_1593_axes_0 = const()[name = string("op_1593_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_1593_keep_dims_0 = const()[name = string("op_1593_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 16, 16, 1]> var_1593_cast_fp16 = reduce_sum(axes = var_1593_axes_0, keep_dims = var_1593_keep_dims_0, x = var_1591_cast_fp16)[name = string("op_1593_cast_fp16")];
            fp16 var_1594_to_fp16 = const()[name = string("op_1594_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 16, 16, 1]> var_1595_cast_fp16 = add(x = var_1593_cast_fp16, y = var_1594_to_fp16)[name = string("op_1595_cast_fp16")];
            fp32 var_1596_epsilon_0 = const()[name = string("op_1596_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 16, 16, 1]> var_1596_cast_fp16 = rsqrt(epsilon = var_1596_epsilon_0, x = var_1595_cast_fp16)[name = string("op_1596_cast_fp16")];
            tensor<fp16, [1, 16, 16, 128]> var_1597_cast_fp16 = mul(x = value_33_cast_fp16, y = var_1596_cast_fp16)[name = string("op_1597_cast_fp16")];
            tensor<int32, [4]> var_1598_perm_0 = const()[name = string("op_1598_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            fp16 _inversed_query_11_y_0_to_fp16 = const()[name = string("_inversed_query_11_y_0_to_fp16"), val = fp16(0x1.6ap-4)];
            tensor<fp16, [1, 16, 16, 128]> var_1598_cast_fp16 = transpose(perm = var_1598_perm_0, x = var_1597_cast_fp16)[name = string("transpose_125")];
            tensor<fp16, [1, 16, 16, 128]> _inversed_query_11_cast_fp16 = mul(x = var_1598_cast_fp16, y = _inversed_query_11_y_0_to_fp16)[name = string("_inversed_query_11_cast_fp16")];
            tensor<fp16, [1, 16, 16, 128]> var_1601_cast_fp16 = mul(x = value_35_cast_fp16, y = value_35_cast_fp16)[name = string("op_1601_cast_fp16")];
            tensor<int32, [1]> var_1603_axes_0 = const()[name = string("op_1603_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_1603_keep_dims_0 = const()[name = string("op_1603_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 16, 16, 1]> var_1603_cast_fp16 = reduce_sum(axes = var_1603_axes_0, keep_dims = var_1603_keep_dims_0, x = var_1601_cast_fp16)[name = string("op_1603_cast_fp16")];
            fp16 var_1604_to_fp16 = const()[name = string("op_1604_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 16, 16, 1]> var_1605_cast_fp16 = add(x = var_1603_cast_fp16, y = var_1604_to_fp16)[name = string("op_1605_cast_fp16")];
            fp32 var_1606_epsilon_0 = const()[name = string("op_1606_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 16, 16, 1]> var_1606_cast_fp16 = rsqrt(epsilon = var_1606_epsilon_0, x = var_1605_cast_fp16)[name = string("op_1606_cast_fp16")];
            tensor<fp16, [1, 16, 16, 128]> var_1607_cast_fp16 = mul(x = value_35_cast_fp16, y = var_1606_cast_fp16)[name = string("op_1607_cast_fp16")];
            tensor<int32, [4]> key_11_perm_0 = const()[name = string("key_11_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<int32, [4]> value_39_perm_0 = const()[name = string("value_39_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<fp16, [16, 1024]> groups_0_2_in_proj_b_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(37582720))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(37595072))))[name = string("groups_0_2_in_proj_b_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16]> linear_17_bias_0_to_fp16 = const()[name = string("linear_17_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(4862656)))];
            tensor<fp16, [16, 16]> linear_17_cast_fp16 = linear(bias = linear_17_bias_0_to_fp16, weight = groups_0_2_in_proj_b_weight_promoted_to_fp16_palettized, x = input_29_cast_fp16)[name = string("linear_17_cast_fp16")];
            tensor<fp16, [16, 16]> var_1612_cast_fp16 = sigmoid(x = linear_17_cast_fp16)[name = string("op_1612_cast_fp16")];
            tensor<int32, [2]> var_1613_perm_0 = const()[name = string("op_1613_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> beta_5_axes_0 = const()[name = string("beta_5_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 16]> var_1613_cast_fp16 = transpose(perm = var_1613_perm_0, x = var_1612_cast_fp16)[name = string("transpose_124")];
            tensor<fp16, [1, 16, 16]> beta_5_cast_fp16 = expand_dims(axes = beta_5_axes_0, x = var_1613_cast_fp16)[name = string("beta_5_cast_fp16")];
            tensor<fp16, [16, 1024]> op_1619_weight_0_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(37595264))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(37607616))))[name = string("op_1619_weight_0_to_fp16_palettized")];
            tensor<fp16, [16]> var_1619_bias_0_to_fp16 = const()[name = string("op_1619_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(37607808)))];
            tensor<fp16, [16, 16]> var_1619_cast_fp16 = linear(bias = var_1619_bias_0_to_fp16, weight = op_1619_weight_0_to_fp16_palettized, x = input_29_cast_fp16)[name = string("op_1619_cast_fp16")];
            tensor<fp16, [16, 16]> var_1620_cast_fp16 = softplus(x = var_1619_cast_fp16)[name = string("op_1620_cast_fp16")];
            tensor<fp16, [16]> var_1616_promoted_to_fp16 = const()[name = string("op_1616_promoted_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(37607936)))];
            tensor<fp16, [16, 16]> var_1621_cast_fp16 = mul(x = var_1616_promoted_to_fp16, y = var_1620_cast_fp16)[name = string("op_1621_cast_fp16")];
            tensor<int32, [2]> var_1622_perm_0 = const()[name = string("op_1622_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> decay_5_axes_0 = const()[name = string("decay_5_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 16]> var_1622_cast_fp16 = transpose(perm = var_1622_perm_0, x = var_1621_cast_fp16)[name = string("transpose_123")];
            tensor<fp16, [1, 16, 16]> decay_5_cast_fp16 = expand_dims(axes = decay_5_axes_0, x = var_1622_cast_fp16)[name = string("decay_5_cast_fp16")];
            tensor<fp16, [2048, 1024]> groups_0_2_in_proj_z_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [2048, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(37608064))), lut = tensor<fp16, [128, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(39180992))))[name = string("groups_0_2_in_proj_z_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 2048]> linear_19_cast_fp16 = linear(bias = linear_3_bias_0_to_fp16, weight = groups_0_2_in_proj_z_weight_promoted_to_fp16_palettized, x = input_29_cast_fp16)[name = string("linear_19_cast_fp16")];
            tensor<int32, [3]> var_1630_begin_0 = const()[name = string("op_1630_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_1630_end_0 = const()[name = string("op_1630_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_1630_end_mask_0 = const()[name = string("op_1630_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1630_squeeze_mask_0 = const()[name = string("op_1630_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1630_cast_fp16 = slice_by_index(begin = var_1630_begin_0, end = var_1630_end_0, end_mask = var_1630_end_mask_0, squeeze_mask = var_1630_squeeze_mask_0, x = decay_5_cast_fp16)[name = string("op_1630_cast_fp16")];
            tensor<fp16, [1, 16]> var_1631_cast_fp16 = exp(x = var_1630_cast_fp16)[name = string("op_1631_cast_fp16")];
            tensor<int32, [1]> var_1632_axes_0 = const()[name = string("op_1632_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1632_cast_fp16 = expand_dims(axes = var_1632_axes_0, x = var_1631_cast_fp16)[name = string("op_1632_cast_fp16")];
            tensor<int32, [1]> var_1633_axes_0 = const()[name = string("op_1633_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_1633_cast_fp16 = expand_dims(axes = var_1633_axes_0, x = var_1632_cast_fp16)[name = string("op_1633_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_129_cast_fp16 = mul(x = rec2, y = var_1633_cast_fp16)[name = string("state_129_cast_fp16")];
            tensor<int32, [4]> key_token_65_begin_0 = const()[name = string("key_token_65_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> key_token_65_end_0 = const()[name = string("key_token_65_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> key_token_65_end_mask_0 = const()[name = string("key_token_65_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_65_squeeze_mask_0 = const()[name = string("key_token_65_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 16, 128]> key_11_cast_fp16 = transpose(perm = key_11_perm_0, x = var_1607_cast_fp16)[name = string("transpose_122")];
            tensor<fp16, [1, 16, 128]> key_token_65_cast_fp16 = slice_by_index(begin = key_token_65_begin_0, end = key_token_65_end_0, end_mask = key_token_65_end_mask_0, squeeze_mask = key_token_65_squeeze_mask_0, x = key_11_cast_fp16)[name = string("key_token_65_cast_fp16")];
            tensor<int32, [4]> value_token_65_begin_0 = const()[name = string("value_token_65_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> value_token_65_end_0 = const()[name = string("value_token_65_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> value_token_65_end_mask_0 = const()[name = string("value_token_65_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_65_squeeze_mask_0 = const()[name = string("value_token_65_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 16, 128]> value_39_cast_fp16 = transpose(perm = value_39_perm_0, x = value_37_cast_fp16)[name = string("transpose_121")];
            tensor<fp16, [1, 16, 128]> value_token_65_cast_fp16 = slice_by_index(begin = value_token_65_begin_0, end = value_token_65_end_0, end_mask = value_token_65_end_mask_0, squeeze_mask = value_token_65_squeeze_mask_0, x = value_39_cast_fp16)[name = string("value_token_65_cast_fp16")];
            tensor<int32, [1]> var_1641_axes_0 = const()[name = string("op_1641_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1641_cast_fp16 = expand_dims(axes = var_1641_axes_0, x = key_token_65_cast_fp16)[name = string("op_1641_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1642_cast_fp16 = mul(x = state_129_cast_fp16, y = var_1641_cast_fp16)[name = string("op_1642_cast_fp16")];
            tensor<int32, [1]> memory_65_axes_0 = const()[name = string("memory_65_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_65_keep_dims_0 = const()[name = string("memory_65_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_65_cast_fp16 = reduce_sum(axes = memory_65_axes_0, keep_dims = memory_65_keep_dims_0, x = var_1642_cast_fp16)[name = string("memory_65_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_1645_cast_fp16 = sub(x = value_token_65_cast_fp16, y = memory_65_cast_fp16)[name = string("op_1645_cast_fp16")];
            tensor<int32, [3]> var_1648_begin_0 = const()[name = string("op_1648_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_1648_end_0 = const()[name = string("op_1648_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_1648_end_mask_0 = const()[name = string("op_1648_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1648_squeeze_mask_0 = const()[name = string("op_1648_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1648_cast_fp16 = slice_by_index(begin = var_1648_begin_0, end = var_1648_end_0, end_mask = var_1648_end_mask_0, squeeze_mask = var_1648_squeeze_mask_0, x = beta_5_cast_fp16)[name = string("op_1648_cast_fp16")];
            tensor<int32, [1]> var_1649_axes_0 = const()[name = string("op_1649_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1649_cast_fp16 = expand_dims(axes = var_1649_axes_0, x = var_1648_cast_fp16)[name = string("op_1649_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_65_cast_fp16 = mul(x = var_1645_cast_fp16, y = var_1649_cast_fp16)[name = string("delta_65_cast_fp16")];
            tensor<int32, [1]> var_1652_axes_0 = const()[name = string("op_1652_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_1652_cast_fp16 = expand_dims(axes = var_1652_axes_0, x = delta_65_cast_fp16)[name = string("op_1652_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1653_cast_fp16 = mul(x = var_1641_cast_fp16, y = var_1652_cast_fp16)[name = string("op_1653_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_131_cast_fp16 = add(x = state_129_cast_fp16, y = var_1653_cast_fp16)[name = string("state_131_cast_fp16")];
            tensor<int32, [4]> var_1657_begin_0 = const()[name = string("op_1657_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> var_1657_end_0 = const()[name = string("op_1657_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> var_1657_end_mask_0 = const()[name = string("op_1657_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_1657_squeeze_mask_0 = const()[name = string("op_1657_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_1657_cast_fp16 = slice_by_index(begin = var_1657_begin_0, end = var_1657_end_0, end_mask = var_1657_end_mask_0, squeeze_mask = var_1657_squeeze_mask_0, x = _inversed_query_11_cast_fp16)[name = string("op_1657_cast_fp16")];
            tensor<int32, [1]> var_1658_axes_0 = const()[name = string("op_1658_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1658_cast_fp16 = expand_dims(axes = var_1658_axes_0, x = var_1657_cast_fp16)[name = string("op_1658_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1659_cast_fp16 = mul(x = state_131_cast_fp16, y = var_1658_cast_fp16)[name = string("op_1659_cast_fp16")];
            tensor<int32, [1]> var_1661_axes_0 = const()[name = string("op_1661_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_1661_keep_dims_0 = const()[name = string("op_1661_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_1661_cast_fp16 = reduce_sum(axes = var_1661_axes_0, keep_dims = var_1661_keep_dims_0, x = var_1659_cast_fp16)[name = string("op_1661_cast_fp16")];
            tensor<int32, [3]> var_1664_begin_0 = const()[name = string("op_1664_begin_0"), val = tensor<int32, [3]>([0, 0, 1])];
            tensor<int32, [3]> var_1664_end_0 = const()[name = string("op_1664_end_0"), val = tensor<int32, [3]>([1, 16, 2])];
            tensor<bool, [3]> var_1664_end_mask_0 = const()[name = string("op_1664_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1664_squeeze_mask_0 = const()[name = string("op_1664_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1664_cast_fp16 = slice_by_index(begin = var_1664_begin_0, end = var_1664_end_0, end_mask = var_1664_end_mask_0, squeeze_mask = var_1664_squeeze_mask_0, x = decay_5_cast_fp16)[name = string("op_1664_cast_fp16")];
            tensor<fp16, [1, 16]> var_1665_cast_fp16 = exp(x = var_1664_cast_fp16)[name = string("op_1665_cast_fp16")];
            tensor<int32, [1]> var_1666_axes_0 = const()[name = string("op_1666_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1666_cast_fp16 = expand_dims(axes = var_1666_axes_0, x = var_1665_cast_fp16)[name = string("op_1666_cast_fp16")];
            tensor<int32, [1]> var_1667_axes_0 = const()[name = string("op_1667_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_1667_cast_fp16 = expand_dims(axes = var_1667_axes_0, x = var_1666_cast_fp16)[name = string("op_1667_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_133_cast_fp16 = mul(x = state_131_cast_fp16, y = var_1667_cast_fp16)[name = string("state_133_cast_fp16")];
            tensor<int32, [4]> key_token_67_begin_0 = const()[name = string("key_token_67_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> key_token_67_end_0 = const()[name = string("key_token_67_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> key_token_67_end_mask_0 = const()[name = string("key_token_67_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_67_squeeze_mask_0 = const()[name = string("key_token_67_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_67_cast_fp16 = slice_by_index(begin = key_token_67_begin_0, end = key_token_67_end_0, end_mask = key_token_67_end_mask_0, squeeze_mask = key_token_67_squeeze_mask_0, x = key_11_cast_fp16)[name = string("key_token_67_cast_fp16")];
            tensor<int32, [4]> value_token_67_begin_0 = const()[name = string("value_token_67_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> value_token_67_end_0 = const()[name = string("value_token_67_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> value_token_67_end_mask_0 = const()[name = string("value_token_67_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_67_squeeze_mask_0 = const()[name = string("value_token_67_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_67_cast_fp16 = slice_by_index(begin = value_token_67_begin_0, end = value_token_67_end_0, end_mask = value_token_67_end_mask_0, squeeze_mask = value_token_67_squeeze_mask_0, x = value_39_cast_fp16)[name = string("value_token_67_cast_fp16")];
            tensor<int32, [1]> var_1675_axes_0 = const()[name = string("op_1675_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1675_cast_fp16 = expand_dims(axes = var_1675_axes_0, x = key_token_67_cast_fp16)[name = string("op_1675_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1676_cast_fp16 = mul(x = state_133_cast_fp16, y = var_1675_cast_fp16)[name = string("op_1676_cast_fp16")];
            tensor<int32, [1]> memory_67_axes_0 = const()[name = string("memory_67_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_67_keep_dims_0 = const()[name = string("memory_67_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_67_cast_fp16 = reduce_sum(axes = memory_67_axes_0, keep_dims = memory_67_keep_dims_0, x = var_1676_cast_fp16)[name = string("memory_67_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_1679_cast_fp16 = sub(x = value_token_67_cast_fp16, y = memory_67_cast_fp16)[name = string("op_1679_cast_fp16")];
            tensor<int32, [3]> var_1682_begin_0 = const()[name = string("op_1682_begin_0"), val = tensor<int32, [3]>([0, 0, 1])];
            tensor<int32, [3]> var_1682_end_0 = const()[name = string("op_1682_end_0"), val = tensor<int32, [3]>([1, 16, 2])];
            tensor<bool, [3]> var_1682_end_mask_0 = const()[name = string("op_1682_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1682_squeeze_mask_0 = const()[name = string("op_1682_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1682_cast_fp16 = slice_by_index(begin = var_1682_begin_0, end = var_1682_end_0, end_mask = var_1682_end_mask_0, squeeze_mask = var_1682_squeeze_mask_0, x = beta_5_cast_fp16)[name = string("op_1682_cast_fp16")];
            tensor<int32, [1]> var_1683_axes_0 = const()[name = string("op_1683_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1683_cast_fp16 = expand_dims(axes = var_1683_axes_0, x = var_1682_cast_fp16)[name = string("op_1683_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_67_cast_fp16 = mul(x = var_1679_cast_fp16, y = var_1683_cast_fp16)[name = string("delta_67_cast_fp16")];
            tensor<int32, [1]> var_1686_axes_0 = const()[name = string("op_1686_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_1686_cast_fp16 = expand_dims(axes = var_1686_axes_0, x = delta_67_cast_fp16)[name = string("op_1686_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1687_cast_fp16 = mul(x = var_1675_cast_fp16, y = var_1686_cast_fp16)[name = string("op_1687_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_135_cast_fp16 = add(x = state_133_cast_fp16, y = var_1687_cast_fp16)[name = string("state_135_cast_fp16")];
            tensor<int32, [4]> var_1691_begin_0 = const()[name = string("op_1691_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> var_1691_end_0 = const()[name = string("op_1691_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> var_1691_end_mask_0 = const()[name = string("op_1691_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_1691_squeeze_mask_0 = const()[name = string("op_1691_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_1691_cast_fp16 = slice_by_index(begin = var_1691_begin_0, end = var_1691_end_0, end_mask = var_1691_end_mask_0, squeeze_mask = var_1691_squeeze_mask_0, x = _inversed_query_11_cast_fp16)[name = string("op_1691_cast_fp16")];
            tensor<int32, [1]> var_1692_axes_0 = const()[name = string("op_1692_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1692_cast_fp16 = expand_dims(axes = var_1692_axes_0, x = var_1691_cast_fp16)[name = string("op_1692_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1693_cast_fp16 = mul(x = state_135_cast_fp16, y = var_1692_cast_fp16)[name = string("op_1693_cast_fp16")];
            tensor<int32, [1]> var_1695_axes_0 = const()[name = string("op_1695_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_1695_keep_dims_0 = const()[name = string("op_1695_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_1695_cast_fp16 = reduce_sum(axes = var_1695_axes_0, keep_dims = var_1695_keep_dims_0, x = var_1693_cast_fp16)[name = string("op_1695_cast_fp16")];
            tensor<int32, [3]> var_1698_begin_0 = const()[name = string("op_1698_begin_0"), val = tensor<int32, [3]>([0, 0, 2])];
            tensor<int32, [3]> var_1698_end_0 = const()[name = string("op_1698_end_0"), val = tensor<int32, [3]>([1, 16, 3])];
            tensor<bool, [3]> var_1698_end_mask_0 = const()[name = string("op_1698_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1698_squeeze_mask_0 = const()[name = string("op_1698_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1698_cast_fp16 = slice_by_index(begin = var_1698_begin_0, end = var_1698_end_0, end_mask = var_1698_end_mask_0, squeeze_mask = var_1698_squeeze_mask_0, x = decay_5_cast_fp16)[name = string("op_1698_cast_fp16")];
            tensor<fp16, [1, 16]> var_1699_cast_fp16 = exp(x = var_1698_cast_fp16)[name = string("op_1699_cast_fp16")];
            tensor<int32, [1]> var_1700_axes_0 = const()[name = string("op_1700_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1700_cast_fp16 = expand_dims(axes = var_1700_axes_0, x = var_1699_cast_fp16)[name = string("op_1700_cast_fp16")];
            tensor<int32, [1]> var_1701_axes_0 = const()[name = string("op_1701_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_1701_cast_fp16 = expand_dims(axes = var_1701_axes_0, x = var_1700_cast_fp16)[name = string("op_1701_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_137_cast_fp16 = mul(x = state_135_cast_fp16, y = var_1701_cast_fp16)[name = string("state_137_cast_fp16")];
            tensor<int32, [4]> key_token_69_begin_0 = const()[name = string("key_token_69_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> key_token_69_end_0 = const()[name = string("key_token_69_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> key_token_69_end_mask_0 = const()[name = string("key_token_69_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_69_squeeze_mask_0 = const()[name = string("key_token_69_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_69_cast_fp16 = slice_by_index(begin = key_token_69_begin_0, end = key_token_69_end_0, end_mask = key_token_69_end_mask_0, squeeze_mask = key_token_69_squeeze_mask_0, x = key_11_cast_fp16)[name = string("key_token_69_cast_fp16")];
            tensor<int32, [4]> value_token_69_begin_0 = const()[name = string("value_token_69_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> value_token_69_end_0 = const()[name = string("value_token_69_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> value_token_69_end_mask_0 = const()[name = string("value_token_69_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_69_squeeze_mask_0 = const()[name = string("value_token_69_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_69_cast_fp16 = slice_by_index(begin = value_token_69_begin_0, end = value_token_69_end_0, end_mask = value_token_69_end_mask_0, squeeze_mask = value_token_69_squeeze_mask_0, x = value_39_cast_fp16)[name = string("value_token_69_cast_fp16")];
            tensor<int32, [1]> var_1709_axes_0 = const()[name = string("op_1709_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1709_cast_fp16 = expand_dims(axes = var_1709_axes_0, x = key_token_69_cast_fp16)[name = string("op_1709_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1710_cast_fp16 = mul(x = state_137_cast_fp16, y = var_1709_cast_fp16)[name = string("op_1710_cast_fp16")];
            tensor<int32, [1]> memory_69_axes_0 = const()[name = string("memory_69_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_69_keep_dims_0 = const()[name = string("memory_69_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_69_cast_fp16 = reduce_sum(axes = memory_69_axes_0, keep_dims = memory_69_keep_dims_0, x = var_1710_cast_fp16)[name = string("memory_69_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_1713_cast_fp16 = sub(x = value_token_69_cast_fp16, y = memory_69_cast_fp16)[name = string("op_1713_cast_fp16")];
            tensor<int32, [3]> var_1716_begin_0 = const()[name = string("op_1716_begin_0"), val = tensor<int32, [3]>([0, 0, 2])];
            tensor<int32, [3]> var_1716_end_0 = const()[name = string("op_1716_end_0"), val = tensor<int32, [3]>([1, 16, 3])];
            tensor<bool, [3]> var_1716_end_mask_0 = const()[name = string("op_1716_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1716_squeeze_mask_0 = const()[name = string("op_1716_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1716_cast_fp16 = slice_by_index(begin = var_1716_begin_0, end = var_1716_end_0, end_mask = var_1716_end_mask_0, squeeze_mask = var_1716_squeeze_mask_0, x = beta_5_cast_fp16)[name = string("op_1716_cast_fp16")];
            tensor<int32, [1]> var_1717_axes_0 = const()[name = string("op_1717_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1717_cast_fp16 = expand_dims(axes = var_1717_axes_0, x = var_1716_cast_fp16)[name = string("op_1717_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_69_cast_fp16 = mul(x = var_1713_cast_fp16, y = var_1717_cast_fp16)[name = string("delta_69_cast_fp16")];
            tensor<int32, [1]> var_1720_axes_0 = const()[name = string("op_1720_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_1720_cast_fp16 = expand_dims(axes = var_1720_axes_0, x = delta_69_cast_fp16)[name = string("op_1720_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1721_cast_fp16 = mul(x = var_1709_cast_fp16, y = var_1720_cast_fp16)[name = string("op_1721_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_139_cast_fp16 = add(x = state_137_cast_fp16, y = var_1721_cast_fp16)[name = string("state_139_cast_fp16")];
            tensor<int32, [4]> var_1725_begin_0 = const()[name = string("op_1725_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> var_1725_end_0 = const()[name = string("op_1725_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> var_1725_end_mask_0 = const()[name = string("op_1725_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_1725_squeeze_mask_0 = const()[name = string("op_1725_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_1725_cast_fp16 = slice_by_index(begin = var_1725_begin_0, end = var_1725_end_0, end_mask = var_1725_end_mask_0, squeeze_mask = var_1725_squeeze_mask_0, x = _inversed_query_11_cast_fp16)[name = string("op_1725_cast_fp16")];
            tensor<int32, [1]> var_1726_axes_0 = const()[name = string("op_1726_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1726_cast_fp16 = expand_dims(axes = var_1726_axes_0, x = var_1725_cast_fp16)[name = string("op_1726_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1727_cast_fp16 = mul(x = state_139_cast_fp16, y = var_1726_cast_fp16)[name = string("op_1727_cast_fp16")];
            tensor<int32, [1]> var_1729_axes_0 = const()[name = string("op_1729_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_1729_keep_dims_0 = const()[name = string("op_1729_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_1729_cast_fp16 = reduce_sum(axes = var_1729_axes_0, keep_dims = var_1729_keep_dims_0, x = var_1727_cast_fp16)[name = string("op_1729_cast_fp16")];
            tensor<int32, [3]> var_1732_begin_0 = const()[name = string("op_1732_begin_0"), val = tensor<int32, [3]>([0, 0, 3])];
            tensor<int32, [3]> var_1732_end_0 = const()[name = string("op_1732_end_0"), val = tensor<int32, [3]>([1, 16, 4])];
            tensor<bool, [3]> var_1732_end_mask_0 = const()[name = string("op_1732_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1732_squeeze_mask_0 = const()[name = string("op_1732_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1732_cast_fp16 = slice_by_index(begin = var_1732_begin_0, end = var_1732_end_0, end_mask = var_1732_end_mask_0, squeeze_mask = var_1732_squeeze_mask_0, x = decay_5_cast_fp16)[name = string("op_1732_cast_fp16")];
            tensor<fp16, [1, 16]> var_1733_cast_fp16 = exp(x = var_1732_cast_fp16)[name = string("op_1733_cast_fp16")];
            tensor<int32, [1]> var_1734_axes_0 = const()[name = string("op_1734_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1734_cast_fp16 = expand_dims(axes = var_1734_axes_0, x = var_1733_cast_fp16)[name = string("op_1734_cast_fp16")];
            tensor<int32, [1]> var_1735_axes_0 = const()[name = string("op_1735_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_1735_cast_fp16 = expand_dims(axes = var_1735_axes_0, x = var_1734_cast_fp16)[name = string("op_1735_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_141_cast_fp16 = mul(x = state_139_cast_fp16, y = var_1735_cast_fp16)[name = string("state_141_cast_fp16")];
            tensor<int32, [4]> key_token_71_begin_0 = const()[name = string("key_token_71_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> key_token_71_end_0 = const()[name = string("key_token_71_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> key_token_71_end_mask_0 = const()[name = string("key_token_71_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_71_squeeze_mask_0 = const()[name = string("key_token_71_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_71_cast_fp16 = slice_by_index(begin = key_token_71_begin_0, end = key_token_71_end_0, end_mask = key_token_71_end_mask_0, squeeze_mask = key_token_71_squeeze_mask_0, x = key_11_cast_fp16)[name = string("key_token_71_cast_fp16")];
            tensor<int32, [4]> value_token_71_begin_0 = const()[name = string("value_token_71_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> value_token_71_end_0 = const()[name = string("value_token_71_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> value_token_71_end_mask_0 = const()[name = string("value_token_71_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_71_squeeze_mask_0 = const()[name = string("value_token_71_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_71_cast_fp16 = slice_by_index(begin = value_token_71_begin_0, end = value_token_71_end_0, end_mask = value_token_71_end_mask_0, squeeze_mask = value_token_71_squeeze_mask_0, x = value_39_cast_fp16)[name = string("value_token_71_cast_fp16")];
            tensor<int32, [1]> var_1743_axes_0 = const()[name = string("op_1743_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1743_cast_fp16 = expand_dims(axes = var_1743_axes_0, x = key_token_71_cast_fp16)[name = string("op_1743_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1744_cast_fp16 = mul(x = state_141_cast_fp16, y = var_1743_cast_fp16)[name = string("op_1744_cast_fp16")];
            tensor<int32, [1]> memory_71_axes_0 = const()[name = string("memory_71_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_71_keep_dims_0 = const()[name = string("memory_71_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_71_cast_fp16 = reduce_sum(axes = memory_71_axes_0, keep_dims = memory_71_keep_dims_0, x = var_1744_cast_fp16)[name = string("memory_71_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_1747_cast_fp16 = sub(x = value_token_71_cast_fp16, y = memory_71_cast_fp16)[name = string("op_1747_cast_fp16")];
            tensor<int32, [3]> var_1750_begin_0 = const()[name = string("op_1750_begin_0"), val = tensor<int32, [3]>([0, 0, 3])];
            tensor<int32, [3]> var_1750_end_0 = const()[name = string("op_1750_end_0"), val = tensor<int32, [3]>([1, 16, 4])];
            tensor<bool, [3]> var_1750_end_mask_0 = const()[name = string("op_1750_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1750_squeeze_mask_0 = const()[name = string("op_1750_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1750_cast_fp16 = slice_by_index(begin = var_1750_begin_0, end = var_1750_end_0, end_mask = var_1750_end_mask_0, squeeze_mask = var_1750_squeeze_mask_0, x = beta_5_cast_fp16)[name = string("op_1750_cast_fp16")];
            tensor<int32, [1]> var_1751_axes_0 = const()[name = string("op_1751_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1751_cast_fp16 = expand_dims(axes = var_1751_axes_0, x = var_1750_cast_fp16)[name = string("op_1751_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_71_cast_fp16 = mul(x = var_1747_cast_fp16, y = var_1751_cast_fp16)[name = string("delta_71_cast_fp16")];
            tensor<int32, [1]> var_1754_axes_0 = const()[name = string("op_1754_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_1754_cast_fp16 = expand_dims(axes = var_1754_axes_0, x = delta_71_cast_fp16)[name = string("op_1754_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1755_cast_fp16 = mul(x = var_1743_cast_fp16, y = var_1754_cast_fp16)[name = string("op_1755_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_143_cast_fp16 = add(x = state_141_cast_fp16, y = var_1755_cast_fp16)[name = string("state_143_cast_fp16")];
            tensor<int32, [4]> var_1759_begin_0 = const()[name = string("op_1759_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> var_1759_end_0 = const()[name = string("op_1759_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> var_1759_end_mask_0 = const()[name = string("op_1759_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_1759_squeeze_mask_0 = const()[name = string("op_1759_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_1759_cast_fp16 = slice_by_index(begin = var_1759_begin_0, end = var_1759_end_0, end_mask = var_1759_end_mask_0, squeeze_mask = var_1759_squeeze_mask_0, x = _inversed_query_11_cast_fp16)[name = string("op_1759_cast_fp16")];
            tensor<int32, [1]> var_1760_axes_0 = const()[name = string("op_1760_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1760_cast_fp16 = expand_dims(axes = var_1760_axes_0, x = var_1759_cast_fp16)[name = string("op_1760_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1761_cast_fp16 = mul(x = state_143_cast_fp16, y = var_1760_cast_fp16)[name = string("op_1761_cast_fp16")];
            tensor<int32, [1]> var_1763_axes_0 = const()[name = string("op_1763_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_1763_keep_dims_0 = const()[name = string("op_1763_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_1763_cast_fp16 = reduce_sum(axes = var_1763_axes_0, keep_dims = var_1763_keep_dims_0, x = var_1761_cast_fp16)[name = string("op_1763_cast_fp16")];
            tensor<int32, [3]> var_1766_begin_0 = const()[name = string("op_1766_begin_0"), val = tensor<int32, [3]>([0, 0, 4])];
            tensor<int32, [3]> var_1766_end_0 = const()[name = string("op_1766_end_0"), val = tensor<int32, [3]>([1, 16, 5])];
            tensor<bool, [3]> var_1766_end_mask_0 = const()[name = string("op_1766_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1766_squeeze_mask_0 = const()[name = string("op_1766_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1766_cast_fp16 = slice_by_index(begin = var_1766_begin_0, end = var_1766_end_0, end_mask = var_1766_end_mask_0, squeeze_mask = var_1766_squeeze_mask_0, x = decay_5_cast_fp16)[name = string("op_1766_cast_fp16")];
            tensor<fp16, [1, 16]> var_1767_cast_fp16 = exp(x = var_1766_cast_fp16)[name = string("op_1767_cast_fp16")];
            tensor<int32, [1]> var_1768_axes_0 = const()[name = string("op_1768_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1768_cast_fp16 = expand_dims(axes = var_1768_axes_0, x = var_1767_cast_fp16)[name = string("op_1768_cast_fp16")];
            tensor<int32, [1]> var_1769_axes_0 = const()[name = string("op_1769_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_1769_cast_fp16 = expand_dims(axes = var_1769_axes_0, x = var_1768_cast_fp16)[name = string("op_1769_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_145_cast_fp16 = mul(x = state_143_cast_fp16, y = var_1769_cast_fp16)[name = string("state_145_cast_fp16")];
            tensor<int32, [4]> key_token_73_begin_0 = const()[name = string("key_token_73_begin_0"), val = tensor<int32, [4]>([0, 0, 4, 0])];
            tensor<int32, [4]> key_token_73_end_0 = const()[name = string("key_token_73_end_0"), val = tensor<int32, [4]>([1, 16, 5, 128])];
            tensor<bool, [4]> key_token_73_end_mask_0 = const()[name = string("key_token_73_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_73_squeeze_mask_0 = const()[name = string("key_token_73_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_73_cast_fp16 = slice_by_index(begin = key_token_73_begin_0, end = key_token_73_end_0, end_mask = key_token_73_end_mask_0, squeeze_mask = key_token_73_squeeze_mask_0, x = key_11_cast_fp16)[name = string("key_token_73_cast_fp16")];
            tensor<int32, [4]> value_token_73_begin_0 = const()[name = string("value_token_73_begin_0"), val = tensor<int32, [4]>([0, 0, 4, 0])];
            tensor<int32, [4]> value_token_73_end_0 = const()[name = string("value_token_73_end_0"), val = tensor<int32, [4]>([1, 16, 5, 128])];
            tensor<bool, [4]> value_token_73_end_mask_0 = const()[name = string("value_token_73_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_73_squeeze_mask_0 = const()[name = string("value_token_73_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_73_cast_fp16 = slice_by_index(begin = value_token_73_begin_0, end = value_token_73_end_0, end_mask = value_token_73_end_mask_0, squeeze_mask = value_token_73_squeeze_mask_0, x = value_39_cast_fp16)[name = string("value_token_73_cast_fp16")];
            tensor<int32, [1]> var_1777_axes_0 = const()[name = string("op_1777_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1777_cast_fp16 = expand_dims(axes = var_1777_axes_0, x = key_token_73_cast_fp16)[name = string("op_1777_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1778_cast_fp16 = mul(x = state_145_cast_fp16, y = var_1777_cast_fp16)[name = string("op_1778_cast_fp16")];
            tensor<int32, [1]> memory_73_axes_0 = const()[name = string("memory_73_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_73_keep_dims_0 = const()[name = string("memory_73_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_73_cast_fp16 = reduce_sum(axes = memory_73_axes_0, keep_dims = memory_73_keep_dims_0, x = var_1778_cast_fp16)[name = string("memory_73_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_1781_cast_fp16 = sub(x = value_token_73_cast_fp16, y = memory_73_cast_fp16)[name = string("op_1781_cast_fp16")];
            tensor<int32, [3]> var_1784_begin_0 = const()[name = string("op_1784_begin_0"), val = tensor<int32, [3]>([0, 0, 4])];
            tensor<int32, [3]> var_1784_end_0 = const()[name = string("op_1784_end_0"), val = tensor<int32, [3]>([1, 16, 5])];
            tensor<bool, [3]> var_1784_end_mask_0 = const()[name = string("op_1784_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1784_squeeze_mask_0 = const()[name = string("op_1784_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1784_cast_fp16 = slice_by_index(begin = var_1784_begin_0, end = var_1784_end_0, end_mask = var_1784_end_mask_0, squeeze_mask = var_1784_squeeze_mask_0, x = beta_5_cast_fp16)[name = string("op_1784_cast_fp16")];
            tensor<int32, [1]> var_1785_axes_0 = const()[name = string("op_1785_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1785_cast_fp16 = expand_dims(axes = var_1785_axes_0, x = var_1784_cast_fp16)[name = string("op_1785_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_73_cast_fp16 = mul(x = var_1781_cast_fp16, y = var_1785_cast_fp16)[name = string("delta_73_cast_fp16")];
            tensor<int32, [1]> var_1788_axes_0 = const()[name = string("op_1788_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_1788_cast_fp16 = expand_dims(axes = var_1788_axes_0, x = delta_73_cast_fp16)[name = string("op_1788_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1789_cast_fp16 = mul(x = var_1777_cast_fp16, y = var_1788_cast_fp16)[name = string("op_1789_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_147_cast_fp16 = add(x = state_145_cast_fp16, y = var_1789_cast_fp16)[name = string("state_147_cast_fp16")];
            tensor<int32, [4]> var_1793_begin_0 = const()[name = string("op_1793_begin_0"), val = tensor<int32, [4]>([0, 0, 4, 0])];
            tensor<int32, [4]> var_1793_end_0 = const()[name = string("op_1793_end_0"), val = tensor<int32, [4]>([1, 16, 5, 128])];
            tensor<bool, [4]> var_1793_end_mask_0 = const()[name = string("op_1793_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_1793_squeeze_mask_0 = const()[name = string("op_1793_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_1793_cast_fp16 = slice_by_index(begin = var_1793_begin_0, end = var_1793_end_0, end_mask = var_1793_end_mask_0, squeeze_mask = var_1793_squeeze_mask_0, x = _inversed_query_11_cast_fp16)[name = string("op_1793_cast_fp16")];
            tensor<int32, [1]> var_1794_axes_0 = const()[name = string("op_1794_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1794_cast_fp16 = expand_dims(axes = var_1794_axes_0, x = var_1793_cast_fp16)[name = string("op_1794_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1795_cast_fp16 = mul(x = state_147_cast_fp16, y = var_1794_cast_fp16)[name = string("op_1795_cast_fp16")];
            tensor<int32, [1]> var_1797_axes_0 = const()[name = string("op_1797_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_1797_keep_dims_0 = const()[name = string("op_1797_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_1797_cast_fp16 = reduce_sum(axes = var_1797_axes_0, keep_dims = var_1797_keep_dims_0, x = var_1795_cast_fp16)[name = string("op_1797_cast_fp16")];
            tensor<int32, [3]> var_1800_begin_0 = const()[name = string("op_1800_begin_0"), val = tensor<int32, [3]>([0, 0, 5])];
            tensor<int32, [3]> var_1800_end_0 = const()[name = string("op_1800_end_0"), val = tensor<int32, [3]>([1, 16, 6])];
            tensor<bool, [3]> var_1800_end_mask_0 = const()[name = string("op_1800_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1800_squeeze_mask_0 = const()[name = string("op_1800_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1800_cast_fp16 = slice_by_index(begin = var_1800_begin_0, end = var_1800_end_0, end_mask = var_1800_end_mask_0, squeeze_mask = var_1800_squeeze_mask_0, x = decay_5_cast_fp16)[name = string("op_1800_cast_fp16")];
            tensor<fp16, [1, 16]> var_1801_cast_fp16 = exp(x = var_1800_cast_fp16)[name = string("op_1801_cast_fp16")];
            tensor<int32, [1]> var_1802_axes_0 = const()[name = string("op_1802_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1802_cast_fp16 = expand_dims(axes = var_1802_axes_0, x = var_1801_cast_fp16)[name = string("op_1802_cast_fp16")];
            tensor<int32, [1]> var_1803_axes_0 = const()[name = string("op_1803_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_1803_cast_fp16 = expand_dims(axes = var_1803_axes_0, x = var_1802_cast_fp16)[name = string("op_1803_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_149_cast_fp16 = mul(x = state_147_cast_fp16, y = var_1803_cast_fp16)[name = string("state_149_cast_fp16")];
            tensor<int32, [4]> key_token_75_begin_0 = const()[name = string("key_token_75_begin_0"), val = tensor<int32, [4]>([0, 0, 5, 0])];
            tensor<int32, [4]> key_token_75_end_0 = const()[name = string("key_token_75_end_0"), val = tensor<int32, [4]>([1, 16, 6, 128])];
            tensor<bool, [4]> key_token_75_end_mask_0 = const()[name = string("key_token_75_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_75_squeeze_mask_0 = const()[name = string("key_token_75_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_75_cast_fp16 = slice_by_index(begin = key_token_75_begin_0, end = key_token_75_end_0, end_mask = key_token_75_end_mask_0, squeeze_mask = key_token_75_squeeze_mask_0, x = key_11_cast_fp16)[name = string("key_token_75_cast_fp16")];
            tensor<int32, [4]> value_token_75_begin_0 = const()[name = string("value_token_75_begin_0"), val = tensor<int32, [4]>([0, 0, 5, 0])];
            tensor<int32, [4]> value_token_75_end_0 = const()[name = string("value_token_75_end_0"), val = tensor<int32, [4]>([1, 16, 6, 128])];
            tensor<bool, [4]> value_token_75_end_mask_0 = const()[name = string("value_token_75_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_75_squeeze_mask_0 = const()[name = string("value_token_75_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_75_cast_fp16 = slice_by_index(begin = value_token_75_begin_0, end = value_token_75_end_0, end_mask = value_token_75_end_mask_0, squeeze_mask = value_token_75_squeeze_mask_0, x = value_39_cast_fp16)[name = string("value_token_75_cast_fp16")];
            tensor<int32, [1]> var_1811_axes_0 = const()[name = string("op_1811_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1811_cast_fp16 = expand_dims(axes = var_1811_axes_0, x = key_token_75_cast_fp16)[name = string("op_1811_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1812_cast_fp16 = mul(x = state_149_cast_fp16, y = var_1811_cast_fp16)[name = string("op_1812_cast_fp16")];
            tensor<int32, [1]> memory_75_axes_0 = const()[name = string("memory_75_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_75_keep_dims_0 = const()[name = string("memory_75_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_75_cast_fp16 = reduce_sum(axes = memory_75_axes_0, keep_dims = memory_75_keep_dims_0, x = var_1812_cast_fp16)[name = string("memory_75_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_1815_cast_fp16 = sub(x = value_token_75_cast_fp16, y = memory_75_cast_fp16)[name = string("op_1815_cast_fp16")];
            tensor<int32, [3]> var_1818_begin_0 = const()[name = string("op_1818_begin_0"), val = tensor<int32, [3]>([0, 0, 5])];
            tensor<int32, [3]> var_1818_end_0 = const()[name = string("op_1818_end_0"), val = tensor<int32, [3]>([1, 16, 6])];
            tensor<bool, [3]> var_1818_end_mask_0 = const()[name = string("op_1818_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1818_squeeze_mask_0 = const()[name = string("op_1818_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1818_cast_fp16 = slice_by_index(begin = var_1818_begin_0, end = var_1818_end_0, end_mask = var_1818_end_mask_0, squeeze_mask = var_1818_squeeze_mask_0, x = beta_5_cast_fp16)[name = string("op_1818_cast_fp16")];
            tensor<int32, [1]> var_1819_axes_0 = const()[name = string("op_1819_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1819_cast_fp16 = expand_dims(axes = var_1819_axes_0, x = var_1818_cast_fp16)[name = string("op_1819_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_75_cast_fp16 = mul(x = var_1815_cast_fp16, y = var_1819_cast_fp16)[name = string("delta_75_cast_fp16")];
            tensor<int32, [1]> var_1822_axes_0 = const()[name = string("op_1822_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_1822_cast_fp16 = expand_dims(axes = var_1822_axes_0, x = delta_75_cast_fp16)[name = string("op_1822_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1823_cast_fp16 = mul(x = var_1811_cast_fp16, y = var_1822_cast_fp16)[name = string("op_1823_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_151_cast_fp16 = add(x = state_149_cast_fp16, y = var_1823_cast_fp16)[name = string("state_151_cast_fp16")];
            tensor<int32, [4]> var_1827_begin_0 = const()[name = string("op_1827_begin_0"), val = tensor<int32, [4]>([0, 0, 5, 0])];
            tensor<int32, [4]> var_1827_end_0 = const()[name = string("op_1827_end_0"), val = tensor<int32, [4]>([1, 16, 6, 128])];
            tensor<bool, [4]> var_1827_end_mask_0 = const()[name = string("op_1827_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_1827_squeeze_mask_0 = const()[name = string("op_1827_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_1827_cast_fp16 = slice_by_index(begin = var_1827_begin_0, end = var_1827_end_0, end_mask = var_1827_end_mask_0, squeeze_mask = var_1827_squeeze_mask_0, x = _inversed_query_11_cast_fp16)[name = string("op_1827_cast_fp16")];
            tensor<int32, [1]> var_1828_axes_0 = const()[name = string("op_1828_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1828_cast_fp16 = expand_dims(axes = var_1828_axes_0, x = var_1827_cast_fp16)[name = string("op_1828_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1829_cast_fp16 = mul(x = state_151_cast_fp16, y = var_1828_cast_fp16)[name = string("op_1829_cast_fp16")];
            tensor<int32, [1]> var_1831_axes_0 = const()[name = string("op_1831_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_1831_keep_dims_0 = const()[name = string("op_1831_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_1831_cast_fp16 = reduce_sum(axes = var_1831_axes_0, keep_dims = var_1831_keep_dims_0, x = var_1829_cast_fp16)[name = string("op_1831_cast_fp16")];
            tensor<int32, [3]> var_1834_begin_0 = const()[name = string("op_1834_begin_0"), val = tensor<int32, [3]>([0, 0, 6])];
            tensor<int32, [3]> var_1834_end_0 = const()[name = string("op_1834_end_0"), val = tensor<int32, [3]>([1, 16, 7])];
            tensor<bool, [3]> var_1834_end_mask_0 = const()[name = string("op_1834_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1834_squeeze_mask_0 = const()[name = string("op_1834_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1834_cast_fp16 = slice_by_index(begin = var_1834_begin_0, end = var_1834_end_0, end_mask = var_1834_end_mask_0, squeeze_mask = var_1834_squeeze_mask_0, x = decay_5_cast_fp16)[name = string("op_1834_cast_fp16")];
            tensor<fp16, [1, 16]> var_1835_cast_fp16 = exp(x = var_1834_cast_fp16)[name = string("op_1835_cast_fp16")];
            tensor<int32, [1]> var_1836_axes_0 = const()[name = string("op_1836_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1836_cast_fp16 = expand_dims(axes = var_1836_axes_0, x = var_1835_cast_fp16)[name = string("op_1836_cast_fp16")];
            tensor<int32, [1]> var_1837_axes_0 = const()[name = string("op_1837_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_1837_cast_fp16 = expand_dims(axes = var_1837_axes_0, x = var_1836_cast_fp16)[name = string("op_1837_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_153_cast_fp16 = mul(x = state_151_cast_fp16, y = var_1837_cast_fp16)[name = string("state_153_cast_fp16")];
            tensor<int32, [4]> key_token_77_begin_0 = const()[name = string("key_token_77_begin_0"), val = tensor<int32, [4]>([0, 0, 6, 0])];
            tensor<int32, [4]> key_token_77_end_0 = const()[name = string("key_token_77_end_0"), val = tensor<int32, [4]>([1, 16, 7, 128])];
            tensor<bool, [4]> key_token_77_end_mask_0 = const()[name = string("key_token_77_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_77_squeeze_mask_0 = const()[name = string("key_token_77_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_77_cast_fp16 = slice_by_index(begin = key_token_77_begin_0, end = key_token_77_end_0, end_mask = key_token_77_end_mask_0, squeeze_mask = key_token_77_squeeze_mask_0, x = key_11_cast_fp16)[name = string("key_token_77_cast_fp16")];
            tensor<int32, [4]> value_token_77_begin_0 = const()[name = string("value_token_77_begin_0"), val = tensor<int32, [4]>([0, 0, 6, 0])];
            tensor<int32, [4]> value_token_77_end_0 = const()[name = string("value_token_77_end_0"), val = tensor<int32, [4]>([1, 16, 7, 128])];
            tensor<bool, [4]> value_token_77_end_mask_0 = const()[name = string("value_token_77_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_77_squeeze_mask_0 = const()[name = string("value_token_77_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_77_cast_fp16 = slice_by_index(begin = value_token_77_begin_0, end = value_token_77_end_0, end_mask = value_token_77_end_mask_0, squeeze_mask = value_token_77_squeeze_mask_0, x = value_39_cast_fp16)[name = string("value_token_77_cast_fp16")];
            tensor<int32, [1]> var_1845_axes_0 = const()[name = string("op_1845_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1845_cast_fp16 = expand_dims(axes = var_1845_axes_0, x = key_token_77_cast_fp16)[name = string("op_1845_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1846_cast_fp16 = mul(x = state_153_cast_fp16, y = var_1845_cast_fp16)[name = string("op_1846_cast_fp16")];
            tensor<int32, [1]> memory_77_axes_0 = const()[name = string("memory_77_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_77_keep_dims_0 = const()[name = string("memory_77_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_77_cast_fp16 = reduce_sum(axes = memory_77_axes_0, keep_dims = memory_77_keep_dims_0, x = var_1846_cast_fp16)[name = string("memory_77_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_1849_cast_fp16 = sub(x = value_token_77_cast_fp16, y = memory_77_cast_fp16)[name = string("op_1849_cast_fp16")];
            tensor<int32, [3]> var_1852_begin_0 = const()[name = string("op_1852_begin_0"), val = tensor<int32, [3]>([0, 0, 6])];
            tensor<int32, [3]> var_1852_end_0 = const()[name = string("op_1852_end_0"), val = tensor<int32, [3]>([1, 16, 7])];
            tensor<bool, [3]> var_1852_end_mask_0 = const()[name = string("op_1852_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1852_squeeze_mask_0 = const()[name = string("op_1852_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1852_cast_fp16 = slice_by_index(begin = var_1852_begin_0, end = var_1852_end_0, end_mask = var_1852_end_mask_0, squeeze_mask = var_1852_squeeze_mask_0, x = beta_5_cast_fp16)[name = string("op_1852_cast_fp16")];
            tensor<int32, [1]> var_1853_axes_0 = const()[name = string("op_1853_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1853_cast_fp16 = expand_dims(axes = var_1853_axes_0, x = var_1852_cast_fp16)[name = string("op_1853_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_77_cast_fp16 = mul(x = var_1849_cast_fp16, y = var_1853_cast_fp16)[name = string("delta_77_cast_fp16")];
            tensor<int32, [1]> var_1856_axes_0 = const()[name = string("op_1856_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_1856_cast_fp16 = expand_dims(axes = var_1856_axes_0, x = delta_77_cast_fp16)[name = string("op_1856_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1857_cast_fp16 = mul(x = var_1845_cast_fp16, y = var_1856_cast_fp16)[name = string("op_1857_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_155_cast_fp16 = add(x = state_153_cast_fp16, y = var_1857_cast_fp16)[name = string("state_155_cast_fp16")];
            tensor<int32, [4]> var_1861_begin_0 = const()[name = string("op_1861_begin_0"), val = tensor<int32, [4]>([0, 0, 6, 0])];
            tensor<int32, [4]> var_1861_end_0 = const()[name = string("op_1861_end_0"), val = tensor<int32, [4]>([1, 16, 7, 128])];
            tensor<bool, [4]> var_1861_end_mask_0 = const()[name = string("op_1861_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_1861_squeeze_mask_0 = const()[name = string("op_1861_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_1861_cast_fp16 = slice_by_index(begin = var_1861_begin_0, end = var_1861_end_0, end_mask = var_1861_end_mask_0, squeeze_mask = var_1861_squeeze_mask_0, x = _inversed_query_11_cast_fp16)[name = string("op_1861_cast_fp16")];
            tensor<int32, [1]> var_1862_axes_0 = const()[name = string("op_1862_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1862_cast_fp16 = expand_dims(axes = var_1862_axes_0, x = var_1861_cast_fp16)[name = string("op_1862_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1863_cast_fp16 = mul(x = state_155_cast_fp16, y = var_1862_cast_fp16)[name = string("op_1863_cast_fp16")];
            tensor<int32, [1]> var_1865_axes_0 = const()[name = string("op_1865_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_1865_keep_dims_0 = const()[name = string("op_1865_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_1865_cast_fp16 = reduce_sum(axes = var_1865_axes_0, keep_dims = var_1865_keep_dims_0, x = var_1863_cast_fp16)[name = string("op_1865_cast_fp16")];
            tensor<int32, [3]> var_1868_begin_0 = const()[name = string("op_1868_begin_0"), val = tensor<int32, [3]>([0, 0, 7])];
            tensor<int32, [3]> var_1868_end_0 = const()[name = string("op_1868_end_0"), val = tensor<int32, [3]>([1, 16, 8])];
            tensor<bool, [3]> var_1868_end_mask_0 = const()[name = string("op_1868_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1868_squeeze_mask_0 = const()[name = string("op_1868_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1868_cast_fp16 = slice_by_index(begin = var_1868_begin_0, end = var_1868_end_0, end_mask = var_1868_end_mask_0, squeeze_mask = var_1868_squeeze_mask_0, x = decay_5_cast_fp16)[name = string("op_1868_cast_fp16")];
            tensor<fp16, [1, 16]> var_1869_cast_fp16 = exp(x = var_1868_cast_fp16)[name = string("op_1869_cast_fp16")];
            tensor<int32, [1]> var_1870_axes_0 = const()[name = string("op_1870_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1870_cast_fp16 = expand_dims(axes = var_1870_axes_0, x = var_1869_cast_fp16)[name = string("op_1870_cast_fp16")];
            tensor<int32, [1]> var_1871_axes_0 = const()[name = string("op_1871_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_1871_cast_fp16 = expand_dims(axes = var_1871_axes_0, x = var_1870_cast_fp16)[name = string("op_1871_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_157_cast_fp16 = mul(x = state_155_cast_fp16, y = var_1871_cast_fp16)[name = string("state_157_cast_fp16")];
            tensor<int32, [4]> key_token_79_begin_0 = const()[name = string("key_token_79_begin_0"), val = tensor<int32, [4]>([0, 0, 7, 0])];
            tensor<int32, [4]> key_token_79_end_0 = const()[name = string("key_token_79_end_0"), val = tensor<int32, [4]>([1, 16, 8, 128])];
            tensor<bool, [4]> key_token_79_end_mask_0 = const()[name = string("key_token_79_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_79_squeeze_mask_0 = const()[name = string("key_token_79_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_79_cast_fp16 = slice_by_index(begin = key_token_79_begin_0, end = key_token_79_end_0, end_mask = key_token_79_end_mask_0, squeeze_mask = key_token_79_squeeze_mask_0, x = key_11_cast_fp16)[name = string("key_token_79_cast_fp16")];
            tensor<int32, [4]> value_token_79_begin_0 = const()[name = string("value_token_79_begin_0"), val = tensor<int32, [4]>([0, 0, 7, 0])];
            tensor<int32, [4]> value_token_79_end_0 = const()[name = string("value_token_79_end_0"), val = tensor<int32, [4]>([1, 16, 8, 128])];
            tensor<bool, [4]> value_token_79_end_mask_0 = const()[name = string("value_token_79_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_79_squeeze_mask_0 = const()[name = string("value_token_79_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_79_cast_fp16 = slice_by_index(begin = value_token_79_begin_0, end = value_token_79_end_0, end_mask = value_token_79_end_mask_0, squeeze_mask = value_token_79_squeeze_mask_0, x = value_39_cast_fp16)[name = string("value_token_79_cast_fp16")];
            tensor<int32, [1]> var_1879_axes_0 = const()[name = string("op_1879_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1879_cast_fp16 = expand_dims(axes = var_1879_axes_0, x = key_token_79_cast_fp16)[name = string("op_1879_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1880_cast_fp16 = mul(x = state_157_cast_fp16, y = var_1879_cast_fp16)[name = string("op_1880_cast_fp16")];
            tensor<int32, [1]> memory_79_axes_0 = const()[name = string("memory_79_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_79_keep_dims_0 = const()[name = string("memory_79_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_79_cast_fp16 = reduce_sum(axes = memory_79_axes_0, keep_dims = memory_79_keep_dims_0, x = var_1880_cast_fp16)[name = string("memory_79_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_1883_cast_fp16 = sub(x = value_token_79_cast_fp16, y = memory_79_cast_fp16)[name = string("op_1883_cast_fp16")];
            tensor<int32, [3]> var_1886_begin_0 = const()[name = string("op_1886_begin_0"), val = tensor<int32, [3]>([0, 0, 7])];
            tensor<int32, [3]> var_1886_end_0 = const()[name = string("op_1886_end_0"), val = tensor<int32, [3]>([1, 16, 8])];
            tensor<bool, [3]> var_1886_end_mask_0 = const()[name = string("op_1886_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1886_squeeze_mask_0 = const()[name = string("op_1886_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1886_cast_fp16 = slice_by_index(begin = var_1886_begin_0, end = var_1886_end_0, end_mask = var_1886_end_mask_0, squeeze_mask = var_1886_squeeze_mask_0, x = beta_5_cast_fp16)[name = string("op_1886_cast_fp16")];
            tensor<int32, [1]> var_1887_axes_0 = const()[name = string("op_1887_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1887_cast_fp16 = expand_dims(axes = var_1887_axes_0, x = var_1886_cast_fp16)[name = string("op_1887_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_79_cast_fp16 = mul(x = var_1883_cast_fp16, y = var_1887_cast_fp16)[name = string("delta_79_cast_fp16")];
            tensor<int32, [1]> var_1890_axes_0 = const()[name = string("op_1890_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_1890_cast_fp16 = expand_dims(axes = var_1890_axes_0, x = delta_79_cast_fp16)[name = string("op_1890_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1891_cast_fp16 = mul(x = var_1879_cast_fp16, y = var_1890_cast_fp16)[name = string("op_1891_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_159_cast_fp16 = add(x = state_157_cast_fp16, y = var_1891_cast_fp16)[name = string("state_159_cast_fp16")];
            tensor<int32, [4]> var_1895_begin_0 = const()[name = string("op_1895_begin_0"), val = tensor<int32, [4]>([0, 0, 7, 0])];
            tensor<int32, [4]> var_1895_end_0 = const()[name = string("op_1895_end_0"), val = tensor<int32, [4]>([1, 16, 8, 128])];
            tensor<bool, [4]> var_1895_end_mask_0 = const()[name = string("op_1895_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_1895_squeeze_mask_0 = const()[name = string("op_1895_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_1895_cast_fp16 = slice_by_index(begin = var_1895_begin_0, end = var_1895_end_0, end_mask = var_1895_end_mask_0, squeeze_mask = var_1895_squeeze_mask_0, x = _inversed_query_11_cast_fp16)[name = string("op_1895_cast_fp16")];
            tensor<int32, [1]> var_1896_axes_0 = const()[name = string("op_1896_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1896_cast_fp16 = expand_dims(axes = var_1896_axes_0, x = var_1895_cast_fp16)[name = string("op_1896_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1897_cast_fp16 = mul(x = state_159_cast_fp16, y = var_1896_cast_fp16)[name = string("op_1897_cast_fp16")];
            tensor<int32, [1]> var_1899_axes_0 = const()[name = string("op_1899_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_1899_keep_dims_0 = const()[name = string("op_1899_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_1899_cast_fp16 = reduce_sum(axes = var_1899_axes_0, keep_dims = var_1899_keep_dims_0, x = var_1897_cast_fp16)[name = string("op_1899_cast_fp16")];
            tensor<int32, [3]> var_1902_begin_0 = const()[name = string("op_1902_begin_0"), val = tensor<int32, [3]>([0, 0, 8])];
            tensor<int32, [3]> var_1902_end_0 = const()[name = string("op_1902_end_0"), val = tensor<int32, [3]>([1, 16, 9])];
            tensor<bool, [3]> var_1902_end_mask_0 = const()[name = string("op_1902_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1902_squeeze_mask_0 = const()[name = string("op_1902_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1902_cast_fp16 = slice_by_index(begin = var_1902_begin_0, end = var_1902_end_0, end_mask = var_1902_end_mask_0, squeeze_mask = var_1902_squeeze_mask_0, x = decay_5_cast_fp16)[name = string("op_1902_cast_fp16")];
            tensor<fp16, [1, 16]> var_1903_cast_fp16 = exp(x = var_1902_cast_fp16)[name = string("op_1903_cast_fp16")];
            tensor<int32, [1]> var_1904_axes_0 = const()[name = string("op_1904_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1904_cast_fp16 = expand_dims(axes = var_1904_axes_0, x = var_1903_cast_fp16)[name = string("op_1904_cast_fp16")];
            tensor<int32, [1]> var_1905_axes_0 = const()[name = string("op_1905_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_1905_cast_fp16 = expand_dims(axes = var_1905_axes_0, x = var_1904_cast_fp16)[name = string("op_1905_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_161_cast_fp16 = mul(x = state_159_cast_fp16, y = var_1905_cast_fp16)[name = string("state_161_cast_fp16")];
            tensor<int32, [4]> key_token_81_begin_0 = const()[name = string("key_token_81_begin_0"), val = tensor<int32, [4]>([0, 0, 8, 0])];
            tensor<int32, [4]> key_token_81_end_0 = const()[name = string("key_token_81_end_0"), val = tensor<int32, [4]>([1, 16, 9, 128])];
            tensor<bool, [4]> key_token_81_end_mask_0 = const()[name = string("key_token_81_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_81_squeeze_mask_0 = const()[name = string("key_token_81_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_81_cast_fp16 = slice_by_index(begin = key_token_81_begin_0, end = key_token_81_end_0, end_mask = key_token_81_end_mask_0, squeeze_mask = key_token_81_squeeze_mask_0, x = key_11_cast_fp16)[name = string("key_token_81_cast_fp16")];
            tensor<int32, [4]> value_token_81_begin_0 = const()[name = string("value_token_81_begin_0"), val = tensor<int32, [4]>([0, 0, 8, 0])];
            tensor<int32, [4]> value_token_81_end_0 = const()[name = string("value_token_81_end_0"), val = tensor<int32, [4]>([1, 16, 9, 128])];
            tensor<bool, [4]> value_token_81_end_mask_0 = const()[name = string("value_token_81_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_81_squeeze_mask_0 = const()[name = string("value_token_81_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_81_cast_fp16 = slice_by_index(begin = value_token_81_begin_0, end = value_token_81_end_0, end_mask = value_token_81_end_mask_0, squeeze_mask = value_token_81_squeeze_mask_0, x = value_39_cast_fp16)[name = string("value_token_81_cast_fp16")];
            tensor<int32, [1]> var_1913_axes_0 = const()[name = string("op_1913_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1913_cast_fp16 = expand_dims(axes = var_1913_axes_0, x = key_token_81_cast_fp16)[name = string("op_1913_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1914_cast_fp16 = mul(x = state_161_cast_fp16, y = var_1913_cast_fp16)[name = string("op_1914_cast_fp16")];
            tensor<int32, [1]> memory_81_axes_0 = const()[name = string("memory_81_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_81_keep_dims_0 = const()[name = string("memory_81_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_81_cast_fp16 = reduce_sum(axes = memory_81_axes_0, keep_dims = memory_81_keep_dims_0, x = var_1914_cast_fp16)[name = string("memory_81_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_1917_cast_fp16 = sub(x = value_token_81_cast_fp16, y = memory_81_cast_fp16)[name = string("op_1917_cast_fp16")];
            tensor<int32, [3]> var_1920_begin_0 = const()[name = string("op_1920_begin_0"), val = tensor<int32, [3]>([0, 0, 8])];
            tensor<int32, [3]> var_1920_end_0 = const()[name = string("op_1920_end_0"), val = tensor<int32, [3]>([1, 16, 9])];
            tensor<bool, [3]> var_1920_end_mask_0 = const()[name = string("op_1920_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1920_squeeze_mask_0 = const()[name = string("op_1920_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1920_cast_fp16 = slice_by_index(begin = var_1920_begin_0, end = var_1920_end_0, end_mask = var_1920_end_mask_0, squeeze_mask = var_1920_squeeze_mask_0, x = beta_5_cast_fp16)[name = string("op_1920_cast_fp16")];
            tensor<int32, [1]> var_1921_axes_0 = const()[name = string("op_1921_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1921_cast_fp16 = expand_dims(axes = var_1921_axes_0, x = var_1920_cast_fp16)[name = string("op_1921_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_81_cast_fp16 = mul(x = var_1917_cast_fp16, y = var_1921_cast_fp16)[name = string("delta_81_cast_fp16")];
            tensor<int32, [1]> var_1924_axes_0 = const()[name = string("op_1924_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_1924_cast_fp16 = expand_dims(axes = var_1924_axes_0, x = delta_81_cast_fp16)[name = string("op_1924_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1925_cast_fp16 = mul(x = var_1913_cast_fp16, y = var_1924_cast_fp16)[name = string("op_1925_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_163_cast_fp16 = add(x = state_161_cast_fp16, y = var_1925_cast_fp16)[name = string("state_163_cast_fp16")];
            tensor<int32, [4]> var_1929_begin_0 = const()[name = string("op_1929_begin_0"), val = tensor<int32, [4]>([0, 0, 8, 0])];
            tensor<int32, [4]> var_1929_end_0 = const()[name = string("op_1929_end_0"), val = tensor<int32, [4]>([1, 16, 9, 128])];
            tensor<bool, [4]> var_1929_end_mask_0 = const()[name = string("op_1929_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_1929_squeeze_mask_0 = const()[name = string("op_1929_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_1929_cast_fp16 = slice_by_index(begin = var_1929_begin_0, end = var_1929_end_0, end_mask = var_1929_end_mask_0, squeeze_mask = var_1929_squeeze_mask_0, x = _inversed_query_11_cast_fp16)[name = string("op_1929_cast_fp16")];
            tensor<int32, [1]> var_1930_axes_0 = const()[name = string("op_1930_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1930_cast_fp16 = expand_dims(axes = var_1930_axes_0, x = var_1929_cast_fp16)[name = string("op_1930_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1931_cast_fp16 = mul(x = state_163_cast_fp16, y = var_1930_cast_fp16)[name = string("op_1931_cast_fp16")];
            tensor<int32, [1]> var_1933_axes_0 = const()[name = string("op_1933_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_1933_keep_dims_0 = const()[name = string("op_1933_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_1933_cast_fp16 = reduce_sum(axes = var_1933_axes_0, keep_dims = var_1933_keep_dims_0, x = var_1931_cast_fp16)[name = string("op_1933_cast_fp16")];
            tensor<int32, [3]> var_1936_begin_0 = const()[name = string("op_1936_begin_0"), val = tensor<int32, [3]>([0, 0, 9])];
            tensor<int32, [3]> var_1936_end_0 = const()[name = string("op_1936_end_0"), val = tensor<int32, [3]>([1, 16, 10])];
            tensor<bool, [3]> var_1936_end_mask_0 = const()[name = string("op_1936_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1936_squeeze_mask_0 = const()[name = string("op_1936_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1936_cast_fp16 = slice_by_index(begin = var_1936_begin_0, end = var_1936_end_0, end_mask = var_1936_end_mask_0, squeeze_mask = var_1936_squeeze_mask_0, x = decay_5_cast_fp16)[name = string("op_1936_cast_fp16")];
            tensor<fp16, [1, 16]> var_1937_cast_fp16 = exp(x = var_1936_cast_fp16)[name = string("op_1937_cast_fp16")];
            tensor<int32, [1]> var_1938_axes_0 = const()[name = string("op_1938_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1938_cast_fp16 = expand_dims(axes = var_1938_axes_0, x = var_1937_cast_fp16)[name = string("op_1938_cast_fp16")];
            tensor<int32, [1]> var_1939_axes_0 = const()[name = string("op_1939_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_1939_cast_fp16 = expand_dims(axes = var_1939_axes_0, x = var_1938_cast_fp16)[name = string("op_1939_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_165_cast_fp16 = mul(x = state_163_cast_fp16, y = var_1939_cast_fp16)[name = string("state_165_cast_fp16")];
            tensor<int32, [4]> key_token_83_begin_0 = const()[name = string("key_token_83_begin_0"), val = tensor<int32, [4]>([0, 0, 9, 0])];
            tensor<int32, [4]> key_token_83_end_0 = const()[name = string("key_token_83_end_0"), val = tensor<int32, [4]>([1, 16, 10, 128])];
            tensor<bool, [4]> key_token_83_end_mask_0 = const()[name = string("key_token_83_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_83_squeeze_mask_0 = const()[name = string("key_token_83_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_83_cast_fp16 = slice_by_index(begin = key_token_83_begin_0, end = key_token_83_end_0, end_mask = key_token_83_end_mask_0, squeeze_mask = key_token_83_squeeze_mask_0, x = key_11_cast_fp16)[name = string("key_token_83_cast_fp16")];
            tensor<int32, [4]> value_token_83_begin_0 = const()[name = string("value_token_83_begin_0"), val = tensor<int32, [4]>([0, 0, 9, 0])];
            tensor<int32, [4]> value_token_83_end_0 = const()[name = string("value_token_83_end_0"), val = tensor<int32, [4]>([1, 16, 10, 128])];
            tensor<bool, [4]> value_token_83_end_mask_0 = const()[name = string("value_token_83_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_83_squeeze_mask_0 = const()[name = string("value_token_83_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_83_cast_fp16 = slice_by_index(begin = value_token_83_begin_0, end = value_token_83_end_0, end_mask = value_token_83_end_mask_0, squeeze_mask = value_token_83_squeeze_mask_0, x = value_39_cast_fp16)[name = string("value_token_83_cast_fp16")];
            tensor<int32, [1]> var_1947_axes_0 = const()[name = string("op_1947_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1947_cast_fp16 = expand_dims(axes = var_1947_axes_0, x = key_token_83_cast_fp16)[name = string("op_1947_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1948_cast_fp16 = mul(x = state_165_cast_fp16, y = var_1947_cast_fp16)[name = string("op_1948_cast_fp16")];
            tensor<int32, [1]> memory_83_axes_0 = const()[name = string("memory_83_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_83_keep_dims_0 = const()[name = string("memory_83_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_83_cast_fp16 = reduce_sum(axes = memory_83_axes_0, keep_dims = memory_83_keep_dims_0, x = var_1948_cast_fp16)[name = string("memory_83_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_1951_cast_fp16 = sub(x = value_token_83_cast_fp16, y = memory_83_cast_fp16)[name = string("op_1951_cast_fp16")];
            tensor<int32, [3]> var_1954_begin_0 = const()[name = string("op_1954_begin_0"), val = tensor<int32, [3]>([0, 0, 9])];
            tensor<int32, [3]> var_1954_end_0 = const()[name = string("op_1954_end_0"), val = tensor<int32, [3]>([1, 16, 10])];
            tensor<bool, [3]> var_1954_end_mask_0 = const()[name = string("op_1954_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1954_squeeze_mask_0 = const()[name = string("op_1954_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1954_cast_fp16 = slice_by_index(begin = var_1954_begin_0, end = var_1954_end_0, end_mask = var_1954_end_mask_0, squeeze_mask = var_1954_squeeze_mask_0, x = beta_5_cast_fp16)[name = string("op_1954_cast_fp16")];
            tensor<int32, [1]> var_1955_axes_0 = const()[name = string("op_1955_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1955_cast_fp16 = expand_dims(axes = var_1955_axes_0, x = var_1954_cast_fp16)[name = string("op_1955_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_83_cast_fp16 = mul(x = var_1951_cast_fp16, y = var_1955_cast_fp16)[name = string("delta_83_cast_fp16")];
            tensor<int32, [1]> var_1958_axes_0 = const()[name = string("op_1958_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_1958_cast_fp16 = expand_dims(axes = var_1958_axes_0, x = delta_83_cast_fp16)[name = string("op_1958_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1959_cast_fp16 = mul(x = var_1947_cast_fp16, y = var_1958_cast_fp16)[name = string("op_1959_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_167_cast_fp16 = add(x = state_165_cast_fp16, y = var_1959_cast_fp16)[name = string("state_167_cast_fp16")];
            tensor<int32, [4]> var_1963_begin_0 = const()[name = string("op_1963_begin_0"), val = tensor<int32, [4]>([0, 0, 9, 0])];
            tensor<int32, [4]> var_1963_end_0 = const()[name = string("op_1963_end_0"), val = tensor<int32, [4]>([1, 16, 10, 128])];
            tensor<bool, [4]> var_1963_end_mask_0 = const()[name = string("op_1963_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_1963_squeeze_mask_0 = const()[name = string("op_1963_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_1963_cast_fp16 = slice_by_index(begin = var_1963_begin_0, end = var_1963_end_0, end_mask = var_1963_end_mask_0, squeeze_mask = var_1963_squeeze_mask_0, x = _inversed_query_11_cast_fp16)[name = string("op_1963_cast_fp16")];
            tensor<int32, [1]> var_1964_axes_0 = const()[name = string("op_1964_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1964_cast_fp16 = expand_dims(axes = var_1964_axes_0, x = var_1963_cast_fp16)[name = string("op_1964_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1965_cast_fp16 = mul(x = state_167_cast_fp16, y = var_1964_cast_fp16)[name = string("op_1965_cast_fp16")];
            tensor<int32, [1]> var_1967_axes_0 = const()[name = string("op_1967_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_1967_keep_dims_0 = const()[name = string("op_1967_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_1967_cast_fp16 = reduce_sum(axes = var_1967_axes_0, keep_dims = var_1967_keep_dims_0, x = var_1965_cast_fp16)[name = string("op_1967_cast_fp16")];
            tensor<int32, [3]> var_1970_begin_0 = const()[name = string("op_1970_begin_0"), val = tensor<int32, [3]>([0, 0, 10])];
            tensor<int32, [3]> var_1970_end_0 = const()[name = string("op_1970_end_0"), val = tensor<int32, [3]>([1, 16, 11])];
            tensor<bool, [3]> var_1970_end_mask_0 = const()[name = string("op_1970_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1970_squeeze_mask_0 = const()[name = string("op_1970_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1970_cast_fp16 = slice_by_index(begin = var_1970_begin_0, end = var_1970_end_0, end_mask = var_1970_end_mask_0, squeeze_mask = var_1970_squeeze_mask_0, x = decay_5_cast_fp16)[name = string("op_1970_cast_fp16")];
            tensor<fp16, [1, 16]> var_1971_cast_fp16 = exp(x = var_1970_cast_fp16)[name = string("op_1971_cast_fp16")];
            tensor<int32, [1]> var_1972_axes_0 = const()[name = string("op_1972_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1972_cast_fp16 = expand_dims(axes = var_1972_axes_0, x = var_1971_cast_fp16)[name = string("op_1972_cast_fp16")];
            tensor<int32, [1]> var_1973_axes_0 = const()[name = string("op_1973_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_1973_cast_fp16 = expand_dims(axes = var_1973_axes_0, x = var_1972_cast_fp16)[name = string("op_1973_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_169_cast_fp16 = mul(x = state_167_cast_fp16, y = var_1973_cast_fp16)[name = string("state_169_cast_fp16")];
            tensor<int32, [4]> key_token_85_begin_0 = const()[name = string("key_token_85_begin_0"), val = tensor<int32, [4]>([0, 0, 10, 0])];
            tensor<int32, [4]> key_token_85_end_0 = const()[name = string("key_token_85_end_0"), val = tensor<int32, [4]>([1, 16, 11, 128])];
            tensor<bool, [4]> key_token_85_end_mask_0 = const()[name = string("key_token_85_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_85_squeeze_mask_0 = const()[name = string("key_token_85_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_85_cast_fp16 = slice_by_index(begin = key_token_85_begin_0, end = key_token_85_end_0, end_mask = key_token_85_end_mask_0, squeeze_mask = key_token_85_squeeze_mask_0, x = key_11_cast_fp16)[name = string("key_token_85_cast_fp16")];
            tensor<int32, [4]> value_token_85_begin_0 = const()[name = string("value_token_85_begin_0"), val = tensor<int32, [4]>([0, 0, 10, 0])];
            tensor<int32, [4]> value_token_85_end_0 = const()[name = string("value_token_85_end_0"), val = tensor<int32, [4]>([1, 16, 11, 128])];
            tensor<bool, [4]> value_token_85_end_mask_0 = const()[name = string("value_token_85_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_85_squeeze_mask_0 = const()[name = string("value_token_85_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_85_cast_fp16 = slice_by_index(begin = value_token_85_begin_0, end = value_token_85_end_0, end_mask = value_token_85_end_mask_0, squeeze_mask = value_token_85_squeeze_mask_0, x = value_39_cast_fp16)[name = string("value_token_85_cast_fp16")];
            tensor<int32, [1]> var_1981_axes_0 = const()[name = string("op_1981_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1981_cast_fp16 = expand_dims(axes = var_1981_axes_0, x = key_token_85_cast_fp16)[name = string("op_1981_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1982_cast_fp16 = mul(x = state_169_cast_fp16, y = var_1981_cast_fp16)[name = string("op_1982_cast_fp16")];
            tensor<int32, [1]> memory_85_axes_0 = const()[name = string("memory_85_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_85_keep_dims_0 = const()[name = string("memory_85_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_85_cast_fp16 = reduce_sum(axes = memory_85_axes_0, keep_dims = memory_85_keep_dims_0, x = var_1982_cast_fp16)[name = string("memory_85_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_1985_cast_fp16 = sub(x = value_token_85_cast_fp16, y = memory_85_cast_fp16)[name = string("op_1985_cast_fp16")];
            tensor<int32, [3]> var_1988_begin_0 = const()[name = string("op_1988_begin_0"), val = tensor<int32, [3]>([0, 0, 10])];
            tensor<int32, [3]> var_1988_end_0 = const()[name = string("op_1988_end_0"), val = tensor<int32, [3]>([1, 16, 11])];
            tensor<bool, [3]> var_1988_end_mask_0 = const()[name = string("op_1988_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1988_squeeze_mask_0 = const()[name = string("op_1988_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1988_cast_fp16 = slice_by_index(begin = var_1988_begin_0, end = var_1988_end_0, end_mask = var_1988_end_mask_0, squeeze_mask = var_1988_squeeze_mask_0, x = beta_5_cast_fp16)[name = string("op_1988_cast_fp16")];
            tensor<int32, [1]> var_1989_axes_0 = const()[name = string("op_1989_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1989_cast_fp16 = expand_dims(axes = var_1989_axes_0, x = var_1988_cast_fp16)[name = string("op_1989_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_85_cast_fp16 = mul(x = var_1985_cast_fp16, y = var_1989_cast_fp16)[name = string("delta_85_cast_fp16")];
            tensor<int32, [1]> var_1992_axes_0 = const()[name = string("op_1992_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_1992_cast_fp16 = expand_dims(axes = var_1992_axes_0, x = delta_85_cast_fp16)[name = string("op_1992_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1993_cast_fp16 = mul(x = var_1981_cast_fp16, y = var_1992_cast_fp16)[name = string("op_1993_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_171_cast_fp16 = add(x = state_169_cast_fp16, y = var_1993_cast_fp16)[name = string("state_171_cast_fp16")];
            tensor<int32, [4]> var_1997_begin_0 = const()[name = string("op_1997_begin_0"), val = tensor<int32, [4]>([0, 0, 10, 0])];
            tensor<int32, [4]> var_1997_end_0 = const()[name = string("op_1997_end_0"), val = tensor<int32, [4]>([1, 16, 11, 128])];
            tensor<bool, [4]> var_1997_end_mask_0 = const()[name = string("op_1997_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_1997_squeeze_mask_0 = const()[name = string("op_1997_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_1997_cast_fp16 = slice_by_index(begin = var_1997_begin_0, end = var_1997_end_0, end_mask = var_1997_end_mask_0, squeeze_mask = var_1997_squeeze_mask_0, x = _inversed_query_11_cast_fp16)[name = string("op_1997_cast_fp16")];
            tensor<int32, [1]> var_1998_axes_0 = const()[name = string("op_1998_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1998_cast_fp16 = expand_dims(axes = var_1998_axes_0, x = var_1997_cast_fp16)[name = string("op_1998_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1999_cast_fp16 = mul(x = state_171_cast_fp16, y = var_1998_cast_fp16)[name = string("op_1999_cast_fp16")];
            tensor<int32, [1]> var_2001_axes_0 = const()[name = string("op_2001_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_2001_keep_dims_0 = const()[name = string("op_2001_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_2001_cast_fp16 = reduce_sum(axes = var_2001_axes_0, keep_dims = var_2001_keep_dims_0, x = var_1999_cast_fp16)[name = string("op_2001_cast_fp16")];
            tensor<int32, [3]> var_2004_begin_0 = const()[name = string("op_2004_begin_0"), val = tensor<int32, [3]>([0, 0, 11])];
            tensor<int32, [3]> var_2004_end_0 = const()[name = string("op_2004_end_0"), val = tensor<int32, [3]>([1, 16, 12])];
            tensor<bool, [3]> var_2004_end_mask_0 = const()[name = string("op_2004_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_2004_squeeze_mask_0 = const()[name = string("op_2004_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_2004_cast_fp16 = slice_by_index(begin = var_2004_begin_0, end = var_2004_end_0, end_mask = var_2004_end_mask_0, squeeze_mask = var_2004_squeeze_mask_0, x = decay_5_cast_fp16)[name = string("op_2004_cast_fp16")];
            tensor<fp16, [1, 16]> var_2005_cast_fp16 = exp(x = var_2004_cast_fp16)[name = string("op_2005_cast_fp16")];
            tensor<int32, [1]> var_2006_axes_0 = const()[name = string("op_2006_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_2006_cast_fp16 = expand_dims(axes = var_2006_axes_0, x = var_2005_cast_fp16)[name = string("op_2006_cast_fp16")];
            tensor<int32, [1]> var_2007_axes_0 = const()[name = string("op_2007_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_2007_cast_fp16 = expand_dims(axes = var_2007_axes_0, x = var_2006_cast_fp16)[name = string("op_2007_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_173_cast_fp16 = mul(x = state_171_cast_fp16, y = var_2007_cast_fp16)[name = string("state_173_cast_fp16")];
            tensor<int32, [4]> key_token_87_begin_0 = const()[name = string("key_token_87_begin_0"), val = tensor<int32, [4]>([0, 0, 11, 0])];
            tensor<int32, [4]> key_token_87_end_0 = const()[name = string("key_token_87_end_0"), val = tensor<int32, [4]>([1, 16, 12, 128])];
            tensor<bool, [4]> key_token_87_end_mask_0 = const()[name = string("key_token_87_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_87_squeeze_mask_0 = const()[name = string("key_token_87_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_87_cast_fp16 = slice_by_index(begin = key_token_87_begin_0, end = key_token_87_end_0, end_mask = key_token_87_end_mask_0, squeeze_mask = key_token_87_squeeze_mask_0, x = key_11_cast_fp16)[name = string("key_token_87_cast_fp16")];
            tensor<int32, [4]> value_token_87_begin_0 = const()[name = string("value_token_87_begin_0"), val = tensor<int32, [4]>([0, 0, 11, 0])];
            tensor<int32, [4]> value_token_87_end_0 = const()[name = string("value_token_87_end_0"), val = tensor<int32, [4]>([1, 16, 12, 128])];
            tensor<bool, [4]> value_token_87_end_mask_0 = const()[name = string("value_token_87_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_87_squeeze_mask_0 = const()[name = string("value_token_87_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_87_cast_fp16 = slice_by_index(begin = value_token_87_begin_0, end = value_token_87_end_0, end_mask = value_token_87_end_mask_0, squeeze_mask = value_token_87_squeeze_mask_0, x = value_39_cast_fp16)[name = string("value_token_87_cast_fp16")];
            tensor<int32, [1]> var_2015_axes_0 = const()[name = string("op_2015_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_2015_cast_fp16 = expand_dims(axes = var_2015_axes_0, x = key_token_87_cast_fp16)[name = string("op_2015_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2016_cast_fp16 = mul(x = state_173_cast_fp16, y = var_2015_cast_fp16)[name = string("op_2016_cast_fp16")];
            tensor<int32, [1]> memory_87_axes_0 = const()[name = string("memory_87_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_87_keep_dims_0 = const()[name = string("memory_87_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_87_cast_fp16 = reduce_sum(axes = memory_87_axes_0, keep_dims = memory_87_keep_dims_0, x = var_2016_cast_fp16)[name = string("memory_87_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_2019_cast_fp16 = sub(x = value_token_87_cast_fp16, y = memory_87_cast_fp16)[name = string("op_2019_cast_fp16")];
            tensor<int32, [3]> var_2022_begin_0 = const()[name = string("op_2022_begin_0"), val = tensor<int32, [3]>([0, 0, 11])];
            tensor<int32, [3]> var_2022_end_0 = const()[name = string("op_2022_end_0"), val = tensor<int32, [3]>([1, 16, 12])];
            tensor<bool, [3]> var_2022_end_mask_0 = const()[name = string("op_2022_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_2022_squeeze_mask_0 = const()[name = string("op_2022_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_2022_cast_fp16 = slice_by_index(begin = var_2022_begin_0, end = var_2022_end_0, end_mask = var_2022_end_mask_0, squeeze_mask = var_2022_squeeze_mask_0, x = beta_5_cast_fp16)[name = string("op_2022_cast_fp16")];
            tensor<int32, [1]> var_2023_axes_0 = const()[name = string("op_2023_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_2023_cast_fp16 = expand_dims(axes = var_2023_axes_0, x = var_2022_cast_fp16)[name = string("op_2023_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_87_cast_fp16 = mul(x = var_2019_cast_fp16, y = var_2023_cast_fp16)[name = string("delta_87_cast_fp16")];
            tensor<int32, [1]> var_2026_axes_0 = const()[name = string("op_2026_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_2026_cast_fp16 = expand_dims(axes = var_2026_axes_0, x = delta_87_cast_fp16)[name = string("op_2026_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2027_cast_fp16 = mul(x = var_2015_cast_fp16, y = var_2026_cast_fp16)[name = string("op_2027_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_175_cast_fp16 = add(x = state_173_cast_fp16, y = var_2027_cast_fp16)[name = string("state_175_cast_fp16")];
            tensor<int32, [4]> var_2031_begin_0 = const()[name = string("op_2031_begin_0"), val = tensor<int32, [4]>([0, 0, 11, 0])];
            tensor<int32, [4]> var_2031_end_0 = const()[name = string("op_2031_end_0"), val = tensor<int32, [4]>([1, 16, 12, 128])];
            tensor<bool, [4]> var_2031_end_mask_0 = const()[name = string("op_2031_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_2031_squeeze_mask_0 = const()[name = string("op_2031_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_2031_cast_fp16 = slice_by_index(begin = var_2031_begin_0, end = var_2031_end_0, end_mask = var_2031_end_mask_0, squeeze_mask = var_2031_squeeze_mask_0, x = _inversed_query_11_cast_fp16)[name = string("op_2031_cast_fp16")];
            tensor<int32, [1]> var_2032_axes_0 = const()[name = string("op_2032_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_2032_cast_fp16 = expand_dims(axes = var_2032_axes_0, x = var_2031_cast_fp16)[name = string("op_2032_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2033_cast_fp16 = mul(x = state_175_cast_fp16, y = var_2032_cast_fp16)[name = string("op_2033_cast_fp16")];
            tensor<int32, [1]> var_2035_axes_0 = const()[name = string("op_2035_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_2035_keep_dims_0 = const()[name = string("op_2035_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_2035_cast_fp16 = reduce_sum(axes = var_2035_axes_0, keep_dims = var_2035_keep_dims_0, x = var_2033_cast_fp16)[name = string("op_2035_cast_fp16")];
            tensor<int32, [3]> var_2038_begin_0 = const()[name = string("op_2038_begin_0"), val = tensor<int32, [3]>([0, 0, 12])];
            tensor<int32, [3]> var_2038_end_0 = const()[name = string("op_2038_end_0"), val = tensor<int32, [3]>([1, 16, 13])];
            tensor<bool, [3]> var_2038_end_mask_0 = const()[name = string("op_2038_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_2038_squeeze_mask_0 = const()[name = string("op_2038_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_2038_cast_fp16 = slice_by_index(begin = var_2038_begin_0, end = var_2038_end_0, end_mask = var_2038_end_mask_0, squeeze_mask = var_2038_squeeze_mask_0, x = decay_5_cast_fp16)[name = string("op_2038_cast_fp16")];
            tensor<fp16, [1, 16]> var_2039_cast_fp16 = exp(x = var_2038_cast_fp16)[name = string("op_2039_cast_fp16")];
            tensor<int32, [1]> var_2040_axes_0 = const()[name = string("op_2040_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_2040_cast_fp16 = expand_dims(axes = var_2040_axes_0, x = var_2039_cast_fp16)[name = string("op_2040_cast_fp16")];
            tensor<int32, [1]> var_2041_axes_0 = const()[name = string("op_2041_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_2041_cast_fp16 = expand_dims(axes = var_2041_axes_0, x = var_2040_cast_fp16)[name = string("op_2041_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_177_cast_fp16 = mul(x = state_175_cast_fp16, y = var_2041_cast_fp16)[name = string("state_177_cast_fp16")];
            tensor<int32, [4]> key_token_89_begin_0 = const()[name = string("key_token_89_begin_0"), val = tensor<int32, [4]>([0, 0, 12, 0])];
            tensor<int32, [4]> key_token_89_end_0 = const()[name = string("key_token_89_end_0"), val = tensor<int32, [4]>([1, 16, 13, 128])];
            tensor<bool, [4]> key_token_89_end_mask_0 = const()[name = string("key_token_89_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_89_squeeze_mask_0 = const()[name = string("key_token_89_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_89_cast_fp16 = slice_by_index(begin = key_token_89_begin_0, end = key_token_89_end_0, end_mask = key_token_89_end_mask_0, squeeze_mask = key_token_89_squeeze_mask_0, x = key_11_cast_fp16)[name = string("key_token_89_cast_fp16")];
            tensor<int32, [4]> value_token_89_begin_0 = const()[name = string("value_token_89_begin_0"), val = tensor<int32, [4]>([0, 0, 12, 0])];
            tensor<int32, [4]> value_token_89_end_0 = const()[name = string("value_token_89_end_0"), val = tensor<int32, [4]>([1, 16, 13, 128])];
            tensor<bool, [4]> value_token_89_end_mask_0 = const()[name = string("value_token_89_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_89_squeeze_mask_0 = const()[name = string("value_token_89_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_89_cast_fp16 = slice_by_index(begin = value_token_89_begin_0, end = value_token_89_end_0, end_mask = value_token_89_end_mask_0, squeeze_mask = value_token_89_squeeze_mask_0, x = value_39_cast_fp16)[name = string("value_token_89_cast_fp16")];
            tensor<int32, [1]> var_2049_axes_0 = const()[name = string("op_2049_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_2049_cast_fp16 = expand_dims(axes = var_2049_axes_0, x = key_token_89_cast_fp16)[name = string("op_2049_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2050_cast_fp16 = mul(x = state_177_cast_fp16, y = var_2049_cast_fp16)[name = string("op_2050_cast_fp16")];
            tensor<int32, [1]> memory_89_axes_0 = const()[name = string("memory_89_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_89_keep_dims_0 = const()[name = string("memory_89_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_89_cast_fp16 = reduce_sum(axes = memory_89_axes_0, keep_dims = memory_89_keep_dims_0, x = var_2050_cast_fp16)[name = string("memory_89_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_2053_cast_fp16 = sub(x = value_token_89_cast_fp16, y = memory_89_cast_fp16)[name = string("op_2053_cast_fp16")];
            tensor<int32, [3]> var_2056_begin_0 = const()[name = string("op_2056_begin_0"), val = tensor<int32, [3]>([0, 0, 12])];
            tensor<int32, [3]> var_2056_end_0 = const()[name = string("op_2056_end_0"), val = tensor<int32, [3]>([1, 16, 13])];
            tensor<bool, [3]> var_2056_end_mask_0 = const()[name = string("op_2056_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_2056_squeeze_mask_0 = const()[name = string("op_2056_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_2056_cast_fp16 = slice_by_index(begin = var_2056_begin_0, end = var_2056_end_0, end_mask = var_2056_end_mask_0, squeeze_mask = var_2056_squeeze_mask_0, x = beta_5_cast_fp16)[name = string("op_2056_cast_fp16")];
            tensor<int32, [1]> var_2057_axes_0 = const()[name = string("op_2057_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_2057_cast_fp16 = expand_dims(axes = var_2057_axes_0, x = var_2056_cast_fp16)[name = string("op_2057_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_89_cast_fp16 = mul(x = var_2053_cast_fp16, y = var_2057_cast_fp16)[name = string("delta_89_cast_fp16")];
            tensor<int32, [1]> var_2060_axes_0 = const()[name = string("op_2060_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_2060_cast_fp16 = expand_dims(axes = var_2060_axes_0, x = delta_89_cast_fp16)[name = string("op_2060_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2061_cast_fp16 = mul(x = var_2049_cast_fp16, y = var_2060_cast_fp16)[name = string("op_2061_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_179_cast_fp16 = add(x = state_177_cast_fp16, y = var_2061_cast_fp16)[name = string("state_179_cast_fp16")];
            tensor<int32, [4]> var_2065_begin_0 = const()[name = string("op_2065_begin_0"), val = tensor<int32, [4]>([0, 0, 12, 0])];
            tensor<int32, [4]> var_2065_end_0 = const()[name = string("op_2065_end_0"), val = tensor<int32, [4]>([1, 16, 13, 128])];
            tensor<bool, [4]> var_2065_end_mask_0 = const()[name = string("op_2065_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_2065_squeeze_mask_0 = const()[name = string("op_2065_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_2065_cast_fp16 = slice_by_index(begin = var_2065_begin_0, end = var_2065_end_0, end_mask = var_2065_end_mask_0, squeeze_mask = var_2065_squeeze_mask_0, x = _inversed_query_11_cast_fp16)[name = string("op_2065_cast_fp16")];
            tensor<int32, [1]> var_2066_axes_0 = const()[name = string("op_2066_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_2066_cast_fp16 = expand_dims(axes = var_2066_axes_0, x = var_2065_cast_fp16)[name = string("op_2066_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2067_cast_fp16 = mul(x = state_179_cast_fp16, y = var_2066_cast_fp16)[name = string("op_2067_cast_fp16")];
            tensor<int32, [1]> var_2069_axes_0 = const()[name = string("op_2069_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_2069_keep_dims_0 = const()[name = string("op_2069_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_2069_cast_fp16 = reduce_sum(axes = var_2069_axes_0, keep_dims = var_2069_keep_dims_0, x = var_2067_cast_fp16)[name = string("op_2069_cast_fp16")];
            tensor<int32, [3]> var_2072_begin_0 = const()[name = string("op_2072_begin_0"), val = tensor<int32, [3]>([0, 0, 13])];
            tensor<int32, [3]> var_2072_end_0 = const()[name = string("op_2072_end_0"), val = tensor<int32, [3]>([1, 16, 14])];
            tensor<bool, [3]> var_2072_end_mask_0 = const()[name = string("op_2072_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_2072_squeeze_mask_0 = const()[name = string("op_2072_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_2072_cast_fp16 = slice_by_index(begin = var_2072_begin_0, end = var_2072_end_0, end_mask = var_2072_end_mask_0, squeeze_mask = var_2072_squeeze_mask_0, x = decay_5_cast_fp16)[name = string("op_2072_cast_fp16")];
            tensor<fp16, [1, 16]> var_2073_cast_fp16 = exp(x = var_2072_cast_fp16)[name = string("op_2073_cast_fp16")];
            tensor<int32, [1]> var_2074_axes_0 = const()[name = string("op_2074_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_2074_cast_fp16 = expand_dims(axes = var_2074_axes_0, x = var_2073_cast_fp16)[name = string("op_2074_cast_fp16")];
            tensor<int32, [1]> var_2075_axes_0 = const()[name = string("op_2075_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_2075_cast_fp16 = expand_dims(axes = var_2075_axes_0, x = var_2074_cast_fp16)[name = string("op_2075_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_181_cast_fp16 = mul(x = state_179_cast_fp16, y = var_2075_cast_fp16)[name = string("state_181_cast_fp16")];
            tensor<int32, [4]> key_token_91_begin_0 = const()[name = string("key_token_91_begin_0"), val = tensor<int32, [4]>([0, 0, 13, 0])];
            tensor<int32, [4]> key_token_91_end_0 = const()[name = string("key_token_91_end_0"), val = tensor<int32, [4]>([1, 16, 14, 128])];
            tensor<bool, [4]> key_token_91_end_mask_0 = const()[name = string("key_token_91_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_91_squeeze_mask_0 = const()[name = string("key_token_91_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_91_cast_fp16 = slice_by_index(begin = key_token_91_begin_0, end = key_token_91_end_0, end_mask = key_token_91_end_mask_0, squeeze_mask = key_token_91_squeeze_mask_0, x = key_11_cast_fp16)[name = string("key_token_91_cast_fp16")];
            tensor<int32, [4]> value_token_91_begin_0 = const()[name = string("value_token_91_begin_0"), val = tensor<int32, [4]>([0, 0, 13, 0])];
            tensor<int32, [4]> value_token_91_end_0 = const()[name = string("value_token_91_end_0"), val = tensor<int32, [4]>([1, 16, 14, 128])];
            tensor<bool, [4]> value_token_91_end_mask_0 = const()[name = string("value_token_91_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_91_squeeze_mask_0 = const()[name = string("value_token_91_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_91_cast_fp16 = slice_by_index(begin = value_token_91_begin_0, end = value_token_91_end_0, end_mask = value_token_91_end_mask_0, squeeze_mask = value_token_91_squeeze_mask_0, x = value_39_cast_fp16)[name = string("value_token_91_cast_fp16")];
            tensor<int32, [1]> var_2083_axes_0 = const()[name = string("op_2083_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_2083_cast_fp16 = expand_dims(axes = var_2083_axes_0, x = key_token_91_cast_fp16)[name = string("op_2083_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2084_cast_fp16 = mul(x = state_181_cast_fp16, y = var_2083_cast_fp16)[name = string("op_2084_cast_fp16")];
            tensor<int32, [1]> memory_91_axes_0 = const()[name = string("memory_91_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_91_keep_dims_0 = const()[name = string("memory_91_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_91_cast_fp16 = reduce_sum(axes = memory_91_axes_0, keep_dims = memory_91_keep_dims_0, x = var_2084_cast_fp16)[name = string("memory_91_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_2087_cast_fp16 = sub(x = value_token_91_cast_fp16, y = memory_91_cast_fp16)[name = string("op_2087_cast_fp16")];
            tensor<int32, [3]> var_2090_begin_0 = const()[name = string("op_2090_begin_0"), val = tensor<int32, [3]>([0, 0, 13])];
            tensor<int32, [3]> var_2090_end_0 = const()[name = string("op_2090_end_0"), val = tensor<int32, [3]>([1, 16, 14])];
            tensor<bool, [3]> var_2090_end_mask_0 = const()[name = string("op_2090_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_2090_squeeze_mask_0 = const()[name = string("op_2090_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_2090_cast_fp16 = slice_by_index(begin = var_2090_begin_0, end = var_2090_end_0, end_mask = var_2090_end_mask_0, squeeze_mask = var_2090_squeeze_mask_0, x = beta_5_cast_fp16)[name = string("op_2090_cast_fp16")];
            tensor<int32, [1]> var_2091_axes_0 = const()[name = string("op_2091_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_2091_cast_fp16 = expand_dims(axes = var_2091_axes_0, x = var_2090_cast_fp16)[name = string("op_2091_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_91_cast_fp16 = mul(x = var_2087_cast_fp16, y = var_2091_cast_fp16)[name = string("delta_91_cast_fp16")];
            tensor<int32, [1]> var_2094_axes_0 = const()[name = string("op_2094_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_2094_cast_fp16 = expand_dims(axes = var_2094_axes_0, x = delta_91_cast_fp16)[name = string("op_2094_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2095_cast_fp16 = mul(x = var_2083_cast_fp16, y = var_2094_cast_fp16)[name = string("op_2095_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_183_cast_fp16 = add(x = state_181_cast_fp16, y = var_2095_cast_fp16)[name = string("state_183_cast_fp16")];
            tensor<int32, [4]> var_2099_begin_0 = const()[name = string("op_2099_begin_0"), val = tensor<int32, [4]>([0, 0, 13, 0])];
            tensor<int32, [4]> var_2099_end_0 = const()[name = string("op_2099_end_0"), val = tensor<int32, [4]>([1, 16, 14, 128])];
            tensor<bool, [4]> var_2099_end_mask_0 = const()[name = string("op_2099_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_2099_squeeze_mask_0 = const()[name = string("op_2099_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_2099_cast_fp16 = slice_by_index(begin = var_2099_begin_0, end = var_2099_end_0, end_mask = var_2099_end_mask_0, squeeze_mask = var_2099_squeeze_mask_0, x = _inversed_query_11_cast_fp16)[name = string("op_2099_cast_fp16")];
            tensor<int32, [1]> var_2100_axes_0 = const()[name = string("op_2100_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_2100_cast_fp16 = expand_dims(axes = var_2100_axes_0, x = var_2099_cast_fp16)[name = string("op_2100_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2101_cast_fp16 = mul(x = state_183_cast_fp16, y = var_2100_cast_fp16)[name = string("op_2101_cast_fp16")];
            tensor<int32, [1]> var_2103_axes_0 = const()[name = string("op_2103_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_2103_keep_dims_0 = const()[name = string("op_2103_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_2103_cast_fp16 = reduce_sum(axes = var_2103_axes_0, keep_dims = var_2103_keep_dims_0, x = var_2101_cast_fp16)[name = string("op_2103_cast_fp16")];
            tensor<int32, [3]> var_2106_begin_0 = const()[name = string("op_2106_begin_0"), val = tensor<int32, [3]>([0, 0, 14])];
            tensor<int32, [3]> var_2106_end_0 = const()[name = string("op_2106_end_0"), val = tensor<int32, [3]>([1, 16, 15])];
            tensor<bool, [3]> var_2106_end_mask_0 = const()[name = string("op_2106_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_2106_squeeze_mask_0 = const()[name = string("op_2106_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_2106_cast_fp16 = slice_by_index(begin = var_2106_begin_0, end = var_2106_end_0, end_mask = var_2106_end_mask_0, squeeze_mask = var_2106_squeeze_mask_0, x = decay_5_cast_fp16)[name = string("op_2106_cast_fp16")];
            tensor<fp16, [1, 16]> var_2107_cast_fp16 = exp(x = var_2106_cast_fp16)[name = string("op_2107_cast_fp16")];
            tensor<int32, [1]> var_2108_axes_0 = const()[name = string("op_2108_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_2108_cast_fp16 = expand_dims(axes = var_2108_axes_0, x = var_2107_cast_fp16)[name = string("op_2108_cast_fp16")];
            tensor<int32, [1]> var_2109_axes_0 = const()[name = string("op_2109_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_2109_cast_fp16 = expand_dims(axes = var_2109_axes_0, x = var_2108_cast_fp16)[name = string("op_2109_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_185_cast_fp16 = mul(x = state_183_cast_fp16, y = var_2109_cast_fp16)[name = string("state_185_cast_fp16")];
            tensor<int32, [4]> key_token_93_begin_0 = const()[name = string("key_token_93_begin_0"), val = tensor<int32, [4]>([0, 0, 14, 0])];
            tensor<int32, [4]> key_token_93_end_0 = const()[name = string("key_token_93_end_0"), val = tensor<int32, [4]>([1, 16, 15, 128])];
            tensor<bool, [4]> key_token_93_end_mask_0 = const()[name = string("key_token_93_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_93_squeeze_mask_0 = const()[name = string("key_token_93_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_93_cast_fp16 = slice_by_index(begin = key_token_93_begin_0, end = key_token_93_end_0, end_mask = key_token_93_end_mask_0, squeeze_mask = key_token_93_squeeze_mask_0, x = key_11_cast_fp16)[name = string("key_token_93_cast_fp16")];
            tensor<int32, [4]> value_token_93_begin_0 = const()[name = string("value_token_93_begin_0"), val = tensor<int32, [4]>([0, 0, 14, 0])];
            tensor<int32, [4]> value_token_93_end_0 = const()[name = string("value_token_93_end_0"), val = tensor<int32, [4]>([1, 16, 15, 128])];
            tensor<bool, [4]> value_token_93_end_mask_0 = const()[name = string("value_token_93_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_93_squeeze_mask_0 = const()[name = string("value_token_93_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_93_cast_fp16 = slice_by_index(begin = value_token_93_begin_0, end = value_token_93_end_0, end_mask = value_token_93_end_mask_0, squeeze_mask = value_token_93_squeeze_mask_0, x = value_39_cast_fp16)[name = string("value_token_93_cast_fp16")];
            tensor<int32, [1]> var_2117_axes_0 = const()[name = string("op_2117_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_2117_cast_fp16 = expand_dims(axes = var_2117_axes_0, x = key_token_93_cast_fp16)[name = string("op_2117_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2118_cast_fp16 = mul(x = state_185_cast_fp16, y = var_2117_cast_fp16)[name = string("op_2118_cast_fp16")];
            tensor<int32, [1]> memory_93_axes_0 = const()[name = string("memory_93_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_93_keep_dims_0 = const()[name = string("memory_93_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_93_cast_fp16 = reduce_sum(axes = memory_93_axes_0, keep_dims = memory_93_keep_dims_0, x = var_2118_cast_fp16)[name = string("memory_93_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_2121_cast_fp16 = sub(x = value_token_93_cast_fp16, y = memory_93_cast_fp16)[name = string("op_2121_cast_fp16")];
            tensor<int32, [3]> var_2124_begin_0 = const()[name = string("op_2124_begin_0"), val = tensor<int32, [3]>([0, 0, 14])];
            tensor<int32, [3]> var_2124_end_0 = const()[name = string("op_2124_end_0"), val = tensor<int32, [3]>([1, 16, 15])];
            tensor<bool, [3]> var_2124_end_mask_0 = const()[name = string("op_2124_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_2124_squeeze_mask_0 = const()[name = string("op_2124_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_2124_cast_fp16 = slice_by_index(begin = var_2124_begin_0, end = var_2124_end_0, end_mask = var_2124_end_mask_0, squeeze_mask = var_2124_squeeze_mask_0, x = beta_5_cast_fp16)[name = string("op_2124_cast_fp16")];
            tensor<int32, [1]> var_2125_axes_0 = const()[name = string("op_2125_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_2125_cast_fp16 = expand_dims(axes = var_2125_axes_0, x = var_2124_cast_fp16)[name = string("op_2125_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_93_cast_fp16 = mul(x = var_2121_cast_fp16, y = var_2125_cast_fp16)[name = string("delta_93_cast_fp16")];
            tensor<int32, [1]> var_2128_axes_0 = const()[name = string("op_2128_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_2128_cast_fp16 = expand_dims(axes = var_2128_axes_0, x = delta_93_cast_fp16)[name = string("op_2128_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2129_cast_fp16 = mul(x = var_2117_cast_fp16, y = var_2128_cast_fp16)[name = string("op_2129_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_187_cast_fp16 = add(x = state_185_cast_fp16, y = var_2129_cast_fp16)[name = string("state_187_cast_fp16")];
            tensor<int32, [4]> var_2133_begin_0 = const()[name = string("op_2133_begin_0"), val = tensor<int32, [4]>([0, 0, 14, 0])];
            tensor<int32, [4]> var_2133_end_0 = const()[name = string("op_2133_end_0"), val = tensor<int32, [4]>([1, 16, 15, 128])];
            tensor<bool, [4]> var_2133_end_mask_0 = const()[name = string("op_2133_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_2133_squeeze_mask_0 = const()[name = string("op_2133_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_2133_cast_fp16 = slice_by_index(begin = var_2133_begin_0, end = var_2133_end_0, end_mask = var_2133_end_mask_0, squeeze_mask = var_2133_squeeze_mask_0, x = _inversed_query_11_cast_fp16)[name = string("op_2133_cast_fp16")];
            tensor<int32, [1]> var_2134_axes_0 = const()[name = string("op_2134_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_2134_cast_fp16 = expand_dims(axes = var_2134_axes_0, x = var_2133_cast_fp16)[name = string("op_2134_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2135_cast_fp16 = mul(x = state_187_cast_fp16, y = var_2134_cast_fp16)[name = string("op_2135_cast_fp16")];
            tensor<int32, [1]> var_2137_axes_0 = const()[name = string("op_2137_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_2137_keep_dims_0 = const()[name = string("op_2137_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_2137_cast_fp16 = reduce_sum(axes = var_2137_axes_0, keep_dims = var_2137_keep_dims_0, x = var_2135_cast_fp16)[name = string("op_2137_cast_fp16")];
            tensor<int32, [3]> var_2140_begin_0 = const()[name = string("op_2140_begin_0"), val = tensor<int32, [3]>([0, 0, 15])];
            tensor<int32, [3]> var_2140_end_0 = const()[name = string("op_2140_end_0"), val = tensor<int32, [3]>([1, 16, 16])];
            tensor<bool, [3]> var_2140_end_mask_0 = const()[name = string("op_2140_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_2140_squeeze_mask_0 = const()[name = string("op_2140_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_2140_cast_fp16 = slice_by_index(begin = var_2140_begin_0, end = var_2140_end_0, end_mask = var_2140_end_mask_0, squeeze_mask = var_2140_squeeze_mask_0, x = decay_5_cast_fp16)[name = string("op_2140_cast_fp16")];
            tensor<fp16, [1, 16]> var_2141_cast_fp16 = exp(x = var_2140_cast_fp16)[name = string("op_2141_cast_fp16")];
            tensor<int32, [1]> var_2142_axes_0 = const()[name = string("op_2142_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_2142_cast_fp16 = expand_dims(axes = var_2142_axes_0, x = var_2141_cast_fp16)[name = string("op_2142_cast_fp16")];
            tensor<int32, [1]> var_2143_axes_0 = const()[name = string("op_2143_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_2143_cast_fp16 = expand_dims(axes = var_2143_axes_0, x = var_2142_cast_fp16)[name = string("op_2143_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_189_cast_fp16 = mul(x = state_187_cast_fp16, y = var_2143_cast_fp16)[name = string("state_189_cast_fp16")];
            tensor<int32, [4]> key_token_95_begin_0 = const()[name = string("key_token_95_begin_0"), val = tensor<int32, [4]>([0, 0, 15, 0])];
            tensor<int32, [4]> key_token_95_end_0 = const()[name = string("key_token_95_end_0"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<bool, [4]> key_token_95_end_mask_0 = const()[name = string("key_token_95_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_95_squeeze_mask_0 = const()[name = string("key_token_95_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_95_cast_fp16 = slice_by_index(begin = key_token_95_begin_0, end = key_token_95_end_0, end_mask = key_token_95_end_mask_0, squeeze_mask = key_token_95_squeeze_mask_0, x = key_11_cast_fp16)[name = string("key_token_95_cast_fp16")];
            tensor<int32, [4]> value_token_95_begin_0 = const()[name = string("value_token_95_begin_0"), val = tensor<int32, [4]>([0, 0, 15, 0])];
            tensor<int32, [4]> value_token_95_end_0 = const()[name = string("value_token_95_end_0"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<bool, [4]> value_token_95_end_mask_0 = const()[name = string("value_token_95_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_95_squeeze_mask_0 = const()[name = string("value_token_95_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_95_cast_fp16 = slice_by_index(begin = value_token_95_begin_0, end = value_token_95_end_0, end_mask = value_token_95_end_mask_0, squeeze_mask = value_token_95_squeeze_mask_0, x = value_39_cast_fp16)[name = string("value_token_95_cast_fp16")];
            tensor<int32, [1]> var_2151_axes_0 = const()[name = string("op_2151_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_2151_cast_fp16 = expand_dims(axes = var_2151_axes_0, x = key_token_95_cast_fp16)[name = string("op_2151_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2152_cast_fp16 = mul(x = state_189_cast_fp16, y = var_2151_cast_fp16)[name = string("op_2152_cast_fp16")];
            tensor<int32, [1]> memory_95_axes_0 = const()[name = string("memory_95_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_95_keep_dims_0 = const()[name = string("memory_95_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_95_cast_fp16 = reduce_sum(axes = memory_95_axes_0, keep_dims = memory_95_keep_dims_0, x = var_2152_cast_fp16)[name = string("memory_95_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_2155_cast_fp16 = sub(x = value_token_95_cast_fp16, y = memory_95_cast_fp16)[name = string("op_2155_cast_fp16")];
            tensor<int32, [3]> var_2158_begin_0 = const()[name = string("op_2158_begin_0"), val = tensor<int32, [3]>([0, 0, 15])];
            tensor<int32, [3]> var_2158_end_0 = const()[name = string("op_2158_end_0"), val = tensor<int32, [3]>([1, 16, 16])];
            tensor<bool, [3]> var_2158_end_mask_0 = const()[name = string("op_2158_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_2158_squeeze_mask_0 = const()[name = string("op_2158_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_2158_cast_fp16 = slice_by_index(begin = var_2158_begin_0, end = var_2158_end_0, end_mask = var_2158_end_mask_0, squeeze_mask = var_2158_squeeze_mask_0, x = beta_5_cast_fp16)[name = string("op_2158_cast_fp16")];
            tensor<int32, [1]> var_2159_axes_0 = const()[name = string("op_2159_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_2159_cast_fp16 = expand_dims(axes = var_2159_axes_0, x = var_2158_cast_fp16)[name = string("op_2159_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_95_cast_fp16 = mul(x = var_2155_cast_fp16, y = var_2159_cast_fp16)[name = string("delta_95_cast_fp16")];
            tensor<int32, [1]> var_2162_axes_0 = const()[name = string("op_2162_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_2162_cast_fp16 = expand_dims(axes = var_2162_axes_0, x = delta_95_cast_fp16)[name = string("op_2162_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2163_cast_fp16 = mul(x = var_2151_cast_fp16, y = var_2162_cast_fp16)[name = string("op_2163_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> rec2_out = add(x = state_189_cast_fp16, y = var_2163_cast_fp16)[name = string("state_191_cast_fp16")];
            tensor<int32, [4]> var_2167_begin_0 = const()[name = string("op_2167_begin_0"), val = tensor<int32, [4]>([0, 0, 15, 0])];
            tensor<int32, [4]> var_2167_end_0 = const()[name = string("op_2167_end_0"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<bool, [4]> var_2167_end_mask_0 = const()[name = string("op_2167_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_2167_squeeze_mask_0 = const()[name = string("op_2167_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_2167_cast_fp16 = slice_by_index(begin = var_2167_begin_0, end = var_2167_end_0, end_mask = var_2167_end_mask_0, squeeze_mask = var_2167_squeeze_mask_0, x = _inversed_query_11_cast_fp16)[name = string("op_2167_cast_fp16")];
            tensor<int32, [1]> var_2168_axes_0 = const()[name = string("op_2168_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_2168_cast_fp16 = expand_dims(axes = var_2168_axes_0, x = var_2167_cast_fp16)[name = string("op_2168_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2169_cast_fp16 = mul(x = rec2_out, y = var_2168_cast_fp16)[name = string("op_2169_cast_fp16")];
            tensor<int32, [1]> var_2171_axes_0 = const()[name = string("op_2171_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_2171_keep_dims_0 = const()[name = string("op_2171_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_2171_cast_fp16 = reduce_sum(axes = var_2171_axes_0, keep_dims = var_2171_keep_dims_0, x = var_2169_cast_fp16)[name = string("op_2171_cast_fp16")];
            int32 attention_21_axis_0 = const()[name = string("attention_21_axis_0"), val = int32(1)];
            tensor<fp16, [1, 16, 16, 128]> attention_21_cast_fp16 = stack(axis = attention_21_axis_0, values = (var_1661_cast_fp16, var_1695_cast_fp16, var_1729_cast_fp16, var_1763_cast_fp16, var_1797_cast_fp16, var_1831_cast_fp16, var_1865_cast_fp16, var_1899_cast_fp16, var_1933_cast_fp16, var_1967_cast_fp16, var_2001_cast_fp16, var_2035_cast_fp16, var_2069_cast_fp16, var_2103_cast_fp16, var_2137_cast_fp16, var_2171_cast_fp16))[name = string("attention_21_cast_fp16")];
            tensor<int32, [2]> var_2174 = const()[name = string("op_2174"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [256, 128]> attention_23_cast_fp16 = reshape(shape = var_2174, x = attention_21_cast_fp16)[name = string("attention_23_cast_fp16")];
            tensor<int32, [2]> var_2176 = const()[name = string("op_2176"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [256, 128]> input_33_cast_fp16 = reshape(shape = var_2176, x = linear_19_cast_fp16)[name = string("input_33_cast_fp16")];
            tensor<fp16, [256, 128]> var_2178_cast_fp16 = mul(x = attention_23_cast_fp16, y = attention_23_cast_fp16)[name = string("op_2178_cast_fp16")];
            tensor<int32, [1]> variance_15_axes_0 = const()[name = string("variance_15_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_15_keep_dims_0 = const()[name = string("variance_15_keep_dims_0"), val = bool(true)];
            tensor<fp16, [256, 1]> variance_15_cast_fp16 = reduce_mean(axes = variance_15_axes_0, keep_dims = variance_15_keep_dims_0, x = var_2178_cast_fp16)[name = string("variance_15_cast_fp16")];
            fp16 var_2181_to_fp16 = const()[name = string("op_2181_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [256, 1]> var_2182_cast_fp16 = add(x = variance_15_cast_fp16, y = var_2181_to_fp16)[name = string("op_2182_cast_fp16")];
            fp32 var_2183_epsilon_0 = const()[name = string("op_2183_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [256, 1]> var_2183_cast_fp16 = rsqrt(epsilon = var_2183_epsilon_0, x = var_2182_cast_fp16)[name = string("op_2183_cast_fp16")];
            tensor<fp16, [256, 128]> attention_25_cast_fp16 = mul(x = attention_23_cast_fp16, y = var_2183_cast_fp16)[name = string("attention_25_cast_fp16")];
            tensor<fp16, [128]> groups_0_2_gated_norm_promoted_to_fp16 = const()[name = string("groups_0_2_gated_norm_promoted_to_fp16"), val = tensor<fp16, [128]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(39197440)))];
            tensor<fp16, [256, 128]> attention_27_cast_fp16 = mul(x = attention_25_cast_fp16, y = groups_0_2_gated_norm_promoted_to_fp16)[name = string("attention_27_cast_fp16")];
            tensor<fp16, [256, 128]> var_2186_cast_fp16 = silu(x = input_33_cast_fp16)[name = string("op_2186_cast_fp16")];
            tensor<fp16, [256, 128]> attention_29_cast_fp16 = mul(x = attention_27_cast_fp16, y = var_2186_cast_fp16)[name = string("attention_29_cast_fp16")];
            tensor<int32, [2]> var_2188 = const()[name = string("op_2188"), val = tensor<int32, [2]>([16, 2048])];
            tensor<fp16, [16, 2048]> input_35_cast_fp16 = reshape(shape = var_2188, x = attention_29_cast_fp16)[name = string("input_35_cast_fp16")];
            tensor<fp16, [1024, 2048]> groups_0_2_out_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 2048]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(39197760))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(40770688))))[name = string("groups_0_2_out_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 1024]> linear_20_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = groups_0_2_out_proj_weight_promoted_to_fp16_palettized, x = input_35_cast_fp16)[name = string("linear_20_cast_fp16")];
            tensor<fp16, [16, 1024]> value_41_cast_fp16 = add(x = value_29_cast_fp16, y = linear_20_cast_fp16)[name = string("value_41_cast_fp16")];
            tensor<fp16, [16, 1024]> var_2193_cast_fp16 = mul(x = value_41_cast_fp16, y = value_41_cast_fp16)[name = string("op_2193_cast_fp16")];
            tensor<int32, [1]> variance_17_axes_0 = const()[name = string("variance_17_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_17_keep_dims_0 = const()[name = string("variance_17_keep_dims_0"), val = bool(true)];
            tensor<fp16, [16, 1]> variance_17_cast_fp16 = reduce_mean(axes = variance_17_axes_0, keep_dims = variance_17_keep_dims_0, x = var_2193_cast_fp16)[name = string("variance_17_cast_fp16")];
            fp16 var_2196_to_fp16 = const()[name = string("op_2196_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [16, 1]> var_2197_cast_fp16 = add(x = variance_17_cast_fp16, y = var_2196_to_fp16)[name = string("op_2197_cast_fp16")];
            fp32 var_2198_epsilon_0 = const()[name = string("op_2198_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [16, 1]> var_2198_cast_fp16 = rsqrt(epsilon = var_2198_epsilon_0, x = var_2197_cast_fp16)[name = string("op_2198_cast_fp16")];
            tensor<fp16, [16, 1024]> var_2199_cast_fp16 = mul(x = value_41_cast_fp16, y = var_2198_cast_fp16)[name = string("op_2199_cast_fp16")];
            tensor<fp16, [1024]> var_2201_to_fp16 = const()[name = string("op_2201_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(40778944)))];
            tensor<fp16, [16, 1024]> input_37_cast_fp16 = mul(x = var_2199_cast_fp16, y = var_2201_to_fp16)[name = string("input_37_cast_fp16")];
            tensor<fp16, [3584, 1024]> groups_0_2_gate_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(40781056))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(43533632))))[name = string("groups_0_2_gate_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 3584]> linear_21_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = groups_0_2_gate_proj_weight_promoted_to_fp16_palettized, x = input_37_cast_fp16)[name = string("linear_21_cast_fp16")];
            tensor<fp16, [16, 3584]> var_2205_cast_fp16 = silu(x = linear_21_cast_fp16)[name = string("op_2205_cast_fp16")];
            tensor<fp16, [3584, 1024]> groups_0_2_up_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(43562368))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(46314944))))[name = string("groups_0_2_up_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 3584]> linear_22_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = groups_0_2_up_proj_weight_promoted_to_fp16_palettized, x = input_37_cast_fp16)[name = string("linear_22_cast_fp16")];
            tensor<fp16, [16, 3584]> input_41_cast_fp16 = mul(x = var_2205_cast_fp16, y = linear_22_cast_fp16)[name = string("input_41_cast_fp16")];
            tensor<fp16, [1024, 3584]> groups_0_2_down_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 3584]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(46343680))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(49096256))))[name = string("groups_0_2_down_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 1024]> linear_23_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = groups_0_2_down_proj_weight_promoted_to_fp16_palettized, x = input_41_cast_fp16)[name = string("linear_23_cast_fp16")];
            tensor<fp16, [16, 1024]> value_43_cast_fp16 = add(x = value_41_cast_fp16, y = linear_23_cast_fp16)[name = string("value_43_cast_fp16")];
            int32 var_2230 = const()[name = string("op_2230"), val = int32(-1)];
            tensor<fp16, [16, 1024]> var_2237_cast_fp16 = mul(x = value_43_cast_fp16, y = value_43_cast_fp16)[name = string("op_2237_cast_fp16")];
            tensor<int32, [1]> variance_19_axes_0 = const()[name = string("variance_19_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_19_keep_dims_0 = const()[name = string("variance_19_keep_dims_0"), val = bool(true)];
            tensor<fp16, [16, 1]> variance_19_cast_fp16 = reduce_mean(axes = variance_19_axes_0, keep_dims = variance_19_keep_dims_0, x = var_2237_cast_fp16)[name = string("variance_19_cast_fp16")];
            fp16 var_2240_to_fp16 = const()[name = string("op_2240_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [16, 1]> var_2241_cast_fp16 = add(x = variance_19_cast_fp16, y = var_2240_to_fp16)[name = string("op_2241_cast_fp16")];
            fp32 var_2242_epsilon_0 = const()[name = string("op_2242_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [16, 1]> var_2242_cast_fp16 = rsqrt(epsilon = var_2242_epsilon_0, x = var_2241_cast_fp16)[name = string("op_2242_cast_fp16")];
            tensor<fp16, [16, 1024]> var_2243_cast_fp16 = mul(x = value_43_cast_fp16, y = var_2242_cast_fp16)[name = string("op_2243_cast_fp16")];
            tensor<fp16, [1024]> var_2245_to_fp16 = const()[name = string("op_2245_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(49104512)))];
            tensor<fp16, [16, 1024]> input_43_cast_fp16 = mul(x = var_2243_cast_fp16, y = var_2245_to_fp16)[name = string("input_43_cast_fp16")];
            tensor<fp16, [4096, 1024]> projections_0_q_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [4096, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(49106624))), lut = tensor<fp16, [256, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(52252416))))[name = string("projections_0_q_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4096]> linear_24_bias_0_to_fp16 = const()[name = string("linear_24_bias_0_to_fp16"), val = tensor<fp16, [4096]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(52285248)))];
            tensor<fp16, [16, 4096]> linear_24_cast_fp16 = linear(bias = linear_24_bias_0_to_fp16, weight = projections_0_q_proj_weight_promoted_to_fp16_palettized, x = input_43_cast_fp16)[name = string("linear_24_cast_fp16")];
            tensor<int32, [3]> var_2249 = const()[name = string("op_2249"), val = tensor<int32, [3]>([16, 8, 512])];
            tensor<fp16, [16, 8, 512]> query_and_gate_1_cast_fp16 = reshape(shape = var_2249, x = linear_24_cast_fp16)[name = string("query_and_gate_1_cast_fp16")];
            tensor<int32, [2]> var_2251_split_sizes_0 = const()[name = string("op_2251_split_sizes_0"), val = tensor<int32, [2]>([256, 256])];
            int32 var_2251_axis_0 = const()[name = string("op_2251_axis_0"), val = int32(-1)];
            tensor<fp16, [16, 8, 256]> var_2251_cast_fp16_0, tensor<fp16, [16, 8, 256]> var_2251_cast_fp16_1 = split(axis = var_2251_axis_0, split_sizes = var_2251_split_sizes_0, x = query_and_gate_1_cast_fp16)[name = string("op_2251_cast_fp16")];
            tensor<fp16, [512, 1024]> projections_0_k_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [512, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(52293504))), lut = tensor<fp16, [32, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(52686784))))[name = string("projections_0_k_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [512]> linear_25_bias_0_to_fp16 = const()[name = string("linear_25_bias_0_to_fp16"), val = tensor<fp16, [512]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(52690944)))];
            tensor<fp16, [16, 512]> linear_25_cast_fp16 = linear(bias = linear_25_bias_0_to_fp16, weight = projections_0_k_proj_weight_promoted_to_fp16_palettized, x = input_43_cast_fp16)[name = string("linear_25_cast_fp16")];
            tensor<int32, [3]> var_2255 = const()[name = string("op_2255"), val = tensor<int32, [3]>([16, 2, 256])];
            tensor<fp16, [16, 2, 256]> value_47_cast_fp16 = reshape(shape = var_2255, x = linear_25_cast_fp16)[name = string("value_47_cast_fp16")];
            tensor<fp16, [512, 1024]> projections_0_v_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [512, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(52692032))), lut = tensor<fp16, [32, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(53085312))))[name = string("projections_0_v_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 512]> linear_26_cast_fp16 = linear(bias = linear_25_bias_0_to_fp16, weight = projections_0_v_proj_weight_promoted_to_fp16_palettized, x = input_43_cast_fp16)[name = string("linear_26_cast_fp16")];
            tensor<int32, [3]> var_2259 = const()[name = string("op_2259"), val = tensor<int32, [3]>([16, 2, 256])];
            tensor<fp16, [16, 2, 256]> value_53_cast_fp16 = reshape(shape = var_2259, x = linear_26_cast_fp16)[name = string("value_53_cast_fp16")];
            tensor<fp16, [16, 8, 256]> var_2261_cast_fp16 = mul(x = var_2251_cast_fp16_0, y = var_2251_cast_fp16_0)[name = string("op_2261_cast_fp16")];
            tensor<int32, [1]> variance_21_axes_0 = const()[name = string("variance_21_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_21_keep_dims_0 = const()[name = string("variance_21_keep_dims_0"), val = bool(true)];
            tensor<fp16, [16, 8, 1]> variance_21_cast_fp16 = reduce_mean(axes = variance_21_axes_0, keep_dims = variance_21_keep_dims_0, x = var_2261_cast_fp16)[name = string("variance_21_cast_fp16")];
            fp16 var_2264_to_fp16 = const()[name = string("op_2264_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [16, 8, 1]> var_2265_cast_fp16 = add(x = variance_21_cast_fp16, y = var_2264_to_fp16)[name = string("op_2265_cast_fp16")];
            fp32 var_2266_epsilon_0 = const()[name = string("op_2266_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [16, 8, 1]> var_2266_cast_fp16 = rsqrt(epsilon = var_2266_epsilon_0, x = var_2265_cast_fp16)[name = string("op_2266_cast_fp16")];
            tensor<fp16, [16, 8, 256]> var_2267_cast_fp16 = mul(x = var_2251_cast_fp16_0, y = var_2266_cast_fp16)[name = string("op_2267_cast_fp16")];
            tensor<fp16, [256]> var_2269_to_fp16 = const()[name = string("op_2269_to_fp16"), val = tensor<fp16, [256]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(53089472)))];
            tensor<fp16, [16, 8, 256]> var_2270_cast_fp16 = mul(x = var_2267_cast_fp16, y = var_2269_to_fp16)[name = string("op_2270_cast_fp16")];
            tensor<int32, [3]> var_2271 = const()[name = string("op_2271"), val = tensor<int32, [3]>([1, 0, 2])];
            tensor<int32, [1]> value_49_axes_0 = const()[name = string("value_49_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [8, 16, 256]> var_2272_cast_fp16 = transpose(perm = var_2271, x = var_2270_cast_fp16)[name = string("transpose_120")];
            tensor<fp16, [1, 8, 16, 256]> value_49_cast_fp16 = expand_dims(axes = value_49_axes_0, x = var_2272_cast_fp16)[name = string("value_49_cast_fp16")];
            tensor<fp16, [16, 2, 256]> var_2274_cast_fp16 = mul(x = value_47_cast_fp16, y = value_47_cast_fp16)[name = string("op_2274_cast_fp16")];
            tensor<int32, [1]> variance_23_axes_0 = const()[name = string("variance_23_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_23_keep_dims_0 = const()[name = string("variance_23_keep_dims_0"), val = bool(true)];
            tensor<fp16, [16, 2, 1]> variance_23_cast_fp16 = reduce_mean(axes = variance_23_axes_0, keep_dims = variance_23_keep_dims_0, x = var_2274_cast_fp16)[name = string("variance_23_cast_fp16")];
            fp16 var_2277_to_fp16 = const()[name = string("op_2277_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [16, 2, 1]> var_2278_cast_fp16 = add(x = variance_23_cast_fp16, y = var_2277_to_fp16)[name = string("op_2278_cast_fp16")];
            fp32 var_2279_epsilon_0 = const()[name = string("op_2279_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [16, 2, 1]> var_2279_cast_fp16 = rsqrt(epsilon = var_2279_epsilon_0, x = var_2278_cast_fp16)[name = string("op_2279_cast_fp16")];
            tensor<fp16, [16, 2, 256]> var_2280_cast_fp16 = mul(x = value_47_cast_fp16, y = var_2279_cast_fp16)[name = string("op_2280_cast_fp16")];
            tensor<fp16, [256]> var_2282_to_fp16 = const()[name = string("op_2282_to_fp16"), val = tensor<fp16, [256]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(53090048)))];
            tensor<fp16, [16, 2, 256]> var_2283_cast_fp16 = mul(x = var_2280_cast_fp16, y = var_2282_to_fp16)[name = string("op_2283_cast_fp16")];
            tensor<int32, [3]> var_2284 = const()[name = string("op_2284"), val = tensor<int32, [3]>([1, 0, 2])];
            tensor<int32, [1]> value_51_axes_0 = const()[name = string("value_51_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [2, 16, 256]> var_2285_cast_fp16 = transpose(perm = var_2284, x = var_2283_cast_fp16)[name = string("transpose_119")];
            tensor<fp16, [1, 2, 16, 256]> value_51_cast_fp16 = expand_dims(axes = value_51_axes_0, x = var_2285_cast_fp16)[name = string("value_51_cast_fp16")];
            tensor<int32, [4]> rotated_1_begin_0 = const()[name = string("rotated_1_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> rotated_1_end_0 = const()[name = string("rotated_1_end_0"), val = tensor<int32, [4]>([1, 8, 16, 64])];
            tensor<bool, [4]> rotated_1_end_mask_0 = const()[name = string("rotated_1_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
            tensor<fp16, [1, 8, 16, 64]> rotated_1_cast_fp16 = slice_by_index(begin = rotated_1_begin_0, end = rotated_1_end_0, end_mask = rotated_1_end_mask_0, x = value_49_cast_fp16)[name = string("rotated_1_cast_fp16")];
            tensor<int32, [4]> passthrough_1_begin_0 = const()[name = string("passthrough_1_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 64])];
            tensor<int32, [4]> passthrough_1_end_0 = const()[name = string("passthrough_1_end_0"), val = tensor<int32, [4]>([1, 8, 16, 256])];
            tensor<bool, [4]> passthrough_1_end_mask_0 = const()[name = string("passthrough_1_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
            tensor<fp16, [1, 8, 16, 192]> passthrough_1_cast_fp16 = slice_by_index(begin = passthrough_1_begin_0, end = passthrough_1_end_0, end_mask = passthrough_1_end_mask_0, x = value_49_cast_fp16)[name = string("passthrough_1_cast_fp16")];
            tensor<int32, [2]> var_2289_split_sizes_0 = const()[name = string("op_2289_split_sizes_0"), val = tensor<int32, [2]>([32, 32])];
            int32 var_2289_axis_0 = const()[name = string("op_2289_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 8, 16, 32]> var_2289_cast_fp16_0, tensor<fp16, [1, 8, 16, 32]> var_2289_cast_fp16_1 = split(axis = var_2289_axis_0, split_sizes = var_2289_split_sizes_0, x = rotated_1_cast_fp16)[name = string("op_2289_cast_fp16")];
            fp16 const_3_promoted_to_fp16 = const()[name = string("const_3_promoted_to_fp16"), val = fp16(-0x1p+0)];
            tensor<fp16, [1, 8, 16, 32]> var_2291_cast_fp16 = mul(x = var_2289_cast_fp16_1, y = const_3_promoted_to_fp16)[name = string("op_2291_cast_fp16")];
            bool rotated_half_1_interleave_0 = const()[name = string("rotated_half_1_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 8, 16, 64]> rotated_half_1_cast_fp16 = concat(axis = var_2230, interleave = rotated_half_1_interleave_0, values = (var_2291_cast_fp16, var_2289_cast_fp16_0))[name = string("rotated_half_1_cast_fp16")];
            tensor<fp16, [1, 8, 16, 64]> var_2294_cast_fp16 = mul(x = rotated_1_cast_fp16, y = cos)[name = string("op_2294_cast_fp16")];
            tensor<fp16, [1, 8, 16, 64]> var_2295_cast_fp16 = mul(x = rotated_half_1_cast_fp16, y = sin)[name = string("op_2295_cast_fp16")];
            tensor<fp16, [1, 8, 16, 64]> var_2296_cast_fp16 = add(x = var_2294_cast_fp16, y = var_2295_cast_fp16)[name = string("op_2296_cast_fp16")];
            bool query_13_interleave_0 = const()[name = string("query_13_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 8, 16, 256]> query_13_cast_fp16 = concat(axis = var_2230, interleave = query_13_interleave_0, values = (var_2296_cast_fp16, passthrough_1_cast_fp16))[name = string("query_13_cast_fp16")];
            tensor<int32, [4]> rotated_3_begin_0 = const()[name = string("rotated_3_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> rotated_3_end_0 = const()[name = string("rotated_3_end_0"), val = tensor<int32, [4]>([1, 2, 16, 64])];
            tensor<bool, [4]> rotated_3_end_mask_0 = const()[name = string("rotated_3_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
            tensor<fp16, [1, 2, 16, 64]> rotated_3_cast_fp16 = slice_by_index(begin = rotated_3_begin_0, end = rotated_3_end_0, end_mask = rotated_3_end_mask_0, x = value_51_cast_fp16)[name = string("rotated_3_cast_fp16")];
            tensor<int32, [4]> passthrough_3_begin_0 = const()[name = string("passthrough_3_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 64])];
            tensor<int32, [4]> passthrough_3_end_0 = const()[name = string("passthrough_3_end_0"), val = tensor<int32, [4]>([1, 2, 16, 256])];
            tensor<bool, [4]> passthrough_3_end_mask_0 = const()[name = string("passthrough_3_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
            tensor<fp16, [1, 2, 16, 192]> passthrough_3_cast_fp16 = slice_by_index(begin = passthrough_3_begin_0, end = passthrough_3_end_0, end_mask = passthrough_3_end_mask_0, x = value_51_cast_fp16)[name = string("passthrough_3_cast_fp16")];
            tensor<int32, [2]> var_2301_split_sizes_0 = const()[name = string("op_2301_split_sizes_0"), val = tensor<int32, [2]>([32, 32])];
            int32 var_2301_axis_0 = const()[name = string("op_2301_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 2, 16, 32]> var_2301_cast_fp16_0, tensor<fp16, [1, 2, 16, 32]> var_2301_cast_fp16_1 = split(axis = var_2301_axis_0, split_sizes = var_2301_split_sizes_0, x = rotated_3_cast_fp16)[name = string("op_2301_cast_fp16")];
            fp16 const_4_promoted_to_fp16 = const()[name = string("const_4_promoted_to_fp16"), val = fp16(-0x1p+0)];
            tensor<fp16, [1, 2, 16, 32]> var_2303_cast_fp16 = mul(x = var_2301_cast_fp16_1, y = const_4_promoted_to_fp16)[name = string("op_2303_cast_fp16")];
            bool rotated_half_3_interleave_0 = const()[name = string("rotated_half_3_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 2, 16, 64]> rotated_half_3_cast_fp16 = concat(axis = var_2230, interleave = rotated_half_3_interleave_0, values = (var_2303_cast_fp16, var_2301_cast_fp16_0))[name = string("rotated_half_3_cast_fp16")];
            tensor<fp16, [1, 2, 16, 64]> var_2306_cast_fp16 = mul(x = rotated_3_cast_fp16, y = cos)[name = string("op_2306_cast_fp16")];
            tensor<fp16, [1, 2, 16, 64]> var_2307_cast_fp16 = mul(x = rotated_half_3_cast_fp16, y = sin)[name = string("op_2307_cast_fp16")];
            tensor<fp16, [1, 2, 16, 64]> var_2308_cast_fp16 = add(x = var_2306_cast_fp16, y = var_2307_cast_fp16)[name = string("op_2308_cast_fp16")];
            bool key_13_interleave_0 = const()[name = string("key_13_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 2, 16, 256]> key_13_cast_fp16 = concat(axis = var_2230, interleave = key_13_interleave_0, values = (var_2308_cast_fp16, passthrough_3_cast_fp16))[name = string("key_13_cast_fp16")];
            tensor<int32, [4]> var_2311 = const()[name = string("op_2311"), val = tensor<int32, [4]>([2, 4, 16, 256])];
            tensor<fp16, [2, 4, 16, 256]> var_2312_cast_fp16 = reshape(shape = var_2311, x = query_13_cast_fp16)[name = string("op_2312_cast_fp16")];
            tensor<int32, [4]> transpose_0_perm_0 = const()[name = string("transpose_0_perm_0"), val = tensor<int32, [4]>([0, 1, 3, 2])];
            tensor<int32, [3]> var_2315 = const()[name = string("op_2315"), val = tensor<int32, [3]>([2, 16, 256])];
            tensor<fp16, [2, 16, 256]> key_15_cast_fp16 = reshape(shape = var_2315, x = key_13_cast_fp16)[name = string("key_15_cast_fp16")];
            tensor<int32, [3]> var_2317 = const()[name = string("op_2317"), val = tensor<int32, [3]>([1, 0, 2])];
            tensor<int32, [2]> var_2319 = const()[name = string("op_2319"), val = tensor<int32, [2]>([16, 2048])];
            tensor<fp16, [16, 2048]> gate_3_cast_fp16 = reshape(shape = var_2319, x = var_2251_cast_fp16_1)[name = string("gate_3_cast_fp16")];
            tensor<int32, [1]> var_2327_axes_0 = const()[name = string("op_2327_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [1, 2, 16, 256]> var_2327_cast_fp16 = expand_dims(axes = var_2327_axes_0, x = key_15_cast_fp16)[name = string("op_2327_cast_fp16")];
            tensor<int32, [1]> var_2329_axes_0 = const()[name = string("op_2329_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [1, 1, 2, 16, 256]> var_2329_cast_fp16 = expand_dims(axes = var_2329_axes_0, x = var_2327_cast_fp16)[name = string("op_2329_cast_fp16")];
            int32 var_2331 = const()[name = string("op_2331"), val = int32(16)];
            tensor<int32, [1]> var_2332 = add(x = current_pos, y = var_2331)[name = string("op_2332")];
            tensor<fp16, [4, 2, 2, 2048, 256]> read_state_0 = read_state(input = kv_cache)[name = string("read_state_0")];
            tensor<int32, [1]> expand_dims_0 = const()[name = string("expand_dims_0"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_1 = const()[name = string("expand_dims_1"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_2 = const()[name = string("expand_dims_2"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_4 = const()[name = string("expand_dims_4"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_5 = const()[name = string("expand_dims_5"), val = tensor<int32, [1]>([1])];
            tensor<int32, [1]> expand_dims_6 = const()[name = string("expand_dims_6"), val = tensor<int32, [1]>([1])];
            int32 concat_2_axis_0 = const()[name = string("concat_2_axis_0"), val = int32(0)];
            bool concat_2_interleave_0 = const()[name = string("concat_2_interleave_0"), val = bool(false)];
            tensor<int32, [5]> concat_2 = concat(axis = concat_2_axis_0, interleave = concat_2_interleave_0, values = (expand_dims_0, expand_dims_1, expand_dims_2, current_pos, expand_dims_4))[name = string("concat_2")];
            tensor<int32, [1]> concat_3_values2_0 = const()[name = string("concat_3_values2_0"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> concat_3_values4_0 = const()[name = string("concat_3_values4_0"), val = tensor<int32, [1]>([0])];
            int32 concat_3_axis_0 = const()[name = string("concat_3_axis_0"), val = int32(0)];
            bool concat_3_interleave_0 = const()[name = string("concat_3_interleave_0"), val = bool(false)];
            tensor<int32, [5]> concat_3 = concat(axis = concat_3_axis_0, interleave = concat_3_interleave_0, values = (expand_dims_5, expand_dims_6, concat_3_values2_0, var_2332, concat_3_values4_0))[name = string("concat_3")];
            tensor<int32, [5]> kv_cache_internal_tensor_assign_1_stride_0 = const()[name = string("kv_cache_internal_tensor_assign_1_stride_0"), val = tensor<int32, [5]>([1, 1, 1, 1, 1])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_1_begin_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_1_begin_mask_0"), val = tensor<bool, [5]>([false, false, false, false, false])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_1_end_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_1_end_mask_0"), val = tensor<bool, [5]>([false, false, true, false, true])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_1_squeeze_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_1_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, false, false, false])];
            tensor<fp16, [4, 2, 2, 2048, 256]> kv_cache_internal_tensor_assign_1_cast_fp16 = slice_update(begin = concat_2, begin_mask = kv_cache_internal_tensor_assign_1_begin_mask_0, end = concat_3, end_mask = kv_cache_internal_tensor_assign_1_end_mask_0, squeeze_mask = kv_cache_internal_tensor_assign_1_squeeze_mask_0, stride = kv_cache_internal_tensor_assign_1_stride_0, update = var_2329_cast_fp16, x = read_state_0)[name = string("kv_cache_internal_tensor_assign_1_cast_fp16")];
            write_state(data = kv_cache_internal_tensor_assign_1_cast_fp16, input = kv_cache)[name = string("coreml_update_state_8_write_state")];
            tensor<fp16, [4, 2, 2, 2048, 256]> coreml_update_state_8 = read_state(input = kv_cache)[name = string("coreml_update_state_8")];
            tensor<int32, [1]> var_2360_axes_0 = const()[name = string("op_2360_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [2, 16, 256]> value_55_cast_fp16 = transpose(perm = var_2317, x = value_53_cast_fp16)[name = string("transpose_118")];
            tensor<fp16, [1, 2, 16, 256]> var_2360_cast_fp16 = expand_dims(axes = var_2360_axes_0, x = value_55_cast_fp16)[name = string("op_2360_cast_fp16")];
            tensor<int32, [1]> var_2362_axes_0 = const()[name = string("op_2362_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [1, 1, 2, 16, 256]> var_2362_cast_fp16 = expand_dims(axes = var_2362_axes_0, x = var_2360_cast_fp16)[name = string("op_2362_cast_fp16")];
            tensor<int32, [1]> expand_dims_8 = const()[name = string("expand_dims_8"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_9 = const()[name = string("expand_dims_9"), val = tensor<int32, [1]>([1])];
            tensor<int32, [1]> expand_dims_10 = const()[name = string("expand_dims_10"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_12 = const()[name = string("expand_dims_12"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_13 = const()[name = string("expand_dims_13"), val = tensor<int32, [1]>([1])];
            tensor<int32, [1]> expand_dims_14 = const()[name = string("expand_dims_14"), val = tensor<int32, [1]>([2])];
            int32 concat_6_axis_0 = const()[name = string("concat_6_axis_0"), val = int32(0)];
            bool concat_6_interleave_0 = const()[name = string("concat_6_interleave_0"), val = bool(false)];
            tensor<int32, [5]> concat_6 = concat(axis = concat_6_axis_0, interleave = concat_6_interleave_0, values = (expand_dims_8, expand_dims_9, expand_dims_10, current_pos, expand_dims_12))[name = string("concat_6")];
            tensor<int32, [1]> concat_7_values2_0 = const()[name = string("concat_7_values2_0"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> concat_7_values4_0 = const()[name = string("concat_7_values4_0"), val = tensor<int32, [1]>([0])];
            int32 concat_7_axis_0 = const()[name = string("concat_7_axis_0"), val = int32(0)];
            bool concat_7_interleave_0 = const()[name = string("concat_7_interleave_0"), val = bool(false)];
            tensor<int32, [5]> concat_7 = concat(axis = concat_7_axis_0, interleave = concat_7_interleave_0, values = (expand_dims_13, expand_dims_14, concat_7_values2_0, var_2332, concat_7_values4_0))[name = string("concat_7")];
            tensor<int32, [5]> kv_cache_internal_tensor_assign_2_stride_0 = const()[name = string("kv_cache_internal_tensor_assign_2_stride_0"), val = tensor<int32, [5]>([1, 1, 1, 1, 1])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_2_begin_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_2_begin_mask_0"), val = tensor<bool, [5]>([false, false, false, false, false])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_2_end_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_2_end_mask_0"), val = tensor<bool, [5]>([false, false, true, false, true])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_2_squeeze_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_2_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, false, false, false])];
            tensor<fp16, [4, 2, 2, 2048, 256]> kv_cache_internal_tensor_assign_2_cast_fp16 = slice_update(begin = concat_6, begin_mask = kv_cache_internal_tensor_assign_2_begin_mask_0, end = concat_7, end_mask = kv_cache_internal_tensor_assign_2_end_mask_0, squeeze_mask = kv_cache_internal_tensor_assign_2_squeeze_mask_0, stride = kv_cache_internal_tensor_assign_2_stride_0, update = var_2362_cast_fp16, x = coreml_update_state_8)[name = string("kv_cache_internal_tensor_assign_2_cast_fp16")];
            write_state(data = kv_cache_internal_tensor_assign_2_cast_fp16, input = kv_cache)[name = string("coreml_update_state_9_write_state")];
            tensor<fp16, [4, 2, 2, 2048, 256]> coreml_update_state_9 = read_state(input = kv_cache)[name = string("coreml_update_state_9")];
            tensor<int32, [5]> var_2394_begin_0 = const()[name = string("op_2394_begin_0"), val = tensor<int32, [5]>([0, 0, 0, 0, 0])];
            tensor<int32, [5]> var_2394_end_0 = const()[name = string("op_2394_end_0"), val = tensor<int32, [5]>([1, 2, 2, 2048, 256])];
            tensor<bool, [5]> var_2394_end_mask_0 = const()[name = string("op_2394_end_mask_0"), val = tensor<bool, [5]>([false, true, true, true, true])];
            tensor<bool, [5]> var_2394_squeeze_mask_0 = const()[name = string("op_2394_squeeze_mask_0"), val = tensor<bool, [5]>([true, false, false, false, false])];
            tensor<fp16, [2, 2, 2048, 256]> var_2394_cast_fp16 = slice_by_index(begin = var_2394_begin_0, end = var_2394_end_0, end_mask = var_2394_end_mask_0, squeeze_mask = var_2394_squeeze_mask_0, x = coreml_update_state_9)[name = string("op_2394_cast_fp16")];
            tensor<int32, [4]> keys_1_begin_0 = const()[name = string("keys_1_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> keys_1_end_0 = const()[name = string("keys_1_end_0"), val = tensor<int32, [4]>([1, 2, 2048, 256])];
            tensor<bool, [4]> keys_1_end_mask_0 = const()[name = string("keys_1_end_mask_0"), val = tensor<bool, [4]>([false, true, true, true])];
            tensor<bool, [4]> keys_1_squeeze_mask_0 = const()[name = string("keys_1_squeeze_mask_0"), val = tensor<bool, [4]>([true, false, false, false])];
            tensor<fp16, [2, 2048, 256]> keys_1_cast_fp16 = slice_by_index(begin = keys_1_begin_0, end = keys_1_end_0, end_mask = keys_1_end_mask_0, squeeze_mask = keys_1_squeeze_mask_0, x = var_2394_cast_fp16)[name = string("keys_1_cast_fp16")];
            tensor<int32, [4]> values_1_begin_0 = const()[name = string("values_1_begin_0"), val = tensor<int32, [4]>([1, 0, 0, 0])];
            tensor<int32, [4]> values_1_end_0 = const()[name = string("values_1_end_0"), val = tensor<int32, [4]>([2, 2, 2048, 256])];
            tensor<bool, [4]> values_1_end_mask_0 = const()[name = string("values_1_end_mask_0"), val = tensor<bool, [4]>([false, true, true, true])];
            tensor<bool, [4]> values_1_squeeze_mask_0 = const()[name = string("values_1_squeeze_mask_0"), val = tensor<bool, [4]>([true, false, false, false])];
            tensor<fp16, [2, 2048, 256]> values_1_cast_fp16 = slice_by_index(begin = values_1_begin_0, end = values_1_end_0, end_mask = values_1_end_mask_0, squeeze_mask = values_1_squeeze_mask_0, x = var_2394_cast_fp16)[name = string("values_1_cast_fp16")];
            int32 var_2412 = const()[name = string("op_2412"), val = int32(1)];
            tensor<int32, [4]> var_2419 = const()[name = string("op_2419"), val = tensor<int32, [4]>([1, 2048, 1, 16])];
            tensor<fp16, [2, 4, 256, 16]> transpose_0_cast_fp16 = transpose(perm = transpose_0_perm_0, x = var_2312_cast_fp16)[name = string("transpose_117")];
            tensor<fp16, [1, 2048, 1, 16]> var_2420_cast_fp16 = reshape(shape = var_2419, x = transpose_0_cast_fp16)[name = string("op_2420_cast_fp16")];
            tensor<int32, [3]> var_2421 = const()[name = string("op_2421"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [4]> var_2424 = const()[name = string("op_2424"), val = tensor<int32, [4]>([1, 512, 1, 2048])];
            tensor<fp16, [2, 256, 2048]> var_2422_cast_fp16 = transpose(perm = var_2421, x = keys_1_cast_fp16)[name = string("transpose_116")];
            tensor<fp16, [1, 512, 1, 2048]> key_native_1_cast_fp16 = reshape(shape = var_2424, x = var_2422_cast_fp16)[name = string("key_native_1_cast_fp16")];
            tensor<int32, [3]> var_2426 = const()[name = string("op_2426"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [4]> var_2429 = const()[name = string("op_2429"), val = tensor<int32, [4]>([1, 512, 1, 2048])];
            tensor<fp16, [2, 256, 2048]> var_2427_cast_fp16 = transpose(perm = var_2426, x = values_1_cast_fp16)[name = string("transpose_115")];
            tensor<fp16, [1, 512, 1, 2048]> var_2430_cast_fp16 = reshape(shape = var_2429, x = var_2427_cast_fp16)[name = string("op_2430_cast_fp16")];
            tensor<int32, [3]> var_2431 = const()[name = string("op_2431"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [1]> mask_native_1_axes_0 = const()[name = string("mask_native_1_axes_0"), val = tensor<int32, [1]>([2])];
            tensor<fp16, [1, 2048, 16]> var_2432_cast_fp16 = transpose(perm = var_2431, x = mask3)[name = string("transpose_114")];
            tensor<fp16, [1, 2048, 1, 16]> mask_native_1_cast_fp16 = expand_dims(axes = mask_native_1_axes_0, x = var_2432_cast_fp16)[name = string("mask_native_1_cast_fp16")];
            tensor<int32, [8]> tile_0 = const()[name = string("tile_0"), val = tensor<int32, [8]>([256, 256, 256, 256, 256, 256, 256, 256])];
            int32 var_2434_axis_0 = const()[name = string("op_2434_axis_0"), val = int32(1)];
            tensor<fp16, [1, 256, 1, 16]> var_2434_cast_fp16_0, tensor<fp16, [1, 256, 1, 16]> var_2434_cast_fp16_1, tensor<fp16, [1, 256, 1, 16]> var_2434_cast_fp16_2, tensor<fp16, [1, 256, 1, 16]> var_2434_cast_fp16_3, tensor<fp16, [1, 256, 1, 16]> var_2434_cast_fp16_4, tensor<fp16, [1, 256, 1, 16]> var_2434_cast_fp16_5, tensor<fp16, [1, 256, 1, 16]> var_2434_cast_fp16_6, tensor<fp16, [1, 256, 1, 16]> var_2434_cast_fp16_7 = split(axis = var_2434_axis_0, split_sizes = tile_0, x = var_2420_cast_fp16)[name = string("op_2434_cast_fp16")];
            tensor<int32, [4]> var_2443_perm_0 = const()[name = string("op_2443_perm_0"), val = tensor<int32, [4]>([0, 3, 2, 1])];
            tensor<int32, [2]> tile_1 = const()[name = string("tile_1"), val = tensor<int32, [2]>([256, 256])];
            int32 var_2444_axis_0 = const()[name = string("op_2444_axis_0"), val = int32(3)];
            tensor<fp16, [1, 2048, 1, 512]> var_2443_cast_fp16 = transpose(perm = var_2443_perm_0, x = key_native_1_cast_fp16)[name = string("transpose_113")];
            tensor<fp16, [1, 2048, 1, 256]> var_2444_cast_fp16_0, tensor<fp16, [1, 2048, 1, 256]> var_2444_cast_fp16_1 = split(axis = var_2444_axis_0, split_sizes = tile_1, x = var_2443_cast_fp16)[name = string("op_2444_cast_fp16")];
            tensor<int32, [2]> tile_2 = const()[name = string("tile_2"), val = tensor<int32, [2]>([256, 256])];
            int32 var_2447_axis_0 = const()[name = string("op_2447_axis_0"), val = int32(1)];
            tensor<fp16, [1, 256, 1, 2048]> var_2447_cast_fp16_0, tensor<fp16, [1, 256, 1, 2048]> var_2447_cast_fp16_1 = split(axis = var_2447_axis_0, split_sizes = tile_2, x = var_2430_cast_fp16)[name = string("op_2447_cast_fp16")];
            string scores_1_equation_0 = const()[name = string("scores_1_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 16]> scores_1_cast_fp16 = einsum(equation = scores_1_equation_0, values = (var_2444_cast_fp16_0, var_2434_cast_fp16_0))[name = string("scores_1_cast_fp16")];
            fp16 var_2452_to_fp16 = const()[name = string("op_2452_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 16]> var_2453_cast_fp16 = mul(x = scores_1_cast_fp16, y = var_2452_to_fp16)[name = string("op_2453_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_2454_cast_fp16 = add(x = var_2453_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_2454_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_2455_cast_fp16 = softmax(axis = var_2412, x = var_2454_cast_fp16)[name = string("op_2455_cast_fp16")];
            tensor<int32, [4]> tile_3 = const()[name = string("tile_3"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_2456_axis_0 = const()[name = string("op_2456_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 16]> var_2456_cast_fp16_0, tensor<fp16, [1, 512, 1, 16]> var_2456_cast_fp16_1, tensor<fp16, [1, 512, 1, 16]> var_2456_cast_fp16_2, tensor<fp16, [1, 512, 1, 16]> var_2456_cast_fp16_3 = split(axis = var_2456_axis_0, split_sizes = tile_3, x = var_2455_cast_fp16)[name = string("op_2456_cast_fp16")];
            tensor<int32, [4]> tile_4 = const()[name = string("tile_4"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_2461_axis_0 = const()[name = string("op_2461_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_2461_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_2461_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_2461_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_2461_cast_fp16_3 = split(axis = var_2461_axis_0, split_sizes = tile_4, x = var_2447_cast_fp16_0)[name = string("op_2461_cast_fp16")];
            fp16 var_2466_to_fp16 = const()[name = string("op_2466_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_2467_cast_fp16 = mul(x = var_2456_cast_fp16_0, y = var_2466_to_fp16)[name = string("op_2467_cast_fp16")];
            string var_2469_equation_0 = const()[name = string("op_2469_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_2469_cast_fp16 = einsum(equation = var_2469_equation_0, values = (var_2461_cast_fp16_0, var_2467_cast_fp16))[name = string("op_2469_cast_fp16")];
            fp16 var_2470_to_fp16 = const()[name = string("op_2470_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_2471_cast_fp16 = mul(x = var_2456_cast_fp16_1, y = var_2470_to_fp16)[name = string("op_2471_cast_fp16")];
            string var_2473_equation_0 = const()[name = string("op_2473_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_2473_cast_fp16 = einsum(equation = var_2473_equation_0, values = (var_2461_cast_fp16_1, var_2471_cast_fp16))[name = string("op_2473_cast_fp16")];
            fp16 var_2474_to_fp16 = const()[name = string("op_2474_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_2475_cast_fp16 = mul(x = var_2456_cast_fp16_2, y = var_2474_to_fp16)[name = string("op_2475_cast_fp16")];
            string var_2477_equation_0 = const()[name = string("op_2477_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_2477_cast_fp16 = einsum(equation = var_2477_equation_0, values = (var_2461_cast_fp16_2, var_2475_cast_fp16))[name = string("op_2477_cast_fp16")];
            fp16 var_2478_to_fp16 = const()[name = string("op_2478_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_2479_cast_fp16 = mul(x = var_2456_cast_fp16_3, y = var_2478_to_fp16)[name = string("op_2479_cast_fp16")];
            string var_2481_equation_0 = const()[name = string("op_2481_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_2481_cast_fp16 = einsum(equation = var_2481_equation_0, values = (var_2461_cast_fp16_3, var_2479_cast_fp16))[name = string("op_2481_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_2482_cast_fp16 = add(x = var_2469_cast_fp16, y = var_2473_cast_fp16)[name = string("op_2482_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_2483_cast_fp16 = add(x = var_2477_cast_fp16, y = var_2481_cast_fp16)[name = string("op_2483_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_2484_cast_fp16 = add(x = var_2482_cast_fp16, y = var_2483_cast_fp16)[name = string("op_2484_cast_fp16")];
            fp16 _inversed_2486_y_0_to_fp16 = const()[name = string("_inversed_2486_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 16]> _inversed_2486_cast_fp16 = mul(x = var_2484_cast_fp16, y = _inversed_2486_y_0_to_fp16)[name = string("_inversed_2486_cast_fp16")];
            string scores_3_equation_0 = const()[name = string("scores_3_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 16]> scores_3_cast_fp16 = einsum(equation = scores_3_equation_0, values = (var_2444_cast_fp16_0, var_2434_cast_fp16_1))[name = string("scores_3_cast_fp16")];
            fp16 var_2489_to_fp16 = const()[name = string("op_2489_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 16]> var_2490_cast_fp16 = mul(x = scores_3_cast_fp16, y = var_2489_to_fp16)[name = string("op_2490_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_2491_cast_fp16 = add(x = var_2490_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_2491_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_2492_cast_fp16 = softmax(axis = var_2412, x = var_2491_cast_fp16)[name = string("op_2492_cast_fp16")];
            tensor<int32, [4]> tile_5 = const()[name = string("tile_5"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_2493_axis_0 = const()[name = string("op_2493_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 16]> var_2493_cast_fp16_0, tensor<fp16, [1, 512, 1, 16]> var_2493_cast_fp16_1, tensor<fp16, [1, 512, 1, 16]> var_2493_cast_fp16_2, tensor<fp16, [1, 512, 1, 16]> var_2493_cast_fp16_3 = split(axis = var_2493_axis_0, split_sizes = tile_5, x = var_2492_cast_fp16)[name = string("op_2493_cast_fp16")];
            tensor<int32, [4]> tile_6 = const()[name = string("tile_6"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_2498_axis_0 = const()[name = string("op_2498_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_2498_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_2498_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_2498_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_2498_cast_fp16_3 = split(axis = var_2498_axis_0, split_sizes = tile_6, x = var_2447_cast_fp16_0)[name = string("op_2498_cast_fp16")];
            fp16 var_2503_to_fp16 = const()[name = string("op_2503_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_2504_cast_fp16 = mul(x = var_2493_cast_fp16_0, y = var_2503_to_fp16)[name = string("op_2504_cast_fp16")];
            string var_2506_equation_0 = const()[name = string("op_2506_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_2506_cast_fp16 = einsum(equation = var_2506_equation_0, values = (var_2498_cast_fp16_0, var_2504_cast_fp16))[name = string("op_2506_cast_fp16")];
            fp16 var_2507_to_fp16 = const()[name = string("op_2507_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_2508_cast_fp16 = mul(x = var_2493_cast_fp16_1, y = var_2507_to_fp16)[name = string("op_2508_cast_fp16")];
            string var_2510_equation_0 = const()[name = string("op_2510_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_2510_cast_fp16 = einsum(equation = var_2510_equation_0, values = (var_2498_cast_fp16_1, var_2508_cast_fp16))[name = string("op_2510_cast_fp16")];
            fp16 var_2511_to_fp16 = const()[name = string("op_2511_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_2512_cast_fp16 = mul(x = var_2493_cast_fp16_2, y = var_2511_to_fp16)[name = string("op_2512_cast_fp16")];
            string var_2514_equation_0 = const()[name = string("op_2514_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_2514_cast_fp16 = einsum(equation = var_2514_equation_0, values = (var_2498_cast_fp16_2, var_2512_cast_fp16))[name = string("op_2514_cast_fp16")];
            fp16 var_2515_to_fp16 = const()[name = string("op_2515_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_2516_cast_fp16 = mul(x = var_2493_cast_fp16_3, y = var_2515_to_fp16)[name = string("op_2516_cast_fp16")];
            string var_2518_equation_0 = const()[name = string("op_2518_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_2518_cast_fp16 = einsum(equation = var_2518_equation_0, values = (var_2498_cast_fp16_3, var_2516_cast_fp16))[name = string("op_2518_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_2519_cast_fp16 = add(x = var_2506_cast_fp16, y = var_2510_cast_fp16)[name = string("op_2519_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_2520_cast_fp16 = add(x = var_2514_cast_fp16, y = var_2518_cast_fp16)[name = string("op_2520_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_2521_cast_fp16 = add(x = var_2519_cast_fp16, y = var_2520_cast_fp16)[name = string("op_2521_cast_fp16")];
            fp16 _inversed_2523_y_0_to_fp16 = const()[name = string("_inversed_2523_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 16]> _inversed_2523_cast_fp16 = mul(x = var_2521_cast_fp16, y = _inversed_2523_y_0_to_fp16)[name = string("_inversed_2523_cast_fp16")];
            string scores_5_equation_0 = const()[name = string("scores_5_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 16]> scores_5_cast_fp16 = einsum(equation = scores_5_equation_0, values = (var_2444_cast_fp16_0, var_2434_cast_fp16_2))[name = string("scores_5_cast_fp16")];
            fp16 var_2526_to_fp16 = const()[name = string("op_2526_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 16]> var_2527_cast_fp16 = mul(x = scores_5_cast_fp16, y = var_2526_to_fp16)[name = string("op_2527_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_2528_cast_fp16 = add(x = var_2527_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_2528_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_2529_cast_fp16 = softmax(axis = var_2412, x = var_2528_cast_fp16)[name = string("op_2529_cast_fp16")];
            tensor<int32, [4]> tile_7 = const()[name = string("tile_7"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_2530_axis_0 = const()[name = string("op_2530_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 16]> var_2530_cast_fp16_0, tensor<fp16, [1, 512, 1, 16]> var_2530_cast_fp16_1, tensor<fp16, [1, 512, 1, 16]> var_2530_cast_fp16_2, tensor<fp16, [1, 512, 1, 16]> var_2530_cast_fp16_3 = split(axis = var_2530_axis_0, split_sizes = tile_7, x = var_2529_cast_fp16)[name = string("op_2530_cast_fp16")];
            tensor<int32, [4]> tile_8 = const()[name = string("tile_8"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_2535_axis_0 = const()[name = string("op_2535_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_2535_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_2535_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_2535_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_2535_cast_fp16_3 = split(axis = var_2535_axis_0, split_sizes = tile_8, x = var_2447_cast_fp16_0)[name = string("op_2535_cast_fp16")];
            fp16 var_2540_to_fp16 = const()[name = string("op_2540_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_2541_cast_fp16 = mul(x = var_2530_cast_fp16_0, y = var_2540_to_fp16)[name = string("op_2541_cast_fp16")];
            string var_2543_equation_0 = const()[name = string("op_2543_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_2543_cast_fp16 = einsum(equation = var_2543_equation_0, values = (var_2535_cast_fp16_0, var_2541_cast_fp16))[name = string("op_2543_cast_fp16")];
            fp16 var_2544_to_fp16 = const()[name = string("op_2544_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_2545_cast_fp16 = mul(x = var_2530_cast_fp16_1, y = var_2544_to_fp16)[name = string("op_2545_cast_fp16")];
            string var_2547_equation_0 = const()[name = string("op_2547_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_2547_cast_fp16 = einsum(equation = var_2547_equation_0, values = (var_2535_cast_fp16_1, var_2545_cast_fp16))[name = string("op_2547_cast_fp16")];
            fp16 var_2548_to_fp16 = const()[name = string("op_2548_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_2549_cast_fp16 = mul(x = var_2530_cast_fp16_2, y = var_2548_to_fp16)[name = string("op_2549_cast_fp16")];
            string var_2551_equation_0 = const()[name = string("op_2551_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_2551_cast_fp16 = einsum(equation = var_2551_equation_0, values = (var_2535_cast_fp16_2, var_2549_cast_fp16))[name = string("op_2551_cast_fp16")];
            fp16 var_2552_to_fp16 = const()[name = string("op_2552_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_2553_cast_fp16 = mul(x = var_2530_cast_fp16_3, y = var_2552_to_fp16)[name = string("op_2553_cast_fp16")];
            string var_2555_equation_0 = const()[name = string("op_2555_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_2555_cast_fp16 = einsum(equation = var_2555_equation_0, values = (var_2535_cast_fp16_3, var_2553_cast_fp16))[name = string("op_2555_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_2556_cast_fp16 = add(x = var_2543_cast_fp16, y = var_2547_cast_fp16)[name = string("op_2556_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_2557_cast_fp16 = add(x = var_2551_cast_fp16, y = var_2555_cast_fp16)[name = string("op_2557_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_2558_cast_fp16 = add(x = var_2556_cast_fp16, y = var_2557_cast_fp16)[name = string("op_2558_cast_fp16")];
            fp16 _inversed_2560_y_0_to_fp16 = const()[name = string("_inversed_2560_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 16]> _inversed_2560_cast_fp16 = mul(x = var_2558_cast_fp16, y = _inversed_2560_y_0_to_fp16)[name = string("_inversed_2560_cast_fp16")];
            string scores_7_equation_0 = const()[name = string("scores_7_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 16]> scores_7_cast_fp16 = einsum(equation = scores_7_equation_0, values = (var_2444_cast_fp16_0, var_2434_cast_fp16_3))[name = string("scores_7_cast_fp16")];
            fp16 var_2563_to_fp16 = const()[name = string("op_2563_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 16]> var_2564_cast_fp16 = mul(x = scores_7_cast_fp16, y = var_2563_to_fp16)[name = string("op_2564_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_2565_cast_fp16 = add(x = var_2564_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_2565_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_2566_cast_fp16 = softmax(axis = var_2412, x = var_2565_cast_fp16)[name = string("op_2566_cast_fp16")];
            tensor<int32, [4]> tile_9 = const()[name = string("tile_9"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_2567_axis_0 = const()[name = string("op_2567_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 16]> var_2567_cast_fp16_0, tensor<fp16, [1, 512, 1, 16]> var_2567_cast_fp16_1, tensor<fp16, [1, 512, 1, 16]> var_2567_cast_fp16_2, tensor<fp16, [1, 512, 1, 16]> var_2567_cast_fp16_3 = split(axis = var_2567_axis_0, split_sizes = tile_9, x = var_2566_cast_fp16)[name = string("op_2567_cast_fp16")];
            tensor<int32, [4]> tile_10 = const()[name = string("tile_10"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_2572_axis_0 = const()[name = string("op_2572_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_2572_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_2572_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_2572_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_2572_cast_fp16_3 = split(axis = var_2572_axis_0, split_sizes = tile_10, x = var_2447_cast_fp16_0)[name = string("op_2572_cast_fp16")];
            fp16 var_2577_to_fp16 = const()[name = string("op_2577_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_2578_cast_fp16 = mul(x = var_2567_cast_fp16_0, y = var_2577_to_fp16)[name = string("op_2578_cast_fp16")];
            string var_2580_equation_0 = const()[name = string("op_2580_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_2580_cast_fp16 = einsum(equation = var_2580_equation_0, values = (var_2572_cast_fp16_0, var_2578_cast_fp16))[name = string("op_2580_cast_fp16")];
            fp16 var_2581_to_fp16 = const()[name = string("op_2581_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_2582_cast_fp16 = mul(x = var_2567_cast_fp16_1, y = var_2581_to_fp16)[name = string("op_2582_cast_fp16")];
            string var_2584_equation_0 = const()[name = string("op_2584_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_2584_cast_fp16 = einsum(equation = var_2584_equation_0, values = (var_2572_cast_fp16_1, var_2582_cast_fp16))[name = string("op_2584_cast_fp16")];
            fp16 var_2585_to_fp16 = const()[name = string("op_2585_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_2586_cast_fp16 = mul(x = var_2567_cast_fp16_2, y = var_2585_to_fp16)[name = string("op_2586_cast_fp16")];
            string var_2588_equation_0 = const()[name = string("op_2588_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_2588_cast_fp16 = einsum(equation = var_2588_equation_0, values = (var_2572_cast_fp16_2, var_2586_cast_fp16))[name = string("op_2588_cast_fp16")];
            fp16 var_2589_to_fp16 = const()[name = string("op_2589_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_2590_cast_fp16 = mul(x = var_2567_cast_fp16_3, y = var_2589_to_fp16)[name = string("op_2590_cast_fp16")];
            string var_2592_equation_0 = const()[name = string("op_2592_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_2592_cast_fp16 = einsum(equation = var_2592_equation_0, values = (var_2572_cast_fp16_3, var_2590_cast_fp16))[name = string("op_2592_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_2593_cast_fp16 = add(x = var_2580_cast_fp16, y = var_2584_cast_fp16)[name = string("op_2593_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_2594_cast_fp16 = add(x = var_2588_cast_fp16, y = var_2592_cast_fp16)[name = string("op_2594_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_2595_cast_fp16 = add(x = var_2593_cast_fp16, y = var_2594_cast_fp16)[name = string("op_2595_cast_fp16")];
            fp16 _inversed_2597_y_0_to_fp16 = const()[name = string("_inversed_2597_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 16]> _inversed_2597_cast_fp16 = mul(x = var_2595_cast_fp16, y = _inversed_2597_y_0_to_fp16)[name = string("_inversed_2597_cast_fp16")];
            string scores_9_equation_0 = const()[name = string("scores_9_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 16]> scores_9_cast_fp16 = einsum(equation = scores_9_equation_0, values = (var_2444_cast_fp16_1, var_2434_cast_fp16_4))[name = string("scores_9_cast_fp16")];
            fp16 var_2600_to_fp16 = const()[name = string("op_2600_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 16]> var_2601_cast_fp16 = mul(x = scores_9_cast_fp16, y = var_2600_to_fp16)[name = string("op_2601_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_2602_cast_fp16 = add(x = var_2601_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_2602_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_2603_cast_fp16 = softmax(axis = var_2412, x = var_2602_cast_fp16)[name = string("op_2603_cast_fp16")];
            tensor<int32, [4]> tile_11 = const()[name = string("tile_11"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_2604_axis_0 = const()[name = string("op_2604_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 16]> var_2604_cast_fp16_0, tensor<fp16, [1, 512, 1, 16]> var_2604_cast_fp16_1, tensor<fp16, [1, 512, 1, 16]> var_2604_cast_fp16_2, tensor<fp16, [1, 512, 1, 16]> var_2604_cast_fp16_3 = split(axis = var_2604_axis_0, split_sizes = tile_11, x = var_2603_cast_fp16)[name = string("op_2604_cast_fp16")];
            tensor<int32, [4]> tile_12 = const()[name = string("tile_12"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_2609_axis_0 = const()[name = string("op_2609_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_2609_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_2609_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_2609_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_2609_cast_fp16_3 = split(axis = var_2609_axis_0, split_sizes = tile_12, x = var_2447_cast_fp16_1)[name = string("op_2609_cast_fp16")];
            fp16 var_2614_to_fp16 = const()[name = string("op_2614_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_2615_cast_fp16 = mul(x = var_2604_cast_fp16_0, y = var_2614_to_fp16)[name = string("op_2615_cast_fp16")];
            string var_2617_equation_0 = const()[name = string("op_2617_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_2617_cast_fp16 = einsum(equation = var_2617_equation_0, values = (var_2609_cast_fp16_0, var_2615_cast_fp16))[name = string("op_2617_cast_fp16")];
            fp16 var_2618_to_fp16 = const()[name = string("op_2618_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_2619_cast_fp16 = mul(x = var_2604_cast_fp16_1, y = var_2618_to_fp16)[name = string("op_2619_cast_fp16")];
            string var_2621_equation_0 = const()[name = string("op_2621_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_2621_cast_fp16 = einsum(equation = var_2621_equation_0, values = (var_2609_cast_fp16_1, var_2619_cast_fp16))[name = string("op_2621_cast_fp16")];
            fp16 var_2622_to_fp16 = const()[name = string("op_2622_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_2623_cast_fp16 = mul(x = var_2604_cast_fp16_2, y = var_2622_to_fp16)[name = string("op_2623_cast_fp16")];
            string var_2625_equation_0 = const()[name = string("op_2625_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_2625_cast_fp16 = einsum(equation = var_2625_equation_0, values = (var_2609_cast_fp16_2, var_2623_cast_fp16))[name = string("op_2625_cast_fp16")];
            fp16 var_2626_to_fp16 = const()[name = string("op_2626_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_2627_cast_fp16 = mul(x = var_2604_cast_fp16_3, y = var_2626_to_fp16)[name = string("op_2627_cast_fp16")];
            string var_2629_equation_0 = const()[name = string("op_2629_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_2629_cast_fp16 = einsum(equation = var_2629_equation_0, values = (var_2609_cast_fp16_3, var_2627_cast_fp16))[name = string("op_2629_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_2630_cast_fp16 = add(x = var_2617_cast_fp16, y = var_2621_cast_fp16)[name = string("op_2630_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_2631_cast_fp16 = add(x = var_2625_cast_fp16, y = var_2629_cast_fp16)[name = string("op_2631_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_2632_cast_fp16 = add(x = var_2630_cast_fp16, y = var_2631_cast_fp16)[name = string("op_2632_cast_fp16")];
            fp16 _inversed_2634_y_0_to_fp16 = const()[name = string("_inversed_2634_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 16]> _inversed_2634_cast_fp16 = mul(x = var_2632_cast_fp16, y = _inversed_2634_y_0_to_fp16)[name = string("_inversed_2634_cast_fp16")];
            string scores_11_equation_0 = const()[name = string("scores_11_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 16]> scores_11_cast_fp16 = einsum(equation = scores_11_equation_0, values = (var_2444_cast_fp16_1, var_2434_cast_fp16_5))[name = string("scores_11_cast_fp16")];
            fp16 var_2637_to_fp16 = const()[name = string("op_2637_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 16]> var_2638_cast_fp16 = mul(x = scores_11_cast_fp16, y = var_2637_to_fp16)[name = string("op_2638_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_2639_cast_fp16 = add(x = var_2638_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_2639_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_2640_cast_fp16 = softmax(axis = var_2412, x = var_2639_cast_fp16)[name = string("op_2640_cast_fp16")];
            tensor<int32, [4]> tile_13 = const()[name = string("tile_13"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_2641_axis_0 = const()[name = string("op_2641_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 16]> var_2641_cast_fp16_0, tensor<fp16, [1, 512, 1, 16]> var_2641_cast_fp16_1, tensor<fp16, [1, 512, 1, 16]> var_2641_cast_fp16_2, tensor<fp16, [1, 512, 1, 16]> var_2641_cast_fp16_3 = split(axis = var_2641_axis_0, split_sizes = tile_13, x = var_2640_cast_fp16)[name = string("op_2641_cast_fp16")];
            tensor<int32, [4]> tile_14 = const()[name = string("tile_14"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_2646_axis_0 = const()[name = string("op_2646_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_2646_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_2646_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_2646_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_2646_cast_fp16_3 = split(axis = var_2646_axis_0, split_sizes = tile_14, x = var_2447_cast_fp16_1)[name = string("op_2646_cast_fp16")];
            fp16 var_2651_to_fp16 = const()[name = string("op_2651_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_2652_cast_fp16 = mul(x = var_2641_cast_fp16_0, y = var_2651_to_fp16)[name = string("op_2652_cast_fp16")];
            string var_2654_equation_0 = const()[name = string("op_2654_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_2654_cast_fp16 = einsum(equation = var_2654_equation_0, values = (var_2646_cast_fp16_0, var_2652_cast_fp16))[name = string("op_2654_cast_fp16")];
            fp16 var_2655_to_fp16 = const()[name = string("op_2655_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_2656_cast_fp16 = mul(x = var_2641_cast_fp16_1, y = var_2655_to_fp16)[name = string("op_2656_cast_fp16")];
            string var_2658_equation_0 = const()[name = string("op_2658_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_2658_cast_fp16 = einsum(equation = var_2658_equation_0, values = (var_2646_cast_fp16_1, var_2656_cast_fp16))[name = string("op_2658_cast_fp16")];
            fp16 var_2659_to_fp16 = const()[name = string("op_2659_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_2660_cast_fp16 = mul(x = var_2641_cast_fp16_2, y = var_2659_to_fp16)[name = string("op_2660_cast_fp16")];
            string var_2662_equation_0 = const()[name = string("op_2662_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_2662_cast_fp16 = einsum(equation = var_2662_equation_0, values = (var_2646_cast_fp16_2, var_2660_cast_fp16))[name = string("op_2662_cast_fp16")];
            fp16 var_2663_to_fp16 = const()[name = string("op_2663_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_2664_cast_fp16 = mul(x = var_2641_cast_fp16_3, y = var_2663_to_fp16)[name = string("op_2664_cast_fp16")];
            string var_2666_equation_0 = const()[name = string("op_2666_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_2666_cast_fp16 = einsum(equation = var_2666_equation_0, values = (var_2646_cast_fp16_3, var_2664_cast_fp16))[name = string("op_2666_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_2667_cast_fp16 = add(x = var_2654_cast_fp16, y = var_2658_cast_fp16)[name = string("op_2667_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_2668_cast_fp16 = add(x = var_2662_cast_fp16, y = var_2666_cast_fp16)[name = string("op_2668_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_2669_cast_fp16 = add(x = var_2667_cast_fp16, y = var_2668_cast_fp16)[name = string("op_2669_cast_fp16")];
            fp16 _inversed_2671_y_0_to_fp16 = const()[name = string("_inversed_2671_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 16]> _inversed_2671_cast_fp16 = mul(x = var_2669_cast_fp16, y = _inversed_2671_y_0_to_fp16)[name = string("_inversed_2671_cast_fp16")];
            string scores_13_equation_0 = const()[name = string("scores_13_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 16]> scores_13_cast_fp16 = einsum(equation = scores_13_equation_0, values = (var_2444_cast_fp16_1, var_2434_cast_fp16_6))[name = string("scores_13_cast_fp16")];
            fp16 var_2674_to_fp16 = const()[name = string("op_2674_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 16]> var_2675_cast_fp16 = mul(x = scores_13_cast_fp16, y = var_2674_to_fp16)[name = string("op_2675_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_2676_cast_fp16 = add(x = var_2675_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_2676_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_2677_cast_fp16 = softmax(axis = var_2412, x = var_2676_cast_fp16)[name = string("op_2677_cast_fp16")];
            tensor<int32, [4]> tile_15 = const()[name = string("tile_15"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_2678_axis_0 = const()[name = string("op_2678_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 16]> var_2678_cast_fp16_0, tensor<fp16, [1, 512, 1, 16]> var_2678_cast_fp16_1, tensor<fp16, [1, 512, 1, 16]> var_2678_cast_fp16_2, tensor<fp16, [1, 512, 1, 16]> var_2678_cast_fp16_3 = split(axis = var_2678_axis_0, split_sizes = tile_15, x = var_2677_cast_fp16)[name = string("op_2678_cast_fp16")];
            tensor<int32, [4]> tile_16 = const()[name = string("tile_16"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_2683_axis_0 = const()[name = string("op_2683_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_2683_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_2683_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_2683_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_2683_cast_fp16_3 = split(axis = var_2683_axis_0, split_sizes = tile_16, x = var_2447_cast_fp16_1)[name = string("op_2683_cast_fp16")];
            fp16 var_2688_to_fp16 = const()[name = string("op_2688_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_2689_cast_fp16 = mul(x = var_2678_cast_fp16_0, y = var_2688_to_fp16)[name = string("op_2689_cast_fp16")];
            string var_2691_equation_0 = const()[name = string("op_2691_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_2691_cast_fp16 = einsum(equation = var_2691_equation_0, values = (var_2683_cast_fp16_0, var_2689_cast_fp16))[name = string("op_2691_cast_fp16")];
            fp16 var_2692_to_fp16 = const()[name = string("op_2692_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_2693_cast_fp16 = mul(x = var_2678_cast_fp16_1, y = var_2692_to_fp16)[name = string("op_2693_cast_fp16")];
            string var_2695_equation_0 = const()[name = string("op_2695_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_2695_cast_fp16 = einsum(equation = var_2695_equation_0, values = (var_2683_cast_fp16_1, var_2693_cast_fp16))[name = string("op_2695_cast_fp16")];
            fp16 var_2696_to_fp16 = const()[name = string("op_2696_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_2697_cast_fp16 = mul(x = var_2678_cast_fp16_2, y = var_2696_to_fp16)[name = string("op_2697_cast_fp16")];
            string var_2699_equation_0 = const()[name = string("op_2699_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_2699_cast_fp16 = einsum(equation = var_2699_equation_0, values = (var_2683_cast_fp16_2, var_2697_cast_fp16))[name = string("op_2699_cast_fp16")];
            fp16 var_2700_to_fp16 = const()[name = string("op_2700_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_2701_cast_fp16 = mul(x = var_2678_cast_fp16_3, y = var_2700_to_fp16)[name = string("op_2701_cast_fp16")];
            string var_2703_equation_0 = const()[name = string("op_2703_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_2703_cast_fp16 = einsum(equation = var_2703_equation_0, values = (var_2683_cast_fp16_3, var_2701_cast_fp16))[name = string("op_2703_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_2704_cast_fp16 = add(x = var_2691_cast_fp16, y = var_2695_cast_fp16)[name = string("op_2704_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_2705_cast_fp16 = add(x = var_2699_cast_fp16, y = var_2703_cast_fp16)[name = string("op_2705_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_2706_cast_fp16 = add(x = var_2704_cast_fp16, y = var_2705_cast_fp16)[name = string("op_2706_cast_fp16")];
            fp16 _inversed_2708_y_0_to_fp16 = const()[name = string("_inversed_2708_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 16]> _inversed_2708_cast_fp16 = mul(x = var_2706_cast_fp16, y = _inversed_2708_y_0_to_fp16)[name = string("_inversed_2708_cast_fp16")];
            string scores_15_equation_0 = const()[name = string("scores_15_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 16]> scores_15_cast_fp16 = einsum(equation = scores_15_equation_0, values = (var_2444_cast_fp16_1, var_2434_cast_fp16_7))[name = string("scores_15_cast_fp16")];
            fp16 var_2711_to_fp16 = const()[name = string("op_2711_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 16]> var_2712_cast_fp16 = mul(x = scores_15_cast_fp16, y = var_2711_to_fp16)[name = string("op_2712_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_2713_cast_fp16 = add(x = var_2712_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_2713_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_2714_cast_fp16 = softmax(axis = var_2412, x = var_2713_cast_fp16)[name = string("op_2714_cast_fp16")];
            tensor<int32, [4]> tile_17 = const()[name = string("tile_17"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_2715_axis_0 = const()[name = string("op_2715_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 16]> var_2715_cast_fp16_0, tensor<fp16, [1, 512, 1, 16]> var_2715_cast_fp16_1, tensor<fp16, [1, 512, 1, 16]> var_2715_cast_fp16_2, tensor<fp16, [1, 512, 1, 16]> var_2715_cast_fp16_3 = split(axis = var_2715_axis_0, split_sizes = tile_17, x = var_2714_cast_fp16)[name = string("op_2715_cast_fp16")];
            tensor<int32, [4]> tile_18 = const()[name = string("tile_18"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_2720_axis_0 = const()[name = string("op_2720_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_2720_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_2720_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_2720_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_2720_cast_fp16_3 = split(axis = var_2720_axis_0, split_sizes = tile_18, x = var_2447_cast_fp16_1)[name = string("op_2720_cast_fp16")];
            fp16 var_2725_to_fp16 = const()[name = string("op_2725_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_2726_cast_fp16 = mul(x = var_2715_cast_fp16_0, y = var_2725_to_fp16)[name = string("op_2726_cast_fp16")];
            string var_2728_equation_0 = const()[name = string("op_2728_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_2728_cast_fp16 = einsum(equation = var_2728_equation_0, values = (var_2720_cast_fp16_0, var_2726_cast_fp16))[name = string("op_2728_cast_fp16")];
            fp16 var_2729_to_fp16 = const()[name = string("op_2729_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_2730_cast_fp16 = mul(x = var_2715_cast_fp16_1, y = var_2729_to_fp16)[name = string("op_2730_cast_fp16")];
            string var_2732_equation_0 = const()[name = string("op_2732_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_2732_cast_fp16 = einsum(equation = var_2732_equation_0, values = (var_2720_cast_fp16_1, var_2730_cast_fp16))[name = string("op_2732_cast_fp16")];
            fp16 var_2733_to_fp16 = const()[name = string("op_2733_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_2734_cast_fp16 = mul(x = var_2715_cast_fp16_2, y = var_2733_to_fp16)[name = string("op_2734_cast_fp16")];
            string var_2736_equation_0 = const()[name = string("op_2736_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_2736_cast_fp16 = einsum(equation = var_2736_equation_0, values = (var_2720_cast_fp16_2, var_2734_cast_fp16))[name = string("op_2736_cast_fp16")];
            fp16 var_2737_to_fp16 = const()[name = string("op_2737_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_2738_cast_fp16 = mul(x = var_2715_cast_fp16_3, y = var_2737_to_fp16)[name = string("op_2738_cast_fp16")];
            string var_2740_equation_0 = const()[name = string("op_2740_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_2740_cast_fp16 = einsum(equation = var_2740_equation_0, values = (var_2720_cast_fp16_3, var_2738_cast_fp16))[name = string("op_2740_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_2741_cast_fp16 = add(x = var_2728_cast_fp16, y = var_2732_cast_fp16)[name = string("op_2741_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_2742_cast_fp16 = add(x = var_2736_cast_fp16, y = var_2740_cast_fp16)[name = string("op_2742_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_2743_cast_fp16 = add(x = var_2741_cast_fp16, y = var_2742_cast_fp16)[name = string("op_2743_cast_fp16")];
            fp16 _inversed_2745_y_0_to_fp16 = const()[name = string("_inversed_2745_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 16]> _inversed_2745_cast_fp16 = mul(x = var_2743_cast_fp16, y = _inversed_2745_y_0_to_fp16)[name = string("_inversed_2745_cast_fp16")];
            bool var_2747_interleave_0 = const()[name = string("op_2747_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 2048, 1, 16]> var_2747_cast_fp16 = concat(axis = var_2412, interleave = var_2747_interleave_0, values = (_inversed_2486_cast_fp16, _inversed_2523_cast_fp16, _inversed_2560_cast_fp16, _inversed_2597_cast_fp16, _inversed_2634_cast_fp16, _inversed_2671_cast_fp16, _inversed_2708_cast_fp16, _inversed_2745_cast_fp16))[name = string("op_2747_cast_fp16")];
            tensor<int32, [4]> var_2748 = const()[name = string("op_2748"), val = tensor<int32, [4]>([2, 4, 256, 16])];
            tensor<fp16, [2, 4, 256, 16]> var_2749_cast_fp16 = reshape(shape = var_2748, x = var_2747_cast_fp16)[name = string("op_2749_cast_fp16")];
            tensor<int32, [4]> transpose_1_perm_0 = const()[name = string("transpose_1_perm_0"), val = tensor<int32, [4]>([3, 0, 1, 2])];
            tensor<int32, [2]> var_2768 = const()[name = string("op_2768"), val = tensor<int32, [2]>([16, 2048])];
            tensor<fp16, [16, 2, 4, 256]> transpose_1_cast_fp16 = transpose(perm = transpose_1_perm_0, x = var_2749_cast_fp16)[name = string("transpose_112")];
            tensor<fp16, [16, 2048]> attention_output_3_cast_fp16 = reshape(shape = var_2768, x = transpose_1_cast_fp16)[name = string("attention_output_3_cast_fp16")];
            tensor<fp16, [16, 2048]> var_2770_cast_fp16 = sigmoid(x = gate_3_cast_fp16)[name = string("op_2770_cast_fp16")];
            tensor<fp16, [16, 2048]> input_45_cast_fp16 = mul(x = attention_output_3_cast_fp16, y = var_2770_cast_fp16)[name = string("input_45_cast_fp16")];
            tensor<fp16, [1024, 2048]> completions_0_o_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 2048]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(53090624))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(54663552))))[name = string("completions_0_o_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 1024]> linear_27_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = completions_0_o_proj_weight_promoted_to_fp16_palettized, x = input_45_cast_fp16)[name = string("linear_27_cast_fp16")];
            tensor<fp16, [16, 1024]> value_57_cast_fp16 = add(x = value_43_cast_fp16, y = linear_27_cast_fp16)[name = string("value_57_cast_fp16")];
            tensor<fp16, [16, 1024]> var_2775_cast_fp16 = mul(x = value_57_cast_fp16, y = value_57_cast_fp16)[name = string("op_2775_cast_fp16")];
            tensor<int32, [1]> variance_25_axes_0 = const()[name = string("variance_25_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_25_keep_dims_0 = const()[name = string("variance_25_keep_dims_0"), val = bool(true)];
            tensor<fp16, [16, 1]> variance_25_cast_fp16 = reduce_mean(axes = variance_25_axes_0, keep_dims = variance_25_keep_dims_0, x = var_2775_cast_fp16)[name = string("variance_25_cast_fp16")];
            fp16 var_2778_to_fp16 = const()[name = string("op_2778_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [16, 1]> var_2779_cast_fp16 = add(x = variance_25_cast_fp16, y = var_2778_to_fp16)[name = string("op_2779_cast_fp16")];
            fp32 var_2780_epsilon_0 = const()[name = string("op_2780_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [16, 1]> var_2780_cast_fp16 = rsqrt(epsilon = var_2780_epsilon_0, x = var_2779_cast_fp16)[name = string("op_2780_cast_fp16")];
            tensor<fp16, [16, 1024]> var_2781_cast_fp16 = mul(x = value_57_cast_fp16, y = var_2780_cast_fp16)[name = string("op_2781_cast_fp16")];
            tensor<fp16, [1024]> var_2783_to_fp16 = const()[name = string("op_2783_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(54671808)))];
            tensor<fp16, [16, 1024]> input_47_cast_fp16 = mul(x = var_2781_cast_fp16, y = var_2783_to_fp16)[name = string("input_47_cast_fp16")];
            tensor<fp16, [3584, 1024]> completions_0_gate_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(54673920))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(57426496))))[name = string("completions_0_gate_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 3584]> linear_28_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = completions_0_gate_proj_weight_promoted_to_fp16_palettized, x = input_47_cast_fp16)[name = string("linear_28_cast_fp16")];
            tensor<fp16, [16, 3584]> var_2787_cast_fp16 = silu(x = linear_28_cast_fp16)[name = string("op_2787_cast_fp16")];
            tensor<fp16, [3584, 1024]> completions_0_up_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(57455232))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(60207808))))[name = string("completions_0_up_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 3584]> linear_29_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = completions_0_up_proj_weight_promoted_to_fp16_palettized, x = input_47_cast_fp16)[name = string("linear_29_cast_fp16")];
            tensor<fp16, [16, 3584]> input_51_cast_fp16 = mul(x = var_2787_cast_fp16, y = linear_29_cast_fp16)[name = string("input_51_cast_fp16")];
            tensor<fp16, [1024, 3584]> completions_0_down_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 3584]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(60236544))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(62989120))))[name = string("completions_0_down_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 1024]> linear_30_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = completions_0_down_proj_weight_promoted_to_fp16_palettized, x = input_51_cast_fp16)[name = string("linear_30_cast_fp16")];
            tensor<fp16, [16, 1024]> value_59_cast_fp16 = add(x = value_57_cast_fp16, y = linear_30_cast_fp16)[name = string("value_59_cast_fp16")];
            int32 var_2821 = const()[name = string("op_2821"), val = int32(-1)];
            tensor<fp16, [16, 1024]> var_2836_cast_fp16 = mul(x = value_59_cast_fp16, y = value_59_cast_fp16)[name = string("op_2836_cast_fp16")];
            tensor<int32, [1]> variance_27_axes_0 = const()[name = string("variance_27_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_27_keep_dims_0 = const()[name = string("variance_27_keep_dims_0"), val = bool(true)];
            tensor<fp16, [16, 1]> variance_27_cast_fp16 = reduce_mean(axes = variance_27_axes_0, keep_dims = variance_27_keep_dims_0, x = var_2836_cast_fp16)[name = string("variance_27_cast_fp16")];
            fp16 var_2839_to_fp16 = const()[name = string("op_2839_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [16, 1]> var_2840_cast_fp16 = add(x = variance_27_cast_fp16, y = var_2839_to_fp16)[name = string("op_2840_cast_fp16")];
            fp32 var_2841_epsilon_0 = const()[name = string("op_2841_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [16, 1]> var_2841_cast_fp16 = rsqrt(epsilon = var_2841_epsilon_0, x = var_2840_cast_fp16)[name = string("op_2841_cast_fp16")];
            tensor<fp16, [16, 1024]> var_2842_cast_fp16 = mul(x = value_59_cast_fp16, y = var_2841_cast_fp16)[name = string("op_2842_cast_fp16")];
            tensor<fp16, [1024]> var_2844_to_fp16 = const()[name = string("op_2844_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(62997376)))];
            tensor<fp16, [16, 1024]> input_53_cast_fp16 = mul(x = var_2842_cast_fp16, y = var_2844_to_fp16)[name = string("input_53_cast_fp16")];
            tensor<fp16, [6144, 1024]> groups_1_0_in_proj_qkv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(62999488))), lut = tensor<fp16, [384, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(67718144))))[name = string("groups_1_0_in_proj_qkv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 6144]> linear_31_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = groups_1_0_in_proj_qkv_weight_promoted_to_fp16_palettized, x = input_53_cast_fp16)[name = string("linear_31_cast_fp16")];
            tensor<int32, [2]> var_2848_perm_0 = const()[name = string("op_2848_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> mixed_7_axes_0 = const()[name = string("mixed_7_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [6144, 16]> var_2848_cast_fp16 = transpose(perm = var_2848_perm_0, x = linear_31_cast_fp16)[name = string("transpose_111")];
            tensor<fp16, [1, 6144, 16]> mixed_7_cast_fp16 = expand_dims(axes = mixed_7_axes_0, x = var_2848_cast_fp16)[name = string("mixed_7_cast_fp16")];
            bool convolution_input_7_interleave_0 = const()[name = string("convolution_input_7_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 6144, 19]> convolution_input_7_cast_fp16 = concat(axis = var_2821, interleave = convolution_input_7_interleave_0, values = (conv4, mixed_7_cast_fp16))[name = string("convolution_input_7_cast_fp16")];
            string input_55_pad_type_0 = const()[name = string("input_55_pad_type_0"), val = string("valid")];
            int32 input_55_groups_0 = const()[name = string("input_55_groups_0"), val = int32(6144)];
            tensor<int32, [1]> input_55_strides_0 = const()[name = string("input_55_strides_0"), val = tensor<int32, [1]>([1])];
            tensor<int32, [2]> input_55_pad_0 = const()[name = string("input_55_pad_0"), val = tensor<int32, [2]>([0, 0])];
            tensor<int32, [1]> input_55_dilations_0 = const()[name = string("input_55_dilations_0"), val = tensor<int32, [1]>([1])];
            tensor<fp16, [6144, 1, 4]> groups_1_0_conv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1, 4]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(67767360))), lut = tensor<fp16, [384, 1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(67785856))))[name = string("groups_1_0_conv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 6144, 16]> input_55_cast_fp16 = conv(dilations = input_55_dilations_0, groups = input_55_groups_0, pad = input_55_pad_0, pad_type = input_55_pad_type_0, strides = input_55_strides_0, weight = groups_1_0_conv_weight_promoted_to_fp16_palettized, x = convolution_input_7_cast_fp16)[name = string("input_55_cast_fp16")];
            tensor<fp16, [1, 6144, 16]> var_2857_cast_fp16 = silu(x = input_55_cast_fp16)[name = string("op_2857_cast_fp16")];
            tensor<int32, [3]> var_2858_perm_0 = const()[name = string("op_2858_perm_0"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [3]> var_2861_begin_0 = const()[name = string("op_2861_begin_0"), val = tensor<int32, [3]>([0, 0, 16])];
            tensor<int32, [3]> var_2861_end_0 = const()[name = string("op_2861_end_0"), val = tensor<int32, [3]>([1, 6144, 19])];
            tensor<bool, [3]> var_2861_end_mask_0 = const()[name = string("op_2861_end_mask_0"), val = tensor<bool, [3]>([true, true, true])];
            tensor<fp16, [1, 6144, 3]> conv4_out = slice_by_index(begin = var_2861_begin_0, end = var_2861_end_0, end_mask = var_2861_end_mask_0, x = convolution_input_7_cast_fp16)[name = string("op_2861_cast_fp16")];
            tensor<int32, [3]> var_2862 = const()[name = string("op_2862"), val = tensor<int32, [3]>([2048, 2048, 2048])];
            int32 var_2863_axis_0 = const()[name = string("op_2863_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 16, 6144]> var_2858_cast_fp16 = transpose(perm = var_2858_perm_0, x = var_2857_cast_fp16)[name = string("transpose_110")];
            tensor<fp16, [1, 16, 2048]> var_2863_cast_fp16_0, tensor<fp16, [1, 16, 2048]> var_2863_cast_fp16_1, tensor<fp16, [1, 16, 2048]> var_2863_cast_fp16_2 = split(axis = var_2863_axis_0, split_sizes = var_2862, x = var_2858_cast_fp16)[name = string("op_2863_cast_fp16")];
            tensor<int32, [4]> var_2867 = const()[name = string("op_2867"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<fp16, [1, 16, 16, 128]> value_63_cast_fp16 = reshape(shape = var_2867, x = var_2863_cast_fp16_0)[name = string("value_63_cast_fp16")];
            tensor<int32, [4]> var_2869 = const()[name = string("op_2869"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<fp16, [1, 16, 16, 128]> value_65_cast_fp16 = reshape(shape = var_2869, x = var_2863_cast_fp16_1)[name = string("value_65_cast_fp16")];
            tensor<int32, [4]> var_2871 = const()[name = string("op_2871"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<fp16, [1, 16, 16, 128]> value_67_cast_fp16 = reshape(shape = var_2871, x = var_2863_cast_fp16_2)[name = string("value_67_cast_fp16")];
            tensor<fp16, [1, 16, 16, 128]> var_2873_cast_fp16 = mul(x = value_63_cast_fp16, y = value_63_cast_fp16)[name = string("op_2873_cast_fp16")];
            tensor<int32, [1]> var_2875_axes_0 = const()[name = string("op_2875_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_2875_keep_dims_0 = const()[name = string("op_2875_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 16, 16, 1]> var_2875_cast_fp16 = reduce_sum(axes = var_2875_axes_0, keep_dims = var_2875_keep_dims_0, x = var_2873_cast_fp16)[name = string("op_2875_cast_fp16")];
            fp16 var_2876_to_fp16 = const()[name = string("op_2876_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 16, 16, 1]> var_2877_cast_fp16 = add(x = var_2875_cast_fp16, y = var_2876_to_fp16)[name = string("op_2877_cast_fp16")];
            fp32 var_2878_epsilon_0 = const()[name = string("op_2878_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 16, 16, 1]> var_2878_cast_fp16 = rsqrt(epsilon = var_2878_epsilon_0, x = var_2877_cast_fp16)[name = string("op_2878_cast_fp16")];
            tensor<fp16, [1, 16, 16, 128]> var_2879_cast_fp16 = mul(x = value_63_cast_fp16, y = var_2878_cast_fp16)[name = string("op_2879_cast_fp16")];
            tensor<int32, [4]> var_2880_perm_0 = const()[name = string("op_2880_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            fp16 _inversed_query_19_y_0_to_fp16 = const()[name = string("_inversed_query_19_y_0_to_fp16"), val = fp16(0x1.6ap-4)];
            tensor<fp16, [1, 16, 16, 128]> var_2880_cast_fp16 = transpose(perm = var_2880_perm_0, x = var_2879_cast_fp16)[name = string("transpose_109")];
            tensor<fp16, [1, 16, 16, 128]> _inversed_query_19_cast_fp16 = mul(x = var_2880_cast_fp16, y = _inversed_query_19_y_0_to_fp16)[name = string("_inversed_query_19_cast_fp16")];
            tensor<fp16, [1, 16, 16, 128]> var_2883_cast_fp16 = mul(x = value_65_cast_fp16, y = value_65_cast_fp16)[name = string("op_2883_cast_fp16")];
            tensor<int32, [1]> var_2885_axes_0 = const()[name = string("op_2885_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_2885_keep_dims_0 = const()[name = string("op_2885_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 16, 16, 1]> var_2885_cast_fp16 = reduce_sum(axes = var_2885_axes_0, keep_dims = var_2885_keep_dims_0, x = var_2883_cast_fp16)[name = string("op_2885_cast_fp16")];
            fp16 var_2886_to_fp16 = const()[name = string("op_2886_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 16, 16, 1]> var_2887_cast_fp16 = add(x = var_2885_cast_fp16, y = var_2886_to_fp16)[name = string("op_2887_cast_fp16")];
            fp32 var_2888_epsilon_0 = const()[name = string("op_2888_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 16, 16, 1]> var_2888_cast_fp16 = rsqrt(epsilon = var_2888_epsilon_0, x = var_2887_cast_fp16)[name = string("op_2888_cast_fp16")];
            tensor<fp16, [1, 16, 16, 128]> var_2889_cast_fp16 = mul(x = value_65_cast_fp16, y = var_2888_cast_fp16)[name = string("op_2889_cast_fp16")];
            tensor<int32, [4]> key_19_perm_0 = const()[name = string("key_19_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<int32, [4]> value_69_perm_0 = const()[name = string("value_69_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<fp16, [16, 1024]> groups_1_0_in_proj_b_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(67835072))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(67847424))))[name = string("groups_1_0_in_proj_b_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16]> linear_32_bias_0_to_fp16 = const()[name = string("linear_32_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(4862656)))];
            tensor<fp16, [16, 16]> linear_32_cast_fp16 = linear(bias = linear_32_bias_0_to_fp16, weight = groups_1_0_in_proj_b_weight_promoted_to_fp16_palettized, x = input_53_cast_fp16)[name = string("linear_32_cast_fp16")];
            tensor<fp16, [16, 16]> var_2894_cast_fp16 = sigmoid(x = linear_32_cast_fp16)[name = string("op_2894_cast_fp16")];
            tensor<int32, [2]> var_2895_perm_0 = const()[name = string("op_2895_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> beta_7_axes_0 = const()[name = string("beta_7_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 16]> var_2895_cast_fp16 = transpose(perm = var_2895_perm_0, x = var_2894_cast_fp16)[name = string("transpose_108")];
            tensor<fp16, [1, 16, 16]> beta_7_cast_fp16 = expand_dims(axes = beta_7_axes_0, x = var_2895_cast_fp16)[name = string("beta_7_cast_fp16")];
            tensor<fp16, [16, 1024]> op_2901_weight_0_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(67847616))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(67859968))))[name = string("op_2901_weight_0_to_fp16_palettized")];
            tensor<fp16, [16]> var_2901_bias_0_to_fp16 = const()[name = string("op_2901_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(67860160)))];
            tensor<fp16, [16, 16]> var_2901_cast_fp16 = linear(bias = var_2901_bias_0_to_fp16, weight = op_2901_weight_0_to_fp16_palettized, x = input_53_cast_fp16)[name = string("op_2901_cast_fp16")];
            tensor<fp16, [16, 16]> var_2902_cast_fp16 = softplus(x = var_2901_cast_fp16)[name = string("op_2902_cast_fp16")];
            tensor<fp16, [16]> var_2898_promoted_to_fp16 = const()[name = string("op_2898_promoted_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(67860288)))];
            tensor<fp16, [16, 16]> var_2903_cast_fp16 = mul(x = var_2898_promoted_to_fp16, y = var_2902_cast_fp16)[name = string("op_2903_cast_fp16")];
            tensor<int32, [2]> var_2904_perm_0 = const()[name = string("op_2904_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> decay_7_axes_0 = const()[name = string("decay_7_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 16]> var_2904_cast_fp16 = transpose(perm = var_2904_perm_0, x = var_2903_cast_fp16)[name = string("transpose_107")];
            tensor<fp16, [1, 16, 16]> decay_7_cast_fp16 = expand_dims(axes = decay_7_axes_0, x = var_2904_cast_fp16)[name = string("decay_7_cast_fp16")];
            tensor<fp16, [2048, 1024]> groups_1_0_in_proj_z_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [2048, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(67860416))), lut = tensor<fp16, [128, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(69433344))))[name = string("groups_1_0_in_proj_z_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 2048]> linear_34_cast_fp16 = linear(bias = linear_3_bias_0_to_fp16, weight = groups_1_0_in_proj_z_weight_promoted_to_fp16_palettized, x = input_53_cast_fp16)[name = string("linear_34_cast_fp16")];
            tensor<int32, [3]> var_2912_begin_0 = const()[name = string("op_2912_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_2912_end_0 = const()[name = string("op_2912_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_2912_end_mask_0 = const()[name = string("op_2912_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_2912_squeeze_mask_0 = const()[name = string("op_2912_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_2912_cast_fp16 = slice_by_index(begin = var_2912_begin_0, end = var_2912_end_0, end_mask = var_2912_end_mask_0, squeeze_mask = var_2912_squeeze_mask_0, x = decay_7_cast_fp16)[name = string("op_2912_cast_fp16")];
            tensor<fp16, [1, 16]> var_2913_cast_fp16 = exp(x = var_2912_cast_fp16)[name = string("op_2913_cast_fp16")];
            tensor<int32, [1]> var_2914_axes_0 = const()[name = string("op_2914_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_2914_cast_fp16 = expand_dims(axes = var_2914_axes_0, x = var_2913_cast_fp16)[name = string("op_2914_cast_fp16")];
            tensor<int32, [1]> var_2915_axes_0 = const()[name = string("op_2915_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_2915_cast_fp16 = expand_dims(axes = var_2915_axes_0, x = var_2914_cast_fp16)[name = string("op_2915_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_193_cast_fp16 = mul(x = rec4, y = var_2915_cast_fp16)[name = string("state_193_cast_fp16")];
            tensor<int32, [4]> key_token_97_begin_0 = const()[name = string("key_token_97_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> key_token_97_end_0 = const()[name = string("key_token_97_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> key_token_97_end_mask_0 = const()[name = string("key_token_97_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_97_squeeze_mask_0 = const()[name = string("key_token_97_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 16, 128]> key_19_cast_fp16 = transpose(perm = key_19_perm_0, x = var_2889_cast_fp16)[name = string("transpose_106")];
            tensor<fp16, [1, 16, 128]> key_token_97_cast_fp16 = slice_by_index(begin = key_token_97_begin_0, end = key_token_97_end_0, end_mask = key_token_97_end_mask_0, squeeze_mask = key_token_97_squeeze_mask_0, x = key_19_cast_fp16)[name = string("key_token_97_cast_fp16")];
            tensor<int32, [4]> value_token_97_begin_0 = const()[name = string("value_token_97_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> value_token_97_end_0 = const()[name = string("value_token_97_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> value_token_97_end_mask_0 = const()[name = string("value_token_97_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_97_squeeze_mask_0 = const()[name = string("value_token_97_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 16, 128]> value_69_cast_fp16 = transpose(perm = value_69_perm_0, x = value_67_cast_fp16)[name = string("transpose_105")];
            tensor<fp16, [1, 16, 128]> value_token_97_cast_fp16 = slice_by_index(begin = value_token_97_begin_0, end = value_token_97_end_0, end_mask = value_token_97_end_mask_0, squeeze_mask = value_token_97_squeeze_mask_0, x = value_69_cast_fp16)[name = string("value_token_97_cast_fp16")];
            tensor<int32, [1]> var_2923_axes_0 = const()[name = string("op_2923_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_2923_cast_fp16 = expand_dims(axes = var_2923_axes_0, x = key_token_97_cast_fp16)[name = string("op_2923_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2924_cast_fp16 = mul(x = state_193_cast_fp16, y = var_2923_cast_fp16)[name = string("op_2924_cast_fp16")];
            tensor<int32, [1]> memory_97_axes_0 = const()[name = string("memory_97_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_97_keep_dims_0 = const()[name = string("memory_97_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_97_cast_fp16 = reduce_sum(axes = memory_97_axes_0, keep_dims = memory_97_keep_dims_0, x = var_2924_cast_fp16)[name = string("memory_97_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_2927_cast_fp16 = sub(x = value_token_97_cast_fp16, y = memory_97_cast_fp16)[name = string("op_2927_cast_fp16")];
            tensor<int32, [3]> var_2930_begin_0 = const()[name = string("op_2930_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_2930_end_0 = const()[name = string("op_2930_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_2930_end_mask_0 = const()[name = string("op_2930_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_2930_squeeze_mask_0 = const()[name = string("op_2930_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_2930_cast_fp16 = slice_by_index(begin = var_2930_begin_0, end = var_2930_end_0, end_mask = var_2930_end_mask_0, squeeze_mask = var_2930_squeeze_mask_0, x = beta_7_cast_fp16)[name = string("op_2930_cast_fp16")];
            tensor<int32, [1]> var_2931_axes_0 = const()[name = string("op_2931_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_2931_cast_fp16 = expand_dims(axes = var_2931_axes_0, x = var_2930_cast_fp16)[name = string("op_2931_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_97_cast_fp16 = mul(x = var_2927_cast_fp16, y = var_2931_cast_fp16)[name = string("delta_97_cast_fp16")];
            tensor<int32, [1]> var_2934_axes_0 = const()[name = string("op_2934_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_2934_cast_fp16 = expand_dims(axes = var_2934_axes_0, x = delta_97_cast_fp16)[name = string("op_2934_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2935_cast_fp16 = mul(x = var_2923_cast_fp16, y = var_2934_cast_fp16)[name = string("op_2935_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_195_cast_fp16 = add(x = state_193_cast_fp16, y = var_2935_cast_fp16)[name = string("state_195_cast_fp16")];
            tensor<int32, [4]> var_2939_begin_0 = const()[name = string("op_2939_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> var_2939_end_0 = const()[name = string("op_2939_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> var_2939_end_mask_0 = const()[name = string("op_2939_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_2939_squeeze_mask_0 = const()[name = string("op_2939_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_2939_cast_fp16 = slice_by_index(begin = var_2939_begin_0, end = var_2939_end_0, end_mask = var_2939_end_mask_0, squeeze_mask = var_2939_squeeze_mask_0, x = _inversed_query_19_cast_fp16)[name = string("op_2939_cast_fp16")];
            tensor<int32, [1]> var_2940_axes_0 = const()[name = string("op_2940_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_2940_cast_fp16 = expand_dims(axes = var_2940_axes_0, x = var_2939_cast_fp16)[name = string("op_2940_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2941_cast_fp16 = mul(x = state_195_cast_fp16, y = var_2940_cast_fp16)[name = string("op_2941_cast_fp16")];
            tensor<int32, [1]> var_2943_axes_0 = const()[name = string("op_2943_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_2943_keep_dims_0 = const()[name = string("op_2943_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_2943_cast_fp16 = reduce_sum(axes = var_2943_axes_0, keep_dims = var_2943_keep_dims_0, x = var_2941_cast_fp16)[name = string("op_2943_cast_fp16")];
            tensor<int32, [3]> var_2946_begin_0 = const()[name = string("op_2946_begin_0"), val = tensor<int32, [3]>([0, 0, 1])];
            tensor<int32, [3]> var_2946_end_0 = const()[name = string("op_2946_end_0"), val = tensor<int32, [3]>([1, 16, 2])];
            tensor<bool, [3]> var_2946_end_mask_0 = const()[name = string("op_2946_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_2946_squeeze_mask_0 = const()[name = string("op_2946_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_2946_cast_fp16 = slice_by_index(begin = var_2946_begin_0, end = var_2946_end_0, end_mask = var_2946_end_mask_0, squeeze_mask = var_2946_squeeze_mask_0, x = decay_7_cast_fp16)[name = string("op_2946_cast_fp16")];
            tensor<fp16, [1, 16]> var_2947_cast_fp16 = exp(x = var_2946_cast_fp16)[name = string("op_2947_cast_fp16")];
            tensor<int32, [1]> var_2948_axes_0 = const()[name = string("op_2948_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_2948_cast_fp16 = expand_dims(axes = var_2948_axes_0, x = var_2947_cast_fp16)[name = string("op_2948_cast_fp16")];
            tensor<int32, [1]> var_2949_axes_0 = const()[name = string("op_2949_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_2949_cast_fp16 = expand_dims(axes = var_2949_axes_0, x = var_2948_cast_fp16)[name = string("op_2949_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_197_cast_fp16 = mul(x = state_195_cast_fp16, y = var_2949_cast_fp16)[name = string("state_197_cast_fp16")];
            tensor<int32, [4]> key_token_99_begin_0 = const()[name = string("key_token_99_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> key_token_99_end_0 = const()[name = string("key_token_99_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> key_token_99_end_mask_0 = const()[name = string("key_token_99_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_99_squeeze_mask_0 = const()[name = string("key_token_99_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_99_cast_fp16 = slice_by_index(begin = key_token_99_begin_0, end = key_token_99_end_0, end_mask = key_token_99_end_mask_0, squeeze_mask = key_token_99_squeeze_mask_0, x = key_19_cast_fp16)[name = string("key_token_99_cast_fp16")];
            tensor<int32, [4]> value_token_99_begin_0 = const()[name = string("value_token_99_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> value_token_99_end_0 = const()[name = string("value_token_99_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> value_token_99_end_mask_0 = const()[name = string("value_token_99_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_99_squeeze_mask_0 = const()[name = string("value_token_99_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_99_cast_fp16 = slice_by_index(begin = value_token_99_begin_0, end = value_token_99_end_0, end_mask = value_token_99_end_mask_0, squeeze_mask = value_token_99_squeeze_mask_0, x = value_69_cast_fp16)[name = string("value_token_99_cast_fp16")];
            tensor<int32, [1]> var_2957_axes_0 = const()[name = string("op_2957_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_2957_cast_fp16 = expand_dims(axes = var_2957_axes_0, x = key_token_99_cast_fp16)[name = string("op_2957_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2958_cast_fp16 = mul(x = state_197_cast_fp16, y = var_2957_cast_fp16)[name = string("op_2958_cast_fp16")];
            tensor<int32, [1]> memory_99_axes_0 = const()[name = string("memory_99_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_99_keep_dims_0 = const()[name = string("memory_99_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_99_cast_fp16 = reduce_sum(axes = memory_99_axes_0, keep_dims = memory_99_keep_dims_0, x = var_2958_cast_fp16)[name = string("memory_99_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_2961_cast_fp16 = sub(x = value_token_99_cast_fp16, y = memory_99_cast_fp16)[name = string("op_2961_cast_fp16")];
            tensor<int32, [3]> var_2964_begin_0 = const()[name = string("op_2964_begin_0"), val = tensor<int32, [3]>([0, 0, 1])];
            tensor<int32, [3]> var_2964_end_0 = const()[name = string("op_2964_end_0"), val = tensor<int32, [3]>([1, 16, 2])];
            tensor<bool, [3]> var_2964_end_mask_0 = const()[name = string("op_2964_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_2964_squeeze_mask_0 = const()[name = string("op_2964_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_2964_cast_fp16 = slice_by_index(begin = var_2964_begin_0, end = var_2964_end_0, end_mask = var_2964_end_mask_0, squeeze_mask = var_2964_squeeze_mask_0, x = beta_7_cast_fp16)[name = string("op_2964_cast_fp16")];
            tensor<int32, [1]> var_2965_axes_0 = const()[name = string("op_2965_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_2965_cast_fp16 = expand_dims(axes = var_2965_axes_0, x = var_2964_cast_fp16)[name = string("op_2965_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_99_cast_fp16 = mul(x = var_2961_cast_fp16, y = var_2965_cast_fp16)[name = string("delta_99_cast_fp16")];
            tensor<int32, [1]> var_2968_axes_0 = const()[name = string("op_2968_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_2968_cast_fp16 = expand_dims(axes = var_2968_axes_0, x = delta_99_cast_fp16)[name = string("op_2968_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2969_cast_fp16 = mul(x = var_2957_cast_fp16, y = var_2968_cast_fp16)[name = string("op_2969_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_199_cast_fp16 = add(x = state_197_cast_fp16, y = var_2969_cast_fp16)[name = string("state_199_cast_fp16")];
            tensor<int32, [4]> var_2973_begin_0 = const()[name = string("op_2973_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> var_2973_end_0 = const()[name = string("op_2973_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> var_2973_end_mask_0 = const()[name = string("op_2973_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_2973_squeeze_mask_0 = const()[name = string("op_2973_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_2973_cast_fp16 = slice_by_index(begin = var_2973_begin_0, end = var_2973_end_0, end_mask = var_2973_end_mask_0, squeeze_mask = var_2973_squeeze_mask_0, x = _inversed_query_19_cast_fp16)[name = string("op_2973_cast_fp16")];
            tensor<int32, [1]> var_2974_axes_0 = const()[name = string("op_2974_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_2974_cast_fp16 = expand_dims(axes = var_2974_axes_0, x = var_2973_cast_fp16)[name = string("op_2974_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2975_cast_fp16 = mul(x = state_199_cast_fp16, y = var_2974_cast_fp16)[name = string("op_2975_cast_fp16")];
            tensor<int32, [1]> var_2977_axes_0 = const()[name = string("op_2977_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_2977_keep_dims_0 = const()[name = string("op_2977_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_2977_cast_fp16 = reduce_sum(axes = var_2977_axes_0, keep_dims = var_2977_keep_dims_0, x = var_2975_cast_fp16)[name = string("op_2977_cast_fp16")];
            tensor<int32, [3]> var_2980_begin_0 = const()[name = string("op_2980_begin_0"), val = tensor<int32, [3]>([0, 0, 2])];
            tensor<int32, [3]> var_2980_end_0 = const()[name = string("op_2980_end_0"), val = tensor<int32, [3]>([1, 16, 3])];
            tensor<bool, [3]> var_2980_end_mask_0 = const()[name = string("op_2980_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_2980_squeeze_mask_0 = const()[name = string("op_2980_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_2980_cast_fp16 = slice_by_index(begin = var_2980_begin_0, end = var_2980_end_0, end_mask = var_2980_end_mask_0, squeeze_mask = var_2980_squeeze_mask_0, x = decay_7_cast_fp16)[name = string("op_2980_cast_fp16")];
            tensor<fp16, [1, 16]> var_2981_cast_fp16 = exp(x = var_2980_cast_fp16)[name = string("op_2981_cast_fp16")];
            tensor<int32, [1]> var_2982_axes_0 = const()[name = string("op_2982_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_2982_cast_fp16 = expand_dims(axes = var_2982_axes_0, x = var_2981_cast_fp16)[name = string("op_2982_cast_fp16")];
            tensor<int32, [1]> var_2983_axes_0 = const()[name = string("op_2983_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_2983_cast_fp16 = expand_dims(axes = var_2983_axes_0, x = var_2982_cast_fp16)[name = string("op_2983_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_201_cast_fp16 = mul(x = state_199_cast_fp16, y = var_2983_cast_fp16)[name = string("state_201_cast_fp16")];
            tensor<int32, [4]> key_token_101_begin_0 = const()[name = string("key_token_101_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> key_token_101_end_0 = const()[name = string("key_token_101_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> key_token_101_end_mask_0 = const()[name = string("key_token_101_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_101_squeeze_mask_0 = const()[name = string("key_token_101_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_101_cast_fp16 = slice_by_index(begin = key_token_101_begin_0, end = key_token_101_end_0, end_mask = key_token_101_end_mask_0, squeeze_mask = key_token_101_squeeze_mask_0, x = key_19_cast_fp16)[name = string("key_token_101_cast_fp16")];
            tensor<int32, [4]> value_token_101_begin_0 = const()[name = string("value_token_101_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> value_token_101_end_0 = const()[name = string("value_token_101_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> value_token_101_end_mask_0 = const()[name = string("value_token_101_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_101_squeeze_mask_0 = const()[name = string("value_token_101_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_101_cast_fp16 = slice_by_index(begin = value_token_101_begin_0, end = value_token_101_end_0, end_mask = value_token_101_end_mask_0, squeeze_mask = value_token_101_squeeze_mask_0, x = value_69_cast_fp16)[name = string("value_token_101_cast_fp16")];
            tensor<int32, [1]> var_2991_axes_0 = const()[name = string("op_2991_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_2991_cast_fp16 = expand_dims(axes = var_2991_axes_0, x = key_token_101_cast_fp16)[name = string("op_2991_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2992_cast_fp16 = mul(x = state_201_cast_fp16, y = var_2991_cast_fp16)[name = string("op_2992_cast_fp16")];
            tensor<int32, [1]> memory_101_axes_0 = const()[name = string("memory_101_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_101_keep_dims_0 = const()[name = string("memory_101_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_101_cast_fp16 = reduce_sum(axes = memory_101_axes_0, keep_dims = memory_101_keep_dims_0, x = var_2992_cast_fp16)[name = string("memory_101_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_2995_cast_fp16 = sub(x = value_token_101_cast_fp16, y = memory_101_cast_fp16)[name = string("op_2995_cast_fp16")];
            tensor<int32, [3]> var_2998_begin_0 = const()[name = string("op_2998_begin_0"), val = tensor<int32, [3]>([0, 0, 2])];
            tensor<int32, [3]> var_2998_end_0 = const()[name = string("op_2998_end_0"), val = tensor<int32, [3]>([1, 16, 3])];
            tensor<bool, [3]> var_2998_end_mask_0 = const()[name = string("op_2998_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_2998_squeeze_mask_0 = const()[name = string("op_2998_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_2998_cast_fp16 = slice_by_index(begin = var_2998_begin_0, end = var_2998_end_0, end_mask = var_2998_end_mask_0, squeeze_mask = var_2998_squeeze_mask_0, x = beta_7_cast_fp16)[name = string("op_2998_cast_fp16")];
            tensor<int32, [1]> var_2999_axes_0 = const()[name = string("op_2999_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_2999_cast_fp16 = expand_dims(axes = var_2999_axes_0, x = var_2998_cast_fp16)[name = string("op_2999_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_101_cast_fp16 = mul(x = var_2995_cast_fp16, y = var_2999_cast_fp16)[name = string("delta_101_cast_fp16")];
            tensor<int32, [1]> var_3002_axes_0 = const()[name = string("op_3002_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_3002_cast_fp16 = expand_dims(axes = var_3002_axes_0, x = delta_101_cast_fp16)[name = string("op_3002_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3003_cast_fp16 = mul(x = var_2991_cast_fp16, y = var_3002_cast_fp16)[name = string("op_3003_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_203_cast_fp16 = add(x = state_201_cast_fp16, y = var_3003_cast_fp16)[name = string("state_203_cast_fp16")];
            tensor<int32, [4]> var_3007_begin_0 = const()[name = string("op_3007_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> var_3007_end_0 = const()[name = string("op_3007_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> var_3007_end_mask_0 = const()[name = string("op_3007_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_3007_squeeze_mask_0 = const()[name = string("op_3007_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_3007_cast_fp16 = slice_by_index(begin = var_3007_begin_0, end = var_3007_end_0, end_mask = var_3007_end_mask_0, squeeze_mask = var_3007_squeeze_mask_0, x = _inversed_query_19_cast_fp16)[name = string("op_3007_cast_fp16")];
            tensor<int32, [1]> var_3008_axes_0 = const()[name = string("op_3008_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_3008_cast_fp16 = expand_dims(axes = var_3008_axes_0, x = var_3007_cast_fp16)[name = string("op_3008_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3009_cast_fp16 = mul(x = state_203_cast_fp16, y = var_3008_cast_fp16)[name = string("op_3009_cast_fp16")];
            tensor<int32, [1]> var_3011_axes_0 = const()[name = string("op_3011_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_3011_keep_dims_0 = const()[name = string("op_3011_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_3011_cast_fp16 = reduce_sum(axes = var_3011_axes_0, keep_dims = var_3011_keep_dims_0, x = var_3009_cast_fp16)[name = string("op_3011_cast_fp16")];
            tensor<int32, [3]> var_3014_begin_0 = const()[name = string("op_3014_begin_0"), val = tensor<int32, [3]>([0, 0, 3])];
            tensor<int32, [3]> var_3014_end_0 = const()[name = string("op_3014_end_0"), val = tensor<int32, [3]>([1, 16, 4])];
            tensor<bool, [3]> var_3014_end_mask_0 = const()[name = string("op_3014_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_3014_squeeze_mask_0 = const()[name = string("op_3014_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_3014_cast_fp16 = slice_by_index(begin = var_3014_begin_0, end = var_3014_end_0, end_mask = var_3014_end_mask_0, squeeze_mask = var_3014_squeeze_mask_0, x = decay_7_cast_fp16)[name = string("op_3014_cast_fp16")];
            tensor<fp16, [1, 16]> var_3015_cast_fp16 = exp(x = var_3014_cast_fp16)[name = string("op_3015_cast_fp16")];
            tensor<int32, [1]> var_3016_axes_0 = const()[name = string("op_3016_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_3016_cast_fp16 = expand_dims(axes = var_3016_axes_0, x = var_3015_cast_fp16)[name = string("op_3016_cast_fp16")];
            tensor<int32, [1]> var_3017_axes_0 = const()[name = string("op_3017_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_3017_cast_fp16 = expand_dims(axes = var_3017_axes_0, x = var_3016_cast_fp16)[name = string("op_3017_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_205_cast_fp16 = mul(x = state_203_cast_fp16, y = var_3017_cast_fp16)[name = string("state_205_cast_fp16")];
            tensor<int32, [4]> key_token_103_begin_0 = const()[name = string("key_token_103_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> key_token_103_end_0 = const()[name = string("key_token_103_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> key_token_103_end_mask_0 = const()[name = string("key_token_103_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_103_squeeze_mask_0 = const()[name = string("key_token_103_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_103_cast_fp16 = slice_by_index(begin = key_token_103_begin_0, end = key_token_103_end_0, end_mask = key_token_103_end_mask_0, squeeze_mask = key_token_103_squeeze_mask_0, x = key_19_cast_fp16)[name = string("key_token_103_cast_fp16")];
            tensor<int32, [4]> value_token_103_begin_0 = const()[name = string("value_token_103_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> value_token_103_end_0 = const()[name = string("value_token_103_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> value_token_103_end_mask_0 = const()[name = string("value_token_103_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_103_squeeze_mask_0 = const()[name = string("value_token_103_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_103_cast_fp16 = slice_by_index(begin = value_token_103_begin_0, end = value_token_103_end_0, end_mask = value_token_103_end_mask_0, squeeze_mask = value_token_103_squeeze_mask_0, x = value_69_cast_fp16)[name = string("value_token_103_cast_fp16")];
            tensor<int32, [1]> var_3025_axes_0 = const()[name = string("op_3025_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_3025_cast_fp16 = expand_dims(axes = var_3025_axes_0, x = key_token_103_cast_fp16)[name = string("op_3025_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3026_cast_fp16 = mul(x = state_205_cast_fp16, y = var_3025_cast_fp16)[name = string("op_3026_cast_fp16")];
            tensor<int32, [1]> memory_103_axes_0 = const()[name = string("memory_103_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_103_keep_dims_0 = const()[name = string("memory_103_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_103_cast_fp16 = reduce_sum(axes = memory_103_axes_0, keep_dims = memory_103_keep_dims_0, x = var_3026_cast_fp16)[name = string("memory_103_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_3029_cast_fp16 = sub(x = value_token_103_cast_fp16, y = memory_103_cast_fp16)[name = string("op_3029_cast_fp16")];
            tensor<int32, [3]> var_3032_begin_0 = const()[name = string("op_3032_begin_0"), val = tensor<int32, [3]>([0, 0, 3])];
            tensor<int32, [3]> var_3032_end_0 = const()[name = string("op_3032_end_0"), val = tensor<int32, [3]>([1, 16, 4])];
            tensor<bool, [3]> var_3032_end_mask_0 = const()[name = string("op_3032_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_3032_squeeze_mask_0 = const()[name = string("op_3032_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_3032_cast_fp16 = slice_by_index(begin = var_3032_begin_0, end = var_3032_end_0, end_mask = var_3032_end_mask_0, squeeze_mask = var_3032_squeeze_mask_0, x = beta_7_cast_fp16)[name = string("op_3032_cast_fp16")];
            tensor<int32, [1]> var_3033_axes_0 = const()[name = string("op_3033_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_3033_cast_fp16 = expand_dims(axes = var_3033_axes_0, x = var_3032_cast_fp16)[name = string("op_3033_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_103_cast_fp16 = mul(x = var_3029_cast_fp16, y = var_3033_cast_fp16)[name = string("delta_103_cast_fp16")];
            tensor<int32, [1]> var_3036_axes_0 = const()[name = string("op_3036_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_3036_cast_fp16 = expand_dims(axes = var_3036_axes_0, x = delta_103_cast_fp16)[name = string("op_3036_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3037_cast_fp16 = mul(x = var_3025_cast_fp16, y = var_3036_cast_fp16)[name = string("op_3037_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_207_cast_fp16 = add(x = state_205_cast_fp16, y = var_3037_cast_fp16)[name = string("state_207_cast_fp16")];
            tensor<int32, [4]> var_3041_begin_0 = const()[name = string("op_3041_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> var_3041_end_0 = const()[name = string("op_3041_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> var_3041_end_mask_0 = const()[name = string("op_3041_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_3041_squeeze_mask_0 = const()[name = string("op_3041_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_3041_cast_fp16 = slice_by_index(begin = var_3041_begin_0, end = var_3041_end_0, end_mask = var_3041_end_mask_0, squeeze_mask = var_3041_squeeze_mask_0, x = _inversed_query_19_cast_fp16)[name = string("op_3041_cast_fp16")];
            tensor<int32, [1]> var_3042_axes_0 = const()[name = string("op_3042_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_3042_cast_fp16 = expand_dims(axes = var_3042_axes_0, x = var_3041_cast_fp16)[name = string("op_3042_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3043_cast_fp16 = mul(x = state_207_cast_fp16, y = var_3042_cast_fp16)[name = string("op_3043_cast_fp16")];
            tensor<int32, [1]> var_3045_axes_0 = const()[name = string("op_3045_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_3045_keep_dims_0 = const()[name = string("op_3045_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_3045_cast_fp16 = reduce_sum(axes = var_3045_axes_0, keep_dims = var_3045_keep_dims_0, x = var_3043_cast_fp16)[name = string("op_3045_cast_fp16")];
            tensor<int32, [3]> var_3048_begin_0 = const()[name = string("op_3048_begin_0"), val = tensor<int32, [3]>([0, 0, 4])];
            tensor<int32, [3]> var_3048_end_0 = const()[name = string("op_3048_end_0"), val = tensor<int32, [3]>([1, 16, 5])];
            tensor<bool, [3]> var_3048_end_mask_0 = const()[name = string("op_3048_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_3048_squeeze_mask_0 = const()[name = string("op_3048_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_3048_cast_fp16 = slice_by_index(begin = var_3048_begin_0, end = var_3048_end_0, end_mask = var_3048_end_mask_0, squeeze_mask = var_3048_squeeze_mask_0, x = decay_7_cast_fp16)[name = string("op_3048_cast_fp16")];
            tensor<fp16, [1, 16]> var_3049_cast_fp16 = exp(x = var_3048_cast_fp16)[name = string("op_3049_cast_fp16")];
            tensor<int32, [1]> var_3050_axes_0 = const()[name = string("op_3050_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_3050_cast_fp16 = expand_dims(axes = var_3050_axes_0, x = var_3049_cast_fp16)[name = string("op_3050_cast_fp16")];
            tensor<int32, [1]> var_3051_axes_0 = const()[name = string("op_3051_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_3051_cast_fp16 = expand_dims(axes = var_3051_axes_0, x = var_3050_cast_fp16)[name = string("op_3051_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_209_cast_fp16 = mul(x = state_207_cast_fp16, y = var_3051_cast_fp16)[name = string("state_209_cast_fp16")];
            tensor<int32, [4]> key_token_105_begin_0 = const()[name = string("key_token_105_begin_0"), val = tensor<int32, [4]>([0, 0, 4, 0])];
            tensor<int32, [4]> key_token_105_end_0 = const()[name = string("key_token_105_end_0"), val = tensor<int32, [4]>([1, 16, 5, 128])];
            tensor<bool, [4]> key_token_105_end_mask_0 = const()[name = string("key_token_105_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_105_squeeze_mask_0 = const()[name = string("key_token_105_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_105_cast_fp16 = slice_by_index(begin = key_token_105_begin_0, end = key_token_105_end_0, end_mask = key_token_105_end_mask_0, squeeze_mask = key_token_105_squeeze_mask_0, x = key_19_cast_fp16)[name = string("key_token_105_cast_fp16")];
            tensor<int32, [4]> value_token_105_begin_0 = const()[name = string("value_token_105_begin_0"), val = tensor<int32, [4]>([0, 0, 4, 0])];
            tensor<int32, [4]> value_token_105_end_0 = const()[name = string("value_token_105_end_0"), val = tensor<int32, [4]>([1, 16, 5, 128])];
            tensor<bool, [4]> value_token_105_end_mask_0 = const()[name = string("value_token_105_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_105_squeeze_mask_0 = const()[name = string("value_token_105_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_105_cast_fp16 = slice_by_index(begin = value_token_105_begin_0, end = value_token_105_end_0, end_mask = value_token_105_end_mask_0, squeeze_mask = value_token_105_squeeze_mask_0, x = value_69_cast_fp16)[name = string("value_token_105_cast_fp16")];
            tensor<int32, [1]> var_3059_axes_0 = const()[name = string("op_3059_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_3059_cast_fp16 = expand_dims(axes = var_3059_axes_0, x = key_token_105_cast_fp16)[name = string("op_3059_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3060_cast_fp16 = mul(x = state_209_cast_fp16, y = var_3059_cast_fp16)[name = string("op_3060_cast_fp16")];
            tensor<int32, [1]> memory_105_axes_0 = const()[name = string("memory_105_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_105_keep_dims_0 = const()[name = string("memory_105_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_105_cast_fp16 = reduce_sum(axes = memory_105_axes_0, keep_dims = memory_105_keep_dims_0, x = var_3060_cast_fp16)[name = string("memory_105_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_3063_cast_fp16 = sub(x = value_token_105_cast_fp16, y = memory_105_cast_fp16)[name = string("op_3063_cast_fp16")];
            tensor<int32, [3]> var_3066_begin_0 = const()[name = string("op_3066_begin_0"), val = tensor<int32, [3]>([0, 0, 4])];
            tensor<int32, [3]> var_3066_end_0 = const()[name = string("op_3066_end_0"), val = tensor<int32, [3]>([1, 16, 5])];
            tensor<bool, [3]> var_3066_end_mask_0 = const()[name = string("op_3066_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_3066_squeeze_mask_0 = const()[name = string("op_3066_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_3066_cast_fp16 = slice_by_index(begin = var_3066_begin_0, end = var_3066_end_0, end_mask = var_3066_end_mask_0, squeeze_mask = var_3066_squeeze_mask_0, x = beta_7_cast_fp16)[name = string("op_3066_cast_fp16")];
            tensor<int32, [1]> var_3067_axes_0 = const()[name = string("op_3067_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_3067_cast_fp16 = expand_dims(axes = var_3067_axes_0, x = var_3066_cast_fp16)[name = string("op_3067_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_105_cast_fp16 = mul(x = var_3063_cast_fp16, y = var_3067_cast_fp16)[name = string("delta_105_cast_fp16")];
            tensor<int32, [1]> var_3070_axes_0 = const()[name = string("op_3070_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_3070_cast_fp16 = expand_dims(axes = var_3070_axes_0, x = delta_105_cast_fp16)[name = string("op_3070_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3071_cast_fp16 = mul(x = var_3059_cast_fp16, y = var_3070_cast_fp16)[name = string("op_3071_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_211_cast_fp16 = add(x = state_209_cast_fp16, y = var_3071_cast_fp16)[name = string("state_211_cast_fp16")];
            tensor<int32, [4]> var_3075_begin_0 = const()[name = string("op_3075_begin_0"), val = tensor<int32, [4]>([0, 0, 4, 0])];
            tensor<int32, [4]> var_3075_end_0 = const()[name = string("op_3075_end_0"), val = tensor<int32, [4]>([1, 16, 5, 128])];
            tensor<bool, [4]> var_3075_end_mask_0 = const()[name = string("op_3075_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_3075_squeeze_mask_0 = const()[name = string("op_3075_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_3075_cast_fp16 = slice_by_index(begin = var_3075_begin_0, end = var_3075_end_0, end_mask = var_3075_end_mask_0, squeeze_mask = var_3075_squeeze_mask_0, x = _inversed_query_19_cast_fp16)[name = string("op_3075_cast_fp16")];
            tensor<int32, [1]> var_3076_axes_0 = const()[name = string("op_3076_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_3076_cast_fp16 = expand_dims(axes = var_3076_axes_0, x = var_3075_cast_fp16)[name = string("op_3076_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3077_cast_fp16 = mul(x = state_211_cast_fp16, y = var_3076_cast_fp16)[name = string("op_3077_cast_fp16")];
            tensor<int32, [1]> var_3079_axes_0 = const()[name = string("op_3079_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_3079_keep_dims_0 = const()[name = string("op_3079_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_3079_cast_fp16 = reduce_sum(axes = var_3079_axes_0, keep_dims = var_3079_keep_dims_0, x = var_3077_cast_fp16)[name = string("op_3079_cast_fp16")];
            tensor<int32, [3]> var_3082_begin_0 = const()[name = string("op_3082_begin_0"), val = tensor<int32, [3]>([0, 0, 5])];
            tensor<int32, [3]> var_3082_end_0 = const()[name = string("op_3082_end_0"), val = tensor<int32, [3]>([1, 16, 6])];
            tensor<bool, [3]> var_3082_end_mask_0 = const()[name = string("op_3082_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_3082_squeeze_mask_0 = const()[name = string("op_3082_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_3082_cast_fp16 = slice_by_index(begin = var_3082_begin_0, end = var_3082_end_0, end_mask = var_3082_end_mask_0, squeeze_mask = var_3082_squeeze_mask_0, x = decay_7_cast_fp16)[name = string("op_3082_cast_fp16")];
            tensor<fp16, [1, 16]> var_3083_cast_fp16 = exp(x = var_3082_cast_fp16)[name = string("op_3083_cast_fp16")];
            tensor<int32, [1]> var_3084_axes_0 = const()[name = string("op_3084_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_3084_cast_fp16 = expand_dims(axes = var_3084_axes_0, x = var_3083_cast_fp16)[name = string("op_3084_cast_fp16")];
            tensor<int32, [1]> var_3085_axes_0 = const()[name = string("op_3085_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_3085_cast_fp16 = expand_dims(axes = var_3085_axes_0, x = var_3084_cast_fp16)[name = string("op_3085_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_213_cast_fp16 = mul(x = state_211_cast_fp16, y = var_3085_cast_fp16)[name = string("state_213_cast_fp16")];
            tensor<int32, [4]> key_token_107_begin_0 = const()[name = string("key_token_107_begin_0"), val = tensor<int32, [4]>([0, 0, 5, 0])];
            tensor<int32, [4]> key_token_107_end_0 = const()[name = string("key_token_107_end_0"), val = tensor<int32, [4]>([1, 16, 6, 128])];
            tensor<bool, [4]> key_token_107_end_mask_0 = const()[name = string("key_token_107_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_107_squeeze_mask_0 = const()[name = string("key_token_107_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_107_cast_fp16 = slice_by_index(begin = key_token_107_begin_0, end = key_token_107_end_0, end_mask = key_token_107_end_mask_0, squeeze_mask = key_token_107_squeeze_mask_0, x = key_19_cast_fp16)[name = string("key_token_107_cast_fp16")];
            tensor<int32, [4]> value_token_107_begin_0 = const()[name = string("value_token_107_begin_0"), val = tensor<int32, [4]>([0, 0, 5, 0])];
            tensor<int32, [4]> value_token_107_end_0 = const()[name = string("value_token_107_end_0"), val = tensor<int32, [4]>([1, 16, 6, 128])];
            tensor<bool, [4]> value_token_107_end_mask_0 = const()[name = string("value_token_107_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_107_squeeze_mask_0 = const()[name = string("value_token_107_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_107_cast_fp16 = slice_by_index(begin = value_token_107_begin_0, end = value_token_107_end_0, end_mask = value_token_107_end_mask_0, squeeze_mask = value_token_107_squeeze_mask_0, x = value_69_cast_fp16)[name = string("value_token_107_cast_fp16")];
            tensor<int32, [1]> var_3093_axes_0 = const()[name = string("op_3093_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_3093_cast_fp16 = expand_dims(axes = var_3093_axes_0, x = key_token_107_cast_fp16)[name = string("op_3093_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3094_cast_fp16 = mul(x = state_213_cast_fp16, y = var_3093_cast_fp16)[name = string("op_3094_cast_fp16")];
            tensor<int32, [1]> memory_107_axes_0 = const()[name = string("memory_107_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_107_keep_dims_0 = const()[name = string("memory_107_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_107_cast_fp16 = reduce_sum(axes = memory_107_axes_0, keep_dims = memory_107_keep_dims_0, x = var_3094_cast_fp16)[name = string("memory_107_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_3097_cast_fp16 = sub(x = value_token_107_cast_fp16, y = memory_107_cast_fp16)[name = string("op_3097_cast_fp16")];
            tensor<int32, [3]> var_3100_begin_0 = const()[name = string("op_3100_begin_0"), val = tensor<int32, [3]>([0, 0, 5])];
            tensor<int32, [3]> var_3100_end_0 = const()[name = string("op_3100_end_0"), val = tensor<int32, [3]>([1, 16, 6])];
            tensor<bool, [3]> var_3100_end_mask_0 = const()[name = string("op_3100_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_3100_squeeze_mask_0 = const()[name = string("op_3100_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_3100_cast_fp16 = slice_by_index(begin = var_3100_begin_0, end = var_3100_end_0, end_mask = var_3100_end_mask_0, squeeze_mask = var_3100_squeeze_mask_0, x = beta_7_cast_fp16)[name = string("op_3100_cast_fp16")];
            tensor<int32, [1]> var_3101_axes_0 = const()[name = string("op_3101_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_3101_cast_fp16 = expand_dims(axes = var_3101_axes_0, x = var_3100_cast_fp16)[name = string("op_3101_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_107_cast_fp16 = mul(x = var_3097_cast_fp16, y = var_3101_cast_fp16)[name = string("delta_107_cast_fp16")];
            tensor<int32, [1]> var_3104_axes_0 = const()[name = string("op_3104_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_3104_cast_fp16 = expand_dims(axes = var_3104_axes_0, x = delta_107_cast_fp16)[name = string("op_3104_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3105_cast_fp16 = mul(x = var_3093_cast_fp16, y = var_3104_cast_fp16)[name = string("op_3105_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_215_cast_fp16 = add(x = state_213_cast_fp16, y = var_3105_cast_fp16)[name = string("state_215_cast_fp16")];
            tensor<int32, [4]> var_3109_begin_0 = const()[name = string("op_3109_begin_0"), val = tensor<int32, [4]>([0, 0, 5, 0])];
            tensor<int32, [4]> var_3109_end_0 = const()[name = string("op_3109_end_0"), val = tensor<int32, [4]>([1, 16, 6, 128])];
            tensor<bool, [4]> var_3109_end_mask_0 = const()[name = string("op_3109_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_3109_squeeze_mask_0 = const()[name = string("op_3109_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_3109_cast_fp16 = slice_by_index(begin = var_3109_begin_0, end = var_3109_end_0, end_mask = var_3109_end_mask_0, squeeze_mask = var_3109_squeeze_mask_0, x = _inversed_query_19_cast_fp16)[name = string("op_3109_cast_fp16")];
            tensor<int32, [1]> var_3110_axes_0 = const()[name = string("op_3110_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_3110_cast_fp16 = expand_dims(axes = var_3110_axes_0, x = var_3109_cast_fp16)[name = string("op_3110_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3111_cast_fp16 = mul(x = state_215_cast_fp16, y = var_3110_cast_fp16)[name = string("op_3111_cast_fp16")];
            tensor<int32, [1]> var_3113_axes_0 = const()[name = string("op_3113_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_3113_keep_dims_0 = const()[name = string("op_3113_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_3113_cast_fp16 = reduce_sum(axes = var_3113_axes_0, keep_dims = var_3113_keep_dims_0, x = var_3111_cast_fp16)[name = string("op_3113_cast_fp16")];
            tensor<int32, [3]> var_3116_begin_0 = const()[name = string("op_3116_begin_0"), val = tensor<int32, [3]>([0, 0, 6])];
            tensor<int32, [3]> var_3116_end_0 = const()[name = string("op_3116_end_0"), val = tensor<int32, [3]>([1, 16, 7])];
            tensor<bool, [3]> var_3116_end_mask_0 = const()[name = string("op_3116_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_3116_squeeze_mask_0 = const()[name = string("op_3116_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_3116_cast_fp16 = slice_by_index(begin = var_3116_begin_0, end = var_3116_end_0, end_mask = var_3116_end_mask_0, squeeze_mask = var_3116_squeeze_mask_0, x = decay_7_cast_fp16)[name = string("op_3116_cast_fp16")];
            tensor<fp16, [1, 16]> var_3117_cast_fp16 = exp(x = var_3116_cast_fp16)[name = string("op_3117_cast_fp16")];
            tensor<int32, [1]> var_3118_axes_0 = const()[name = string("op_3118_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_3118_cast_fp16 = expand_dims(axes = var_3118_axes_0, x = var_3117_cast_fp16)[name = string("op_3118_cast_fp16")];
            tensor<int32, [1]> var_3119_axes_0 = const()[name = string("op_3119_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_3119_cast_fp16 = expand_dims(axes = var_3119_axes_0, x = var_3118_cast_fp16)[name = string("op_3119_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_217_cast_fp16 = mul(x = state_215_cast_fp16, y = var_3119_cast_fp16)[name = string("state_217_cast_fp16")];
            tensor<int32, [4]> key_token_109_begin_0 = const()[name = string("key_token_109_begin_0"), val = tensor<int32, [4]>([0, 0, 6, 0])];
            tensor<int32, [4]> key_token_109_end_0 = const()[name = string("key_token_109_end_0"), val = tensor<int32, [4]>([1, 16, 7, 128])];
            tensor<bool, [4]> key_token_109_end_mask_0 = const()[name = string("key_token_109_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_109_squeeze_mask_0 = const()[name = string("key_token_109_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_109_cast_fp16 = slice_by_index(begin = key_token_109_begin_0, end = key_token_109_end_0, end_mask = key_token_109_end_mask_0, squeeze_mask = key_token_109_squeeze_mask_0, x = key_19_cast_fp16)[name = string("key_token_109_cast_fp16")];
            tensor<int32, [4]> value_token_109_begin_0 = const()[name = string("value_token_109_begin_0"), val = tensor<int32, [4]>([0, 0, 6, 0])];
            tensor<int32, [4]> value_token_109_end_0 = const()[name = string("value_token_109_end_0"), val = tensor<int32, [4]>([1, 16, 7, 128])];
            tensor<bool, [4]> value_token_109_end_mask_0 = const()[name = string("value_token_109_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_109_squeeze_mask_0 = const()[name = string("value_token_109_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_109_cast_fp16 = slice_by_index(begin = value_token_109_begin_0, end = value_token_109_end_0, end_mask = value_token_109_end_mask_0, squeeze_mask = value_token_109_squeeze_mask_0, x = value_69_cast_fp16)[name = string("value_token_109_cast_fp16")];
            tensor<int32, [1]> var_3127_axes_0 = const()[name = string("op_3127_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_3127_cast_fp16 = expand_dims(axes = var_3127_axes_0, x = key_token_109_cast_fp16)[name = string("op_3127_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3128_cast_fp16 = mul(x = state_217_cast_fp16, y = var_3127_cast_fp16)[name = string("op_3128_cast_fp16")];
            tensor<int32, [1]> memory_109_axes_0 = const()[name = string("memory_109_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_109_keep_dims_0 = const()[name = string("memory_109_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_109_cast_fp16 = reduce_sum(axes = memory_109_axes_0, keep_dims = memory_109_keep_dims_0, x = var_3128_cast_fp16)[name = string("memory_109_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_3131_cast_fp16 = sub(x = value_token_109_cast_fp16, y = memory_109_cast_fp16)[name = string("op_3131_cast_fp16")];
            tensor<int32, [3]> var_3134_begin_0 = const()[name = string("op_3134_begin_0"), val = tensor<int32, [3]>([0, 0, 6])];
            tensor<int32, [3]> var_3134_end_0 = const()[name = string("op_3134_end_0"), val = tensor<int32, [3]>([1, 16, 7])];
            tensor<bool, [3]> var_3134_end_mask_0 = const()[name = string("op_3134_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_3134_squeeze_mask_0 = const()[name = string("op_3134_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_3134_cast_fp16 = slice_by_index(begin = var_3134_begin_0, end = var_3134_end_0, end_mask = var_3134_end_mask_0, squeeze_mask = var_3134_squeeze_mask_0, x = beta_7_cast_fp16)[name = string("op_3134_cast_fp16")];
            tensor<int32, [1]> var_3135_axes_0 = const()[name = string("op_3135_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_3135_cast_fp16 = expand_dims(axes = var_3135_axes_0, x = var_3134_cast_fp16)[name = string("op_3135_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_109_cast_fp16 = mul(x = var_3131_cast_fp16, y = var_3135_cast_fp16)[name = string("delta_109_cast_fp16")];
            tensor<int32, [1]> var_3138_axes_0 = const()[name = string("op_3138_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_3138_cast_fp16 = expand_dims(axes = var_3138_axes_0, x = delta_109_cast_fp16)[name = string("op_3138_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3139_cast_fp16 = mul(x = var_3127_cast_fp16, y = var_3138_cast_fp16)[name = string("op_3139_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_219_cast_fp16 = add(x = state_217_cast_fp16, y = var_3139_cast_fp16)[name = string("state_219_cast_fp16")];
            tensor<int32, [4]> var_3143_begin_0 = const()[name = string("op_3143_begin_0"), val = tensor<int32, [4]>([0, 0, 6, 0])];
            tensor<int32, [4]> var_3143_end_0 = const()[name = string("op_3143_end_0"), val = tensor<int32, [4]>([1, 16, 7, 128])];
            tensor<bool, [4]> var_3143_end_mask_0 = const()[name = string("op_3143_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_3143_squeeze_mask_0 = const()[name = string("op_3143_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_3143_cast_fp16 = slice_by_index(begin = var_3143_begin_0, end = var_3143_end_0, end_mask = var_3143_end_mask_0, squeeze_mask = var_3143_squeeze_mask_0, x = _inversed_query_19_cast_fp16)[name = string("op_3143_cast_fp16")];
            tensor<int32, [1]> var_3144_axes_0 = const()[name = string("op_3144_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_3144_cast_fp16 = expand_dims(axes = var_3144_axes_0, x = var_3143_cast_fp16)[name = string("op_3144_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3145_cast_fp16 = mul(x = state_219_cast_fp16, y = var_3144_cast_fp16)[name = string("op_3145_cast_fp16")];
            tensor<int32, [1]> var_3147_axes_0 = const()[name = string("op_3147_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_3147_keep_dims_0 = const()[name = string("op_3147_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_3147_cast_fp16 = reduce_sum(axes = var_3147_axes_0, keep_dims = var_3147_keep_dims_0, x = var_3145_cast_fp16)[name = string("op_3147_cast_fp16")];
            tensor<int32, [3]> var_3150_begin_0 = const()[name = string("op_3150_begin_0"), val = tensor<int32, [3]>([0, 0, 7])];
            tensor<int32, [3]> var_3150_end_0 = const()[name = string("op_3150_end_0"), val = tensor<int32, [3]>([1, 16, 8])];
            tensor<bool, [3]> var_3150_end_mask_0 = const()[name = string("op_3150_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_3150_squeeze_mask_0 = const()[name = string("op_3150_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_3150_cast_fp16 = slice_by_index(begin = var_3150_begin_0, end = var_3150_end_0, end_mask = var_3150_end_mask_0, squeeze_mask = var_3150_squeeze_mask_0, x = decay_7_cast_fp16)[name = string("op_3150_cast_fp16")];
            tensor<fp16, [1, 16]> var_3151_cast_fp16 = exp(x = var_3150_cast_fp16)[name = string("op_3151_cast_fp16")];
            tensor<int32, [1]> var_3152_axes_0 = const()[name = string("op_3152_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_3152_cast_fp16 = expand_dims(axes = var_3152_axes_0, x = var_3151_cast_fp16)[name = string("op_3152_cast_fp16")];
            tensor<int32, [1]> var_3153_axes_0 = const()[name = string("op_3153_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_3153_cast_fp16 = expand_dims(axes = var_3153_axes_0, x = var_3152_cast_fp16)[name = string("op_3153_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_221_cast_fp16 = mul(x = state_219_cast_fp16, y = var_3153_cast_fp16)[name = string("state_221_cast_fp16")];
            tensor<int32, [4]> key_token_111_begin_0 = const()[name = string("key_token_111_begin_0"), val = tensor<int32, [4]>([0, 0, 7, 0])];
            tensor<int32, [4]> key_token_111_end_0 = const()[name = string("key_token_111_end_0"), val = tensor<int32, [4]>([1, 16, 8, 128])];
            tensor<bool, [4]> key_token_111_end_mask_0 = const()[name = string("key_token_111_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_111_squeeze_mask_0 = const()[name = string("key_token_111_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_111_cast_fp16 = slice_by_index(begin = key_token_111_begin_0, end = key_token_111_end_0, end_mask = key_token_111_end_mask_0, squeeze_mask = key_token_111_squeeze_mask_0, x = key_19_cast_fp16)[name = string("key_token_111_cast_fp16")];
            tensor<int32, [4]> value_token_111_begin_0 = const()[name = string("value_token_111_begin_0"), val = tensor<int32, [4]>([0, 0, 7, 0])];
            tensor<int32, [4]> value_token_111_end_0 = const()[name = string("value_token_111_end_0"), val = tensor<int32, [4]>([1, 16, 8, 128])];
            tensor<bool, [4]> value_token_111_end_mask_0 = const()[name = string("value_token_111_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_111_squeeze_mask_0 = const()[name = string("value_token_111_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_111_cast_fp16 = slice_by_index(begin = value_token_111_begin_0, end = value_token_111_end_0, end_mask = value_token_111_end_mask_0, squeeze_mask = value_token_111_squeeze_mask_0, x = value_69_cast_fp16)[name = string("value_token_111_cast_fp16")];
            tensor<int32, [1]> var_3161_axes_0 = const()[name = string("op_3161_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_3161_cast_fp16 = expand_dims(axes = var_3161_axes_0, x = key_token_111_cast_fp16)[name = string("op_3161_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3162_cast_fp16 = mul(x = state_221_cast_fp16, y = var_3161_cast_fp16)[name = string("op_3162_cast_fp16")];
            tensor<int32, [1]> memory_111_axes_0 = const()[name = string("memory_111_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_111_keep_dims_0 = const()[name = string("memory_111_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_111_cast_fp16 = reduce_sum(axes = memory_111_axes_0, keep_dims = memory_111_keep_dims_0, x = var_3162_cast_fp16)[name = string("memory_111_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_3165_cast_fp16 = sub(x = value_token_111_cast_fp16, y = memory_111_cast_fp16)[name = string("op_3165_cast_fp16")];
            tensor<int32, [3]> var_3168_begin_0 = const()[name = string("op_3168_begin_0"), val = tensor<int32, [3]>([0, 0, 7])];
            tensor<int32, [3]> var_3168_end_0 = const()[name = string("op_3168_end_0"), val = tensor<int32, [3]>([1, 16, 8])];
            tensor<bool, [3]> var_3168_end_mask_0 = const()[name = string("op_3168_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_3168_squeeze_mask_0 = const()[name = string("op_3168_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_3168_cast_fp16 = slice_by_index(begin = var_3168_begin_0, end = var_3168_end_0, end_mask = var_3168_end_mask_0, squeeze_mask = var_3168_squeeze_mask_0, x = beta_7_cast_fp16)[name = string("op_3168_cast_fp16")];
            tensor<int32, [1]> var_3169_axes_0 = const()[name = string("op_3169_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_3169_cast_fp16 = expand_dims(axes = var_3169_axes_0, x = var_3168_cast_fp16)[name = string("op_3169_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_111_cast_fp16 = mul(x = var_3165_cast_fp16, y = var_3169_cast_fp16)[name = string("delta_111_cast_fp16")];
            tensor<int32, [1]> var_3172_axes_0 = const()[name = string("op_3172_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_3172_cast_fp16 = expand_dims(axes = var_3172_axes_0, x = delta_111_cast_fp16)[name = string("op_3172_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3173_cast_fp16 = mul(x = var_3161_cast_fp16, y = var_3172_cast_fp16)[name = string("op_3173_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_223_cast_fp16 = add(x = state_221_cast_fp16, y = var_3173_cast_fp16)[name = string("state_223_cast_fp16")];
            tensor<int32, [4]> var_3177_begin_0 = const()[name = string("op_3177_begin_0"), val = tensor<int32, [4]>([0, 0, 7, 0])];
            tensor<int32, [4]> var_3177_end_0 = const()[name = string("op_3177_end_0"), val = tensor<int32, [4]>([1, 16, 8, 128])];
            tensor<bool, [4]> var_3177_end_mask_0 = const()[name = string("op_3177_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_3177_squeeze_mask_0 = const()[name = string("op_3177_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_3177_cast_fp16 = slice_by_index(begin = var_3177_begin_0, end = var_3177_end_0, end_mask = var_3177_end_mask_0, squeeze_mask = var_3177_squeeze_mask_0, x = _inversed_query_19_cast_fp16)[name = string("op_3177_cast_fp16")];
            tensor<int32, [1]> var_3178_axes_0 = const()[name = string("op_3178_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_3178_cast_fp16 = expand_dims(axes = var_3178_axes_0, x = var_3177_cast_fp16)[name = string("op_3178_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3179_cast_fp16 = mul(x = state_223_cast_fp16, y = var_3178_cast_fp16)[name = string("op_3179_cast_fp16")];
            tensor<int32, [1]> var_3181_axes_0 = const()[name = string("op_3181_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_3181_keep_dims_0 = const()[name = string("op_3181_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_3181_cast_fp16 = reduce_sum(axes = var_3181_axes_0, keep_dims = var_3181_keep_dims_0, x = var_3179_cast_fp16)[name = string("op_3181_cast_fp16")];
            tensor<int32, [3]> var_3184_begin_0 = const()[name = string("op_3184_begin_0"), val = tensor<int32, [3]>([0, 0, 8])];
            tensor<int32, [3]> var_3184_end_0 = const()[name = string("op_3184_end_0"), val = tensor<int32, [3]>([1, 16, 9])];
            tensor<bool, [3]> var_3184_end_mask_0 = const()[name = string("op_3184_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_3184_squeeze_mask_0 = const()[name = string("op_3184_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_3184_cast_fp16 = slice_by_index(begin = var_3184_begin_0, end = var_3184_end_0, end_mask = var_3184_end_mask_0, squeeze_mask = var_3184_squeeze_mask_0, x = decay_7_cast_fp16)[name = string("op_3184_cast_fp16")];
            tensor<fp16, [1, 16]> var_3185_cast_fp16 = exp(x = var_3184_cast_fp16)[name = string("op_3185_cast_fp16")];
            tensor<int32, [1]> var_3186_axes_0 = const()[name = string("op_3186_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_3186_cast_fp16 = expand_dims(axes = var_3186_axes_0, x = var_3185_cast_fp16)[name = string("op_3186_cast_fp16")];
            tensor<int32, [1]> var_3187_axes_0 = const()[name = string("op_3187_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_3187_cast_fp16 = expand_dims(axes = var_3187_axes_0, x = var_3186_cast_fp16)[name = string("op_3187_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_225_cast_fp16 = mul(x = state_223_cast_fp16, y = var_3187_cast_fp16)[name = string("state_225_cast_fp16")];
            tensor<int32, [4]> key_token_113_begin_0 = const()[name = string("key_token_113_begin_0"), val = tensor<int32, [4]>([0, 0, 8, 0])];
            tensor<int32, [4]> key_token_113_end_0 = const()[name = string("key_token_113_end_0"), val = tensor<int32, [4]>([1, 16, 9, 128])];
            tensor<bool, [4]> key_token_113_end_mask_0 = const()[name = string("key_token_113_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_113_squeeze_mask_0 = const()[name = string("key_token_113_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_113_cast_fp16 = slice_by_index(begin = key_token_113_begin_0, end = key_token_113_end_0, end_mask = key_token_113_end_mask_0, squeeze_mask = key_token_113_squeeze_mask_0, x = key_19_cast_fp16)[name = string("key_token_113_cast_fp16")];
            tensor<int32, [4]> value_token_113_begin_0 = const()[name = string("value_token_113_begin_0"), val = tensor<int32, [4]>([0, 0, 8, 0])];
            tensor<int32, [4]> value_token_113_end_0 = const()[name = string("value_token_113_end_0"), val = tensor<int32, [4]>([1, 16, 9, 128])];
            tensor<bool, [4]> value_token_113_end_mask_0 = const()[name = string("value_token_113_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_113_squeeze_mask_0 = const()[name = string("value_token_113_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_113_cast_fp16 = slice_by_index(begin = value_token_113_begin_0, end = value_token_113_end_0, end_mask = value_token_113_end_mask_0, squeeze_mask = value_token_113_squeeze_mask_0, x = value_69_cast_fp16)[name = string("value_token_113_cast_fp16")];
            tensor<int32, [1]> var_3195_axes_0 = const()[name = string("op_3195_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_3195_cast_fp16 = expand_dims(axes = var_3195_axes_0, x = key_token_113_cast_fp16)[name = string("op_3195_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3196_cast_fp16 = mul(x = state_225_cast_fp16, y = var_3195_cast_fp16)[name = string("op_3196_cast_fp16")];
            tensor<int32, [1]> memory_113_axes_0 = const()[name = string("memory_113_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_113_keep_dims_0 = const()[name = string("memory_113_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_113_cast_fp16 = reduce_sum(axes = memory_113_axes_0, keep_dims = memory_113_keep_dims_0, x = var_3196_cast_fp16)[name = string("memory_113_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_3199_cast_fp16 = sub(x = value_token_113_cast_fp16, y = memory_113_cast_fp16)[name = string("op_3199_cast_fp16")];
            tensor<int32, [3]> var_3202_begin_0 = const()[name = string("op_3202_begin_0"), val = tensor<int32, [3]>([0, 0, 8])];
            tensor<int32, [3]> var_3202_end_0 = const()[name = string("op_3202_end_0"), val = tensor<int32, [3]>([1, 16, 9])];
            tensor<bool, [3]> var_3202_end_mask_0 = const()[name = string("op_3202_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_3202_squeeze_mask_0 = const()[name = string("op_3202_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_3202_cast_fp16 = slice_by_index(begin = var_3202_begin_0, end = var_3202_end_0, end_mask = var_3202_end_mask_0, squeeze_mask = var_3202_squeeze_mask_0, x = beta_7_cast_fp16)[name = string("op_3202_cast_fp16")];
            tensor<int32, [1]> var_3203_axes_0 = const()[name = string("op_3203_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_3203_cast_fp16 = expand_dims(axes = var_3203_axes_0, x = var_3202_cast_fp16)[name = string("op_3203_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_113_cast_fp16 = mul(x = var_3199_cast_fp16, y = var_3203_cast_fp16)[name = string("delta_113_cast_fp16")];
            tensor<int32, [1]> var_3206_axes_0 = const()[name = string("op_3206_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_3206_cast_fp16 = expand_dims(axes = var_3206_axes_0, x = delta_113_cast_fp16)[name = string("op_3206_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3207_cast_fp16 = mul(x = var_3195_cast_fp16, y = var_3206_cast_fp16)[name = string("op_3207_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_227_cast_fp16 = add(x = state_225_cast_fp16, y = var_3207_cast_fp16)[name = string("state_227_cast_fp16")];
            tensor<int32, [4]> var_3211_begin_0 = const()[name = string("op_3211_begin_0"), val = tensor<int32, [4]>([0, 0, 8, 0])];
            tensor<int32, [4]> var_3211_end_0 = const()[name = string("op_3211_end_0"), val = tensor<int32, [4]>([1, 16, 9, 128])];
            tensor<bool, [4]> var_3211_end_mask_0 = const()[name = string("op_3211_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_3211_squeeze_mask_0 = const()[name = string("op_3211_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_3211_cast_fp16 = slice_by_index(begin = var_3211_begin_0, end = var_3211_end_0, end_mask = var_3211_end_mask_0, squeeze_mask = var_3211_squeeze_mask_0, x = _inversed_query_19_cast_fp16)[name = string("op_3211_cast_fp16")];
            tensor<int32, [1]> var_3212_axes_0 = const()[name = string("op_3212_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_3212_cast_fp16 = expand_dims(axes = var_3212_axes_0, x = var_3211_cast_fp16)[name = string("op_3212_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3213_cast_fp16 = mul(x = state_227_cast_fp16, y = var_3212_cast_fp16)[name = string("op_3213_cast_fp16")];
            tensor<int32, [1]> var_3215_axes_0 = const()[name = string("op_3215_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_3215_keep_dims_0 = const()[name = string("op_3215_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_3215_cast_fp16 = reduce_sum(axes = var_3215_axes_0, keep_dims = var_3215_keep_dims_0, x = var_3213_cast_fp16)[name = string("op_3215_cast_fp16")];
            tensor<int32, [3]> var_3218_begin_0 = const()[name = string("op_3218_begin_0"), val = tensor<int32, [3]>([0, 0, 9])];
            tensor<int32, [3]> var_3218_end_0 = const()[name = string("op_3218_end_0"), val = tensor<int32, [3]>([1, 16, 10])];
            tensor<bool, [3]> var_3218_end_mask_0 = const()[name = string("op_3218_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_3218_squeeze_mask_0 = const()[name = string("op_3218_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_3218_cast_fp16 = slice_by_index(begin = var_3218_begin_0, end = var_3218_end_0, end_mask = var_3218_end_mask_0, squeeze_mask = var_3218_squeeze_mask_0, x = decay_7_cast_fp16)[name = string("op_3218_cast_fp16")];
            tensor<fp16, [1, 16]> var_3219_cast_fp16 = exp(x = var_3218_cast_fp16)[name = string("op_3219_cast_fp16")];
            tensor<int32, [1]> var_3220_axes_0 = const()[name = string("op_3220_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_3220_cast_fp16 = expand_dims(axes = var_3220_axes_0, x = var_3219_cast_fp16)[name = string("op_3220_cast_fp16")];
            tensor<int32, [1]> var_3221_axes_0 = const()[name = string("op_3221_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_3221_cast_fp16 = expand_dims(axes = var_3221_axes_0, x = var_3220_cast_fp16)[name = string("op_3221_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_229_cast_fp16 = mul(x = state_227_cast_fp16, y = var_3221_cast_fp16)[name = string("state_229_cast_fp16")];
            tensor<int32, [4]> key_token_115_begin_0 = const()[name = string("key_token_115_begin_0"), val = tensor<int32, [4]>([0, 0, 9, 0])];
            tensor<int32, [4]> key_token_115_end_0 = const()[name = string("key_token_115_end_0"), val = tensor<int32, [4]>([1, 16, 10, 128])];
            tensor<bool, [4]> key_token_115_end_mask_0 = const()[name = string("key_token_115_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_115_squeeze_mask_0 = const()[name = string("key_token_115_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_115_cast_fp16 = slice_by_index(begin = key_token_115_begin_0, end = key_token_115_end_0, end_mask = key_token_115_end_mask_0, squeeze_mask = key_token_115_squeeze_mask_0, x = key_19_cast_fp16)[name = string("key_token_115_cast_fp16")];
            tensor<int32, [4]> value_token_115_begin_0 = const()[name = string("value_token_115_begin_0"), val = tensor<int32, [4]>([0, 0, 9, 0])];
            tensor<int32, [4]> value_token_115_end_0 = const()[name = string("value_token_115_end_0"), val = tensor<int32, [4]>([1, 16, 10, 128])];
            tensor<bool, [4]> value_token_115_end_mask_0 = const()[name = string("value_token_115_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_115_squeeze_mask_0 = const()[name = string("value_token_115_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_115_cast_fp16 = slice_by_index(begin = value_token_115_begin_0, end = value_token_115_end_0, end_mask = value_token_115_end_mask_0, squeeze_mask = value_token_115_squeeze_mask_0, x = value_69_cast_fp16)[name = string("value_token_115_cast_fp16")];
            tensor<int32, [1]> var_3229_axes_0 = const()[name = string("op_3229_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_3229_cast_fp16 = expand_dims(axes = var_3229_axes_0, x = key_token_115_cast_fp16)[name = string("op_3229_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3230_cast_fp16 = mul(x = state_229_cast_fp16, y = var_3229_cast_fp16)[name = string("op_3230_cast_fp16")];
            tensor<int32, [1]> memory_115_axes_0 = const()[name = string("memory_115_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_115_keep_dims_0 = const()[name = string("memory_115_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_115_cast_fp16 = reduce_sum(axes = memory_115_axes_0, keep_dims = memory_115_keep_dims_0, x = var_3230_cast_fp16)[name = string("memory_115_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_3233_cast_fp16 = sub(x = value_token_115_cast_fp16, y = memory_115_cast_fp16)[name = string("op_3233_cast_fp16")];
            tensor<int32, [3]> var_3236_begin_0 = const()[name = string("op_3236_begin_0"), val = tensor<int32, [3]>([0, 0, 9])];
            tensor<int32, [3]> var_3236_end_0 = const()[name = string("op_3236_end_0"), val = tensor<int32, [3]>([1, 16, 10])];
            tensor<bool, [3]> var_3236_end_mask_0 = const()[name = string("op_3236_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_3236_squeeze_mask_0 = const()[name = string("op_3236_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_3236_cast_fp16 = slice_by_index(begin = var_3236_begin_0, end = var_3236_end_0, end_mask = var_3236_end_mask_0, squeeze_mask = var_3236_squeeze_mask_0, x = beta_7_cast_fp16)[name = string("op_3236_cast_fp16")];
            tensor<int32, [1]> var_3237_axes_0 = const()[name = string("op_3237_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_3237_cast_fp16 = expand_dims(axes = var_3237_axes_0, x = var_3236_cast_fp16)[name = string("op_3237_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_115_cast_fp16 = mul(x = var_3233_cast_fp16, y = var_3237_cast_fp16)[name = string("delta_115_cast_fp16")];
            tensor<int32, [1]> var_3240_axes_0 = const()[name = string("op_3240_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_3240_cast_fp16 = expand_dims(axes = var_3240_axes_0, x = delta_115_cast_fp16)[name = string("op_3240_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3241_cast_fp16 = mul(x = var_3229_cast_fp16, y = var_3240_cast_fp16)[name = string("op_3241_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_231_cast_fp16 = add(x = state_229_cast_fp16, y = var_3241_cast_fp16)[name = string("state_231_cast_fp16")];
            tensor<int32, [4]> var_3245_begin_0 = const()[name = string("op_3245_begin_0"), val = tensor<int32, [4]>([0, 0, 9, 0])];
            tensor<int32, [4]> var_3245_end_0 = const()[name = string("op_3245_end_0"), val = tensor<int32, [4]>([1, 16, 10, 128])];
            tensor<bool, [4]> var_3245_end_mask_0 = const()[name = string("op_3245_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_3245_squeeze_mask_0 = const()[name = string("op_3245_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_3245_cast_fp16 = slice_by_index(begin = var_3245_begin_0, end = var_3245_end_0, end_mask = var_3245_end_mask_0, squeeze_mask = var_3245_squeeze_mask_0, x = _inversed_query_19_cast_fp16)[name = string("op_3245_cast_fp16")];
            tensor<int32, [1]> var_3246_axes_0 = const()[name = string("op_3246_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_3246_cast_fp16 = expand_dims(axes = var_3246_axes_0, x = var_3245_cast_fp16)[name = string("op_3246_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3247_cast_fp16 = mul(x = state_231_cast_fp16, y = var_3246_cast_fp16)[name = string("op_3247_cast_fp16")];
            tensor<int32, [1]> var_3249_axes_0 = const()[name = string("op_3249_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_3249_keep_dims_0 = const()[name = string("op_3249_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_3249_cast_fp16 = reduce_sum(axes = var_3249_axes_0, keep_dims = var_3249_keep_dims_0, x = var_3247_cast_fp16)[name = string("op_3249_cast_fp16")];
            tensor<int32, [3]> var_3252_begin_0 = const()[name = string("op_3252_begin_0"), val = tensor<int32, [3]>([0, 0, 10])];
            tensor<int32, [3]> var_3252_end_0 = const()[name = string("op_3252_end_0"), val = tensor<int32, [3]>([1, 16, 11])];
            tensor<bool, [3]> var_3252_end_mask_0 = const()[name = string("op_3252_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_3252_squeeze_mask_0 = const()[name = string("op_3252_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_3252_cast_fp16 = slice_by_index(begin = var_3252_begin_0, end = var_3252_end_0, end_mask = var_3252_end_mask_0, squeeze_mask = var_3252_squeeze_mask_0, x = decay_7_cast_fp16)[name = string("op_3252_cast_fp16")];
            tensor<fp16, [1, 16]> var_3253_cast_fp16 = exp(x = var_3252_cast_fp16)[name = string("op_3253_cast_fp16")];
            tensor<int32, [1]> var_3254_axes_0 = const()[name = string("op_3254_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_3254_cast_fp16 = expand_dims(axes = var_3254_axes_0, x = var_3253_cast_fp16)[name = string("op_3254_cast_fp16")];
            tensor<int32, [1]> var_3255_axes_0 = const()[name = string("op_3255_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_3255_cast_fp16 = expand_dims(axes = var_3255_axes_0, x = var_3254_cast_fp16)[name = string("op_3255_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_233_cast_fp16 = mul(x = state_231_cast_fp16, y = var_3255_cast_fp16)[name = string("state_233_cast_fp16")];
            tensor<int32, [4]> key_token_117_begin_0 = const()[name = string("key_token_117_begin_0"), val = tensor<int32, [4]>([0, 0, 10, 0])];
            tensor<int32, [4]> key_token_117_end_0 = const()[name = string("key_token_117_end_0"), val = tensor<int32, [4]>([1, 16, 11, 128])];
            tensor<bool, [4]> key_token_117_end_mask_0 = const()[name = string("key_token_117_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_117_squeeze_mask_0 = const()[name = string("key_token_117_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_117_cast_fp16 = slice_by_index(begin = key_token_117_begin_0, end = key_token_117_end_0, end_mask = key_token_117_end_mask_0, squeeze_mask = key_token_117_squeeze_mask_0, x = key_19_cast_fp16)[name = string("key_token_117_cast_fp16")];
            tensor<int32, [4]> value_token_117_begin_0 = const()[name = string("value_token_117_begin_0"), val = tensor<int32, [4]>([0, 0, 10, 0])];
            tensor<int32, [4]> value_token_117_end_0 = const()[name = string("value_token_117_end_0"), val = tensor<int32, [4]>([1, 16, 11, 128])];
            tensor<bool, [4]> value_token_117_end_mask_0 = const()[name = string("value_token_117_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_117_squeeze_mask_0 = const()[name = string("value_token_117_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_117_cast_fp16 = slice_by_index(begin = value_token_117_begin_0, end = value_token_117_end_0, end_mask = value_token_117_end_mask_0, squeeze_mask = value_token_117_squeeze_mask_0, x = value_69_cast_fp16)[name = string("value_token_117_cast_fp16")];
            tensor<int32, [1]> var_3263_axes_0 = const()[name = string("op_3263_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_3263_cast_fp16 = expand_dims(axes = var_3263_axes_0, x = key_token_117_cast_fp16)[name = string("op_3263_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3264_cast_fp16 = mul(x = state_233_cast_fp16, y = var_3263_cast_fp16)[name = string("op_3264_cast_fp16")];
            tensor<int32, [1]> memory_117_axes_0 = const()[name = string("memory_117_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_117_keep_dims_0 = const()[name = string("memory_117_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_117_cast_fp16 = reduce_sum(axes = memory_117_axes_0, keep_dims = memory_117_keep_dims_0, x = var_3264_cast_fp16)[name = string("memory_117_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_3267_cast_fp16 = sub(x = value_token_117_cast_fp16, y = memory_117_cast_fp16)[name = string("op_3267_cast_fp16")];
            tensor<int32, [3]> var_3270_begin_0 = const()[name = string("op_3270_begin_0"), val = tensor<int32, [3]>([0, 0, 10])];
            tensor<int32, [3]> var_3270_end_0 = const()[name = string("op_3270_end_0"), val = tensor<int32, [3]>([1, 16, 11])];
            tensor<bool, [3]> var_3270_end_mask_0 = const()[name = string("op_3270_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_3270_squeeze_mask_0 = const()[name = string("op_3270_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_3270_cast_fp16 = slice_by_index(begin = var_3270_begin_0, end = var_3270_end_0, end_mask = var_3270_end_mask_0, squeeze_mask = var_3270_squeeze_mask_0, x = beta_7_cast_fp16)[name = string("op_3270_cast_fp16")];
            tensor<int32, [1]> var_3271_axes_0 = const()[name = string("op_3271_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_3271_cast_fp16 = expand_dims(axes = var_3271_axes_0, x = var_3270_cast_fp16)[name = string("op_3271_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_117_cast_fp16 = mul(x = var_3267_cast_fp16, y = var_3271_cast_fp16)[name = string("delta_117_cast_fp16")];
            tensor<int32, [1]> var_3274_axes_0 = const()[name = string("op_3274_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_3274_cast_fp16 = expand_dims(axes = var_3274_axes_0, x = delta_117_cast_fp16)[name = string("op_3274_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3275_cast_fp16 = mul(x = var_3263_cast_fp16, y = var_3274_cast_fp16)[name = string("op_3275_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_235_cast_fp16 = add(x = state_233_cast_fp16, y = var_3275_cast_fp16)[name = string("state_235_cast_fp16")];
            tensor<int32, [4]> var_3279_begin_0 = const()[name = string("op_3279_begin_0"), val = tensor<int32, [4]>([0, 0, 10, 0])];
            tensor<int32, [4]> var_3279_end_0 = const()[name = string("op_3279_end_0"), val = tensor<int32, [4]>([1, 16, 11, 128])];
            tensor<bool, [4]> var_3279_end_mask_0 = const()[name = string("op_3279_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_3279_squeeze_mask_0 = const()[name = string("op_3279_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_3279_cast_fp16 = slice_by_index(begin = var_3279_begin_0, end = var_3279_end_0, end_mask = var_3279_end_mask_0, squeeze_mask = var_3279_squeeze_mask_0, x = _inversed_query_19_cast_fp16)[name = string("op_3279_cast_fp16")];
            tensor<int32, [1]> var_3280_axes_0 = const()[name = string("op_3280_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_3280_cast_fp16 = expand_dims(axes = var_3280_axes_0, x = var_3279_cast_fp16)[name = string("op_3280_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3281_cast_fp16 = mul(x = state_235_cast_fp16, y = var_3280_cast_fp16)[name = string("op_3281_cast_fp16")];
            tensor<int32, [1]> var_3283_axes_0 = const()[name = string("op_3283_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_3283_keep_dims_0 = const()[name = string("op_3283_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_3283_cast_fp16 = reduce_sum(axes = var_3283_axes_0, keep_dims = var_3283_keep_dims_0, x = var_3281_cast_fp16)[name = string("op_3283_cast_fp16")];
            tensor<int32, [3]> var_3286_begin_0 = const()[name = string("op_3286_begin_0"), val = tensor<int32, [3]>([0, 0, 11])];
            tensor<int32, [3]> var_3286_end_0 = const()[name = string("op_3286_end_0"), val = tensor<int32, [3]>([1, 16, 12])];
            tensor<bool, [3]> var_3286_end_mask_0 = const()[name = string("op_3286_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_3286_squeeze_mask_0 = const()[name = string("op_3286_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_3286_cast_fp16 = slice_by_index(begin = var_3286_begin_0, end = var_3286_end_0, end_mask = var_3286_end_mask_0, squeeze_mask = var_3286_squeeze_mask_0, x = decay_7_cast_fp16)[name = string("op_3286_cast_fp16")];
            tensor<fp16, [1, 16]> var_3287_cast_fp16 = exp(x = var_3286_cast_fp16)[name = string("op_3287_cast_fp16")];
            tensor<int32, [1]> var_3288_axes_0 = const()[name = string("op_3288_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_3288_cast_fp16 = expand_dims(axes = var_3288_axes_0, x = var_3287_cast_fp16)[name = string("op_3288_cast_fp16")];
            tensor<int32, [1]> var_3289_axes_0 = const()[name = string("op_3289_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_3289_cast_fp16 = expand_dims(axes = var_3289_axes_0, x = var_3288_cast_fp16)[name = string("op_3289_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_237_cast_fp16 = mul(x = state_235_cast_fp16, y = var_3289_cast_fp16)[name = string("state_237_cast_fp16")];
            tensor<int32, [4]> key_token_119_begin_0 = const()[name = string("key_token_119_begin_0"), val = tensor<int32, [4]>([0, 0, 11, 0])];
            tensor<int32, [4]> key_token_119_end_0 = const()[name = string("key_token_119_end_0"), val = tensor<int32, [4]>([1, 16, 12, 128])];
            tensor<bool, [4]> key_token_119_end_mask_0 = const()[name = string("key_token_119_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_119_squeeze_mask_0 = const()[name = string("key_token_119_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_119_cast_fp16 = slice_by_index(begin = key_token_119_begin_0, end = key_token_119_end_0, end_mask = key_token_119_end_mask_0, squeeze_mask = key_token_119_squeeze_mask_0, x = key_19_cast_fp16)[name = string("key_token_119_cast_fp16")];
            tensor<int32, [4]> value_token_119_begin_0 = const()[name = string("value_token_119_begin_0"), val = tensor<int32, [4]>([0, 0, 11, 0])];
            tensor<int32, [4]> value_token_119_end_0 = const()[name = string("value_token_119_end_0"), val = tensor<int32, [4]>([1, 16, 12, 128])];
            tensor<bool, [4]> value_token_119_end_mask_0 = const()[name = string("value_token_119_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_119_squeeze_mask_0 = const()[name = string("value_token_119_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_119_cast_fp16 = slice_by_index(begin = value_token_119_begin_0, end = value_token_119_end_0, end_mask = value_token_119_end_mask_0, squeeze_mask = value_token_119_squeeze_mask_0, x = value_69_cast_fp16)[name = string("value_token_119_cast_fp16")];
            tensor<int32, [1]> var_3297_axes_0 = const()[name = string("op_3297_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_3297_cast_fp16 = expand_dims(axes = var_3297_axes_0, x = key_token_119_cast_fp16)[name = string("op_3297_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3298_cast_fp16 = mul(x = state_237_cast_fp16, y = var_3297_cast_fp16)[name = string("op_3298_cast_fp16")];
            tensor<int32, [1]> memory_119_axes_0 = const()[name = string("memory_119_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_119_keep_dims_0 = const()[name = string("memory_119_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_119_cast_fp16 = reduce_sum(axes = memory_119_axes_0, keep_dims = memory_119_keep_dims_0, x = var_3298_cast_fp16)[name = string("memory_119_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_3301_cast_fp16 = sub(x = value_token_119_cast_fp16, y = memory_119_cast_fp16)[name = string("op_3301_cast_fp16")];
            tensor<int32, [3]> var_3304_begin_0 = const()[name = string("op_3304_begin_0"), val = tensor<int32, [3]>([0, 0, 11])];
            tensor<int32, [3]> var_3304_end_0 = const()[name = string("op_3304_end_0"), val = tensor<int32, [3]>([1, 16, 12])];
            tensor<bool, [3]> var_3304_end_mask_0 = const()[name = string("op_3304_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_3304_squeeze_mask_0 = const()[name = string("op_3304_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_3304_cast_fp16 = slice_by_index(begin = var_3304_begin_0, end = var_3304_end_0, end_mask = var_3304_end_mask_0, squeeze_mask = var_3304_squeeze_mask_0, x = beta_7_cast_fp16)[name = string("op_3304_cast_fp16")];
            tensor<int32, [1]> var_3305_axes_0 = const()[name = string("op_3305_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_3305_cast_fp16 = expand_dims(axes = var_3305_axes_0, x = var_3304_cast_fp16)[name = string("op_3305_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_119_cast_fp16 = mul(x = var_3301_cast_fp16, y = var_3305_cast_fp16)[name = string("delta_119_cast_fp16")];
            tensor<int32, [1]> var_3308_axes_0 = const()[name = string("op_3308_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_3308_cast_fp16 = expand_dims(axes = var_3308_axes_0, x = delta_119_cast_fp16)[name = string("op_3308_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3309_cast_fp16 = mul(x = var_3297_cast_fp16, y = var_3308_cast_fp16)[name = string("op_3309_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_239_cast_fp16 = add(x = state_237_cast_fp16, y = var_3309_cast_fp16)[name = string("state_239_cast_fp16")];
            tensor<int32, [4]> var_3313_begin_0 = const()[name = string("op_3313_begin_0"), val = tensor<int32, [4]>([0, 0, 11, 0])];
            tensor<int32, [4]> var_3313_end_0 = const()[name = string("op_3313_end_0"), val = tensor<int32, [4]>([1, 16, 12, 128])];
            tensor<bool, [4]> var_3313_end_mask_0 = const()[name = string("op_3313_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_3313_squeeze_mask_0 = const()[name = string("op_3313_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_3313_cast_fp16 = slice_by_index(begin = var_3313_begin_0, end = var_3313_end_0, end_mask = var_3313_end_mask_0, squeeze_mask = var_3313_squeeze_mask_0, x = _inversed_query_19_cast_fp16)[name = string("op_3313_cast_fp16")];
            tensor<int32, [1]> var_3314_axes_0 = const()[name = string("op_3314_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_3314_cast_fp16 = expand_dims(axes = var_3314_axes_0, x = var_3313_cast_fp16)[name = string("op_3314_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3315_cast_fp16 = mul(x = state_239_cast_fp16, y = var_3314_cast_fp16)[name = string("op_3315_cast_fp16")];
            tensor<int32, [1]> var_3317_axes_0 = const()[name = string("op_3317_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_3317_keep_dims_0 = const()[name = string("op_3317_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_3317_cast_fp16 = reduce_sum(axes = var_3317_axes_0, keep_dims = var_3317_keep_dims_0, x = var_3315_cast_fp16)[name = string("op_3317_cast_fp16")];
            tensor<int32, [3]> var_3320_begin_0 = const()[name = string("op_3320_begin_0"), val = tensor<int32, [3]>([0, 0, 12])];
            tensor<int32, [3]> var_3320_end_0 = const()[name = string("op_3320_end_0"), val = tensor<int32, [3]>([1, 16, 13])];
            tensor<bool, [3]> var_3320_end_mask_0 = const()[name = string("op_3320_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_3320_squeeze_mask_0 = const()[name = string("op_3320_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_3320_cast_fp16 = slice_by_index(begin = var_3320_begin_0, end = var_3320_end_0, end_mask = var_3320_end_mask_0, squeeze_mask = var_3320_squeeze_mask_0, x = decay_7_cast_fp16)[name = string("op_3320_cast_fp16")];
            tensor<fp16, [1, 16]> var_3321_cast_fp16 = exp(x = var_3320_cast_fp16)[name = string("op_3321_cast_fp16")];
            tensor<int32, [1]> var_3322_axes_0 = const()[name = string("op_3322_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_3322_cast_fp16 = expand_dims(axes = var_3322_axes_0, x = var_3321_cast_fp16)[name = string("op_3322_cast_fp16")];
            tensor<int32, [1]> var_3323_axes_0 = const()[name = string("op_3323_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_3323_cast_fp16 = expand_dims(axes = var_3323_axes_0, x = var_3322_cast_fp16)[name = string("op_3323_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_241_cast_fp16 = mul(x = state_239_cast_fp16, y = var_3323_cast_fp16)[name = string("state_241_cast_fp16")];
            tensor<int32, [4]> key_token_121_begin_0 = const()[name = string("key_token_121_begin_0"), val = tensor<int32, [4]>([0, 0, 12, 0])];
            tensor<int32, [4]> key_token_121_end_0 = const()[name = string("key_token_121_end_0"), val = tensor<int32, [4]>([1, 16, 13, 128])];
            tensor<bool, [4]> key_token_121_end_mask_0 = const()[name = string("key_token_121_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_121_squeeze_mask_0 = const()[name = string("key_token_121_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_121_cast_fp16 = slice_by_index(begin = key_token_121_begin_0, end = key_token_121_end_0, end_mask = key_token_121_end_mask_0, squeeze_mask = key_token_121_squeeze_mask_0, x = key_19_cast_fp16)[name = string("key_token_121_cast_fp16")];
            tensor<int32, [4]> value_token_121_begin_0 = const()[name = string("value_token_121_begin_0"), val = tensor<int32, [4]>([0, 0, 12, 0])];
            tensor<int32, [4]> value_token_121_end_0 = const()[name = string("value_token_121_end_0"), val = tensor<int32, [4]>([1, 16, 13, 128])];
            tensor<bool, [4]> value_token_121_end_mask_0 = const()[name = string("value_token_121_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_121_squeeze_mask_0 = const()[name = string("value_token_121_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_121_cast_fp16 = slice_by_index(begin = value_token_121_begin_0, end = value_token_121_end_0, end_mask = value_token_121_end_mask_0, squeeze_mask = value_token_121_squeeze_mask_0, x = value_69_cast_fp16)[name = string("value_token_121_cast_fp16")];
            tensor<int32, [1]> var_3331_axes_0 = const()[name = string("op_3331_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_3331_cast_fp16 = expand_dims(axes = var_3331_axes_0, x = key_token_121_cast_fp16)[name = string("op_3331_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3332_cast_fp16 = mul(x = state_241_cast_fp16, y = var_3331_cast_fp16)[name = string("op_3332_cast_fp16")];
            tensor<int32, [1]> memory_121_axes_0 = const()[name = string("memory_121_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_121_keep_dims_0 = const()[name = string("memory_121_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_121_cast_fp16 = reduce_sum(axes = memory_121_axes_0, keep_dims = memory_121_keep_dims_0, x = var_3332_cast_fp16)[name = string("memory_121_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_3335_cast_fp16 = sub(x = value_token_121_cast_fp16, y = memory_121_cast_fp16)[name = string("op_3335_cast_fp16")];
            tensor<int32, [3]> var_3338_begin_0 = const()[name = string("op_3338_begin_0"), val = tensor<int32, [3]>([0, 0, 12])];
            tensor<int32, [3]> var_3338_end_0 = const()[name = string("op_3338_end_0"), val = tensor<int32, [3]>([1, 16, 13])];
            tensor<bool, [3]> var_3338_end_mask_0 = const()[name = string("op_3338_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_3338_squeeze_mask_0 = const()[name = string("op_3338_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_3338_cast_fp16 = slice_by_index(begin = var_3338_begin_0, end = var_3338_end_0, end_mask = var_3338_end_mask_0, squeeze_mask = var_3338_squeeze_mask_0, x = beta_7_cast_fp16)[name = string("op_3338_cast_fp16")];
            tensor<int32, [1]> var_3339_axes_0 = const()[name = string("op_3339_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_3339_cast_fp16 = expand_dims(axes = var_3339_axes_0, x = var_3338_cast_fp16)[name = string("op_3339_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_121_cast_fp16 = mul(x = var_3335_cast_fp16, y = var_3339_cast_fp16)[name = string("delta_121_cast_fp16")];
            tensor<int32, [1]> var_3342_axes_0 = const()[name = string("op_3342_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_3342_cast_fp16 = expand_dims(axes = var_3342_axes_0, x = delta_121_cast_fp16)[name = string("op_3342_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3343_cast_fp16 = mul(x = var_3331_cast_fp16, y = var_3342_cast_fp16)[name = string("op_3343_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_243_cast_fp16 = add(x = state_241_cast_fp16, y = var_3343_cast_fp16)[name = string("state_243_cast_fp16")];
            tensor<int32, [4]> var_3347_begin_0 = const()[name = string("op_3347_begin_0"), val = tensor<int32, [4]>([0, 0, 12, 0])];
            tensor<int32, [4]> var_3347_end_0 = const()[name = string("op_3347_end_0"), val = tensor<int32, [4]>([1, 16, 13, 128])];
            tensor<bool, [4]> var_3347_end_mask_0 = const()[name = string("op_3347_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_3347_squeeze_mask_0 = const()[name = string("op_3347_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_3347_cast_fp16 = slice_by_index(begin = var_3347_begin_0, end = var_3347_end_0, end_mask = var_3347_end_mask_0, squeeze_mask = var_3347_squeeze_mask_0, x = _inversed_query_19_cast_fp16)[name = string("op_3347_cast_fp16")];
            tensor<int32, [1]> var_3348_axes_0 = const()[name = string("op_3348_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_3348_cast_fp16 = expand_dims(axes = var_3348_axes_0, x = var_3347_cast_fp16)[name = string("op_3348_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3349_cast_fp16 = mul(x = state_243_cast_fp16, y = var_3348_cast_fp16)[name = string("op_3349_cast_fp16")];
            tensor<int32, [1]> var_3351_axes_0 = const()[name = string("op_3351_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_3351_keep_dims_0 = const()[name = string("op_3351_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_3351_cast_fp16 = reduce_sum(axes = var_3351_axes_0, keep_dims = var_3351_keep_dims_0, x = var_3349_cast_fp16)[name = string("op_3351_cast_fp16")];
            tensor<int32, [3]> var_3354_begin_0 = const()[name = string("op_3354_begin_0"), val = tensor<int32, [3]>([0, 0, 13])];
            tensor<int32, [3]> var_3354_end_0 = const()[name = string("op_3354_end_0"), val = tensor<int32, [3]>([1, 16, 14])];
            tensor<bool, [3]> var_3354_end_mask_0 = const()[name = string("op_3354_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_3354_squeeze_mask_0 = const()[name = string("op_3354_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_3354_cast_fp16 = slice_by_index(begin = var_3354_begin_0, end = var_3354_end_0, end_mask = var_3354_end_mask_0, squeeze_mask = var_3354_squeeze_mask_0, x = decay_7_cast_fp16)[name = string("op_3354_cast_fp16")];
            tensor<fp16, [1, 16]> var_3355_cast_fp16 = exp(x = var_3354_cast_fp16)[name = string("op_3355_cast_fp16")];
            tensor<int32, [1]> var_3356_axes_0 = const()[name = string("op_3356_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_3356_cast_fp16 = expand_dims(axes = var_3356_axes_0, x = var_3355_cast_fp16)[name = string("op_3356_cast_fp16")];
            tensor<int32, [1]> var_3357_axes_0 = const()[name = string("op_3357_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_3357_cast_fp16 = expand_dims(axes = var_3357_axes_0, x = var_3356_cast_fp16)[name = string("op_3357_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_245_cast_fp16 = mul(x = state_243_cast_fp16, y = var_3357_cast_fp16)[name = string("state_245_cast_fp16")];
            tensor<int32, [4]> key_token_123_begin_0 = const()[name = string("key_token_123_begin_0"), val = tensor<int32, [4]>([0, 0, 13, 0])];
            tensor<int32, [4]> key_token_123_end_0 = const()[name = string("key_token_123_end_0"), val = tensor<int32, [4]>([1, 16, 14, 128])];
            tensor<bool, [4]> key_token_123_end_mask_0 = const()[name = string("key_token_123_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_123_squeeze_mask_0 = const()[name = string("key_token_123_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_123_cast_fp16 = slice_by_index(begin = key_token_123_begin_0, end = key_token_123_end_0, end_mask = key_token_123_end_mask_0, squeeze_mask = key_token_123_squeeze_mask_0, x = key_19_cast_fp16)[name = string("key_token_123_cast_fp16")];
            tensor<int32, [4]> value_token_123_begin_0 = const()[name = string("value_token_123_begin_0"), val = tensor<int32, [4]>([0, 0, 13, 0])];
            tensor<int32, [4]> value_token_123_end_0 = const()[name = string("value_token_123_end_0"), val = tensor<int32, [4]>([1, 16, 14, 128])];
            tensor<bool, [4]> value_token_123_end_mask_0 = const()[name = string("value_token_123_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_123_squeeze_mask_0 = const()[name = string("value_token_123_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_123_cast_fp16 = slice_by_index(begin = value_token_123_begin_0, end = value_token_123_end_0, end_mask = value_token_123_end_mask_0, squeeze_mask = value_token_123_squeeze_mask_0, x = value_69_cast_fp16)[name = string("value_token_123_cast_fp16")];
            tensor<int32, [1]> var_3365_axes_0 = const()[name = string("op_3365_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_3365_cast_fp16 = expand_dims(axes = var_3365_axes_0, x = key_token_123_cast_fp16)[name = string("op_3365_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3366_cast_fp16 = mul(x = state_245_cast_fp16, y = var_3365_cast_fp16)[name = string("op_3366_cast_fp16")];
            tensor<int32, [1]> memory_123_axes_0 = const()[name = string("memory_123_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_123_keep_dims_0 = const()[name = string("memory_123_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_123_cast_fp16 = reduce_sum(axes = memory_123_axes_0, keep_dims = memory_123_keep_dims_0, x = var_3366_cast_fp16)[name = string("memory_123_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_3369_cast_fp16 = sub(x = value_token_123_cast_fp16, y = memory_123_cast_fp16)[name = string("op_3369_cast_fp16")];
            tensor<int32, [3]> var_3372_begin_0 = const()[name = string("op_3372_begin_0"), val = tensor<int32, [3]>([0, 0, 13])];
            tensor<int32, [3]> var_3372_end_0 = const()[name = string("op_3372_end_0"), val = tensor<int32, [3]>([1, 16, 14])];
            tensor<bool, [3]> var_3372_end_mask_0 = const()[name = string("op_3372_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_3372_squeeze_mask_0 = const()[name = string("op_3372_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_3372_cast_fp16 = slice_by_index(begin = var_3372_begin_0, end = var_3372_end_0, end_mask = var_3372_end_mask_0, squeeze_mask = var_3372_squeeze_mask_0, x = beta_7_cast_fp16)[name = string("op_3372_cast_fp16")];
            tensor<int32, [1]> var_3373_axes_0 = const()[name = string("op_3373_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_3373_cast_fp16 = expand_dims(axes = var_3373_axes_0, x = var_3372_cast_fp16)[name = string("op_3373_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_123_cast_fp16 = mul(x = var_3369_cast_fp16, y = var_3373_cast_fp16)[name = string("delta_123_cast_fp16")];
            tensor<int32, [1]> var_3376_axes_0 = const()[name = string("op_3376_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_3376_cast_fp16 = expand_dims(axes = var_3376_axes_0, x = delta_123_cast_fp16)[name = string("op_3376_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3377_cast_fp16 = mul(x = var_3365_cast_fp16, y = var_3376_cast_fp16)[name = string("op_3377_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_247_cast_fp16 = add(x = state_245_cast_fp16, y = var_3377_cast_fp16)[name = string("state_247_cast_fp16")];
            tensor<int32, [4]> var_3381_begin_0 = const()[name = string("op_3381_begin_0"), val = tensor<int32, [4]>([0, 0, 13, 0])];
            tensor<int32, [4]> var_3381_end_0 = const()[name = string("op_3381_end_0"), val = tensor<int32, [4]>([1, 16, 14, 128])];
            tensor<bool, [4]> var_3381_end_mask_0 = const()[name = string("op_3381_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_3381_squeeze_mask_0 = const()[name = string("op_3381_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_3381_cast_fp16 = slice_by_index(begin = var_3381_begin_0, end = var_3381_end_0, end_mask = var_3381_end_mask_0, squeeze_mask = var_3381_squeeze_mask_0, x = _inversed_query_19_cast_fp16)[name = string("op_3381_cast_fp16")];
            tensor<int32, [1]> var_3382_axes_0 = const()[name = string("op_3382_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_3382_cast_fp16 = expand_dims(axes = var_3382_axes_0, x = var_3381_cast_fp16)[name = string("op_3382_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3383_cast_fp16 = mul(x = state_247_cast_fp16, y = var_3382_cast_fp16)[name = string("op_3383_cast_fp16")];
            tensor<int32, [1]> var_3385_axes_0 = const()[name = string("op_3385_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_3385_keep_dims_0 = const()[name = string("op_3385_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_3385_cast_fp16 = reduce_sum(axes = var_3385_axes_0, keep_dims = var_3385_keep_dims_0, x = var_3383_cast_fp16)[name = string("op_3385_cast_fp16")];
            tensor<int32, [3]> var_3388_begin_0 = const()[name = string("op_3388_begin_0"), val = tensor<int32, [3]>([0, 0, 14])];
            tensor<int32, [3]> var_3388_end_0 = const()[name = string("op_3388_end_0"), val = tensor<int32, [3]>([1, 16, 15])];
            tensor<bool, [3]> var_3388_end_mask_0 = const()[name = string("op_3388_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_3388_squeeze_mask_0 = const()[name = string("op_3388_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_3388_cast_fp16 = slice_by_index(begin = var_3388_begin_0, end = var_3388_end_0, end_mask = var_3388_end_mask_0, squeeze_mask = var_3388_squeeze_mask_0, x = decay_7_cast_fp16)[name = string("op_3388_cast_fp16")];
            tensor<fp16, [1, 16]> var_3389_cast_fp16 = exp(x = var_3388_cast_fp16)[name = string("op_3389_cast_fp16")];
            tensor<int32, [1]> var_3390_axes_0 = const()[name = string("op_3390_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_3390_cast_fp16 = expand_dims(axes = var_3390_axes_0, x = var_3389_cast_fp16)[name = string("op_3390_cast_fp16")];
            tensor<int32, [1]> var_3391_axes_0 = const()[name = string("op_3391_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_3391_cast_fp16 = expand_dims(axes = var_3391_axes_0, x = var_3390_cast_fp16)[name = string("op_3391_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_249_cast_fp16 = mul(x = state_247_cast_fp16, y = var_3391_cast_fp16)[name = string("state_249_cast_fp16")];
            tensor<int32, [4]> key_token_125_begin_0 = const()[name = string("key_token_125_begin_0"), val = tensor<int32, [4]>([0, 0, 14, 0])];
            tensor<int32, [4]> key_token_125_end_0 = const()[name = string("key_token_125_end_0"), val = tensor<int32, [4]>([1, 16, 15, 128])];
            tensor<bool, [4]> key_token_125_end_mask_0 = const()[name = string("key_token_125_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_125_squeeze_mask_0 = const()[name = string("key_token_125_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_125_cast_fp16 = slice_by_index(begin = key_token_125_begin_0, end = key_token_125_end_0, end_mask = key_token_125_end_mask_0, squeeze_mask = key_token_125_squeeze_mask_0, x = key_19_cast_fp16)[name = string("key_token_125_cast_fp16")];
            tensor<int32, [4]> value_token_125_begin_0 = const()[name = string("value_token_125_begin_0"), val = tensor<int32, [4]>([0, 0, 14, 0])];
            tensor<int32, [4]> value_token_125_end_0 = const()[name = string("value_token_125_end_0"), val = tensor<int32, [4]>([1, 16, 15, 128])];
            tensor<bool, [4]> value_token_125_end_mask_0 = const()[name = string("value_token_125_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_125_squeeze_mask_0 = const()[name = string("value_token_125_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_125_cast_fp16 = slice_by_index(begin = value_token_125_begin_0, end = value_token_125_end_0, end_mask = value_token_125_end_mask_0, squeeze_mask = value_token_125_squeeze_mask_0, x = value_69_cast_fp16)[name = string("value_token_125_cast_fp16")];
            tensor<int32, [1]> var_3399_axes_0 = const()[name = string("op_3399_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_3399_cast_fp16 = expand_dims(axes = var_3399_axes_0, x = key_token_125_cast_fp16)[name = string("op_3399_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3400_cast_fp16 = mul(x = state_249_cast_fp16, y = var_3399_cast_fp16)[name = string("op_3400_cast_fp16")];
            tensor<int32, [1]> memory_125_axes_0 = const()[name = string("memory_125_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_125_keep_dims_0 = const()[name = string("memory_125_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_125_cast_fp16 = reduce_sum(axes = memory_125_axes_0, keep_dims = memory_125_keep_dims_0, x = var_3400_cast_fp16)[name = string("memory_125_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_3403_cast_fp16 = sub(x = value_token_125_cast_fp16, y = memory_125_cast_fp16)[name = string("op_3403_cast_fp16")];
            tensor<int32, [3]> var_3406_begin_0 = const()[name = string("op_3406_begin_0"), val = tensor<int32, [3]>([0, 0, 14])];
            tensor<int32, [3]> var_3406_end_0 = const()[name = string("op_3406_end_0"), val = tensor<int32, [3]>([1, 16, 15])];
            tensor<bool, [3]> var_3406_end_mask_0 = const()[name = string("op_3406_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_3406_squeeze_mask_0 = const()[name = string("op_3406_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_3406_cast_fp16 = slice_by_index(begin = var_3406_begin_0, end = var_3406_end_0, end_mask = var_3406_end_mask_0, squeeze_mask = var_3406_squeeze_mask_0, x = beta_7_cast_fp16)[name = string("op_3406_cast_fp16")];
            tensor<int32, [1]> var_3407_axes_0 = const()[name = string("op_3407_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_3407_cast_fp16 = expand_dims(axes = var_3407_axes_0, x = var_3406_cast_fp16)[name = string("op_3407_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_125_cast_fp16 = mul(x = var_3403_cast_fp16, y = var_3407_cast_fp16)[name = string("delta_125_cast_fp16")];
            tensor<int32, [1]> var_3410_axes_0 = const()[name = string("op_3410_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_3410_cast_fp16 = expand_dims(axes = var_3410_axes_0, x = delta_125_cast_fp16)[name = string("op_3410_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3411_cast_fp16 = mul(x = var_3399_cast_fp16, y = var_3410_cast_fp16)[name = string("op_3411_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_251_cast_fp16 = add(x = state_249_cast_fp16, y = var_3411_cast_fp16)[name = string("state_251_cast_fp16")];
            tensor<int32, [4]> var_3415_begin_0 = const()[name = string("op_3415_begin_0"), val = tensor<int32, [4]>([0, 0, 14, 0])];
            tensor<int32, [4]> var_3415_end_0 = const()[name = string("op_3415_end_0"), val = tensor<int32, [4]>([1, 16, 15, 128])];
            tensor<bool, [4]> var_3415_end_mask_0 = const()[name = string("op_3415_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_3415_squeeze_mask_0 = const()[name = string("op_3415_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_3415_cast_fp16 = slice_by_index(begin = var_3415_begin_0, end = var_3415_end_0, end_mask = var_3415_end_mask_0, squeeze_mask = var_3415_squeeze_mask_0, x = _inversed_query_19_cast_fp16)[name = string("op_3415_cast_fp16")];
            tensor<int32, [1]> var_3416_axes_0 = const()[name = string("op_3416_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_3416_cast_fp16 = expand_dims(axes = var_3416_axes_0, x = var_3415_cast_fp16)[name = string("op_3416_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3417_cast_fp16 = mul(x = state_251_cast_fp16, y = var_3416_cast_fp16)[name = string("op_3417_cast_fp16")];
            tensor<int32, [1]> var_3419_axes_0 = const()[name = string("op_3419_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_3419_keep_dims_0 = const()[name = string("op_3419_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_3419_cast_fp16 = reduce_sum(axes = var_3419_axes_0, keep_dims = var_3419_keep_dims_0, x = var_3417_cast_fp16)[name = string("op_3419_cast_fp16")];
            tensor<int32, [3]> var_3422_begin_0 = const()[name = string("op_3422_begin_0"), val = tensor<int32, [3]>([0, 0, 15])];
            tensor<int32, [3]> var_3422_end_0 = const()[name = string("op_3422_end_0"), val = tensor<int32, [3]>([1, 16, 16])];
            tensor<bool, [3]> var_3422_end_mask_0 = const()[name = string("op_3422_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_3422_squeeze_mask_0 = const()[name = string("op_3422_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_3422_cast_fp16 = slice_by_index(begin = var_3422_begin_0, end = var_3422_end_0, end_mask = var_3422_end_mask_0, squeeze_mask = var_3422_squeeze_mask_0, x = decay_7_cast_fp16)[name = string("op_3422_cast_fp16")];
            tensor<fp16, [1, 16]> var_3423_cast_fp16 = exp(x = var_3422_cast_fp16)[name = string("op_3423_cast_fp16")];
            tensor<int32, [1]> var_3424_axes_0 = const()[name = string("op_3424_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_3424_cast_fp16 = expand_dims(axes = var_3424_axes_0, x = var_3423_cast_fp16)[name = string("op_3424_cast_fp16")];
            tensor<int32, [1]> var_3425_axes_0 = const()[name = string("op_3425_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_3425_cast_fp16 = expand_dims(axes = var_3425_axes_0, x = var_3424_cast_fp16)[name = string("op_3425_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_253_cast_fp16 = mul(x = state_251_cast_fp16, y = var_3425_cast_fp16)[name = string("state_253_cast_fp16")];
            tensor<int32, [4]> key_token_127_begin_0 = const()[name = string("key_token_127_begin_0"), val = tensor<int32, [4]>([0, 0, 15, 0])];
            tensor<int32, [4]> key_token_127_end_0 = const()[name = string("key_token_127_end_0"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<bool, [4]> key_token_127_end_mask_0 = const()[name = string("key_token_127_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_127_squeeze_mask_0 = const()[name = string("key_token_127_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_127_cast_fp16 = slice_by_index(begin = key_token_127_begin_0, end = key_token_127_end_0, end_mask = key_token_127_end_mask_0, squeeze_mask = key_token_127_squeeze_mask_0, x = key_19_cast_fp16)[name = string("key_token_127_cast_fp16")];
            tensor<int32, [4]> value_token_127_begin_0 = const()[name = string("value_token_127_begin_0"), val = tensor<int32, [4]>([0, 0, 15, 0])];
            tensor<int32, [4]> value_token_127_end_0 = const()[name = string("value_token_127_end_0"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<bool, [4]> value_token_127_end_mask_0 = const()[name = string("value_token_127_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_127_squeeze_mask_0 = const()[name = string("value_token_127_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_127_cast_fp16 = slice_by_index(begin = value_token_127_begin_0, end = value_token_127_end_0, end_mask = value_token_127_end_mask_0, squeeze_mask = value_token_127_squeeze_mask_0, x = value_69_cast_fp16)[name = string("value_token_127_cast_fp16")];
            tensor<int32, [1]> var_3433_axes_0 = const()[name = string("op_3433_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_3433_cast_fp16 = expand_dims(axes = var_3433_axes_0, x = key_token_127_cast_fp16)[name = string("op_3433_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3434_cast_fp16 = mul(x = state_253_cast_fp16, y = var_3433_cast_fp16)[name = string("op_3434_cast_fp16")];
            tensor<int32, [1]> memory_127_axes_0 = const()[name = string("memory_127_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_127_keep_dims_0 = const()[name = string("memory_127_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_127_cast_fp16 = reduce_sum(axes = memory_127_axes_0, keep_dims = memory_127_keep_dims_0, x = var_3434_cast_fp16)[name = string("memory_127_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_3437_cast_fp16 = sub(x = value_token_127_cast_fp16, y = memory_127_cast_fp16)[name = string("op_3437_cast_fp16")];
            tensor<int32, [3]> var_3440_begin_0 = const()[name = string("op_3440_begin_0"), val = tensor<int32, [3]>([0, 0, 15])];
            tensor<int32, [3]> var_3440_end_0 = const()[name = string("op_3440_end_0"), val = tensor<int32, [3]>([1, 16, 16])];
            tensor<bool, [3]> var_3440_end_mask_0 = const()[name = string("op_3440_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_3440_squeeze_mask_0 = const()[name = string("op_3440_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_3440_cast_fp16 = slice_by_index(begin = var_3440_begin_0, end = var_3440_end_0, end_mask = var_3440_end_mask_0, squeeze_mask = var_3440_squeeze_mask_0, x = beta_7_cast_fp16)[name = string("op_3440_cast_fp16")];
            tensor<int32, [1]> var_3441_axes_0 = const()[name = string("op_3441_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_3441_cast_fp16 = expand_dims(axes = var_3441_axes_0, x = var_3440_cast_fp16)[name = string("op_3441_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_127_cast_fp16 = mul(x = var_3437_cast_fp16, y = var_3441_cast_fp16)[name = string("delta_127_cast_fp16")];
            tensor<int32, [1]> var_3444_axes_0 = const()[name = string("op_3444_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_3444_cast_fp16 = expand_dims(axes = var_3444_axes_0, x = delta_127_cast_fp16)[name = string("op_3444_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3445_cast_fp16 = mul(x = var_3433_cast_fp16, y = var_3444_cast_fp16)[name = string("op_3445_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> rec4_out = add(x = state_253_cast_fp16, y = var_3445_cast_fp16)[name = string("state_255_cast_fp16")];
            tensor<int32, [4]> var_3449_begin_0 = const()[name = string("op_3449_begin_0"), val = tensor<int32, [4]>([0, 0, 15, 0])];
            tensor<int32, [4]> var_3449_end_0 = const()[name = string("op_3449_end_0"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<bool, [4]> var_3449_end_mask_0 = const()[name = string("op_3449_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_3449_squeeze_mask_0 = const()[name = string("op_3449_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_3449_cast_fp16 = slice_by_index(begin = var_3449_begin_0, end = var_3449_end_0, end_mask = var_3449_end_mask_0, squeeze_mask = var_3449_squeeze_mask_0, x = _inversed_query_19_cast_fp16)[name = string("op_3449_cast_fp16")];
            tensor<int32, [1]> var_3450_axes_0 = const()[name = string("op_3450_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_3450_cast_fp16 = expand_dims(axes = var_3450_axes_0, x = var_3449_cast_fp16)[name = string("op_3450_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3451_cast_fp16 = mul(x = rec4_out, y = var_3450_cast_fp16)[name = string("op_3451_cast_fp16")];
            tensor<int32, [1]> var_3453_axes_0 = const()[name = string("op_3453_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_3453_keep_dims_0 = const()[name = string("op_3453_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_3453_cast_fp16 = reduce_sum(axes = var_3453_axes_0, keep_dims = var_3453_keep_dims_0, x = var_3451_cast_fp16)[name = string("op_3453_cast_fp16")];
            int32 attention_31_axis_0 = const()[name = string("attention_31_axis_0"), val = int32(1)];
            tensor<fp16, [1, 16, 16, 128]> attention_31_cast_fp16 = stack(axis = attention_31_axis_0, values = (var_2943_cast_fp16, var_2977_cast_fp16, var_3011_cast_fp16, var_3045_cast_fp16, var_3079_cast_fp16, var_3113_cast_fp16, var_3147_cast_fp16, var_3181_cast_fp16, var_3215_cast_fp16, var_3249_cast_fp16, var_3283_cast_fp16, var_3317_cast_fp16, var_3351_cast_fp16, var_3385_cast_fp16, var_3419_cast_fp16, var_3453_cast_fp16))[name = string("attention_31_cast_fp16")];
            tensor<int32, [2]> var_3456 = const()[name = string("op_3456"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [256, 128]> attention_33_cast_fp16 = reshape(shape = var_3456, x = attention_31_cast_fp16)[name = string("attention_33_cast_fp16")];
            tensor<int32, [2]> var_3458 = const()[name = string("op_3458"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [256, 128]> input_57_cast_fp16 = reshape(shape = var_3458, x = linear_34_cast_fp16)[name = string("input_57_cast_fp16")];
            tensor<fp16, [256, 128]> var_3460_cast_fp16 = mul(x = attention_33_cast_fp16, y = attention_33_cast_fp16)[name = string("op_3460_cast_fp16")];
            tensor<int32, [1]> variance_29_axes_0 = const()[name = string("variance_29_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_29_keep_dims_0 = const()[name = string("variance_29_keep_dims_0"), val = bool(true)];
            tensor<fp16, [256, 1]> variance_29_cast_fp16 = reduce_mean(axes = variance_29_axes_0, keep_dims = variance_29_keep_dims_0, x = var_3460_cast_fp16)[name = string("variance_29_cast_fp16")];
            fp16 var_3463_to_fp16 = const()[name = string("op_3463_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [256, 1]> var_3464_cast_fp16 = add(x = variance_29_cast_fp16, y = var_3463_to_fp16)[name = string("op_3464_cast_fp16")];
            fp32 var_3465_epsilon_0 = const()[name = string("op_3465_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [256, 1]> var_3465_cast_fp16 = rsqrt(epsilon = var_3465_epsilon_0, x = var_3464_cast_fp16)[name = string("op_3465_cast_fp16")];
            tensor<fp16, [256, 128]> attention_35_cast_fp16 = mul(x = attention_33_cast_fp16, y = var_3465_cast_fp16)[name = string("attention_35_cast_fp16")];
            tensor<fp16, [128]> groups_1_0_gated_norm_promoted_to_fp16 = const()[name = string("groups_1_0_gated_norm_promoted_to_fp16"), val = tensor<fp16, [128]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(69449792)))];
            tensor<fp16, [256, 128]> attention_37_cast_fp16 = mul(x = attention_35_cast_fp16, y = groups_1_0_gated_norm_promoted_to_fp16)[name = string("attention_37_cast_fp16")];
            tensor<fp16, [256, 128]> var_3468_cast_fp16 = silu(x = input_57_cast_fp16)[name = string("op_3468_cast_fp16")];
            tensor<fp16, [256, 128]> attention_39_cast_fp16 = mul(x = attention_37_cast_fp16, y = var_3468_cast_fp16)[name = string("attention_39_cast_fp16")];
            tensor<int32, [2]> var_3470 = const()[name = string("op_3470"), val = tensor<int32, [2]>([16, 2048])];
            tensor<fp16, [16, 2048]> input_59_cast_fp16 = reshape(shape = var_3470, x = attention_39_cast_fp16)[name = string("input_59_cast_fp16")];
            tensor<fp16, [1024, 2048]> groups_1_0_out_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 2048]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(69450112))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(71023040))))[name = string("groups_1_0_out_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 1024]> linear_35_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = groups_1_0_out_proj_weight_promoted_to_fp16_palettized, x = input_59_cast_fp16)[name = string("linear_35_cast_fp16")];
            tensor<fp16, [16, 1024]> value_71_cast_fp16 = add(x = value_59_cast_fp16, y = linear_35_cast_fp16)[name = string("value_71_cast_fp16")];
            tensor<fp16, [16, 1024]> var_3475_cast_fp16 = mul(x = value_71_cast_fp16, y = value_71_cast_fp16)[name = string("op_3475_cast_fp16")];
            tensor<int32, [1]> variance_31_axes_0 = const()[name = string("variance_31_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_31_keep_dims_0 = const()[name = string("variance_31_keep_dims_0"), val = bool(true)];
            tensor<fp16, [16, 1]> variance_31_cast_fp16 = reduce_mean(axes = variance_31_axes_0, keep_dims = variance_31_keep_dims_0, x = var_3475_cast_fp16)[name = string("variance_31_cast_fp16")];
            fp16 var_3478_to_fp16 = const()[name = string("op_3478_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [16, 1]> var_3479_cast_fp16 = add(x = variance_31_cast_fp16, y = var_3478_to_fp16)[name = string("op_3479_cast_fp16")];
            fp32 var_3480_epsilon_0 = const()[name = string("op_3480_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [16, 1]> var_3480_cast_fp16 = rsqrt(epsilon = var_3480_epsilon_0, x = var_3479_cast_fp16)[name = string("op_3480_cast_fp16")];
            tensor<fp16, [16, 1024]> var_3481_cast_fp16 = mul(x = value_71_cast_fp16, y = var_3480_cast_fp16)[name = string("op_3481_cast_fp16")];
            tensor<fp16, [1024]> var_3483_to_fp16 = const()[name = string("op_3483_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(71031296)))];
            tensor<fp16, [16, 1024]> input_61_cast_fp16 = mul(x = var_3481_cast_fp16, y = var_3483_to_fp16)[name = string("input_61_cast_fp16")];
            tensor<fp16, [3584, 1024]> groups_1_0_gate_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(71033408))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(73785984))))[name = string("groups_1_0_gate_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 3584]> linear_36_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = groups_1_0_gate_proj_weight_promoted_to_fp16_palettized, x = input_61_cast_fp16)[name = string("linear_36_cast_fp16")];
            tensor<fp16, [16, 3584]> var_3487_cast_fp16 = silu(x = linear_36_cast_fp16)[name = string("op_3487_cast_fp16")];
            tensor<fp16, [3584, 1024]> groups_1_0_up_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(73814720))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(76567296))))[name = string("groups_1_0_up_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 3584]> linear_37_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = groups_1_0_up_proj_weight_promoted_to_fp16_palettized, x = input_61_cast_fp16)[name = string("linear_37_cast_fp16")];
            tensor<fp16, [16, 3584]> input_65_cast_fp16 = mul(x = var_3487_cast_fp16, y = linear_37_cast_fp16)[name = string("input_65_cast_fp16")];
            tensor<fp16, [1024, 3584]> groups_1_0_down_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 3584]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(76596032))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(79348608))))[name = string("groups_1_0_down_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 1024]> linear_38_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = groups_1_0_down_proj_weight_promoted_to_fp16_palettized, x = input_65_cast_fp16)[name = string("linear_38_cast_fp16")];
            tensor<fp16, [16, 1024]> value_73_cast_fp16 = add(x = value_71_cast_fp16, y = linear_38_cast_fp16)[name = string("value_73_cast_fp16")];
            int32 var_3525 = const()[name = string("op_3525"), val = int32(-1)];
            tensor<fp16, [16, 1024]> var_3540_cast_fp16 = mul(x = value_73_cast_fp16, y = value_73_cast_fp16)[name = string("op_3540_cast_fp16")];
            tensor<int32, [1]> variance_33_axes_0 = const()[name = string("variance_33_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_33_keep_dims_0 = const()[name = string("variance_33_keep_dims_0"), val = bool(true)];
            tensor<fp16, [16, 1]> variance_33_cast_fp16 = reduce_mean(axes = variance_33_axes_0, keep_dims = variance_33_keep_dims_0, x = var_3540_cast_fp16)[name = string("variance_33_cast_fp16")];
            fp16 var_3543_to_fp16 = const()[name = string("op_3543_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [16, 1]> var_3544_cast_fp16 = add(x = variance_33_cast_fp16, y = var_3543_to_fp16)[name = string("op_3544_cast_fp16")];
            fp32 var_3545_epsilon_0 = const()[name = string("op_3545_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [16, 1]> var_3545_cast_fp16 = rsqrt(epsilon = var_3545_epsilon_0, x = var_3544_cast_fp16)[name = string("op_3545_cast_fp16")];
            tensor<fp16, [16, 1024]> var_3546_cast_fp16 = mul(x = value_73_cast_fp16, y = var_3545_cast_fp16)[name = string("op_3546_cast_fp16")];
            tensor<fp16, [1024]> var_3548_to_fp16 = const()[name = string("op_3548_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(79356864)))];
            tensor<fp16, [16, 1024]> input_67_cast_fp16 = mul(x = var_3546_cast_fp16, y = var_3548_to_fp16)[name = string("input_67_cast_fp16")];
            tensor<fp16, [6144, 1024]> groups_1_1_in_proj_qkv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(79358976))), lut = tensor<fp16, [384, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(84077632))))[name = string("groups_1_1_in_proj_qkv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 6144]> linear_39_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = groups_1_1_in_proj_qkv_weight_promoted_to_fp16_palettized, x = input_67_cast_fp16)[name = string("linear_39_cast_fp16")];
            tensor<int32, [2]> var_3552_perm_0 = const()[name = string("op_3552_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> mixed_9_axes_0 = const()[name = string("mixed_9_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [6144, 16]> var_3552_cast_fp16 = transpose(perm = var_3552_perm_0, x = linear_39_cast_fp16)[name = string("transpose_104")];
            tensor<fp16, [1, 6144, 16]> mixed_9_cast_fp16 = expand_dims(axes = mixed_9_axes_0, x = var_3552_cast_fp16)[name = string("mixed_9_cast_fp16")];
            bool convolution_input_9_interleave_0 = const()[name = string("convolution_input_9_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 6144, 19]> convolution_input_9_cast_fp16 = concat(axis = var_3525, interleave = convolution_input_9_interleave_0, values = (conv5, mixed_9_cast_fp16))[name = string("convolution_input_9_cast_fp16")];
            string input_69_pad_type_0 = const()[name = string("input_69_pad_type_0"), val = string("valid")];
            int32 input_69_groups_0 = const()[name = string("input_69_groups_0"), val = int32(6144)];
            tensor<int32, [1]> input_69_strides_0 = const()[name = string("input_69_strides_0"), val = tensor<int32, [1]>([1])];
            tensor<int32, [2]> input_69_pad_0 = const()[name = string("input_69_pad_0"), val = tensor<int32, [2]>([0, 0])];
            tensor<int32, [1]> input_69_dilations_0 = const()[name = string("input_69_dilations_0"), val = tensor<int32, [1]>([1])];
            tensor<fp16, [6144, 1, 4]> groups_1_1_conv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1, 4]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(84126848))), lut = tensor<fp16, [384, 1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(84145344))))[name = string("groups_1_1_conv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 6144, 16]> input_69_cast_fp16 = conv(dilations = input_69_dilations_0, groups = input_69_groups_0, pad = input_69_pad_0, pad_type = input_69_pad_type_0, strides = input_69_strides_0, weight = groups_1_1_conv_weight_promoted_to_fp16_palettized, x = convolution_input_9_cast_fp16)[name = string("input_69_cast_fp16")];
            tensor<fp16, [1, 6144, 16]> var_3561_cast_fp16 = silu(x = input_69_cast_fp16)[name = string("op_3561_cast_fp16")];
            tensor<int32, [3]> var_3562_perm_0 = const()[name = string("op_3562_perm_0"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [3]> var_3565_begin_0 = const()[name = string("op_3565_begin_0"), val = tensor<int32, [3]>([0, 0, 16])];
            tensor<int32, [3]> var_3565_end_0 = const()[name = string("op_3565_end_0"), val = tensor<int32, [3]>([1, 6144, 19])];
            tensor<bool, [3]> var_3565_end_mask_0 = const()[name = string("op_3565_end_mask_0"), val = tensor<bool, [3]>([true, true, true])];
            tensor<fp16, [1, 6144, 3]> conv5_out = slice_by_index(begin = var_3565_begin_0, end = var_3565_end_0, end_mask = var_3565_end_mask_0, x = convolution_input_9_cast_fp16)[name = string("op_3565_cast_fp16")];
            tensor<int32, [3]> var_3566 = const()[name = string("op_3566"), val = tensor<int32, [3]>([2048, 2048, 2048])];
            int32 var_3567_axis_0 = const()[name = string("op_3567_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 16, 6144]> var_3562_cast_fp16 = transpose(perm = var_3562_perm_0, x = var_3561_cast_fp16)[name = string("transpose_103")];
            tensor<fp16, [1, 16, 2048]> var_3567_cast_fp16_0, tensor<fp16, [1, 16, 2048]> var_3567_cast_fp16_1, tensor<fp16, [1, 16, 2048]> var_3567_cast_fp16_2 = split(axis = var_3567_axis_0, split_sizes = var_3566, x = var_3562_cast_fp16)[name = string("op_3567_cast_fp16")];
            tensor<int32, [4]> var_3571 = const()[name = string("op_3571"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<fp16, [1, 16, 16, 128]> value_77_cast_fp16 = reshape(shape = var_3571, x = var_3567_cast_fp16_0)[name = string("value_77_cast_fp16")];
            tensor<int32, [4]> var_3573 = const()[name = string("op_3573"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<fp16, [1, 16, 16, 128]> value_79_cast_fp16 = reshape(shape = var_3573, x = var_3567_cast_fp16_1)[name = string("value_79_cast_fp16")];
            tensor<int32, [4]> var_3575 = const()[name = string("op_3575"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<fp16, [1, 16, 16, 128]> value_81_cast_fp16 = reshape(shape = var_3575, x = var_3567_cast_fp16_2)[name = string("value_81_cast_fp16")];
            tensor<fp16, [1, 16, 16, 128]> var_3577_cast_fp16 = mul(x = value_77_cast_fp16, y = value_77_cast_fp16)[name = string("op_3577_cast_fp16")];
            tensor<int32, [1]> var_3579_axes_0 = const()[name = string("op_3579_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_3579_keep_dims_0 = const()[name = string("op_3579_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 16, 16, 1]> var_3579_cast_fp16 = reduce_sum(axes = var_3579_axes_0, keep_dims = var_3579_keep_dims_0, x = var_3577_cast_fp16)[name = string("op_3579_cast_fp16")];
            fp16 var_3580_to_fp16 = const()[name = string("op_3580_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 16, 16, 1]> var_3581_cast_fp16 = add(x = var_3579_cast_fp16, y = var_3580_to_fp16)[name = string("op_3581_cast_fp16")];
            fp32 var_3582_epsilon_0 = const()[name = string("op_3582_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 16, 16, 1]> var_3582_cast_fp16 = rsqrt(epsilon = var_3582_epsilon_0, x = var_3581_cast_fp16)[name = string("op_3582_cast_fp16")];
            tensor<fp16, [1, 16, 16, 128]> var_3583_cast_fp16 = mul(x = value_77_cast_fp16, y = var_3582_cast_fp16)[name = string("op_3583_cast_fp16")];
            tensor<int32, [4]> var_3584_perm_0 = const()[name = string("op_3584_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            fp16 _inversed_query_23_y_0_to_fp16 = const()[name = string("_inversed_query_23_y_0_to_fp16"), val = fp16(0x1.6ap-4)];
            tensor<fp16, [1, 16, 16, 128]> var_3584_cast_fp16 = transpose(perm = var_3584_perm_0, x = var_3583_cast_fp16)[name = string("transpose_102")];
            tensor<fp16, [1, 16, 16, 128]> _inversed_query_23_cast_fp16 = mul(x = var_3584_cast_fp16, y = _inversed_query_23_y_0_to_fp16)[name = string("_inversed_query_23_cast_fp16")];
            tensor<fp16, [1, 16, 16, 128]> var_3587_cast_fp16 = mul(x = value_79_cast_fp16, y = value_79_cast_fp16)[name = string("op_3587_cast_fp16")];
            tensor<int32, [1]> var_3589_axes_0 = const()[name = string("op_3589_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_3589_keep_dims_0 = const()[name = string("op_3589_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 16, 16, 1]> var_3589_cast_fp16 = reduce_sum(axes = var_3589_axes_0, keep_dims = var_3589_keep_dims_0, x = var_3587_cast_fp16)[name = string("op_3589_cast_fp16")];
            fp16 var_3590_to_fp16 = const()[name = string("op_3590_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 16, 16, 1]> var_3591_cast_fp16 = add(x = var_3589_cast_fp16, y = var_3590_to_fp16)[name = string("op_3591_cast_fp16")];
            fp32 var_3592_epsilon_0 = const()[name = string("op_3592_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 16, 16, 1]> var_3592_cast_fp16 = rsqrt(epsilon = var_3592_epsilon_0, x = var_3591_cast_fp16)[name = string("op_3592_cast_fp16")];
            tensor<fp16, [1, 16, 16, 128]> var_3593_cast_fp16 = mul(x = value_79_cast_fp16, y = var_3592_cast_fp16)[name = string("op_3593_cast_fp16")];
            tensor<int32, [4]> key_23_perm_0 = const()[name = string("key_23_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<int32, [4]> value_83_perm_0 = const()[name = string("value_83_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<fp16, [16, 1024]> groups_1_1_in_proj_b_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(84194560))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(84206912))))[name = string("groups_1_1_in_proj_b_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16]> linear_40_bias_0_to_fp16 = const()[name = string("linear_40_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(4862656)))];
            tensor<fp16, [16, 16]> linear_40_cast_fp16 = linear(bias = linear_40_bias_0_to_fp16, weight = groups_1_1_in_proj_b_weight_promoted_to_fp16_palettized, x = input_67_cast_fp16)[name = string("linear_40_cast_fp16")];
            tensor<fp16, [16, 16]> var_3598_cast_fp16 = sigmoid(x = linear_40_cast_fp16)[name = string("op_3598_cast_fp16")];
            tensor<int32, [2]> var_3599_perm_0 = const()[name = string("op_3599_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> beta_9_axes_0 = const()[name = string("beta_9_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 16]> var_3599_cast_fp16 = transpose(perm = var_3599_perm_0, x = var_3598_cast_fp16)[name = string("transpose_101")];
            tensor<fp16, [1, 16, 16]> beta_9_cast_fp16 = expand_dims(axes = beta_9_axes_0, x = var_3599_cast_fp16)[name = string("beta_9_cast_fp16")];
            tensor<fp16, [16, 1024]> op_3605_weight_0_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(84207104))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(84219456))))[name = string("op_3605_weight_0_to_fp16_palettized")];
            tensor<fp16, [16]> var_3605_bias_0_to_fp16 = const()[name = string("op_3605_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(84219648)))];
            tensor<fp16, [16, 16]> var_3605_cast_fp16 = linear(bias = var_3605_bias_0_to_fp16, weight = op_3605_weight_0_to_fp16_palettized, x = input_67_cast_fp16)[name = string("op_3605_cast_fp16")];
            tensor<fp16, [16, 16]> var_3606_cast_fp16 = softplus(x = var_3605_cast_fp16)[name = string("op_3606_cast_fp16")];
            tensor<fp16, [16]> var_3602_promoted_to_fp16 = const()[name = string("op_3602_promoted_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(84219776)))];
            tensor<fp16, [16, 16]> var_3607_cast_fp16 = mul(x = var_3602_promoted_to_fp16, y = var_3606_cast_fp16)[name = string("op_3607_cast_fp16")];
            tensor<int32, [2]> var_3608_perm_0 = const()[name = string("op_3608_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> decay_9_axes_0 = const()[name = string("decay_9_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 16]> var_3608_cast_fp16 = transpose(perm = var_3608_perm_0, x = var_3607_cast_fp16)[name = string("transpose_100")];
            tensor<fp16, [1, 16, 16]> decay_9_cast_fp16 = expand_dims(axes = decay_9_axes_0, x = var_3608_cast_fp16)[name = string("decay_9_cast_fp16")];
            tensor<fp16, [2048, 1024]> groups_1_1_in_proj_z_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [2048, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(84219904))), lut = tensor<fp16, [128, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(85792832))))[name = string("groups_1_1_in_proj_z_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 2048]> linear_42_cast_fp16 = linear(bias = linear_3_bias_0_to_fp16, weight = groups_1_1_in_proj_z_weight_promoted_to_fp16_palettized, x = input_67_cast_fp16)[name = string("linear_42_cast_fp16")];
            tensor<int32, [3]> var_3616_begin_0 = const()[name = string("op_3616_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_3616_end_0 = const()[name = string("op_3616_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_3616_end_mask_0 = const()[name = string("op_3616_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_3616_squeeze_mask_0 = const()[name = string("op_3616_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_3616_cast_fp16 = slice_by_index(begin = var_3616_begin_0, end = var_3616_end_0, end_mask = var_3616_end_mask_0, squeeze_mask = var_3616_squeeze_mask_0, x = decay_9_cast_fp16)[name = string("op_3616_cast_fp16")];
            tensor<fp16, [1, 16]> var_3617_cast_fp16 = exp(x = var_3616_cast_fp16)[name = string("op_3617_cast_fp16")];
            tensor<int32, [1]> var_3618_axes_0 = const()[name = string("op_3618_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_3618_cast_fp16 = expand_dims(axes = var_3618_axes_0, x = var_3617_cast_fp16)[name = string("op_3618_cast_fp16")];
            tensor<int32, [1]> var_3619_axes_0 = const()[name = string("op_3619_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_3619_cast_fp16 = expand_dims(axes = var_3619_axes_0, x = var_3618_cast_fp16)[name = string("op_3619_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_257_cast_fp16 = mul(x = rec5, y = var_3619_cast_fp16)[name = string("state_257_cast_fp16")];
            tensor<int32, [4]> key_token_129_begin_0 = const()[name = string("key_token_129_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> key_token_129_end_0 = const()[name = string("key_token_129_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> key_token_129_end_mask_0 = const()[name = string("key_token_129_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_129_squeeze_mask_0 = const()[name = string("key_token_129_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 16, 128]> key_23_cast_fp16 = transpose(perm = key_23_perm_0, x = var_3593_cast_fp16)[name = string("transpose_99")];
            tensor<fp16, [1, 16, 128]> key_token_129_cast_fp16 = slice_by_index(begin = key_token_129_begin_0, end = key_token_129_end_0, end_mask = key_token_129_end_mask_0, squeeze_mask = key_token_129_squeeze_mask_0, x = key_23_cast_fp16)[name = string("key_token_129_cast_fp16")];
            tensor<int32, [4]> value_token_129_begin_0 = const()[name = string("value_token_129_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> value_token_129_end_0 = const()[name = string("value_token_129_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> value_token_129_end_mask_0 = const()[name = string("value_token_129_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_129_squeeze_mask_0 = const()[name = string("value_token_129_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 16, 128]> value_83_cast_fp16 = transpose(perm = value_83_perm_0, x = value_81_cast_fp16)[name = string("transpose_98")];
            tensor<fp16, [1, 16, 128]> value_token_129_cast_fp16 = slice_by_index(begin = value_token_129_begin_0, end = value_token_129_end_0, end_mask = value_token_129_end_mask_0, squeeze_mask = value_token_129_squeeze_mask_0, x = value_83_cast_fp16)[name = string("value_token_129_cast_fp16")];
            tensor<int32, [1]> var_3627_axes_0 = const()[name = string("op_3627_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_3627_cast_fp16 = expand_dims(axes = var_3627_axes_0, x = key_token_129_cast_fp16)[name = string("op_3627_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3628_cast_fp16 = mul(x = state_257_cast_fp16, y = var_3627_cast_fp16)[name = string("op_3628_cast_fp16")];
            tensor<int32, [1]> memory_129_axes_0 = const()[name = string("memory_129_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_129_keep_dims_0 = const()[name = string("memory_129_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_129_cast_fp16 = reduce_sum(axes = memory_129_axes_0, keep_dims = memory_129_keep_dims_0, x = var_3628_cast_fp16)[name = string("memory_129_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_3631_cast_fp16 = sub(x = value_token_129_cast_fp16, y = memory_129_cast_fp16)[name = string("op_3631_cast_fp16")];
            tensor<int32, [3]> var_3634_begin_0 = const()[name = string("op_3634_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_3634_end_0 = const()[name = string("op_3634_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_3634_end_mask_0 = const()[name = string("op_3634_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_3634_squeeze_mask_0 = const()[name = string("op_3634_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_3634_cast_fp16 = slice_by_index(begin = var_3634_begin_0, end = var_3634_end_0, end_mask = var_3634_end_mask_0, squeeze_mask = var_3634_squeeze_mask_0, x = beta_9_cast_fp16)[name = string("op_3634_cast_fp16")];
            tensor<int32, [1]> var_3635_axes_0 = const()[name = string("op_3635_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_3635_cast_fp16 = expand_dims(axes = var_3635_axes_0, x = var_3634_cast_fp16)[name = string("op_3635_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_129_cast_fp16 = mul(x = var_3631_cast_fp16, y = var_3635_cast_fp16)[name = string("delta_129_cast_fp16")];
            tensor<int32, [1]> var_3638_axes_0 = const()[name = string("op_3638_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_3638_cast_fp16 = expand_dims(axes = var_3638_axes_0, x = delta_129_cast_fp16)[name = string("op_3638_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3639_cast_fp16 = mul(x = var_3627_cast_fp16, y = var_3638_cast_fp16)[name = string("op_3639_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_259_cast_fp16 = add(x = state_257_cast_fp16, y = var_3639_cast_fp16)[name = string("state_259_cast_fp16")];
            tensor<int32, [4]> var_3643_begin_0 = const()[name = string("op_3643_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> var_3643_end_0 = const()[name = string("op_3643_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> var_3643_end_mask_0 = const()[name = string("op_3643_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_3643_squeeze_mask_0 = const()[name = string("op_3643_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_3643_cast_fp16 = slice_by_index(begin = var_3643_begin_0, end = var_3643_end_0, end_mask = var_3643_end_mask_0, squeeze_mask = var_3643_squeeze_mask_0, x = _inversed_query_23_cast_fp16)[name = string("op_3643_cast_fp16")];
            tensor<int32, [1]> var_3644_axes_0 = const()[name = string("op_3644_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_3644_cast_fp16 = expand_dims(axes = var_3644_axes_0, x = var_3643_cast_fp16)[name = string("op_3644_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3645_cast_fp16 = mul(x = state_259_cast_fp16, y = var_3644_cast_fp16)[name = string("op_3645_cast_fp16")];
            tensor<int32, [1]> var_3647_axes_0 = const()[name = string("op_3647_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_3647_keep_dims_0 = const()[name = string("op_3647_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_3647_cast_fp16 = reduce_sum(axes = var_3647_axes_0, keep_dims = var_3647_keep_dims_0, x = var_3645_cast_fp16)[name = string("op_3647_cast_fp16")];
            tensor<int32, [3]> var_3650_begin_0 = const()[name = string("op_3650_begin_0"), val = tensor<int32, [3]>([0, 0, 1])];
            tensor<int32, [3]> var_3650_end_0 = const()[name = string("op_3650_end_0"), val = tensor<int32, [3]>([1, 16, 2])];
            tensor<bool, [3]> var_3650_end_mask_0 = const()[name = string("op_3650_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_3650_squeeze_mask_0 = const()[name = string("op_3650_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_3650_cast_fp16 = slice_by_index(begin = var_3650_begin_0, end = var_3650_end_0, end_mask = var_3650_end_mask_0, squeeze_mask = var_3650_squeeze_mask_0, x = decay_9_cast_fp16)[name = string("op_3650_cast_fp16")];
            tensor<fp16, [1, 16]> var_3651_cast_fp16 = exp(x = var_3650_cast_fp16)[name = string("op_3651_cast_fp16")];
            tensor<int32, [1]> var_3652_axes_0 = const()[name = string("op_3652_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_3652_cast_fp16 = expand_dims(axes = var_3652_axes_0, x = var_3651_cast_fp16)[name = string("op_3652_cast_fp16")];
            tensor<int32, [1]> var_3653_axes_0 = const()[name = string("op_3653_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_3653_cast_fp16 = expand_dims(axes = var_3653_axes_0, x = var_3652_cast_fp16)[name = string("op_3653_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_261_cast_fp16 = mul(x = state_259_cast_fp16, y = var_3653_cast_fp16)[name = string("state_261_cast_fp16")];
            tensor<int32, [4]> key_token_131_begin_0 = const()[name = string("key_token_131_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> key_token_131_end_0 = const()[name = string("key_token_131_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> key_token_131_end_mask_0 = const()[name = string("key_token_131_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_131_squeeze_mask_0 = const()[name = string("key_token_131_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_131_cast_fp16 = slice_by_index(begin = key_token_131_begin_0, end = key_token_131_end_0, end_mask = key_token_131_end_mask_0, squeeze_mask = key_token_131_squeeze_mask_0, x = key_23_cast_fp16)[name = string("key_token_131_cast_fp16")];
            tensor<int32, [4]> value_token_131_begin_0 = const()[name = string("value_token_131_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> value_token_131_end_0 = const()[name = string("value_token_131_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> value_token_131_end_mask_0 = const()[name = string("value_token_131_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_131_squeeze_mask_0 = const()[name = string("value_token_131_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_131_cast_fp16 = slice_by_index(begin = value_token_131_begin_0, end = value_token_131_end_0, end_mask = value_token_131_end_mask_0, squeeze_mask = value_token_131_squeeze_mask_0, x = value_83_cast_fp16)[name = string("value_token_131_cast_fp16")];
            tensor<int32, [1]> var_3661_axes_0 = const()[name = string("op_3661_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_3661_cast_fp16 = expand_dims(axes = var_3661_axes_0, x = key_token_131_cast_fp16)[name = string("op_3661_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3662_cast_fp16 = mul(x = state_261_cast_fp16, y = var_3661_cast_fp16)[name = string("op_3662_cast_fp16")];
            tensor<int32, [1]> memory_131_axes_0 = const()[name = string("memory_131_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_131_keep_dims_0 = const()[name = string("memory_131_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_131_cast_fp16 = reduce_sum(axes = memory_131_axes_0, keep_dims = memory_131_keep_dims_0, x = var_3662_cast_fp16)[name = string("memory_131_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_3665_cast_fp16 = sub(x = value_token_131_cast_fp16, y = memory_131_cast_fp16)[name = string("op_3665_cast_fp16")];
            tensor<int32, [3]> var_3668_begin_0 = const()[name = string("op_3668_begin_0"), val = tensor<int32, [3]>([0, 0, 1])];
            tensor<int32, [3]> var_3668_end_0 = const()[name = string("op_3668_end_0"), val = tensor<int32, [3]>([1, 16, 2])];
            tensor<bool, [3]> var_3668_end_mask_0 = const()[name = string("op_3668_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_3668_squeeze_mask_0 = const()[name = string("op_3668_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_3668_cast_fp16 = slice_by_index(begin = var_3668_begin_0, end = var_3668_end_0, end_mask = var_3668_end_mask_0, squeeze_mask = var_3668_squeeze_mask_0, x = beta_9_cast_fp16)[name = string("op_3668_cast_fp16")];
            tensor<int32, [1]> var_3669_axes_0 = const()[name = string("op_3669_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_3669_cast_fp16 = expand_dims(axes = var_3669_axes_0, x = var_3668_cast_fp16)[name = string("op_3669_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_131_cast_fp16 = mul(x = var_3665_cast_fp16, y = var_3669_cast_fp16)[name = string("delta_131_cast_fp16")];
            tensor<int32, [1]> var_3672_axes_0 = const()[name = string("op_3672_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_3672_cast_fp16 = expand_dims(axes = var_3672_axes_0, x = delta_131_cast_fp16)[name = string("op_3672_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3673_cast_fp16 = mul(x = var_3661_cast_fp16, y = var_3672_cast_fp16)[name = string("op_3673_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_263_cast_fp16 = add(x = state_261_cast_fp16, y = var_3673_cast_fp16)[name = string("state_263_cast_fp16")];
            tensor<int32, [4]> var_3677_begin_0 = const()[name = string("op_3677_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> var_3677_end_0 = const()[name = string("op_3677_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> var_3677_end_mask_0 = const()[name = string("op_3677_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_3677_squeeze_mask_0 = const()[name = string("op_3677_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_3677_cast_fp16 = slice_by_index(begin = var_3677_begin_0, end = var_3677_end_0, end_mask = var_3677_end_mask_0, squeeze_mask = var_3677_squeeze_mask_0, x = _inversed_query_23_cast_fp16)[name = string("op_3677_cast_fp16")];
            tensor<int32, [1]> var_3678_axes_0 = const()[name = string("op_3678_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_3678_cast_fp16 = expand_dims(axes = var_3678_axes_0, x = var_3677_cast_fp16)[name = string("op_3678_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3679_cast_fp16 = mul(x = state_263_cast_fp16, y = var_3678_cast_fp16)[name = string("op_3679_cast_fp16")];
            tensor<int32, [1]> var_3681_axes_0 = const()[name = string("op_3681_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_3681_keep_dims_0 = const()[name = string("op_3681_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_3681_cast_fp16 = reduce_sum(axes = var_3681_axes_0, keep_dims = var_3681_keep_dims_0, x = var_3679_cast_fp16)[name = string("op_3681_cast_fp16")];
            tensor<int32, [3]> var_3684_begin_0 = const()[name = string("op_3684_begin_0"), val = tensor<int32, [3]>([0, 0, 2])];
            tensor<int32, [3]> var_3684_end_0 = const()[name = string("op_3684_end_0"), val = tensor<int32, [3]>([1, 16, 3])];
            tensor<bool, [3]> var_3684_end_mask_0 = const()[name = string("op_3684_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_3684_squeeze_mask_0 = const()[name = string("op_3684_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_3684_cast_fp16 = slice_by_index(begin = var_3684_begin_0, end = var_3684_end_0, end_mask = var_3684_end_mask_0, squeeze_mask = var_3684_squeeze_mask_0, x = decay_9_cast_fp16)[name = string("op_3684_cast_fp16")];
            tensor<fp16, [1, 16]> var_3685_cast_fp16 = exp(x = var_3684_cast_fp16)[name = string("op_3685_cast_fp16")];
            tensor<int32, [1]> var_3686_axes_0 = const()[name = string("op_3686_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_3686_cast_fp16 = expand_dims(axes = var_3686_axes_0, x = var_3685_cast_fp16)[name = string("op_3686_cast_fp16")];
            tensor<int32, [1]> var_3687_axes_0 = const()[name = string("op_3687_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_3687_cast_fp16 = expand_dims(axes = var_3687_axes_0, x = var_3686_cast_fp16)[name = string("op_3687_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_265_cast_fp16 = mul(x = state_263_cast_fp16, y = var_3687_cast_fp16)[name = string("state_265_cast_fp16")];
            tensor<int32, [4]> key_token_133_begin_0 = const()[name = string("key_token_133_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> key_token_133_end_0 = const()[name = string("key_token_133_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> key_token_133_end_mask_0 = const()[name = string("key_token_133_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_133_squeeze_mask_0 = const()[name = string("key_token_133_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_133_cast_fp16 = slice_by_index(begin = key_token_133_begin_0, end = key_token_133_end_0, end_mask = key_token_133_end_mask_0, squeeze_mask = key_token_133_squeeze_mask_0, x = key_23_cast_fp16)[name = string("key_token_133_cast_fp16")];
            tensor<int32, [4]> value_token_133_begin_0 = const()[name = string("value_token_133_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> value_token_133_end_0 = const()[name = string("value_token_133_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> value_token_133_end_mask_0 = const()[name = string("value_token_133_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_133_squeeze_mask_0 = const()[name = string("value_token_133_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_133_cast_fp16 = slice_by_index(begin = value_token_133_begin_0, end = value_token_133_end_0, end_mask = value_token_133_end_mask_0, squeeze_mask = value_token_133_squeeze_mask_0, x = value_83_cast_fp16)[name = string("value_token_133_cast_fp16")];
            tensor<int32, [1]> var_3695_axes_0 = const()[name = string("op_3695_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_3695_cast_fp16 = expand_dims(axes = var_3695_axes_0, x = key_token_133_cast_fp16)[name = string("op_3695_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3696_cast_fp16 = mul(x = state_265_cast_fp16, y = var_3695_cast_fp16)[name = string("op_3696_cast_fp16")];
            tensor<int32, [1]> memory_133_axes_0 = const()[name = string("memory_133_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_133_keep_dims_0 = const()[name = string("memory_133_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_133_cast_fp16 = reduce_sum(axes = memory_133_axes_0, keep_dims = memory_133_keep_dims_0, x = var_3696_cast_fp16)[name = string("memory_133_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_3699_cast_fp16 = sub(x = value_token_133_cast_fp16, y = memory_133_cast_fp16)[name = string("op_3699_cast_fp16")];
            tensor<int32, [3]> var_3702_begin_0 = const()[name = string("op_3702_begin_0"), val = tensor<int32, [3]>([0, 0, 2])];
            tensor<int32, [3]> var_3702_end_0 = const()[name = string("op_3702_end_0"), val = tensor<int32, [3]>([1, 16, 3])];
            tensor<bool, [3]> var_3702_end_mask_0 = const()[name = string("op_3702_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_3702_squeeze_mask_0 = const()[name = string("op_3702_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_3702_cast_fp16 = slice_by_index(begin = var_3702_begin_0, end = var_3702_end_0, end_mask = var_3702_end_mask_0, squeeze_mask = var_3702_squeeze_mask_0, x = beta_9_cast_fp16)[name = string("op_3702_cast_fp16")];
            tensor<int32, [1]> var_3703_axes_0 = const()[name = string("op_3703_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_3703_cast_fp16 = expand_dims(axes = var_3703_axes_0, x = var_3702_cast_fp16)[name = string("op_3703_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_133_cast_fp16 = mul(x = var_3699_cast_fp16, y = var_3703_cast_fp16)[name = string("delta_133_cast_fp16")];
            tensor<int32, [1]> var_3706_axes_0 = const()[name = string("op_3706_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_3706_cast_fp16 = expand_dims(axes = var_3706_axes_0, x = delta_133_cast_fp16)[name = string("op_3706_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3707_cast_fp16 = mul(x = var_3695_cast_fp16, y = var_3706_cast_fp16)[name = string("op_3707_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_267_cast_fp16 = add(x = state_265_cast_fp16, y = var_3707_cast_fp16)[name = string("state_267_cast_fp16")];
            tensor<int32, [4]> var_3711_begin_0 = const()[name = string("op_3711_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> var_3711_end_0 = const()[name = string("op_3711_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> var_3711_end_mask_0 = const()[name = string("op_3711_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_3711_squeeze_mask_0 = const()[name = string("op_3711_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_3711_cast_fp16 = slice_by_index(begin = var_3711_begin_0, end = var_3711_end_0, end_mask = var_3711_end_mask_0, squeeze_mask = var_3711_squeeze_mask_0, x = _inversed_query_23_cast_fp16)[name = string("op_3711_cast_fp16")];
            tensor<int32, [1]> var_3712_axes_0 = const()[name = string("op_3712_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_3712_cast_fp16 = expand_dims(axes = var_3712_axes_0, x = var_3711_cast_fp16)[name = string("op_3712_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3713_cast_fp16 = mul(x = state_267_cast_fp16, y = var_3712_cast_fp16)[name = string("op_3713_cast_fp16")];
            tensor<int32, [1]> var_3715_axes_0 = const()[name = string("op_3715_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_3715_keep_dims_0 = const()[name = string("op_3715_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_3715_cast_fp16 = reduce_sum(axes = var_3715_axes_0, keep_dims = var_3715_keep_dims_0, x = var_3713_cast_fp16)[name = string("op_3715_cast_fp16")];
            tensor<int32, [3]> var_3718_begin_0 = const()[name = string("op_3718_begin_0"), val = tensor<int32, [3]>([0, 0, 3])];
            tensor<int32, [3]> var_3718_end_0 = const()[name = string("op_3718_end_0"), val = tensor<int32, [3]>([1, 16, 4])];
            tensor<bool, [3]> var_3718_end_mask_0 = const()[name = string("op_3718_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_3718_squeeze_mask_0 = const()[name = string("op_3718_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_3718_cast_fp16 = slice_by_index(begin = var_3718_begin_0, end = var_3718_end_0, end_mask = var_3718_end_mask_0, squeeze_mask = var_3718_squeeze_mask_0, x = decay_9_cast_fp16)[name = string("op_3718_cast_fp16")];
            tensor<fp16, [1, 16]> var_3719_cast_fp16 = exp(x = var_3718_cast_fp16)[name = string("op_3719_cast_fp16")];
            tensor<int32, [1]> var_3720_axes_0 = const()[name = string("op_3720_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_3720_cast_fp16 = expand_dims(axes = var_3720_axes_0, x = var_3719_cast_fp16)[name = string("op_3720_cast_fp16")];
            tensor<int32, [1]> var_3721_axes_0 = const()[name = string("op_3721_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_3721_cast_fp16 = expand_dims(axes = var_3721_axes_0, x = var_3720_cast_fp16)[name = string("op_3721_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_269_cast_fp16 = mul(x = state_267_cast_fp16, y = var_3721_cast_fp16)[name = string("state_269_cast_fp16")];
            tensor<int32, [4]> key_token_135_begin_0 = const()[name = string("key_token_135_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> key_token_135_end_0 = const()[name = string("key_token_135_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> key_token_135_end_mask_0 = const()[name = string("key_token_135_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_135_squeeze_mask_0 = const()[name = string("key_token_135_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_135_cast_fp16 = slice_by_index(begin = key_token_135_begin_0, end = key_token_135_end_0, end_mask = key_token_135_end_mask_0, squeeze_mask = key_token_135_squeeze_mask_0, x = key_23_cast_fp16)[name = string("key_token_135_cast_fp16")];
            tensor<int32, [4]> value_token_135_begin_0 = const()[name = string("value_token_135_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> value_token_135_end_0 = const()[name = string("value_token_135_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> value_token_135_end_mask_0 = const()[name = string("value_token_135_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_135_squeeze_mask_0 = const()[name = string("value_token_135_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_135_cast_fp16 = slice_by_index(begin = value_token_135_begin_0, end = value_token_135_end_0, end_mask = value_token_135_end_mask_0, squeeze_mask = value_token_135_squeeze_mask_0, x = value_83_cast_fp16)[name = string("value_token_135_cast_fp16")];
            tensor<int32, [1]> var_3729_axes_0 = const()[name = string("op_3729_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_3729_cast_fp16 = expand_dims(axes = var_3729_axes_0, x = key_token_135_cast_fp16)[name = string("op_3729_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3730_cast_fp16 = mul(x = state_269_cast_fp16, y = var_3729_cast_fp16)[name = string("op_3730_cast_fp16")];
            tensor<int32, [1]> memory_135_axes_0 = const()[name = string("memory_135_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_135_keep_dims_0 = const()[name = string("memory_135_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_135_cast_fp16 = reduce_sum(axes = memory_135_axes_0, keep_dims = memory_135_keep_dims_0, x = var_3730_cast_fp16)[name = string("memory_135_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_3733_cast_fp16 = sub(x = value_token_135_cast_fp16, y = memory_135_cast_fp16)[name = string("op_3733_cast_fp16")];
            tensor<int32, [3]> var_3736_begin_0 = const()[name = string("op_3736_begin_0"), val = tensor<int32, [3]>([0, 0, 3])];
            tensor<int32, [3]> var_3736_end_0 = const()[name = string("op_3736_end_0"), val = tensor<int32, [3]>([1, 16, 4])];
            tensor<bool, [3]> var_3736_end_mask_0 = const()[name = string("op_3736_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_3736_squeeze_mask_0 = const()[name = string("op_3736_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_3736_cast_fp16 = slice_by_index(begin = var_3736_begin_0, end = var_3736_end_0, end_mask = var_3736_end_mask_0, squeeze_mask = var_3736_squeeze_mask_0, x = beta_9_cast_fp16)[name = string("op_3736_cast_fp16")];
            tensor<int32, [1]> var_3737_axes_0 = const()[name = string("op_3737_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_3737_cast_fp16 = expand_dims(axes = var_3737_axes_0, x = var_3736_cast_fp16)[name = string("op_3737_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_135_cast_fp16 = mul(x = var_3733_cast_fp16, y = var_3737_cast_fp16)[name = string("delta_135_cast_fp16")];
            tensor<int32, [1]> var_3740_axes_0 = const()[name = string("op_3740_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_3740_cast_fp16 = expand_dims(axes = var_3740_axes_0, x = delta_135_cast_fp16)[name = string("op_3740_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3741_cast_fp16 = mul(x = var_3729_cast_fp16, y = var_3740_cast_fp16)[name = string("op_3741_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_271_cast_fp16 = add(x = state_269_cast_fp16, y = var_3741_cast_fp16)[name = string("state_271_cast_fp16")];
            tensor<int32, [4]> var_3745_begin_0 = const()[name = string("op_3745_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> var_3745_end_0 = const()[name = string("op_3745_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> var_3745_end_mask_0 = const()[name = string("op_3745_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_3745_squeeze_mask_0 = const()[name = string("op_3745_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_3745_cast_fp16 = slice_by_index(begin = var_3745_begin_0, end = var_3745_end_0, end_mask = var_3745_end_mask_0, squeeze_mask = var_3745_squeeze_mask_0, x = _inversed_query_23_cast_fp16)[name = string("op_3745_cast_fp16")];
            tensor<int32, [1]> var_3746_axes_0 = const()[name = string("op_3746_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_3746_cast_fp16 = expand_dims(axes = var_3746_axes_0, x = var_3745_cast_fp16)[name = string("op_3746_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3747_cast_fp16 = mul(x = state_271_cast_fp16, y = var_3746_cast_fp16)[name = string("op_3747_cast_fp16")];
            tensor<int32, [1]> var_3749_axes_0 = const()[name = string("op_3749_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_3749_keep_dims_0 = const()[name = string("op_3749_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_3749_cast_fp16 = reduce_sum(axes = var_3749_axes_0, keep_dims = var_3749_keep_dims_0, x = var_3747_cast_fp16)[name = string("op_3749_cast_fp16")];
            tensor<int32, [3]> var_3752_begin_0 = const()[name = string("op_3752_begin_0"), val = tensor<int32, [3]>([0, 0, 4])];
            tensor<int32, [3]> var_3752_end_0 = const()[name = string("op_3752_end_0"), val = tensor<int32, [3]>([1, 16, 5])];
            tensor<bool, [3]> var_3752_end_mask_0 = const()[name = string("op_3752_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_3752_squeeze_mask_0 = const()[name = string("op_3752_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_3752_cast_fp16 = slice_by_index(begin = var_3752_begin_0, end = var_3752_end_0, end_mask = var_3752_end_mask_0, squeeze_mask = var_3752_squeeze_mask_0, x = decay_9_cast_fp16)[name = string("op_3752_cast_fp16")];
            tensor<fp16, [1, 16]> var_3753_cast_fp16 = exp(x = var_3752_cast_fp16)[name = string("op_3753_cast_fp16")];
            tensor<int32, [1]> var_3754_axes_0 = const()[name = string("op_3754_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_3754_cast_fp16 = expand_dims(axes = var_3754_axes_0, x = var_3753_cast_fp16)[name = string("op_3754_cast_fp16")];
            tensor<int32, [1]> var_3755_axes_0 = const()[name = string("op_3755_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_3755_cast_fp16 = expand_dims(axes = var_3755_axes_0, x = var_3754_cast_fp16)[name = string("op_3755_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_273_cast_fp16 = mul(x = state_271_cast_fp16, y = var_3755_cast_fp16)[name = string("state_273_cast_fp16")];
            tensor<int32, [4]> key_token_137_begin_0 = const()[name = string("key_token_137_begin_0"), val = tensor<int32, [4]>([0, 0, 4, 0])];
            tensor<int32, [4]> key_token_137_end_0 = const()[name = string("key_token_137_end_0"), val = tensor<int32, [4]>([1, 16, 5, 128])];
            tensor<bool, [4]> key_token_137_end_mask_0 = const()[name = string("key_token_137_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_137_squeeze_mask_0 = const()[name = string("key_token_137_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_137_cast_fp16 = slice_by_index(begin = key_token_137_begin_0, end = key_token_137_end_0, end_mask = key_token_137_end_mask_0, squeeze_mask = key_token_137_squeeze_mask_0, x = key_23_cast_fp16)[name = string("key_token_137_cast_fp16")];
            tensor<int32, [4]> value_token_137_begin_0 = const()[name = string("value_token_137_begin_0"), val = tensor<int32, [4]>([0, 0, 4, 0])];
            tensor<int32, [4]> value_token_137_end_0 = const()[name = string("value_token_137_end_0"), val = tensor<int32, [4]>([1, 16, 5, 128])];
            tensor<bool, [4]> value_token_137_end_mask_0 = const()[name = string("value_token_137_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_137_squeeze_mask_0 = const()[name = string("value_token_137_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_137_cast_fp16 = slice_by_index(begin = value_token_137_begin_0, end = value_token_137_end_0, end_mask = value_token_137_end_mask_0, squeeze_mask = value_token_137_squeeze_mask_0, x = value_83_cast_fp16)[name = string("value_token_137_cast_fp16")];
            tensor<int32, [1]> var_3763_axes_0 = const()[name = string("op_3763_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_3763_cast_fp16 = expand_dims(axes = var_3763_axes_0, x = key_token_137_cast_fp16)[name = string("op_3763_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3764_cast_fp16 = mul(x = state_273_cast_fp16, y = var_3763_cast_fp16)[name = string("op_3764_cast_fp16")];
            tensor<int32, [1]> memory_137_axes_0 = const()[name = string("memory_137_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_137_keep_dims_0 = const()[name = string("memory_137_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_137_cast_fp16 = reduce_sum(axes = memory_137_axes_0, keep_dims = memory_137_keep_dims_0, x = var_3764_cast_fp16)[name = string("memory_137_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_3767_cast_fp16 = sub(x = value_token_137_cast_fp16, y = memory_137_cast_fp16)[name = string("op_3767_cast_fp16")];
            tensor<int32, [3]> var_3770_begin_0 = const()[name = string("op_3770_begin_0"), val = tensor<int32, [3]>([0, 0, 4])];
            tensor<int32, [3]> var_3770_end_0 = const()[name = string("op_3770_end_0"), val = tensor<int32, [3]>([1, 16, 5])];
            tensor<bool, [3]> var_3770_end_mask_0 = const()[name = string("op_3770_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_3770_squeeze_mask_0 = const()[name = string("op_3770_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_3770_cast_fp16 = slice_by_index(begin = var_3770_begin_0, end = var_3770_end_0, end_mask = var_3770_end_mask_0, squeeze_mask = var_3770_squeeze_mask_0, x = beta_9_cast_fp16)[name = string("op_3770_cast_fp16")];
            tensor<int32, [1]> var_3771_axes_0 = const()[name = string("op_3771_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_3771_cast_fp16 = expand_dims(axes = var_3771_axes_0, x = var_3770_cast_fp16)[name = string("op_3771_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_137_cast_fp16 = mul(x = var_3767_cast_fp16, y = var_3771_cast_fp16)[name = string("delta_137_cast_fp16")];
            tensor<int32, [1]> var_3774_axes_0 = const()[name = string("op_3774_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_3774_cast_fp16 = expand_dims(axes = var_3774_axes_0, x = delta_137_cast_fp16)[name = string("op_3774_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3775_cast_fp16 = mul(x = var_3763_cast_fp16, y = var_3774_cast_fp16)[name = string("op_3775_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_275_cast_fp16 = add(x = state_273_cast_fp16, y = var_3775_cast_fp16)[name = string("state_275_cast_fp16")];
            tensor<int32, [4]> var_3779_begin_0 = const()[name = string("op_3779_begin_0"), val = tensor<int32, [4]>([0, 0, 4, 0])];
            tensor<int32, [4]> var_3779_end_0 = const()[name = string("op_3779_end_0"), val = tensor<int32, [4]>([1, 16, 5, 128])];
            tensor<bool, [4]> var_3779_end_mask_0 = const()[name = string("op_3779_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_3779_squeeze_mask_0 = const()[name = string("op_3779_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_3779_cast_fp16 = slice_by_index(begin = var_3779_begin_0, end = var_3779_end_0, end_mask = var_3779_end_mask_0, squeeze_mask = var_3779_squeeze_mask_0, x = _inversed_query_23_cast_fp16)[name = string("op_3779_cast_fp16")];
            tensor<int32, [1]> var_3780_axes_0 = const()[name = string("op_3780_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_3780_cast_fp16 = expand_dims(axes = var_3780_axes_0, x = var_3779_cast_fp16)[name = string("op_3780_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3781_cast_fp16 = mul(x = state_275_cast_fp16, y = var_3780_cast_fp16)[name = string("op_3781_cast_fp16")];
            tensor<int32, [1]> var_3783_axes_0 = const()[name = string("op_3783_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_3783_keep_dims_0 = const()[name = string("op_3783_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_3783_cast_fp16 = reduce_sum(axes = var_3783_axes_0, keep_dims = var_3783_keep_dims_0, x = var_3781_cast_fp16)[name = string("op_3783_cast_fp16")];
            tensor<int32, [3]> var_3786_begin_0 = const()[name = string("op_3786_begin_0"), val = tensor<int32, [3]>([0, 0, 5])];
            tensor<int32, [3]> var_3786_end_0 = const()[name = string("op_3786_end_0"), val = tensor<int32, [3]>([1, 16, 6])];
            tensor<bool, [3]> var_3786_end_mask_0 = const()[name = string("op_3786_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_3786_squeeze_mask_0 = const()[name = string("op_3786_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_3786_cast_fp16 = slice_by_index(begin = var_3786_begin_0, end = var_3786_end_0, end_mask = var_3786_end_mask_0, squeeze_mask = var_3786_squeeze_mask_0, x = decay_9_cast_fp16)[name = string("op_3786_cast_fp16")];
            tensor<fp16, [1, 16]> var_3787_cast_fp16 = exp(x = var_3786_cast_fp16)[name = string("op_3787_cast_fp16")];
            tensor<int32, [1]> var_3788_axes_0 = const()[name = string("op_3788_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_3788_cast_fp16 = expand_dims(axes = var_3788_axes_0, x = var_3787_cast_fp16)[name = string("op_3788_cast_fp16")];
            tensor<int32, [1]> var_3789_axes_0 = const()[name = string("op_3789_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_3789_cast_fp16 = expand_dims(axes = var_3789_axes_0, x = var_3788_cast_fp16)[name = string("op_3789_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_277_cast_fp16 = mul(x = state_275_cast_fp16, y = var_3789_cast_fp16)[name = string("state_277_cast_fp16")];
            tensor<int32, [4]> key_token_139_begin_0 = const()[name = string("key_token_139_begin_0"), val = tensor<int32, [4]>([0, 0, 5, 0])];
            tensor<int32, [4]> key_token_139_end_0 = const()[name = string("key_token_139_end_0"), val = tensor<int32, [4]>([1, 16, 6, 128])];
            tensor<bool, [4]> key_token_139_end_mask_0 = const()[name = string("key_token_139_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_139_squeeze_mask_0 = const()[name = string("key_token_139_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_139_cast_fp16 = slice_by_index(begin = key_token_139_begin_0, end = key_token_139_end_0, end_mask = key_token_139_end_mask_0, squeeze_mask = key_token_139_squeeze_mask_0, x = key_23_cast_fp16)[name = string("key_token_139_cast_fp16")];
            tensor<int32, [4]> value_token_139_begin_0 = const()[name = string("value_token_139_begin_0"), val = tensor<int32, [4]>([0, 0, 5, 0])];
            tensor<int32, [4]> value_token_139_end_0 = const()[name = string("value_token_139_end_0"), val = tensor<int32, [4]>([1, 16, 6, 128])];
            tensor<bool, [4]> value_token_139_end_mask_0 = const()[name = string("value_token_139_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_139_squeeze_mask_0 = const()[name = string("value_token_139_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_139_cast_fp16 = slice_by_index(begin = value_token_139_begin_0, end = value_token_139_end_0, end_mask = value_token_139_end_mask_0, squeeze_mask = value_token_139_squeeze_mask_0, x = value_83_cast_fp16)[name = string("value_token_139_cast_fp16")];
            tensor<int32, [1]> var_3797_axes_0 = const()[name = string("op_3797_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_3797_cast_fp16 = expand_dims(axes = var_3797_axes_0, x = key_token_139_cast_fp16)[name = string("op_3797_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3798_cast_fp16 = mul(x = state_277_cast_fp16, y = var_3797_cast_fp16)[name = string("op_3798_cast_fp16")];
            tensor<int32, [1]> memory_139_axes_0 = const()[name = string("memory_139_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_139_keep_dims_0 = const()[name = string("memory_139_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_139_cast_fp16 = reduce_sum(axes = memory_139_axes_0, keep_dims = memory_139_keep_dims_0, x = var_3798_cast_fp16)[name = string("memory_139_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_3801_cast_fp16 = sub(x = value_token_139_cast_fp16, y = memory_139_cast_fp16)[name = string("op_3801_cast_fp16")];
            tensor<int32, [3]> var_3804_begin_0 = const()[name = string("op_3804_begin_0"), val = tensor<int32, [3]>([0, 0, 5])];
            tensor<int32, [3]> var_3804_end_0 = const()[name = string("op_3804_end_0"), val = tensor<int32, [3]>([1, 16, 6])];
            tensor<bool, [3]> var_3804_end_mask_0 = const()[name = string("op_3804_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_3804_squeeze_mask_0 = const()[name = string("op_3804_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_3804_cast_fp16 = slice_by_index(begin = var_3804_begin_0, end = var_3804_end_0, end_mask = var_3804_end_mask_0, squeeze_mask = var_3804_squeeze_mask_0, x = beta_9_cast_fp16)[name = string("op_3804_cast_fp16")];
            tensor<int32, [1]> var_3805_axes_0 = const()[name = string("op_3805_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_3805_cast_fp16 = expand_dims(axes = var_3805_axes_0, x = var_3804_cast_fp16)[name = string("op_3805_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_139_cast_fp16 = mul(x = var_3801_cast_fp16, y = var_3805_cast_fp16)[name = string("delta_139_cast_fp16")];
            tensor<int32, [1]> var_3808_axes_0 = const()[name = string("op_3808_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_3808_cast_fp16 = expand_dims(axes = var_3808_axes_0, x = delta_139_cast_fp16)[name = string("op_3808_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3809_cast_fp16 = mul(x = var_3797_cast_fp16, y = var_3808_cast_fp16)[name = string("op_3809_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_279_cast_fp16 = add(x = state_277_cast_fp16, y = var_3809_cast_fp16)[name = string("state_279_cast_fp16")];
            tensor<int32, [4]> var_3813_begin_0 = const()[name = string("op_3813_begin_0"), val = tensor<int32, [4]>([0, 0, 5, 0])];
            tensor<int32, [4]> var_3813_end_0 = const()[name = string("op_3813_end_0"), val = tensor<int32, [4]>([1, 16, 6, 128])];
            tensor<bool, [4]> var_3813_end_mask_0 = const()[name = string("op_3813_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_3813_squeeze_mask_0 = const()[name = string("op_3813_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_3813_cast_fp16 = slice_by_index(begin = var_3813_begin_0, end = var_3813_end_0, end_mask = var_3813_end_mask_0, squeeze_mask = var_3813_squeeze_mask_0, x = _inversed_query_23_cast_fp16)[name = string("op_3813_cast_fp16")];
            tensor<int32, [1]> var_3814_axes_0 = const()[name = string("op_3814_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_3814_cast_fp16 = expand_dims(axes = var_3814_axes_0, x = var_3813_cast_fp16)[name = string("op_3814_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3815_cast_fp16 = mul(x = state_279_cast_fp16, y = var_3814_cast_fp16)[name = string("op_3815_cast_fp16")];
            tensor<int32, [1]> var_3817_axes_0 = const()[name = string("op_3817_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_3817_keep_dims_0 = const()[name = string("op_3817_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_3817_cast_fp16 = reduce_sum(axes = var_3817_axes_0, keep_dims = var_3817_keep_dims_0, x = var_3815_cast_fp16)[name = string("op_3817_cast_fp16")];
            tensor<int32, [3]> var_3820_begin_0 = const()[name = string("op_3820_begin_0"), val = tensor<int32, [3]>([0, 0, 6])];
            tensor<int32, [3]> var_3820_end_0 = const()[name = string("op_3820_end_0"), val = tensor<int32, [3]>([1, 16, 7])];
            tensor<bool, [3]> var_3820_end_mask_0 = const()[name = string("op_3820_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_3820_squeeze_mask_0 = const()[name = string("op_3820_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_3820_cast_fp16 = slice_by_index(begin = var_3820_begin_0, end = var_3820_end_0, end_mask = var_3820_end_mask_0, squeeze_mask = var_3820_squeeze_mask_0, x = decay_9_cast_fp16)[name = string("op_3820_cast_fp16")];
            tensor<fp16, [1, 16]> var_3821_cast_fp16 = exp(x = var_3820_cast_fp16)[name = string("op_3821_cast_fp16")];
            tensor<int32, [1]> var_3822_axes_0 = const()[name = string("op_3822_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_3822_cast_fp16 = expand_dims(axes = var_3822_axes_0, x = var_3821_cast_fp16)[name = string("op_3822_cast_fp16")];
            tensor<int32, [1]> var_3823_axes_0 = const()[name = string("op_3823_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_3823_cast_fp16 = expand_dims(axes = var_3823_axes_0, x = var_3822_cast_fp16)[name = string("op_3823_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_281_cast_fp16 = mul(x = state_279_cast_fp16, y = var_3823_cast_fp16)[name = string("state_281_cast_fp16")];
            tensor<int32, [4]> key_token_141_begin_0 = const()[name = string("key_token_141_begin_0"), val = tensor<int32, [4]>([0, 0, 6, 0])];
            tensor<int32, [4]> key_token_141_end_0 = const()[name = string("key_token_141_end_0"), val = tensor<int32, [4]>([1, 16, 7, 128])];
            tensor<bool, [4]> key_token_141_end_mask_0 = const()[name = string("key_token_141_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_141_squeeze_mask_0 = const()[name = string("key_token_141_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_141_cast_fp16 = slice_by_index(begin = key_token_141_begin_0, end = key_token_141_end_0, end_mask = key_token_141_end_mask_0, squeeze_mask = key_token_141_squeeze_mask_0, x = key_23_cast_fp16)[name = string("key_token_141_cast_fp16")];
            tensor<int32, [4]> value_token_141_begin_0 = const()[name = string("value_token_141_begin_0"), val = tensor<int32, [4]>([0, 0, 6, 0])];
            tensor<int32, [4]> value_token_141_end_0 = const()[name = string("value_token_141_end_0"), val = tensor<int32, [4]>([1, 16, 7, 128])];
            tensor<bool, [4]> value_token_141_end_mask_0 = const()[name = string("value_token_141_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_141_squeeze_mask_0 = const()[name = string("value_token_141_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_141_cast_fp16 = slice_by_index(begin = value_token_141_begin_0, end = value_token_141_end_0, end_mask = value_token_141_end_mask_0, squeeze_mask = value_token_141_squeeze_mask_0, x = value_83_cast_fp16)[name = string("value_token_141_cast_fp16")];
            tensor<int32, [1]> var_3831_axes_0 = const()[name = string("op_3831_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_3831_cast_fp16 = expand_dims(axes = var_3831_axes_0, x = key_token_141_cast_fp16)[name = string("op_3831_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3832_cast_fp16 = mul(x = state_281_cast_fp16, y = var_3831_cast_fp16)[name = string("op_3832_cast_fp16")];
            tensor<int32, [1]> memory_141_axes_0 = const()[name = string("memory_141_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_141_keep_dims_0 = const()[name = string("memory_141_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_141_cast_fp16 = reduce_sum(axes = memory_141_axes_0, keep_dims = memory_141_keep_dims_0, x = var_3832_cast_fp16)[name = string("memory_141_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_3835_cast_fp16 = sub(x = value_token_141_cast_fp16, y = memory_141_cast_fp16)[name = string("op_3835_cast_fp16")];
            tensor<int32, [3]> var_3838_begin_0 = const()[name = string("op_3838_begin_0"), val = tensor<int32, [3]>([0, 0, 6])];
            tensor<int32, [3]> var_3838_end_0 = const()[name = string("op_3838_end_0"), val = tensor<int32, [3]>([1, 16, 7])];
            tensor<bool, [3]> var_3838_end_mask_0 = const()[name = string("op_3838_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_3838_squeeze_mask_0 = const()[name = string("op_3838_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_3838_cast_fp16 = slice_by_index(begin = var_3838_begin_0, end = var_3838_end_0, end_mask = var_3838_end_mask_0, squeeze_mask = var_3838_squeeze_mask_0, x = beta_9_cast_fp16)[name = string("op_3838_cast_fp16")];
            tensor<int32, [1]> var_3839_axes_0 = const()[name = string("op_3839_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_3839_cast_fp16 = expand_dims(axes = var_3839_axes_0, x = var_3838_cast_fp16)[name = string("op_3839_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_141_cast_fp16 = mul(x = var_3835_cast_fp16, y = var_3839_cast_fp16)[name = string("delta_141_cast_fp16")];
            tensor<int32, [1]> var_3842_axes_0 = const()[name = string("op_3842_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_3842_cast_fp16 = expand_dims(axes = var_3842_axes_0, x = delta_141_cast_fp16)[name = string("op_3842_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3843_cast_fp16 = mul(x = var_3831_cast_fp16, y = var_3842_cast_fp16)[name = string("op_3843_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_283_cast_fp16 = add(x = state_281_cast_fp16, y = var_3843_cast_fp16)[name = string("state_283_cast_fp16")];
            tensor<int32, [4]> var_3847_begin_0 = const()[name = string("op_3847_begin_0"), val = tensor<int32, [4]>([0, 0, 6, 0])];
            tensor<int32, [4]> var_3847_end_0 = const()[name = string("op_3847_end_0"), val = tensor<int32, [4]>([1, 16, 7, 128])];
            tensor<bool, [4]> var_3847_end_mask_0 = const()[name = string("op_3847_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_3847_squeeze_mask_0 = const()[name = string("op_3847_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_3847_cast_fp16 = slice_by_index(begin = var_3847_begin_0, end = var_3847_end_0, end_mask = var_3847_end_mask_0, squeeze_mask = var_3847_squeeze_mask_0, x = _inversed_query_23_cast_fp16)[name = string("op_3847_cast_fp16")];
            tensor<int32, [1]> var_3848_axes_0 = const()[name = string("op_3848_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_3848_cast_fp16 = expand_dims(axes = var_3848_axes_0, x = var_3847_cast_fp16)[name = string("op_3848_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3849_cast_fp16 = mul(x = state_283_cast_fp16, y = var_3848_cast_fp16)[name = string("op_3849_cast_fp16")];
            tensor<int32, [1]> var_3851_axes_0 = const()[name = string("op_3851_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_3851_keep_dims_0 = const()[name = string("op_3851_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_3851_cast_fp16 = reduce_sum(axes = var_3851_axes_0, keep_dims = var_3851_keep_dims_0, x = var_3849_cast_fp16)[name = string("op_3851_cast_fp16")];
            tensor<int32, [3]> var_3854_begin_0 = const()[name = string("op_3854_begin_0"), val = tensor<int32, [3]>([0, 0, 7])];
            tensor<int32, [3]> var_3854_end_0 = const()[name = string("op_3854_end_0"), val = tensor<int32, [3]>([1, 16, 8])];
            tensor<bool, [3]> var_3854_end_mask_0 = const()[name = string("op_3854_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_3854_squeeze_mask_0 = const()[name = string("op_3854_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_3854_cast_fp16 = slice_by_index(begin = var_3854_begin_0, end = var_3854_end_0, end_mask = var_3854_end_mask_0, squeeze_mask = var_3854_squeeze_mask_0, x = decay_9_cast_fp16)[name = string("op_3854_cast_fp16")];
            tensor<fp16, [1, 16]> var_3855_cast_fp16 = exp(x = var_3854_cast_fp16)[name = string("op_3855_cast_fp16")];
            tensor<int32, [1]> var_3856_axes_0 = const()[name = string("op_3856_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_3856_cast_fp16 = expand_dims(axes = var_3856_axes_0, x = var_3855_cast_fp16)[name = string("op_3856_cast_fp16")];
            tensor<int32, [1]> var_3857_axes_0 = const()[name = string("op_3857_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_3857_cast_fp16 = expand_dims(axes = var_3857_axes_0, x = var_3856_cast_fp16)[name = string("op_3857_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_285_cast_fp16 = mul(x = state_283_cast_fp16, y = var_3857_cast_fp16)[name = string("state_285_cast_fp16")];
            tensor<int32, [4]> key_token_143_begin_0 = const()[name = string("key_token_143_begin_0"), val = tensor<int32, [4]>([0, 0, 7, 0])];
            tensor<int32, [4]> key_token_143_end_0 = const()[name = string("key_token_143_end_0"), val = tensor<int32, [4]>([1, 16, 8, 128])];
            tensor<bool, [4]> key_token_143_end_mask_0 = const()[name = string("key_token_143_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_143_squeeze_mask_0 = const()[name = string("key_token_143_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_143_cast_fp16 = slice_by_index(begin = key_token_143_begin_0, end = key_token_143_end_0, end_mask = key_token_143_end_mask_0, squeeze_mask = key_token_143_squeeze_mask_0, x = key_23_cast_fp16)[name = string("key_token_143_cast_fp16")];
            tensor<int32, [4]> value_token_143_begin_0 = const()[name = string("value_token_143_begin_0"), val = tensor<int32, [4]>([0, 0, 7, 0])];
            tensor<int32, [4]> value_token_143_end_0 = const()[name = string("value_token_143_end_0"), val = tensor<int32, [4]>([1, 16, 8, 128])];
            tensor<bool, [4]> value_token_143_end_mask_0 = const()[name = string("value_token_143_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_143_squeeze_mask_0 = const()[name = string("value_token_143_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_143_cast_fp16 = slice_by_index(begin = value_token_143_begin_0, end = value_token_143_end_0, end_mask = value_token_143_end_mask_0, squeeze_mask = value_token_143_squeeze_mask_0, x = value_83_cast_fp16)[name = string("value_token_143_cast_fp16")];
            tensor<int32, [1]> var_3865_axes_0 = const()[name = string("op_3865_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_3865_cast_fp16 = expand_dims(axes = var_3865_axes_0, x = key_token_143_cast_fp16)[name = string("op_3865_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3866_cast_fp16 = mul(x = state_285_cast_fp16, y = var_3865_cast_fp16)[name = string("op_3866_cast_fp16")];
            tensor<int32, [1]> memory_143_axes_0 = const()[name = string("memory_143_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_143_keep_dims_0 = const()[name = string("memory_143_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_143_cast_fp16 = reduce_sum(axes = memory_143_axes_0, keep_dims = memory_143_keep_dims_0, x = var_3866_cast_fp16)[name = string("memory_143_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_3869_cast_fp16 = sub(x = value_token_143_cast_fp16, y = memory_143_cast_fp16)[name = string("op_3869_cast_fp16")];
            tensor<int32, [3]> var_3872_begin_0 = const()[name = string("op_3872_begin_0"), val = tensor<int32, [3]>([0, 0, 7])];
            tensor<int32, [3]> var_3872_end_0 = const()[name = string("op_3872_end_0"), val = tensor<int32, [3]>([1, 16, 8])];
            tensor<bool, [3]> var_3872_end_mask_0 = const()[name = string("op_3872_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_3872_squeeze_mask_0 = const()[name = string("op_3872_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_3872_cast_fp16 = slice_by_index(begin = var_3872_begin_0, end = var_3872_end_0, end_mask = var_3872_end_mask_0, squeeze_mask = var_3872_squeeze_mask_0, x = beta_9_cast_fp16)[name = string("op_3872_cast_fp16")];
            tensor<int32, [1]> var_3873_axes_0 = const()[name = string("op_3873_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_3873_cast_fp16 = expand_dims(axes = var_3873_axes_0, x = var_3872_cast_fp16)[name = string("op_3873_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_143_cast_fp16 = mul(x = var_3869_cast_fp16, y = var_3873_cast_fp16)[name = string("delta_143_cast_fp16")];
            tensor<int32, [1]> var_3876_axes_0 = const()[name = string("op_3876_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_3876_cast_fp16 = expand_dims(axes = var_3876_axes_0, x = delta_143_cast_fp16)[name = string("op_3876_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3877_cast_fp16 = mul(x = var_3865_cast_fp16, y = var_3876_cast_fp16)[name = string("op_3877_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_287_cast_fp16 = add(x = state_285_cast_fp16, y = var_3877_cast_fp16)[name = string("state_287_cast_fp16")];
            tensor<int32, [4]> var_3881_begin_0 = const()[name = string("op_3881_begin_0"), val = tensor<int32, [4]>([0, 0, 7, 0])];
            tensor<int32, [4]> var_3881_end_0 = const()[name = string("op_3881_end_0"), val = tensor<int32, [4]>([1, 16, 8, 128])];
            tensor<bool, [4]> var_3881_end_mask_0 = const()[name = string("op_3881_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_3881_squeeze_mask_0 = const()[name = string("op_3881_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_3881_cast_fp16 = slice_by_index(begin = var_3881_begin_0, end = var_3881_end_0, end_mask = var_3881_end_mask_0, squeeze_mask = var_3881_squeeze_mask_0, x = _inversed_query_23_cast_fp16)[name = string("op_3881_cast_fp16")];
            tensor<int32, [1]> var_3882_axes_0 = const()[name = string("op_3882_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_3882_cast_fp16 = expand_dims(axes = var_3882_axes_0, x = var_3881_cast_fp16)[name = string("op_3882_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3883_cast_fp16 = mul(x = state_287_cast_fp16, y = var_3882_cast_fp16)[name = string("op_3883_cast_fp16")];
            tensor<int32, [1]> var_3885_axes_0 = const()[name = string("op_3885_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_3885_keep_dims_0 = const()[name = string("op_3885_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_3885_cast_fp16 = reduce_sum(axes = var_3885_axes_0, keep_dims = var_3885_keep_dims_0, x = var_3883_cast_fp16)[name = string("op_3885_cast_fp16")];
            tensor<int32, [3]> var_3888_begin_0 = const()[name = string("op_3888_begin_0"), val = tensor<int32, [3]>([0, 0, 8])];
            tensor<int32, [3]> var_3888_end_0 = const()[name = string("op_3888_end_0"), val = tensor<int32, [3]>([1, 16, 9])];
            tensor<bool, [3]> var_3888_end_mask_0 = const()[name = string("op_3888_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_3888_squeeze_mask_0 = const()[name = string("op_3888_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_3888_cast_fp16 = slice_by_index(begin = var_3888_begin_0, end = var_3888_end_0, end_mask = var_3888_end_mask_0, squeeze_mask = var_3888_squeeze_mask_0, x = decay_9_cast_fp16)[name = string("op_3888_cast_fp16")];
            tensor<fp16, [1, 16]> var_3889_cast_fp16 = exp(x = var_3888_cast_fp16)[name = string("op_3889_cast_fp16")];
            tensor<int32, [1]> var_3890_axes_0 = const()[name = string("op_3890_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_3890_cast_fp16 = expand_dims(axes = var_3890_axes_0, x = var_3889_cast_fp16)[name = string("op_3890_cast_fp16")];
            tensor<int32, [1]> var_3891_axes_0 = const()[name = string("op_3891_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_3891_cast_fp16 = expand_dims(axes = var_3891_axes_0, x = var_3890_cast_fp16)[name = string("op_3891_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_289_cast_fp16 = mul(x = state_287_cast_fp16, y = var_3891_cast_fp16)[name = string("state_289_cast_fp16")];
            tensor<int32, [4]> key_token_145_begin_0 = const()[name = string("key_token_145_begin_0"), val = tensor<int32, [4]>([0, 0, 8, 0])];
            tensor<int32, [4]> key_token_145_end_0 = const()[name = string("key_token_145_end_0"), val = tensor<int32, [4]>([1, 16, 9, 128])];
            tensor<bool, [4]> key_token_145_end_mask_0 = const()[name = string("key_token_145_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_145_squeeze_mask_0 = const()[name = string("key_token_145_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_145_cast_fp16 = slice_by_index(begin = key_token_145_begin_0, end = key_token_145_end_0, end_mask = key_token_145_end_mask_0, squeeze_mask = key_token_145_squeeze_mask_0, x = key_23_cast_fp16)[name = string("key_token_145_cast_fp16")];
            tensor<int32, [4]> value_token_145_begin_0 = const()[name = string("value_token_145_begin_0"), val = tensor<int32, [4]>([0, 0, 8, 0])];
            tensor<int32, [4]> value_token_145_end_0 = const()[name = string("value_token_145_end_0"), val = tensor<int32, [4]>([1, 16, 9, 128])];
            tensor<bool, [4]> value_token_145_end_mask_0 = const()[name = string("value_token_145_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_145_squeeze_mask_0 = const()[name = string("value_token_145_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_145_cast_fp16 = slice_by_index(begin = value_token_145_begin_0, end = value_token_145_end_0, end_mask = value_token_145_end_mask_0, squeeze_mask = value_token_145_squeeze_mask_0, x = value_83_cast_fp16)[name = string("value_token_145_cast_fp16")];
            tensor<int32, [1]> var_3899_axes_0 = const()[name = string("op_3899_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_3899_cast_fp16 = expand_dims(axes = var_3899_axes_0, x = key_token_145_cast_fp16)[name = string("op_3899_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3900_cast_fp16 = mul(x = state_289_cast_fp16, y = var_3899_cast_fp16)[name = string("op_3900_cast_fp16")];
            tensor<int32, [1]> memory_145_axes_0 = const()[name = string("memory_145_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_145_keep_dims_0 = const()[name = string("memory_145_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_145_cast_fp16 = reduce_sum(axes = memory_145_axes_0, keep_dims = memory_145_keep_dims_0, x = var_3900_cast_fp16)[name = string("memory_145_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_3903_cast_fp16 = sub(x = value_token_145_cast_fp16, y = memory_145_cast_fp16)[name = string("op_3903_cast_fp16")];
            tensor<int32, [3]> var_3906_begin_0 = const()[name = string("op_3906_begin_0"), val = tensor<int32, [3]>([0, 0, 8])];
            tensor<int32, [3]> var_3906_end_0 = const()[name = string("op_3906_end_0"), val = tensor<int32, [3]>([1, 16, 9])];
            tensor<bool, [3]> var_3906_end_mask_0 = const()[name = string("op_3906_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_3906_squeeze_mask_0 = const()[name = string("op_3906_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_3906_cast_fp16 = slice_by_index(begin = var_3906_begin_0, end = var_3906_end_0, end_mask = var_3906_end_mask_0, squeeze_mask = var_3906_squeeze_mask_0, x = beta_9_cast_fp16)[name = string("op_3906_cast_fp16")];
            tensor<int32, [1]> var_3907_axes_0 = const()[name = string("op_3907_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_3907_cast_fp16 = expand_dims(axes = var_3907_axes_0, x = var_3906_cast_fp16)[name = string("op_3907_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_145_cast_fp16 = mul(x = var_3903_cast_fp16, y = var_3907_cast_fp16)[name = string("delta_145_cast_fp16")];
            tensor<int32, [1]> var_3910_axes_0 = const()[name = string("op_3910_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_3910_cast_fp16 = expand_dims(axes = var_3910_axes_0, x = delta_145_cast_fp16)[name = string("op_3910_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3911_cast_fp16 = mul(x = var_3899_cast_fp16, y = var_3910_cast_fp16)[name = string("op_3911_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_291_cast_fp16 = add(x = state_289_cast_fp16, y = var_3911_cast_fp16)[name = string("state_291_cast_fp16")];
            tensor<int32, [4]> var_3915_begin_0 = const()[name = string("op_3915_begin_0"), val = tensor<int32, [4]>([0, 0, 8, 0])];
            tensor<int32, [4]> var_3915_end_0 = const()[name = string("op_3915_end_0"), val = tensor<int32, [4]>([1, 16, 9, 128])];
            tensor<bool, [4]> var_3915_end_mask_0 = const()[name = string("op_3915_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_3915_squeeze_mask_0 = const()[name = string("op_3915_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_3915_cast_fp16 = slice_by_index(begin = var_3915_begin_0, end = var_3915_end_0, end_mask = var_3915_end_mask_0, squeeze_mask = var_3915_squeeze_mask_0, x = _inversed_query_23_cast_fp16)[name = string("op_3915_cast_fp16")];
            tensor<int32, [1]> var_3916_axes_0 = const()[name = string("op_3916_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_3916_cast_fp16 = expand_dims(axes = var_3916_axes_0, x = var_3915_cast_fp16)[name = string("op_3916_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3917_cast_fp16 = mul(x = state_291_cast_fp16, y = var_3916_cast_fp16)[name = string("op_3917_cast_fp16")];
            tensor<int32, [1]> var_3919_axes_0 = const()[name = string("op_3919_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_3919_keep_dims_0 = const()[name = string("op_3919_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_3919_cast_fp16 = reduce_sum(axes = var_3919_axes_0, keep_dims = var_3919_keep_dims_0, x = var_3917_cast_fp16)[name = string("op_3919_cast_fp16")];
            tensor<int32, [3]> var_3922_begin_0 = const()[name = string("op_3922_begin_0"), val = tensor<int32, [3]>([0, 0, 9])];
            tensor<int32, [3]> var_3922_end_0 = const()[name = string("op_3922_end_0"), val = tensor<int32, [3]>([1, 16, 10])];
            tensor<bool, [3]> var_3922_end_mask_0 = const()[name = string("op_3922_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_3922_squeeze_mask_0 = const()[name = string("op_3922_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_3922_cast_fp16 = slice_by_index(begin = var_3922_begin_0, end = var_3922_end_0, end_mask = var_3922_end_mask_0, squeeze_mask = var_3922_squeeze_mask_0, x = decay_9_cast_fp16)[name = string("op_3922_cast_fp16")];
            tensor<fp16, [1, 16]> var_3923_cast_fp16 = exp(x = var_3922_cast_fp16)[name = string("op_3923_cast_fp16")];
            tensor<int32, [1]> var_3924_axes_0 = const()[name = string("op_3924_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_3924_cast_fp16 = expand_dims(axes = var_3924_axes_0, x = var_3923_cast_fp16)[name = string("op_3924_cast_fp16")];
            tensor<int32, [1]> var_3925_axes_0 = const()[name = string("op_3925_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_3925_cast_fp16 = expand_dims(axes = var_3925_axes_0, x = var_3924_cast_fp16)[name = string("op_3925_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_293_cast_fp16 = mul(x = state_291_cast_fp16, y = var_3925_cast_fp16)[name = string("state_293_cast_fp16")];
            tensor<int32, [4]> key_token_147_begin_0 = const()[name = string("key_token_147_begin_0"), val = tensor<int32, [4]>([0, 0, 9, 0])];
            tensor<int32, [4]> key_token_147_end_0 = const()[name = string("key_token_147_end_0"), val = tensor<int32, [4]>([1, 16, 10, 128])];
            tensor<bool, [4]> key_token_147_end_mask_0 = const()[name = string("key_token_147_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_147_squeeze_mask_0 = const()[name = string("key_token_147_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_147_cast_fp16 = slice_by_index(begin = key_token_147_begin_0, end = key_token_147_end_0, end_mask = key_token_147_end_mask_0, squeeze_mask = key_token_147_squeeze_mask_0, x = key_23_cast_fp16)[name = string("key_token_147_cast_fp16")];
            tensor<int32, [4]> value_token_147_begin_0 = const()[name = string("value_token_147_begin_0"), val = tensor<int32, [4]>([0, 0, 9, 0])];
            tensor<int32, [4]> value_token_147_end_0 = const()[name = string("value_token_147_end_0"), val = tensor<int32, [4]>([1, 16, 10, 128])];
            tensor<bool, [4]> value_token_147_end_mask_0 = const()[name = string("value_token_147_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_147_squeeze_mask_0 = const()[name = string("value_token_147_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_147_cast_fp16 = slice_by_index(begin = value_token_147_begin_0, end = value_token_147_end_0, end_mask = value_token_147_end_mask_0, squeeze_mask = value_token_147_squeeze_mask_0, x = value_83_cast_fp16)[name = string("value_token_147_cast_fp16")];
            tensor<int32, [1]> var_3933_axes_0 = const()[name = string("op_3933_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_3933_cast_fp16 = expand_dims(axes = var_3933_axes_0, x = key_token_147_cast_fp16)[name = string("op_3933_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3934_cast_fp16 = mul(x = state_293_cast_fp16, y = var_3933_cast_fp16)[name = string("op_3934_cast_fp16")];
            tensor<int32, [1]> memory_147_axes_0 = const()[name = string("memory_147_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_147_keep_dims_0 = const()[name = string("memory_147_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_147_cast_fp16 = reduce_sum(axes = memory_147_axes_0, keep_dims = memory_147_keep_dims_0, x = var_3934_cast_fp16)[name = string("memory_147_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_3937_cast_fp16 = sub(x = value_token_147_cast_fp16, y = memory_147_cast_fp16)[name = string("op_3937_cast_fp16")];
            tensor<int32, [3]> var_3940_begin_0 = const()[name = string("op_3940_begin_0"), val = tensor<int32, [3]>([0, 0, 9])];
            tensor<int32, [3]> var_3940_end_0 = const()[name = string("op_3940_end_0"), val = tensor<int32, [3]>([1, 16, 10])];
            tensor<bool, [3]> var_3940_end_mask_0 = const()[name = string("op_3940_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_3940_squeeze_mask_0 = const()[name = string("op_3940_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_3940_cast_fp16 = slice_by_index(begin = var_3940_begin_0, end = var_3940_end_0, end_mask = var_3940_end_mask_0, squeeze_mask = var_3940_squeeze_mask_0, x = beta_9_cast_fp16)[name = string("op_3940_cast_fp16")];
            tensor<int32, [1]> var_3941_axes_0 = const()[name = string("op_3941_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_3941_cast_fp16 = expand_dims(axes = var_3941_axes_0, x = var_3940_cast_fp16)[name = string("op_3941_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_147_cast_fp16 = mul(x = var_3937_cast_fp16, y = var_3941_cast_fp16)[name = string("delta_147_cast_fp16")];
            tensor<int32, [1]> var_3944_axes_0 = const()[name = string("op_3944_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_3944_cast_fp16 = expand_dims(axes = var_3944_axes_0, x = delta_147_cast_fp16)[name = string("op_3944_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3945_cast_fp16 = mul(x = var_3933_cast_fp16, y = var_3944_cast_fp16)[name = string("op_3945_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_295_cast_fp16 = add(x = state_293_cast_fp16, y = var_3945_cast_fp16)[name = string("state_295_cast_fp16")];
            tensor<int32, [4]> var_3949_begin_0 = const()[name = string("op_3949_begin_0"), val = tensor<int32, [4]>([0, 0, 9, 0])];
            tensor<int32, [4]> var_3949_end_0 = const()[name = string("op_3949_end_0"), val = tensor<int32, [4]>([1, 16, 10, 128])];
            tensor<bool, [4]> var_3949_end_mask_0 = const()[name = string("op_3949_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_3949_squeeze_mask_0 = const()[name = string("op_3949_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_3949_cast_fp16 = slice_by_index(begin = var_3949_begin_0, end = var_3949_end_0, end_mask = var_3949_end_mask_0, squeeze_mask = var_3949_squeeze_mask_0, x = _inversed_query_23_cast_fp16)[name = string("op_3949_cast_fp16")];
            tensor<int32, [1]> var_3950_axes_0 = const()[name = string("op_3950_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_3950_cast_fp16 = expand_dims(axes = var_3950_axes_0, x = var_3949_cast_fp16)[name = string("op_3950_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3951_cast_fp16 = mul(x = state_295_cast_fp16, y = var_3950_cast_fp16)[name = string("op_3951_cast_fp16")];
            tensor<int32, [1]> var_3953_axes_0 = const()[name = string("op_3953_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_3953_keep_dims_0 = const()[name = string("op_3953_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_3953_cast_fp16 = reduce_sum(axes = var_3953_axes_0, keep_dims = var_3953_keep_dims_0, x = var_3951_cast_fp16)[name = string("op_3953_cast_fp16")];
            tensor<int32, [3]> var_3956_begin_0 = const()[name = string("op_3956_begin_0"), val = tensor<int32, [3]>([0, 0, 10])];
            tensor<int32, [3]> var_3956_end_0 = const()[name = string("op_3956_end_0"), val = tensor<int32, [3]>([1, 16, 11])];
            tensor<bool, [3]> var_3956_end_mask_0 = const()[name = string("op_3956_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_3956_squeeze_mask_0 = const()[name = string("op_3956_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_3956_cast_fp16 = slice_by_index(begin = var_3956_begin_0, end = var_3956_end_0, end_mask = var_3956_end_mask_0, squeeze_mask = var_3956_squeeze_mask_0, x = decay_9_cast_fp16)[name = string("op_3956_cast_fp16")];
            tensor<fp16, [1, 16]> var_3957_cast_fp16 = exp(x = var_3956_cast_fp16)[name = string("op_3957_cast_fp16")];
            tensor<int32, [1]> var_3958_axes_0 = const()[name = string("op_3958_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_3958_cast_fp16 = expand_dims(axes = var_3958_axes_0, x = var_3957_cast_fp16)[name = string("op_3958_cast_fp16")];
            tensor<int32, [1]> var_3959_axes_0 = const()[name = string("op_3959_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_3959_cast_fp16 = expand_dims(axes = var_3959_axes_0, x = var_3958_cast_fp16)[name = string("op_3959_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_297_cast_fp16 = mul(x = state_295_cast_fp16, y = var_3959_cast_fp16)[name = string("state_297_cast_fp16")];
            tensor<int32, [4]> key_token_149_begin_0 = const()[name = string("key_token_149_begin_0"), val = tensor<int32, [4]>([0, 0, 10, 0])];
            tensor<int32, [4]> key_token_149_end_0 = const()[name = string("key_token_149_end_0"), val = tensor<int32, [4]>([1, 16, 11, 128])];
            tensor<bool, [4]> key_token_149_end_mask_0 = const()[name = string("key_token_149_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_149_squeeze_mask_0 = const()[name = string("key_token_149_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_149_cast_fp16 = slice_by_index(begin = key_token_149_begin_0, end = key_token_149_end_0, end_mask = key_token_149_end_mask_0, squeeze_mask = key_token_149_squeeze_mask_0, x = key_23_cast_fp16)[name = string("key_token_149_cast_fp16")];
            tensor<int32, [4]> value_token_149_begin_0 = const()[name = string("value_token_149_begin_0"), val = tensor<int32, [4]>([0, 0, 10, 0])];
            tensor<int32, [4]> value_token_149_end_0 = const()[name = string("value_token_149_end_0"), val = tensor<int32, [4]>([1, 16, 11, 128])];
            tensor<bool, [4]> value_token_149_end_mask_0 = const()[name = string("value_token_149_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_149_squeeze_mask_0 = const()[name = string("value_token_149_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_149_cast_fp16 = slice_by_index(begin = value_token_149_begin_0, end = value_token_149_end_0, end_mask = value_token_149_end_mask_0, squeeze_mask = value_token_149_squeeze_mask_0, x = value_83_cast_fp16)[name = string("value_token_149_cast_fp16")];
            tensor<int32, [1]> var_3967_axes_0 = const()[name = string("op_3967_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_3967_cast_fp16 = expand_dims(axes = var_3967_axes_0, x = key_token_149_cast_fp16)[name = string("op_3967_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3968_cast_fp16 = mul(x = state_297_cast_fp16, y = var_3967_cast_fp16)[name = string("op_3968_cast_fp16")];
            tensor<int32, [1]> memory_149_axes_0 = const()[name = string("memory_149_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_149_keep_dims_0 = const()[name = string("memory_149_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_149_cast_fp16 = reduce_sum(axes = memory_149_axes_0, keep_dims = memory_149_keep_dims_0, x = var_3968_cast_fp16)[name = string("memory_149_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_3971_cast_fp16 = sub(x = value_token_149_cast_fp16, y = memory_149_cast_fp16)[name = string("op_3971_cast_fp16")];
            tensor<int32, [3]> var_3974_begin_0 = const()[name = string("op_3974_begin_0"), val = tensor<int32, [3]>([0, 0, 10])];
            tensor<int32, [3]> var_3974_end_0 = const()[name = string("op_3974_end_0"), val = tensor<int32, [3]>([1, 16, 11])];
            tensor<bool, [3]> var_3974_end_mask_0 = const()[name = string("op_3974_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_3974_squeeze_mask_0 = const()[name = string("op_3974_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_3974_cast_fp16 = slice_by_index(begin = var_3974_begin_0, end = var_3974_end_0, end_mask = var_3974_end_mask_0, squeeze_mask = var_3974_squeeze_mask_0, x = beta_9_cast_fp16)[name = string("op_3974_cast_fp16")];
            tensor<int32, [1]> var_3975_axes_0 = const()[name = string("op_3975_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_3975_cast_fp16 = expand_dims(axes = var_3975_axes_0, x = var_3974_cast_fp16)[name = string("op_3975_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_149_cast_fp16 = mul(x = var_3971_cast_fp16, y = var_3975_cast_fp16)[name = string("delta_149_cast_fp16")];
            tensor<int32, [1]> var_3978_axes_0 = const()[name = string("op_3978_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_3978_cast_fp16 = expand_dims(axes = var_3978_axes_0, x = delta_149_cast_fp16)[name = string("op_3978_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3979_cast_fp16 = mul(x = var_3967_cast_fp16, y = var_3978_cast_fp16)[name = string("op_3979_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_299_cast_fp16 = add(x = state_297_cast_fp16, y = var_3979_cast_fp16)[name = string("state_299_cast_fp16")];
            tensor<int32, [4]> var_3983_begin_0 = const()[name = string("op_3983_begin_0"), val = tensor<int32, [4]>([0, 0, 10, 0])];
            tensor<int32, [4]> var_3983_end_0 = const()[name = string("op_3983_end_0"), val = tensor<int32, [4]>([1, 16, 11, 128])];
            tensor<bool, [4]> var_3983_end_mask_0 = const()[name = string("op_3983_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_3983_squeeze_mask_0 = const()[name = string("op_3983_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_3983_cast_fp16 = slice_by_index(begin = var_3983_begin_0, end = var_3983_end_0, end_mask = var_3983_end_mask_0, squeeze_mask = var_3983_squeeze_mask_0, x = _inversed_query_23_cast_fp16)[name = string("op_3983_cast_fp16")];
            tensor<int32, [1]> var_3984_axes_0 = const()[name = string("op_3984_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_3984_cast_fp16 = expand_dims(axes = var_3984_axes_0, x = var_3983_cast_fp16)[name = string("op_3984_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3985_cast_fp16 = mul(x = state_299_cast_fp16, y = var_3984_cast_fp16)[name = string("op_3985_cast_fp16")];
            tensor<int32, [1]> var_3987_axes_0 = const()[name = string("op_3987_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_3987_keep_dims_0 = const()[name = string("op_3987_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_3987_cast_fp16 = reduce_sum(axes = var_3987_axes_0, keep_dims = var_3987_keep_dims_0, x = var_3985_cast_fp16)[name = string("op_3987_cast_fp16")];
            tensor<int32, [3]> var_3990_begin_0 = const()[name = string("op_3990_begin_0"), val = tensor<int32, [3]>([0, 0, 11])];
            tensor<int32, [3]> var_3990_end_0 = const()[name = string("op_3990_end_0"), val = tensor<int32, [3]>([1, 16, 12])];
            tensor<bool, [3]> var_3990_end_mask_0 = const()[name = string("op_3990_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_3990_squeeze_mask_0 = const()[name = string("op_3990_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_3990_cast_fp16 = slice_by_index(begin = var_3990_begin_0, end = var_3990_end_0, end_mask = var_3990_end_mask_0, squeeze_mask = var_3990_squeeze_mask_0, x = decay_9_cast_fp16)[name = string("op_3990_cast_fp16")];
            tensor<fp16, [1, 16]> var_3991_cast_fp16 = exp(x = var_3990_cast_fp16)[name = string("op_3991_cast_fp16")];
            tensor<int32, [1]> var_3992_axes_0 = const()[name = string("op_3992_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_3992_cast_fp16 = expand_dims(axes = var_3992_axes_0, x = var_3991_cast_fp16)[name = string("op_3992_cast_fp16")];
            tensor<int32, [1]> var_3993_axes_0 = const()[name = string("op_3993_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_3993_cast_fp16 = expand_dims(axes = var_3993_axes_0, x = var_3992_cast_fp16)[name = string("op_3993_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_301_cast_fp16 = mul(x = state_299_cast_fp16, y = var_3993_cast_fp16)[name = string("state_301_cast_fp16")];
            tensor<int32, [4]> key_token_151_begin_0 = const()[name = string("key_token_151_begin_0"), val = tensor<int32, [4]>([0, 0, 11, 0])];
            tensor<int32, [4]> key_token_151_end_0 = const()[name = string("key_token_151_end_0"), val = tensor<int32, [4]>([1, 16, 12, 128])];
            tensor<bool, [4]> key_token_151_end_mask_0 = const()[name = string("key_token_151_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_151_squeeze_mask_0 = const()[name = string("key_token_151_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_151_cast_fp16 = slice_by_index(begin = key_token_151_begin_0, end = key_token_151_end_0, end_mask = key_token_151_end_mask_0, squeeze_mask = key_token_151_squeeze_mask_0, x = key_23_cast_fp16)[name = string("key_token_151_cast_fp16")];
            tensor<int32, [4]> value_token_151_begin_0 = const()[name = string("value_token_151_begin_0"), val = tensor<int32, [4]>([0, 0, 11, 0])];
            tensor<int32, [4]> value_token_151_end_0 = const()[name = string("value_token_151_end_0"), val = tensor<int32, [4]>([1, 16, 12, 128])];
            tensor<bool, [4]> value_token_151_end_mask_0 = const()[name = string("value_token_151_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_151_squeeze_mask_0 = const()[name = string("value_token_151_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_151_cast_fp16 = slice_by_index(begin = value_token_151_begin_0, end = value_token_151_end_0, end_mask = value_token_151_end_mask_0, squeeze_mask = value_token_151_squeeze_mask_0, x = value_83_cast_fp16)[name = string("value_token_151_cast_fp16")];
            tensor<int32, [1]> var_4001_axes_0 = const()[name = string("op_4001_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_4001_cast_fp16 = expand_dims(axes = var_4001_axes_0, x = key_token_151_cast_fp16)[name = string("op_4001_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4002_cast_fp16 = mul(x = state_301_cast_fp16, y = var_4001_cast_fp16)[name = string("op_4002_cast_fp16")];
            tensor<int32, [1]> memory_151_axes_0 = const()[name = string("memory_151_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_151_keep_dims_0 = const()[name = string("memory_151_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_151_cast_fp16 = reduce_sum(axes = memory_151_axes_0, keep_dims = memory_151_keep_dims_0, x = var_4002_cast_fp16)[name = string("memory_151_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_4005_cast_fp16 = sub(x = value_token_151_cast_fp16, y = memory_151_cast_fp16)[name = string("op_4005_cast_fp16")];
            tensor<int32, [3]> var_4008_begin_0 = const()[name = string("op_4008_begin_0"), val = tensor<int32, [3]>([0, 0, 11])];
            tensor<int32, [3]> var_4008_end_0 = const()[name = string("op_4008_end_0"), val = tensor<int32, [3]>([1, 16, 12])];
            tensor<bool, [3]> var_4008_end_mask_0 = const()[name = string("op_4008_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_4008_squeeze_mask_0 = const()[name = string("op_4008_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_4008_cast_fp16 = slice_by_index(begin = var_4008_begin_0, end = var_4008_end_0, end_mask = var_4008_end_mask_0, squeeze_mask = var_4008_squeeze_mask_0, x = beta_9_cast_fp16)[name = string("op_4008_cast_fp16")];
            tensor<int32, [1]> var_4009_axes_0 = const()[name = string("op_4009_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_4009_cast_fp16 = expand_dims(axes = var_4009_axes_0, x = var_4008_cast_fp16)[name = string("op_4009_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_151_cast_fp16 = mul(x = var_4005_cast_fp16, y = var_4009_cast_fp16)[name = string("delta_151_cast_fp16")];
            tensor<int32, [1]> var_4012_axes_0 = const()[name = string("op_4012_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_4012_cast_fp16 = expand_dims(axes = var_4012_axes_0, x = delta_151_cast_fp16)[name = string("op_4012_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4013_cast_fp16 = mul(x = var_4001_cast_fp16, y = var_4012_cast_fp16)[name = string("op_4013_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_303_cast_fp16 = add(x = state_301_cast_fp16, y = var_4013_cast_fp16)[name = string("state_303_cast_fp16")];
            tensor<int32, [4]> var_4017_begin_0 = const()[name = string("op_4017_begin_0"), val = tensor<int32, [4]>([0, 0, 11, 0])];
            tensor<int32, [4]> var_4017_end_0 = const()[name = string("op_4017_end_0"), val = tensor<int32, [4]>([1, 16, 12, 128])];
            tensor<bool, [4]> var_4017_end_mask_0 = const()[name = string("op_4017_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_4017_squeeze_mask_0 = const()[name = string("op_4017_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_4017_cast_fp16 = slice_by_index(begin = var_4017_begin_0, end = var_4017_end_0, end_mask = var_4017_end_mask_0, squeeze_mask = var_4017_squeeze_mask_0, x = _inversed_query_23_cast_fp16)[name = string("op_4017_cast_fp16")];
            tensor<int32, [1]> var_4018_axes_0 = const()[name = string("op_4018_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_4018_cast_fp16 = expand_dims(axes = var_4018_axes_0, x = var_4017_cast_fp16)[name = string("op_4018_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4019_cast_fp16 = mul(x = state_303_cast_fp16, y = var_4018_cast_fp16)[name = string("op_4019_cast_fp16")];
            tensor<int32, [1]> var_4021_axes_0 = const()[name = string("op_4021_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_4021_keep_dims_0 = const()[name = string("op_4021_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_4021_cast_fp16 = reduce_sum(axes = var_4021_axes_0, keep_dims = var_4021_keep_dims_0, x = var_4019_cast_fp16)[name = string("op_4021_cast_fp16")];
            tensor<int32, [3]> var_4024_begin_0 = const()[name = string("op_4024_begin_0"), val = tensor<int32, [3]>([0, 0, 12])];
            tensor<int32, [3]> var_4024_end_0 = const()[name = string("op_4024_end_0"), val = tensor<int32, [3]>([1, 16, 13])];
            tensor<bool, [3]> var_4024_end_mask_0 = const()[name = string("op_4024_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_4024_squeeze_mask_0 = const()[name = string("op_4024_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_4024_cast_fp16 = slice_by_index(begin = var_4024_begin_0, end = var_4024_end_0, end_mask = var_4024_end_mask_0, squeeze_mask = var_4024_squeeze_mask_0, x = decay_9_cast_fp16)[name = string("op_4024_cast_fp16")];
            tensor<fp16, [1, 16]> var_4025_cast_fp16 = exp(x = var_4024_cast_fp16)[name = string("op_4025_cast_fp16")];
            tensor<int32, [1]> var_4026_axes_0 = const()[name = string("op_4026_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_4026_cast_fp16 = expand_dims(axes = var_4026_axes_0, x = var_4025_cast_fp16)[name = string("op_4026_cast_fp16")];
            tensor<int32, [1]> var_4027_axes_0 = const()[name = string("op_4027_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_4027_cast_fp16 = expand_dims(axes = var_4027_axes_0, x = var_4026_cast_fp16)[name = string("op_4027_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_305_cast_fp16 = mul(x = state_303_cast_fp16, y = var_4027_cast_fp16)[name = string("state_305_cast_fp16")];
            tensor<int32, [4]> key_token_153_begin_0 = const()[name = string("key_token_153_begin_0"), val = tensor<int32, [4]>([0, 0, 12, 0])];
            tensor<int32, [4]> key_token_153_end_0 = const()[name = string("key_token_153_end_0"), val = tensor<int32, [4]>([1, 16, 13, 128])];
            tensor<bool, [4]> key_token_153_end_mask_0 = const()[name = string("key_token_153_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_153_squeeze_mask_0 = const()[name = string("key_token_153_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_153_cast_fp16 = slice_by_index(begin = key_token_153_begin_0, end = key_token_153_end_0, end_mask = key_token_153_end_mask_0, squeeze_mask = key_token_153_squeeze_mask_0, x = key_23_cast_fp16)[name = string("key_token_153_cast_fp16")];
            tensor<int32, [4]> value_token_153_begin_0 = const()[name = string("value_token_153_begin_0"), val = tensor<int32, [4]>([0, 0, 12, 0])];
            tensor<int32, [4]> value_token_153_end_0 = const()[name = string("value_token_153_end_0"), val = tensor<int32, [4]>([1, 16, 13, 128])];
            tensor<bool, [4]> value_token_153_end_mask_0 = const()[name = string("value_token_153_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_153_squeeze_mask_0 = const()[name = string("value_token_153_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_153_cast_fp16 = slice_by_index(begin = value_token_153_begin_0, end = value_token_153_end_0, end_mask = value_token_153_end_mask_0, squeeze_mask = value_token_153_squeeze_mask_0, x = value_83_cast_fp16)[name = string("value_token_153_cast_fp16")];
            tensor<int32, [1]> var_4035_axes_0 = const()[name = string("op_4035_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_4035_cast_fp16 = expand_dims(axes = var_4035_axes_0, x = key_token_153_cast_fp16)[name = string("op_4035_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4036_cast_fp16 = mul(x = state_305_cast_fp16, y = var_4035_cast_fp16)[name = string("op_4036_cast_fp16")];
            tensor<int32, [1]> memory_153_axes_0 = const()[name = string("memory_153_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_153_keep_dims_0 = const()[name = string("memory_153_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_153_cast_fp16 = reduce_sum(axes = memory_153_axes_0, keep_dims = memory_153_keep_dims_0, x = var_4036_cast_fp16)[name = string("memory_153_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_4039_cast_fp16 = sub(x = value_token_153_cast_fp16, y = memory_153_cast_fp16)[name = string("op_4039_cast_fp16")];
            tensor<int32, [3]> var_4042_begin_0 = const()[name = string("op_4042_begin_0"), val = tensor<int32, [3]>([0, 0, 12])];
            tensor<int32, [3]> var_4042_end_0 = const()[name = string("op_4042_end_0"), val = tensor<int32, [3]>([1, 16, 13])];
            tensor<bool, [3]> var_4042_end_mask_0 = const()[name = string("op_4042_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_4042_squeeze_mask_0 = const()[name = string("op_4042_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_4042_cast_fp16 = slice_by_index(begin = var_4042_begin_0, end = var_4042_end_0, end_mask = var_4042_end_mask_0, squeeze_mask = var_4042_squeeze_mask_0, x = beta_9_cast_fp16)[name = string("op_4042_cast_fp16")];
            tensor<int32, [1]> var_4043_axes_0 = const()[name = string("op_4043_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_4043_cast_fp16 = expand_dims(axes = var_4043_axes_0, x = var_4042_cast_fp16)[name = string("op_4043_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_153_cast_fp16 = mul(x = var_4039_cast_fp16, y = var_4043_cast_fp16)[name = string("delta_153_cast_fp16")];
            tensor<int32, [1]> var_4046_axes_0 = const()[name = string("op_4046_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_4046_cast_fp16 = expand_dims(axes = var_4046_axes_0, x = delta_153_cast_fp16)[name = string("op_4046_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4047_cast_fp16 = mul(x = var_4035_cast_fp16, y = var_4046_cast_fp16)[name = string("op_4047_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_307_cast_fp16 = add(x = state_305_cast_fp16, y = var_4047_cast_fp16)[name = string("state_307_cast_fp16")];
            tensor<int32, [4]> var_4051_begin_0 = const()[name = string("op_4051_begin_0"), val = tensor<int32, [4]>([0, 0, 12, 0])];
            tensor<int32, [4]> var_4051_end_0 = const()[name = string("op_4051_end_0"), val = tensor<int32, [4]>([1, 16, 13, 128])];
            tensor<bool, [4]> var_4051_end_mask_0 = const()[name = string("op_4051_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_4051_squeeze_mask_0 = const()[name = string("op_4051_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_4051_cast_fp16 = slice_by_index(begin = var_4051_begin_0, end = var_4051_end_0, end_mask = var_4051_end_mask_0, squeeze_mask = var_4051_squeeze_mask_0, x = _inversed_query_23_cast_fp16)[name = string("op_4051_cast_fp16")];
            tensor<int32, [1]> var_4052_axes_0 = const()[name = string("op_4052_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_4052_cast_fp16 = expand_dims(axes = var_4052_axes_0, x = var_4051_cast_fp16)[name = string("op_4052_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4053_cast_fp16 = mul(x = state_307_cast_fp16, y = var_4052_cast_fp16)[name = string("op_4053_cast_fp16")];
            tensor<int32, [1]> var_4055_axes_0 = const()[name = string("op_4055_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_4055_keep_dims_0 = const()[name = string("op_4055_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_4055_cast_fp16 = reduce_sum(axes = var_4055_axes_0, keep_dims = var_4055_keep_dims_0, x = var_4053_cast_fp16)[name = string("op_4055_cast_fp16")];
            tensor<int32, [3]> var_4058_begin_0 = const()[name = string("op_4058_begin_0"), val = tensor<int32, [3]>([0, 0, 13])];
            tensor<int32, [3]> var_4058_end_0 = const()[name = string("op_4058_end_0"), val = tensor<int32, [3]>([1, 16, 14])];
            tensor<bool, [3]> var_4058_end_mask_0 = const()[name = string("op_4058_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_4058_squeeze_mask_0 = const()[name = string("op_4058_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_4058_cast_fp16 = slice_by_index(begin = var_4058_begin_0, end = var_4058_end_0, end_mask = var_4058_end_mask_0, squeeze_mask = var_4058_squeeze_mask_0, x = decay_9_cast_fp16)[name = string("op_4058_cast_fp16")];
            tensor<fp16, [1, 16]> var_4059_cast_fp16 = exp(x = var_4058_cast_fp16)[name = string("op_4059_cast_fp16")];
            tensor<int32, [1]> var_4060_axes_0 = const()[name = string("op_4060_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_4060_cast_fp16 = expand_dims(axes = var_4060_axes_0, x = var_4059_cast_fp16)[name = string("op_4060_cast_fp16")];
            tensor<int32, [1]> var_4061_axes_0 = const()[name = string("op_4061_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_4061_cast_fp16 = expand_dims(axes = var_4061_axes_0, x = var_4060_cast_fp16)[name = string("op_4061_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_309_cast_fp16 = mul(x = state_307_cast_fp16, y = var_4061_cast_fp16)[name = string("state_309_cast_fp16")];
            tensor<int32, [4]> key_token_155_begin_0 = const()[name = string("key_token_155_begin_0"), val = tensor<int32, [4]>([0, 0, 13, 0])];
            tensor<int32, [4]> key_token_155_end_0 = const()[name = string("key_token_155_end_0"), val = tensor<int32, [4]>([1, 16, 14, 128])];
            tensor<bool, [4]> key_token_155_end_mask_0 = const()[name = string("key_token_155_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_155_squeeze_mask_0 = const()[name = string("key_token_155_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_155_cast_fp16 = slice_by_index(begin = key_token_155_begin_0, end = key_token_155_end_0, end_mask = key_token_155_end_mask_0, squeeze_mask = key_token_155_squeeze_mask_0, x = key_23_cast_fp16)[name = string("key_token_155_cast_fp16")];
            tensor<int32, [4]> value_token_155_begin_0 = const()[name = string("value_token_155_begin_0"), val = tensor<int32, [4]>([0, 0, 13, 0])];
            tensor<int32, [4]> value_token_155_end_0 = const()[name = string("value_token_155_end_0"), val = tensor<int32, [4]>([1, 16, 14, 128])];
            tensor<bool, [4]> value_token_155_end_mask_0 = const()[name = string("value_token_155_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_155_squeeze_mask_0 = const()[name = string("value_token_155_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_155_cast_fp16 = slice_by_index(begin = value_token_155_begin_0, end = value_token_155_end_0, end_mask = value_token_155_end_mask_0, squeeze_mask = value_token_155_squeeze_mask_0, x = value_83_cast_fp16)[name = string("value_token_155_cast_fp16")];
            tensor<int32, [1]> var_4069_axes_0 = const()[name = string("op_4069_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_4069_cast_fp16 = expand_dims(axes = var_4069_axes_0, x = key_token_155_cast_fp16)[name = string("op_4069_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4070_cast_fp16 = mul(x = state_309_cast_fp16, y = var_4069_cast_fp16)[name = string("op_4070_cast_fp16")];
            tensor<int32, [1]> memory_155_axes_0 = const()[name = string("memory_155_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_155_keep_dims_0 = const()[name = string("memory_155_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_155_cast_fp16 = reduce_sum(axes = memory_155_axes_0, keep_dims = memory_155_keep_dims_0, x = var_4070_cast_fp16)[name = string("memory_155_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_4073_cast_fp16 = sub(x = value_token_155_cast_fp16, y = memory_155_cast_fp16)[name = string("op_4073_cast_fp16")];
            tensor<int32, [3]> var_4076_begin_0 = const()[name = string("op_4076_begin_0"), val = tensor<int32, [3]>([0, 0, 13])];
            tensor<int32, [3]> var_4076_end_0 = const()[name = string("op_4076_end_0"), val = tensor<int32, [3]>([1, 16, 14])];
            tensor<bool, [3]> var_4076_end_mask_0 = const()[name = string("op_4076_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_4076_squeeze_mask_0 = const()[name = string("op_4076_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_4076_cast_fp16 = slice_by_index(begin = var_4076_begin_0, end = var_4076_end_0, end_mask = var_4076_end_mask_0, squeeze_mask = var_4076_squeeze_mask_0, x = beta_9_cast_fp16)[name = string("op_4076_cast_fp16")];
            tensor<int32, [1]> var_4077_axes_0 = const()[name = string("op_4077_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_4077_cast_fp16 = expand_dims(axes = var_4077_axes_0, x = var_4076_cast_fp16)[name = string("op_4077_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_155_cast_fp16 = mul(x = var_4073_cast_fp16, y = var_4077_cast_fp16)[name = string("delta_155_cast_fp16")];
            tensor<int32, [1]> var_4080_axes_0 = const()[name = string("op_4080_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_4080_cast_fp16 = expand_dims(axes = var_4080_axes_0, x = delta_155_cast_fp16)[name = string("op_4080_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4081_cast_fp16 = mul(x = var_4069_cast_fp16, y = var_4080_cast_fp16)[name = string("op_4081_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_311_cast_fp16 = add(x = state_309_cast_fp16, y = var_4081_cast_fp16)[name = string("state_311_cast_fp16")];
            tensor<int32, [4]> var_4085_begin_0 = const()[name = string("op_4085_begin_0"), val = tensor<int32, [4]>([0, 0, 13, 0])];
            tensor<int32, [4]> var_4085_end_0 = const()[name = string("op_4085_end_0"), val = tensor<int32, [4]>([1, 16, 14, 128])];
            tensor<bool, [4]> var_4085_end_mask_0 = const()[name = string("op_4085_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_4085_squeeze_mask_0 = const()[name = string("op_4085_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_4085_cast_fp16 = slice_by_index(begin = var_4085_begin_0, end = var_4085_end_0, end_mask = var_4085_end_mask_0, squeeze_mask = var_4085_squeeze_mask_0, x = _inversed_query_23_cast_fp16)[name = string("op_4085_cast_fp16")];
            tensor<int32, [1]> var_4086_axes_0 = const()[name = string("op_4086_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_4086_cast_fp16 = expand_dims(axes = var_4086_axes_0, x = var_4085_cast_fp16)[name = string("op_4086_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4087_cast_fp16 = mul(x = state_311_cast_fp16, y = var_4086_cast_fp16)[name = string("op_4087_cast_fp16")];
            tensor<int32, [1]> var_4089_axes_0 = const()[name = string("op_4089_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_4089_keep_dims_0 = const()[name = string("op_4089_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_4089_cast_fp16 = reduce_sum(axes = var_4089_axes_0, keep_dims = var_4089_keep_dims_0, x = var_4087_cast_fp16)[name = string("op_4089_cast_fp16")];
            tensor<int32, [3]> var_4092_begin_0 = const()[name = string("op_4092_begin_0"), val = tensor<int32, [3]>([0, 0, 14])];
            tensor<int32, [3]> var_4092_end_0 = const()[name = string("op_4092_end_0"), val = tensor<int32, [3]>([1, 16, 15])];
            tensor<bool, [3]> var_4092_end_mask_0 = const()[name = string("op_4092_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_4092_squeeze_mask_0 = const()[name = string("op_4092_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_4092_cast_fp16 = slice_by_index(begin = var_4092_begin_0, end = var_4092_end_0, end_mask = var_4092_end_mask_0, squeeze_mask = var_4092_squeeze_mask_0, x = decay_9_cast_fp16)[name = string("op_4092_cast_fp16")];
            tensor<fp16, [1, 16]> var_4093_cast_fp16 = exp(x = var_4092_cast_fp16)[name = string("op_4093_cast_fp16")];
            tensor<int32, [1]> var_4094_axes_0 = const()[name = string("op_4094_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_4094_cast_fp16 = expand_dims(axes = var_4094_axes_0, x = var_4093_cast_fp16)[name = string("op_4094_cast_fp16")];
            tensor<int32, [1]> var_4095_axes_0 = const()[name = string("op_4095_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_4095_cast_fp16 = expand_dims(axes = var_4095_axes_0, x = var_4094_cast_fp16)[name = string("op_4095_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_313_cast_fp16 = mul(x = state_311_cast_fp16, y = var_4095_cast_fp16)[name = string("state_313_cast_fp16")];
            tensor<int32, [4]> key_token_157_begin_0 = const()[name = string("key_token_157_begin_0"), val = tensor<int32, [4]>([0, 0, 14, 0])];
            tensor<int32, [4]> key_token_157_end_0 = const()[name = string("key_token_157_end_0"), val = tensor<int32, [4]>([1, 16, 15, 128])];
            tensor<bool, [4]> key_token_157_end_mask_0 = const()[name = string("key_token_157_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_157_squeeze_mask_0 = const()[name = string("key_token_157_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_157_cast_fp16 = slice_by_index(begin = key_token_157_begin_0, end = key_token_157_end_0, end_mask = key_token_157_end_mask_0, squeeze_mask = key_token_157_squeeze_mask_0, x = key_23_cast_fp16)[name = string("key_token_157_cast_fp16")];
            tensor<int32, [4]> value_token_157_begin_0 = const()[name = string("value_token_157_begin_0"), val = tensor<int32, [4]>([0, 0, 14, 0])];
            tensor<int32, [4]> value_token_157_end_0 = const()[name = string("value_token_157_end_0"), val = tensor<int32, [4]>([1, 16, 15, 128])];
            tensor<bool, [4]> value_token_157_end_mask_0 = const()[name = string("value_token_157_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_157_squeeze_mask_0 = const()[name = string("value_token_157_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_157_cast_fp16 = slice_by_index(begin = value_token_157_begin_0, end = value_token_157_end_0, end_mask = value_token_157_end_mask_0, squeeze_mask = value_token_157_squeeze_mask_0, x = value_83_cast_fp16)[name = string("value_token_157_cast_fp16")];
            tensor<int32, [1]> var_4103_axes_0 = const()[name = string("op_4103_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_4103_cast_fp16 = expand_dims(axes = var_4103_axes_0, x = key_token_157_cast_fp16)[name = string("op_4103_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4104_cast_fp16 = mul(x = state_313_cast_fp16, y = var_4103_cast_fp16)[name = string("op_4104_cast_fp16")];
            tensor<int32, [1]> memory_157_axes_0 = const()[name = string("memory_157_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_157_keep_dims_0 = const()[name = string("memory_157_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_157_cast_fp16 = reduce_sum(axes = memory_157_axes_0, keep_dims = memory_157_keep_dims_0, x = var_4104_cast_fp16)[name = string("memory_157_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_4107_cast_fp16 = sub(x = value_token_157_cast_fp16, y = memory_157_cast_fp16)[name = string("op_4107_cast_fp16")];
            tensor<int32, [3]> var_4110_begin_0 = const()[name = string("op_4110_begin_0"), val = tensor<int32, [3]>([0, 0, 14])];
            tensor<int32, [3]> var_4110_end_0 = const()[name = string("op_4110_end_0"), val = tensor<int32, [3]>([1, 16, 15])];
            tensor<bool, [3]> var_4110_end_mask_0 = const()[name = string("op_4110_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_4110_squeeze_mask_0 = const()[name = string("op_4110_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_4110_cast_fp16 = slice_by_index(begin = var_4110_begin_0, end = var_4110_end_0, end_mask = var_4110_end_mask_0, squeeze_mask = var_4110_squeeze_mask_0, x = beta_9_cast_fp16)[name = string("op_4110_cast_fp16")];
            tensor<int32, [1]> var_4111_axes_0 = const()[name = string("op_4111_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_4111_cast_fp16 = expand_dims(axes = var_4111_axes_0, x = var_4110_cast_fp16)[name = string("op_4111_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_157_cast_fp16 = mul(x = var_4107_cast_fp16, y = var_4111_cast_fp16)[name = string("delta_157_cast_fp16")];
            tensor<int32, [1]> var_4114_axes_0 = const()[name = string("op_4114_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_4114_cast_fp16 = expand_dims(axes = var_4114_axes_0, x = delta_157_cast_fp16)[name = string("op_4114_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4115_cast_fp16 = mul(x = var_4103_cast_fp16, y = var_4114_cast_fp16)[name = string("op_4115_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_315_cast_fp16 = add(x = state_313_cast_fp16, y = var_4115_cast_fp16)[name = string("state_315_cast_fp16")];
            tensor<int32, [4]> var_4119_begin_0 = const()[name = string("op_4119_begin_0"), val = tensor<int32, [4]>([0, 0, 14, 0])];
            tensor<int32, [4]> var_4119_end_0 = const()[name = string("op_4119_end_0"), val = tensor<int32, [4]>([1, 16, 15, 128])];
            tensor<bool, [4]> var_4119_end_mask_0 = const()[name = string("op_4119_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_4119_squeeze_mask_0 = const()[name = string("op_4119_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_4119_cast_fp16 = slice_by_index(begin = var_4119_begin_0, end = var_4119_end_0, end_mask = var_4119_end_mask_0, squeeze_mask = var_4119_squeeze_mask_0, x = _inversed_query_23_cast_fp16)[name = string("op_4119_cast_fp16")];
            tensor<int32, [1]> var_4120_axes_0 = const()[name = string("op_4120_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_4120_cast_fp16 = expand_dims(axes = var_4120_axes_0, x = var_4119_cast_fp16)[name = string("op_4120_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4121_cast_fp16 = mul(x = state_315_cast_fp16, y = var_4120_cast_fp16)[name = string("op_4121_cast_fp16")];
            tensor<int32, [1]> var_4123_axes_0 = const()[name = string("op_4123_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_4123_keep_dims_0 = const()[name = string("op_4123_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_4123_cast_fp16 = reduce_sum(axes = var_4123_axes_0, keep_dims = var_4123_keep_dims_0, x = var_4121_cast_fp16)[name = string("op_4123_cast_fp16")];
            tensor<int32, [3]> var_4126_begin_0 = const()[name = string("op_4126_begin_0"), val = tensor<int32, [3]>([0, 0, 15])];
            tensor<int32, [3]> var_4126_end_0 = const()[name = string("op_4126_end_0"), val = tensor<int32, [3]>([1, 16, 16])];
            tensor<bool, [3]> var_4126_end_mask_0 = const()[name = string("op_4126_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_4126_squeeze_mask_0 = const()[name = string("op_4126_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_4126_cast_fp16 = slice_by_index(begin = var_4126_begin_0, end = var_4126_end_0, end_mask = var_4126_end_mask_0, squeeze_mask = var_4126_squeeze_mask_0, x = decay_9_cast_fp16)[name = string("op_4126_cast_fp16")];
            tensor<fp16, [1, 16]> var_4127_cast_fp16 = exp(x = var_4126_cast_fp16)[name = string("op_4127_cast_fp16")];
            tensor<int32, [1]> var_4128_axes_0 = const()[name = string("op_4128_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_4128_cast_fp16 = expand_dims(axes = var_4128_axes_0, x = var_4127_cast_fp16)[name = string("op_4128_cast_fp16")];
            tensor<int32, [1]> var_4129_axes_0 = const()[name = string("op_4129_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_4129_cast_fp16 = expand_dims(axes = var_4129_axes_0, x = var_4128_cast_fp16)[name = string("op_4129_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_317_cast_fp16 = mul(x = state_315_cast_fp16, y = var_4129_cast_fp16)[name = string("state_317_cast_fp16")];
            tensor<int32, [4]> key_token_159_begin_0 = const()[name = string("key_token_159_begin_0"), val = tensor<int32, [4]>([0, 0, 15, 0])];
            tensor<int32, [4]> key_token_159_end_0 = const()[name = string("key_token_159_end_0"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<bool, [4]> key_token_159_end_mask_0 = const()[name = string("key_token_159_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_159_squeeze_mask_0 = const()[name = string("key_token_159_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_159_cast_fp16 = slice_by_index(begin = key_token_159_begin_0, end = key_token_159_end_0, end_mask = key_token_159_end_mask_0, squeeze_mask = key_token_159_squeeze_mask_0, x = key_23_cast_fp16)[name = string("key_token_159_cast_fp16")];
            tensor<int32, [4]> value_token_159_begin_0 = const()[name = string("value_token_159_begin_0"), val = tensor<int32, [4]>([0, 0, 15, 0])];
            tensor<int32, [4]> value_token_159_end_0 = const()[name = string("value_token_159_end_0"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<bool, [4]> value_token_159_end_mask_0 = const()[name = string("value_token_159_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_159_squeeze_mask_0 = const()[name = string("value_token_159_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_159_cast_fp16 = slice_by_index(begin = value_token_159_begin_0, end = value_token_159_end_0, end_mask = value_token_159_end_mask_0, squeeze_mask = value_token_159_squeeze_mask_0, x = value_83_cast_fp16)[name = string("value_token_159_cast_fp16")];
            tensor<int32, [1]> var_4137_axes_0 = const()[name = string("op_4137_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_4137_cast_fp16 = expand_dims(axes = var_4137_axes_0, x = key_token_159_cast_fp16)[name = string("op_4137_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4138_cast_fp16 = mul(x = state_317_cast_fp16, y = var_4137_cast_fp16)[name = string("op_4138_cast_fp16")];
            tensor<int32, [1]> memory_159_axes_0 = const()[name = string("memory_159_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_159_keep_dims_0 = const()[name = string("memory_159_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_159_cast_fp16 = reduce_sum(axes = memory_159_axes_0, keep_dims = memory_159_keep_dims_0, x = var_4138_cast_fp16)[name = string("memory_159_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_4141_cast_fp16 = sub(x = value_token_159_cast_fp16, y = memory_159_cast_fp16)[name = string("op_4141_cast_fp16")];
            tensor<int32, [3]> var_4144_begin_0 = const()[name = string("op_4144_begin_0"), val = tensor<int32, [3]>([0, 0, 15])];
            tensor<int32, [3]> var_4144_end_0 = const()[name = string("op_4144_end_0"), val = tensor<int32, [3]>([1, 16, 16])];
            tensor<bool, [3]> var_4144_end_mask_0 = const()[name = string("op_4144_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_4144_squeeze_mask_0 = const()[name = string("op_4144_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_4144_cast_fp16 = slice_by_index(begin = var_4144_begin_0, end = var_4144_end_0, end_mask = var_4144_end_mask_0, squeeze_mask = var_4144_squeeze_mask_0, x = beta_9_cast_fp16)[name = string("op_4144_cast_fp16")];
            tensor<int32, [1]> var_4145_axes_0 = const()[name = string("op_4145_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_4145_cast_fp16 = expand_dims(axes = var_4145_axes_0, x = var_4144_cast_fp16)[name = string("op_4145_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_159_cast_fp16 = mul(x = var_4141_cast_fp16, y = var_4145_cast_fp16)[name = string("delta_159_cast_fp16")];
            tensor<int32, [1]> var_4148_axes_0 = const()[name = string("op_4148_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_4148_cast_fp16 = expand_dims(axes = var_4148_axes_0, x = delta_159_cast_fp16)[name = string("op_4148_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4149_cast_fp16 = mul(x = var_4137_cast_fp16, y = var_4148_cast_fp16)[name = string("op_4149_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> rec5_out = add(x = state_317_cast_fp16, y = var_4149_cast_fp16)[name = string("state_319_cast_fp16")];
            tensor<int32, [4]> var_4153_begin_0 = const()[name = string("op_4153_begin_0"), val = tensor<int32, [4]>([0, 0, 15, 0])];
            tensor<int32, [4]> var_4153_end_0 = const()[name = string("op_4153_end_0"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<bool, [4]> var_4153_end_mask_0 = const()[name = string("op_4153_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_4153_squeeze_mask_0 = const()[name = string("op_4153_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_4153_cast_fp16 = slice_by_index(begin = var_4153_begin_0, end = var_4153_end_0, end_mask = var_4153_end_mask_0, squeeze_mask = var_4153_squeeze_mask_0, x = _inversed_query_23_cast_fp16)[name = string("op_4153_cast_fp16")];
            tensor<int32, [1]> var_4154_axes_0 = const()[name = string("op_4154_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_4154_cast_fp16 = expand_dims(axes = var_4154_axes_0, x = var_4153_cast_fp16)[name = string("op_4154_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4155_cast_fp16 = mul(x = rec5_out, y = var_4154_cast_fp16)[name = string("op_4155_cast_fp16")];
            tensor<int32, [1]> var_4157_axes_0 = const()[name = string("op_4157_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_4157_keep_dims_0 = const()[name = string("op_4157_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_4157_cast_fp16 = reduce_sum(axes = var_4157_axes_0, keep_dims = var_4157_keep_dims_0, x = var_4155_cast_fp16)[name = string("op_4157_cast_fp16")];
            int32 attention_41_axis_0 = const()[name = string("attention_41_axis_0"), val = int32(1)];
            tensor<fp16, [1, 16, 16, 128]> attention_41_cast_fp16 = stack(axis = attention_41_axis_0, values = (var_3647_cast_fp16, var_3681_cast_fp16, var_3715_cast_fp16, var_3749_cast_fp16, var_3783_cast_fp16, var_3817_cast_fp16, var_3851_cast_fp16, var_3885_cast_fp16, var_3919_cast_fp16, var_3953_cast_fp16, var_3987_cast_fp16, var_4021_cast_fp16, var_4055_cast_fp16, var_4089_cast_fp16, var_4123_cast_fp16, var_4157_cast_fp16))[name = string("attention_41_cast_fp16")];
            tensor<int32, [2]> var_4160 = const()[name = string("op_4160"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [256, 128]> attention_43_cast_fp16 = reshape(shape = var_4160, x = attention_41_cast_fp16)[name = string("attention_43_cast_fp16")];
            tensor<int32, [2]> var_4162 = const()[name = string("op_4162"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [256, 128]> input_71_cast_fp16 = reshape(shape = var_4162, x = linear_42_cast_fp16)[name = string("input_71_cast_fp16")];
            tensor<fp16, [256, 128]> var_4164_cast_fp16 = mul(x = attention_43_cast_fp16, y = attention_43_cast_fp16)[name = string("op_4164_cast_fp16")];
            tensor<int32, [1]> variance_35_axes_0 = const()[name = string("variance_35_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_35_keep_dims_0 = const()[name = string("variance_35_keep_dims_0"), val = bool(true)];
            tensor<fp16, [256, 1]> variance_35_cast_fp16 = reduce_mean(axes = variance_35_axes_0, keep_dims = variance_35_keep_dims_0, x = var_4164_cast_fp16)[name = string("variance_35_cast_fp16")];
            fp16 var_4167_to_fp16 = const()[name = string("op_4167_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [256, 1]> var_4168_cast_fp16 = add(x = variance_35_cast_fp16, y = var_4167_to_fp16)[name = string("op_4168_cast_fp16")];
            fp32 var_4169_epsilon_0 = const()[name = string("op_4169_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [256, 1]> var_4169_cast_fp16 = rsqrt(epsilon = var_4169_epsilon_0, x = var_4168_cast_fp16)[name = string("op_4169_cast_fp16")];
            tensor<fp16, [256, 128]> attention_45_cast_fp16 = mul(x = attention_43_cast_fp16, y = var_4169_cast_fp16)[name = string("attention_45_cast_fp16")];
            tensor<fp16, [128]> groups_1_1_gated_norm_promoted_to_fp16 = const()[name = string("groups_1_1_gated_norm_promoted_to_fp16"), val = tensor<fp16, [128]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(85809280)))];
            tensor<fp16, [256, 128]> attention_47_cast_fp16 = mul(x = attention_45_cast_fp16, y = groups_1_1_gated_norm_promoted_to_fp16)[name = string("attention_47_cast_fp16")];
            tensor<fp16, [256, 128]> var_4172_cast_fp16 = silu(x = input_71_cast_fp16)[name = string("op_4172_cast_fp16")];
            tensor<fp16, [256, 128]> attention_49_cast_fp16 = mul(x = attention_47_cast_fp16, y = var_4172_cast_fp16)[name = string("attention_49_cast_fp16")];
            tensor<int32, [2]> var_4174 = const()[name = string("op_4174"), val = tensor<int32, [2]>([16, 2048])];
            tensor<fp16, [16, 2048]> input_73_cast_fp16 = reshape(shape = var_4174, x = attention_49_cast_fp16)[name = string("input_73_cast_fp16")];
            tensor<fp16, [1024, 2048]> groups_1_1_out_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 2048]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(85809600))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(87382528))))[name = string("groups_1_1_out_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 1024]> linear_43_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = groups_1_1_out_proj_weight_promoted_to_fp16_palettized, x = input_73_cast_fp16)[name = string("linear_43_cast_fp16")];
            tensor<fp16, [16, 1024]> value_85_cast_fp16 = add(x = value_73_cast_fp16, y = linear_43_cast_fp16)[name = string("value_85_cast_fp16")];
            tensor<fp16, [16, 1024]> var_4179_cast_fp16 = mul(x = value_85_cast_fp16, y = value_85_cast_fp16)[name = string("op_4179_cast_fp16")];
            tensor<int32, [1]> variance_37_axes_0 = const()[name = string("variance_37_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_37_keep_dims_0 = const()[name = string("variance_37_keep_dims_0"), val = bool(true)];
            tensor<fp16, [16, 1]> variance_37_cast_fp16 = reduce_mean(axes = variance_37_axes_0, keep_dims = variance_37_keep_dims_0, x = var_4179_cast_fp16)[name = string("variance_37_cast_fp16")];
            fp16 var_4182_to_fp16 = const()[name = string("op_4182_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [16, 1]> var_4183_cast_fp16 = add(x = variance_37_cast_fp16, y = var_4182_to_fp16)[name = string("op_4183_cast_fp16")];
            fp32 var_4184_epsilon_0 = const()[name = string("op_4184_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [16, 1]> var_4184_cast_fp16 = rsqrt(epsilon = var_4184_epsilon_0, x = var_4183_cast_fp16)[name = string("op_4184_cast_fp16")];
            tensor<fp16, [16, 1024]> var_4185_cast_fp16 = mul(x = value_85_cast_fp16, y = var_4184_cast_fp16)[name = string("op_4185_cast_fp16")];
            tensor<fp16, [1024]> var_4187_to_fp16 = const()[name = string("op_4187_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(87390784)))];
            tensor<fp16, [16, 1024]> input_75_cast_fp16 = mul(x = var_4185_cast_fp16, y = var_4187_to_fp16)[name = string("input_75_cast_fp16")];
            tensor<fp16, [3584, 1024]> groups_1_1_gate_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(87392896))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(90145472))))[name = string("groups_1_1_gate_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 3584]> linear_44_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = groups_1_1_gate_proj_weight_promoted_to_fp16_palettized, x = input_75_cast_fp16)[name = string("linear_44_cast_fp16")];
            tensor<fp16, [16, 3584]> var_4191_cast_fp16 = silu(x = linear_44_cast_fp16)[name = string("op_4191_cast_fp16")];
            tensor<fp16, [3584, 1024]> groups_1_1_up_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(90174208))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(92926784))))[name = string("groups_1_1_up_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 3584]> linear_45_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = groups_1_1_up_proj_weight_promoted_to_fp16_palettized, x = input_75_cast_fp16)[name = string("linear_45_cast_fp16")];
            tensor<fp16, [16, 3584]> input_79_cast_fp16 = mul(x = var_4191_cast_fp16, y = linear_45_cast_fp16)[name = string("input_79_cast_fp16")];
            tensor<fp16, [1024, 3584]> groups_1_1_down_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 3584]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(92955520))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(95708096))))[name = string("groups_1_1_down_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 1024]> linear_46_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = groups_1_1_down_proj_weight_promoted_to_fp16_palettized, x = input_79_cast_fp16)[name = string("linear_46_cast_fp16")];
            tensor<fp16, [16, 1024]> value_87_cast_fp16 = add(x = value_85_cast_fp16, y = linear_46_cast_fp16)[name = string("value_87_cast_fp16")];
            int32 var_4229 = const()[name = string("op_4229"), val = int32(-1)];
            tensor<fp16, [16, 1024]> var_4244_cast_fp16 = mul(x = value_87_cast_fp16, y = value_87_cast_fp16)[name = string("op_4244_cast_fp16")];
            tensor<int32, [1]> variance_39_axes_0 = const()[name = string("variance_39_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_39_keep_dims_0 = const()[name = string("variance_39_keep_dims_0"), val = bool(true)];
            tensor<fp16, [16, 1]> variance_39_cast_fp16 = reduce_mean(axes = variance_39_axes_0, keep_dims = variance_39_keep_dims_0, x = var_4244_cast_fp16)[name = string("variance_39_cast_fp16")];
            fp16 var_4247_to_fp16 = const()[name = string("op_4247_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [16, 1]> var_4248_cast_fp16 = add(x = variance_39_cast_fp16, y = var_4247_to_fp16)[name = string("op_4248_cast_fp16")];
            fp32 var_4249_epsilon_0 = const()[name = string("op_4249_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [16, 1]> var_4249_cast_fp16 = rsqrt(epsilon = var_4249_epsilon_0, x = var_4248_cast_fp16)[name = string("op_4249_cast_fp16")];
            tensor<fp16, [16, 1024]> var_4250_cast_fp16 = mul(x = value_87_cast_fp16, y = var_4249_cast_fp16)[name = string("op_4250_cast_fp16")];
            tensor<fp16, [1024]> var_4252_to_fp16 = const()[name = string("op_4252_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(95716352)))];
            tensor<fp16, [16, 1024]> input_81_cast_fp16 = mul(x = var_4250_cast_fp16, y = var_4252_to_fp16)[name = string("input_81_cast_fp16")];
            tensor<fp16, [6144, 1024]> groups_1_2_in_proj_qkv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(95718464))), lut = tensor<fp16, [384, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(100437120))))[name = string("groups_1_2_in_proj_qkv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 6144]> linear_47_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = groups_1_2_in_proj_qkv_weight_promoted_to_fp16_palettized, x = input_81_cast_fp16)[name = string("linear_47_cast_fp16")];
            tensor<int32, [2]> var_4256_perm_0 = const()[name = string("op_4256_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> mixed_11_axes_0 = const()[name = string("mixed_11_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [6144, 16]> var_4256_cast_fp16 = transpose(perm = var_4256_perm_0, x = linear_47_cast_fp16)[name = string("transpose_97")];
            tensor<fp16, [1, 6144, 16]> mixed_11_cast_fp16 = expand_dims(axes = mixed_11_axes_0, x = var_4256_cast_fp16)[name = string("mixed_11_cast_fp16")];
            bool convolution_input_11_interleave_0 = const()[name = string("convolution_input_11_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 6144, 19]> convolution_input_11_cast_fp16 = concat(axis = var_4229, interleave = convolution_input_11_interleave_0, values = (conv6, mixed_11_cast_fp16))[name = string("convolution_input_11_cast_fp16")];
            string input_83_pad_type_0 = const()[name = string("input_83_pad_type_0"), val = string("valid")];
            int32 input_83_groups_0 = const()[name = string("input_83_groups_0"), val = int32(6144)];
            tensor<int32, [1]> input_83_strides_0 = const()[name = string("input_83_strides_0"), val = tensor<int32, [1]>([1])];
            tensor<int32, [2]> input_83_pad_0 = const()[name = string("input_83_pad_0"), val = tensor<int32, [2]>([0, 0])];
            tensor<int32, [1]> input_83_dilations_0 = const()[name = string("input_83_dilations_0"), val = tensor<int32, [1]>([1])];
            tensor<fp16, [6144, 1, 4]> groups_1_2_conv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1, 4]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(100486336))), lut = tensor<fp16, [384, 1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(100504832))))[name = string("groups_1_2_conv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 6144, 16]> input_83_cast_fp16 = conv(dilations = input_83_dilations_0, groups = input_83_groups_0, pad = input_83_pad_0, pad_type = input_83_pad_type_0, strides = input_83_strides_0, weight = groups_1_2_conv_weight_promoted_to_fp16_palettized, x = convolution_input_11_cast_fp16)[name = string("input_83_cast_fp16")];
            tensor<fp16, [1, 6144, 16]> var_4265_cast_fp16 = silu(x = input_83_cast_fp16)[name = string("op_4265_cast_fp16")];
            tensor<int32, [3]> var_4266_perm_0 = const()[name = string("op_4266_perm_0"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [3]> var_4269_begin_0 = const()[name = string("op_4269_begin_0"), val = tensor<int32, [3]>([0, 0, 16])];
            tensor<int32, [3]> var_4269_end_0 = const()[name = string("op_4269_end_0"), val = tensor<int32, [3]>([1, 6144, 19])];
            tensor<bool, [3]> var_4269_end_mask_0 = const()[name = string("op_4269_end_mask_0"), val = tensor<bool, [3]>([true, true, true])];
            tensor<fp16, [1, 6144, 3]> conv6_out = slice_by_index(begin = var_4269_begin_0, end = var_4269_end_0, end_mask = var_4269_end_mask_0, x = convolution_input_11_cast_fp16)[name = string("op_4269_cast_fp16")];
            tensor<int32, [3]> var_4270 = const()[name = string("op_4270"), val = tensor<int32, [3]>([2048, 2048, 2048])];
            int32 var_4271_axis_0 = const()[name = string("op_4271_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 16, 6144]> var_4266_cast_fp16 = transpose(perm = var_4266_perm_0, x = var_4265_cast_fp16)[name = string("transpose_96")];
            tensor<fp16, [1, 16, 2048]> var_4271_cast_fp16_0, tensor<fp16, [1, 16, 2048]> var_4271_cast_fp16_1, tensor<fp16, [1, 16, 2048]> var_4271_cast_fp16_2 = split(axis = var_4271_axis_0, split_sizes = var_4270, x = var_4266_cast_fp16)[name = string("op_4271_cast_fp16")];
            tensor<int32, [4]> var_4275 = const()[name = string("op_4275"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<fp16, [1, 16, 16, 128]> value_91_cast_fp16 = reshape(shape = var_4275, x = var_4271_cast_fp16_0)[name = string("value_91_cast_fp16")];
            tensor<int32, [4]> var_4277 = const()[name = string("op_4277"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<fp16, [1, 16, 16, 128]> value_93_cast_fp16 = reshape(shape = var_4277, x = var_4271_cast_fp16_1)[name = string("value_93_cast_fp16")];
            tensor<int32, [4]> var_4279 = const()[name = string("op_4279"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<fp16, [1, 16, 16, 128]> value_95_cast_fp16 = reshape(shape = var_4279, x = var_4271_cast_fp16_2)[name = string("value_95_cast_fp16")];
            tensor<fp16, [1, 16, 16, 128]> var_4281_cast_fp16 = mul(x = value_91_cast_fp16, y = value_91_cast_fp16)[name = string("op_4281_cast_fp16")];
            tensor<int32, [1]> var_4283_axes_0 = const()[name = string("op_4283_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_4283_keep_dims_0 = const()[name = string("op_4283_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 16, 16, 1]> var_4283_cast_fp16 = reduce_sum(axes = var_4283_axes_0, keep_dims = var_4283_keep_dims_0, x = var_4281_cast_fp16)[name = string("op_4283_cast_fp16")];
            fp16 var_4284_to_fp16 = const()[name = string("op_4284_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 16, 16, 1]> var_4285_cast_fp16 = add(x = var_4283_cast_fp16, y = var_4284_to_fp16)[name = string("op_4285_cast_fp16")];
            fp32 var_4286_epsilon_0 = const()[name = string("op_4286_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 16, 16, 1]> var_4286_cast_fp16 = rsqrt(epsilon = var_4286_epsilon_0, x = var_4285_cast_fp16)[name = string("op_4286_cast_fp16")];
            tensor<fp16, [1, 16, 16, 128]> var_4287_cast_fp16 = mul(x = value_91_cast_fp16, y = var_4286_cast_fp16)[name = string("op_4287_cast_fp16")];
            tensor<int32, [4]> var_4288_perm_0 = const()[name = string("op_4288_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            fp16 _inversed_query_27_y_0_to_fp16 = const()[name = string("_inversed_query_27_y_0_to_fp16"), val = fp16(0x1.6ap-4)];
            tensor<fp16, [1, 16, 16, 128]> var_4288_cast_fp16 = transpose(perm = var_4288_perm_0, x = var_4287_cast_fp16)[name = string("transpose_95")];
            tensor<fp16, [1, 16, 16, 128]> _inversed_query_27_cast_fp16 = mul(x = var_4288_cast_fp16, y = _inversed_query_27_y_0_to_fp16)[name = string("_inversed_query_27_cast_fp16")];
            tensor<fp16, [1, 16, 16, 128]> var_4291_cast_fp16 = mul(x = value_93_cast_fp16, y = value_93_cast_fp16)[name = string("op_4291_cast_fp16")];
            tensor<int32, [1]> var_4293_axes_0 = const()[name = string("op_4293_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_4293_keep_dims_0 = const()[name = string("op_4293_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 16, 16, 1]> var_4293_cast_fp16 = reduce_sum(axes = var_4293_axes_0, keep_dims = var_4293_keep_dims_0, x = var_4291_cast_fp16)[name = string("op_4293_cast_fp16")];
            fp16 var_4294_to_fp16 = const()[name = string("op_4294_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 16, 16, 1]> var_4295_cast_fp16 = add(x = var_4293_cast_fp16, y = var_4294_to_fp16)[name = string("op_4295_cast_fp16")];
            fp32 var_4296_epsilon_0 = const()[name = string("op_4296_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 16, 16, 1]> var_4296_cast_fp16 = rsqrt(epsilon = var_4296_epsilon_0, x = var_4295_cast_fp16)[name = string("op_4296_cast_fp16")];
            tensor<fp16, [1, 16, 16, 128]> var_4297_cast_fp16 = mul(x = value_93_cast_fp16, y = var_4296_cast_fp16)[name = string("op_4297_cast_fp16")];
            tensor<int32, [4]> key_27_perm_0 = const()[name = string("key_27_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<int32, [4]> value_97_perm_0 = const()[name = string("value_97_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<fp16, [16, 1024]> groups_1_2_in_proj_b_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(100554048))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(100566400))))[name = string("groups_1_2_in_proj_b_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16]> linear_48_bias_0_to_fp16 = const()[name = string("linear_48_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(4862656)))];
            tensor<fp16, [16, 16]> linear_48_cast_fp16 = linear(bias = linear_48_bias_0_to_fp16, weight = groups_1_2_in_proj_b_weight_promoted_to_fp16_palettized, x = input_81_cast_fp16)[name = string("linear_48_cast_fp16")];
            tensor<fp16, [16, 16]> var_4302_cast_fp16 = sigmoid(x = linear_48_cast_fp16)[name = string("op_4302_cast_fp16")];
            tensor<int32, [2]> var_4303_perm_0 = const()[name = string("op_4303_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> beta_11_axes_0 = const()[name = string("beta_11_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 16]> var_4303_cast_fp16 = transpose(perm = var_4303_perm_0, x = var_4302_cast_fp16)[name = string("transpose_94")];
            tensor<fp16, [1, 16, 16]> beta_11_cast_fp16 = expand_dims(axes = beta_11_axes_0, x = var_4303_cast_fp16)[name = string("beta_11_cast_fp16")];
            tensor<fp16, [16, 1024]> op_4309_weight_0_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(100566592))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(100578944))))[name = string("op_4309_weight_0_to_fp16_palettized")];
            tensor<fp16, [16]> var_4309_bias_0_to_fp16 = const()[name = string("op_4309_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(100579136)))];
            tensor<fp16, [16, 16]> var_4309_cast_fp16 = linear(bias = var_4309_bias_0_to_fp16, weight = op_4309_weight_0_to_fp16_palettized, x = input_81_cast_fp16)[name = string("op_4309_cast_fp16")];
            tensor<fp16, [16, 16]> var_4310_cast_fp16 = softplus(x = var_4309_cast_fp16)[name = string("op_4310_cast_fp16")];
            tensor<fp16, [16]> var_4306_promoted_to_fp16 = const()[name = string("op_4306_promoted_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(100579264)))];
            tensor<fp16, [16, 16]> var_4311_cast_fp16 = mul(x = var_4306_promoted_to_fp16, y = var_4310_cast_fp16)[name = string("op_4311_cast_fp16")];
            tensor<int32, [2]> var_4312_perm_0 = const()[name = string("op_4312_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> decay_11_axes_0 = const()[name = string("decay_11_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 16]> var_4312_cast_fp16 = transpose(perm = var_4312_perm_0, x = var_4311_cast_fp16)[name = string("transpose_93")];
            tensor<fp16, [1, 16, 16]> decay_11_cast_fp16 = expand_dims(axes = decay_11_axes_0, x = var_4312_cast_fp16)[name = string("decay_11_cast_fp16")];
            tensor<fp16, [2048, 1024]> groups_1_2_in_proj_z_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [2048, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(100579392))), lut = tensor<fp16, [128, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(102152320))))[name = string("groups_1_2_in_proj_z_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 2048]> linear_50_cast_fp16 = linear(bias = linear_3_bias_0_to_fp16, weight = groups_1_2_in_proj_z_weight_promoted_to_fp16_palettized, x = input_81_cast_fp16)[name = string("linear_50_cast_fp16")];
            tensor<int32, [3]> var_4320_begin_0 = const()[name = string("op_4320_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_4320_end_0 = const()[name = string("op_4320_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_4320_end_mask_0 = const()[name = string("op_4320_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_4320_squeeze_mask_0 = const()[name = string("op_4320_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_4320_cast_fp16 = slice_by_index(begin = var_4320_begin_0, end = var_4320_end_0, end_mask = var_4320_end_mask_0, squeeze_mask = var_4320_squeeze_mask_0, x = decay_11_cast_fp16)[name = string("op_4320_cast_fp16")];
            tensor<fp16, [1, 16]> var_4321_cast_fp16 = exp(x = var_4320_cast_fp16)[name = string("op_4321_cast_fp16")];
            tensor<int32, [1]> var_4322_axes_0 = const()[name = string("op_4322_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_4322_cast_fp16 = expand_dims(axes = var_4322_axes_0, x = var_4321_cast_fp16)[name = string("op_4322_cast_fp16")];
            tensor<int32, [1]> var_4323_axes_0 = const()[name = string("op_4323_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_4323_cast_fp16 = expand_dims(axes = var_4323_axes_0, x = var_4322_cast_fp16)[name = string("op_4323_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_321_cast_fp16 = mul(x = rec6, y = var_4323_cast_fp16)[name = string("state_321_cast_fp16")];
            tensor<int32, [4]> key_token_161_begin_0 = const()[name = string("key_token_161_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> key_token_161_end_0 = const()[name = string("key_token_161_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> key_token_161_end_mask_0 = const()[name = string("key_token_161_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_161_squeeze_mask_0 = const()[name = string("key_token_161_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 16, 128]> key_27_cast_fp16 = transpose(perm = key_27_perm_0, x = var_4297_cast_fp16)[name = string("transpose_92")];
            tensor<fp16, [1, 16, 128]> key_token_161_cast_fp16 = slice_by_index(begin = key_token_161_begin_0, end = key_token_161_end_0, end_mask = key_token_161_end_mask_0, squeeze_mask = key_token_161_squeeze_mask_0, x = key_27_cast_fp16)[name = string("key_token_161_cast_fp16")];
            tensor<int32, [4]> value_token_161_begin_0 = const()[name = string("value_token_161_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> value_token_161_end_0 = const()[name = string("value_token_161_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> value_token_161_end_mask_0 = const()[name = string("value_token_161_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_161_squeeze_mask_0 = const()[name = string("value_token_161_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 16, 128]> value_97_cast_fp16 = transpose(perm = value_97_perm_0, x = value_95_cast_fp16)[name = string("transpose_91")];
            tensor<fp16, [1, 16, 128]> value_token_161_cast_fp16 = slice_by_index(begin = value_token_161_begin_0, end = value_token_161_end_0, end_mask = value_token_161_end_mask_0, squeeze_mask = value_token_161_squeeze_mask_0, x = value_97_cast_fp16)[name = string("value_token_161_cast_fp16")];
            tensor<int32, [1]> var_4331_axes_0 = const()[name = string("op_4331_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_4331_cast_fp16 = expand_dims(axes = var_4331_axes_0, x = key_token_161_cast_fp16)[name = string("op_4331_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4332_cast_fp16 = mul(x = state_321_cast_fp16, y = var_4331_cast_fp16)[name = string("op_4332_cast_fp16")];
            tensor<int32, [1]> memory_161_axes_0 = const()[name = string("memory_161_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_161_keep_dims_0 = const()[name = string("memory_161_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_161_cast_fp16 = reduce_sum(axes = memory_161_axes_0, keep_dims = memory_161_keep_dims_0, x = var_4332_cast_fp16)[name = string("memory_161_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_4335_cast_fp16 = sub(x = value_token_161_cast_fp16, y = memory_161_cast_fp16)[name = string("op_4335_cast_fp16")];
            tensor<int32, [3]> var_4338_begin_0 = const()[name = string("op_4338_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_4338_end_0 = const()[name = string("op_4338_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_4338_end_mask_0 = const()[name = string("op_4338_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_4338_squeeze_mask_0 = const()[name = string("op_4338_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_4338_cast_fp16 = slice_by_index(begin = var_4338_begin_0, end = var_4338_end_0, end_mask = var_4338_end_mask_0, squeeze_mask = var_4338_squeeze_mask_0, x = beta_11_cast_fp16)[name = string("op_4338_cast_fp16")];
            tensor<int32, [1]> var_4339_axes_0 = const()[name = string("op_4339_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_4339_cast_fp16 = expand_dims(axes = var_4339_axes_0, x = var_4338_cast_fp16)[name = string("op_4339_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_161_cast_fp16 = mul(x = var_4335_cast_fp16, y = var_4339_cast_fp16)[name = string("delta_161_cast_fp16")];
            tensor<int32, [1]> var_4342_axes_0 = const()[name = string("op_4342_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_4342_cast_fp16 = expand_dims(axes = var_4342_axes_0, x = delta_161_cast_fp16)[name = string("op_4342_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4343_cast_fp16 = mul(x = var_4331_cast_fp16, y = var_4342_cast_fp16)[name = string("op_4343_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_323_cast_fp16 = add(x = state_321_cast_fp16, y = var_4343_cast_fp16)[name = string("state_323_cast_fp16")];
            tensor<int32, [4]> var_4347_begin_0 = const()[name = string("op_4347_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> var_4347_end_0 = const()[name = string("op_4347_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> var_4347_end_mask_0 = const()[name = string("op_4347_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_4347_squeeze_mask_0 = const()[name = string("op_4347_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_4347_cast_fp16 = slice_by_index(begin = var_4347_begin_0, end = var_4347_end_0, end_mask = var_4347_end_mask_0, squeeze_mask = var_4347_squeeze_mask_0, x = _inversed_query_27_cast_fp16)[name = string("op_4347_cast_fp16")];
            tensor<int32, [1]> var_4348_axes_0 = const()[name = string("op_4348_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_4348_cast_fp16 = expand_dims(axes = var_4348_axes_0, x = var_4347_cast_fp16)[name = string("op_4348_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4349_cast_fp16 = mul(x = state_323_cast_fp16, y = var_4348_cast_fp16)[name = string("op_4349_cast_fp16")];
            tensor<int32, [1]> var_4351_axes_0 = const()[name = string("op_4351_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_4351_keep_dims_0 = const()[name = string("op_4351_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_4351_cast_fp16 = reduce_sum(axes = var_4351_axes_0, keep_dims = var_4351_keep_dims_0, x = var_4349_cast_fp16)[name = string("op_4351_cast_fp16")];
            tensor<int32, [3]> var_4354_begin_0 = const()[name = string("op_4354_begin_0"), val = tensor<int32, [3]>([0, 0, 1])];
            tensor<int32, [3]> var_4354_end_0 = const()[name = string("op_4354_end_0"), val = tensor<int32, [3]>([1, 16, 2])];
            tensor<bool, [3]> var_4354_end_mask_0 = const()[name = string("op_4354_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_4354_squeeze_mask_0 = const()[name = string("op_4354_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_4354_cast_fp16 = slice_by_index(begin = var_4354_begin_0, end = var_4354_end_0, end_mask = var_4354_end_mask_0, squeeze_mask = var_4354_squeeze_mask_0, x = decay_11_cast_fp16)[name = string("op_4354_cast_fp16")];
            tensor<fp16, [1, 16]> var_4355_cast_fp16 = exp(x = var_4354_cast_fp16)[name = string("op_4355_cast_fp16")];
            tensor<int32, [1]> var_4356_axes_0 = const()[name = string("op_4356_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_4356_cast_fp16 = expand_dims(axes = var_4356_axes_0, x = var_4355_cast_fp16)[name = string("op_4356_cast_fp16")];
            tensor<int32, [1]> var_4357_axes_0 = const()[name = string("op_4357_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_4357_cast_fp16 = expand_dims(axes = var_4357_axes_0, x = var_4356_cast_fp16)[name = string("op_4357_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_325_cast_fp16 = mul(x = state_323_cast_fp16, y = var_4357_cast_fp16)[name = string("state_325_cast_fp16")];
            tensor<int32, [4]> key_token_163_begin_0 = const()[name = string("key_token_163_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> key_token_163_end_0 = const()[name = string("key_token_163_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> key_token_163_end_mask_0 = const()[name = string("key_token_163_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_163_squeeze_mask_0 = const()[name = string("key_token_163_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_163_cast_fp16 = slice_by_index(begin = key_token_163_begin_0, end = key_token_163_end_0, end_mask = key_token_163_end_mask_0, squeeze_mask = key_token_163_squeeze_mask_0, x = key_27_cast_fp16)[name = string("key_token_163_cast_fp16")];
            tensor<int32, [4]> value_token_163_begin_0 = const()[name = string("value_token_163_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> value_token_163_end_0 = const()[name = string("value_token_163_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> value_token_163_end_mask_0 = const()[name = string("value_token_163_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_163_squeeze_mask_0 = const()[name = string("value_token_163_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_163_cast_fp16 = slice_by_index(begin = value_token_163_begin_0, end = value_token_163_end_0, end_mask = value_token_163_end_mask_0, squeeze_mask = value_token_163_squeeze_mask_0, x = value_97_cast_fp16)[name = string("value_token_163_cast_fp16")];
            tensor<int32, [1]> var_4365_axes_0 = const()[name = string("op_4365_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_4365_cast_fp16 = expand_dims(axes = var_4365_axes_0, x = key_token_163_cast_fp16)[name = string("op_4365_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4366_cast_fp16 = mul(x = state_325_cast_fp16, y = var_4365_cast_fp16)[name = string("op_4366_cast_fp16")];
            tensor<int32, [1]> memory_163_axes_0 = const()[name = string("memory_163_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_163_keep_dims_0 = const()[name = string("memory_163_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_163_cast_fp16 = reduce_sum(axes = memory_163_axes_0, keep_dims = memory_163_keep_dims_0, x = var_4366_cast_fp16)[name = string("memory_163_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_4369_cast_fp16 = sub(x = value_token_163_cast_fp16, y = memory_163_cast_fp16)[name = string("op_4369_cast_fp16")];
            tensor<int32, [3]> var_4372_begin_0 = const()[name = string("op_4372_begin_0"), val = tensor<int32, [3]>([0, 0, 1])];
            tensor<int32, [3]> var_4372_end_0 = const()[name = string("op_4372_end_0"), val = tensor<int32, [3]>([1, 16, 2])];
            tensor<bool, [3]> var_4372_end_mask_0 = const()[name = string("op_4372_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_4372_squeeze_mask_0 = const()[name = string("op_4372_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_4372_cast_fp16 = slice_by_index(begin = var_4372_begin_0, end = var_4372_end_0, end_mask = var_4372_end_mask_0, squeeze_mask = var_4372_squeeze_mask_0, x = beta_11_cast_fp16)[name = string("op_4372_cast_fp16")];
            tensor<int32, [1]> var_4373_axes_0 = const()[name = string("op_4373_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_4373_cast_fp16 = expand_dims(axes = var_4373_axes_0, x = var_4372_cast_fp16)[name = string("op_4373_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_163_cast_fp16 = mul(x = var_4369_cast_fp16, y = var_4373_cast_fp16)[name = string("delta_163_cast_fp16")];
            tensor<int32, [1]> var_4376_axes_0 = const()[name = string("op_4376_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_4376_cast_fp16 = expand_dims(axes = var_4376_axes_0, x = delta_163_cast_fp16)[name = string("op_4376_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4377_cast_fp16 = mul(x = var_4365_cast_fp16, y = var_4376_cast_fp16)[name = string("op_4377_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_327_cast_fp16 = add(x = state_325_cast_fp16, y = var_4377_cast_fp16)[name = string("state_327_cast_fp16")];
            tensor<int32, [4]> var_4381_begin_0 = const()[name = string("op_4381_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> var_4381_end_0 = const()[name = string("op_4381_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> var_4381_end_mask_0 = const()[name = string("op_4381_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_4381_squeeze_mask_0 = const()[name = string("op_4381_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_4381_cast_fp16 = slice_by_index(begin = var_4381_begin_0, end = var_4381_end_0, end_mask = var_4381_end_mask_0, squeeze_mask = var_4381_squeeze_mask_0, x = _inversed_query_27_cast_fp16)[name = string("op_4381_cast_fp16")];
            tensor<int32, [1]> var_4382_axes_0 = const()[name = string("op_4382_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_4382_cast_fp16 = expand_dims(axes = var_4382_axes_0, x = var_4381_cast_fp16)[name = string("op_4382_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4383_cast_fp16 = mul(x = state_327_cast_fp16, y = var_4382_cast_fp16)[name = string("op_4383_cast_fp16")];
            tensor<int32, [1]> var_4385_axes_0 = const()[name = string("op_4385_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_4385_keep_dims_0 = const()[name = string("op_4385_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_4385_cast_fp16 = reduce_sum(axes = var_4385_axes_0, keep_dims = var_4385_keep_dims_0, x = var_4383_cast_fp16)[name = string("op_4385_cast_fp16")];
            tensor<int32, [3]> var_4388_begin_0 = const()[name = string("op_4388_begin_0"), val = tensor<int32, [3]>([0, 0, 2])];
            tensor<int32, [3]> var_4388_end_0 = const()[name = string("op_4388_end_0"), val = tensor<int32, [3]>([1, 16, 3])];
            tensor<bool, [3]> var_4388_end_mask_0 = const()[name = string("op_4388_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_4388_squeeze_mask_0 = const()[name = string("op_4388_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_4388_cast_fp16 = slice_by_index(begin = var_4388_begin_0, end = var_4388_end_0, end_mask = var_4388_end_mask_0, squeeze_mask = var_4388_squeeze_mask_0, x = decay_11_cast_fp16)[name = string("op_4388_cast_fp16")];
            tensor<fp16, [1, 16]> var_4389_cast_fp16 = exp(x = var_4388_cast_fp16)[name = string("op_4389_cast_fp16")];
            tensor<int32, [1]> var_4390_axes_0 = const()[name = string("op_4390_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_4390_cast_fp16 = expand_dims(axes = var_4390_axes_0, x = var_4389_cast_fp16)[name = string("op_4390_cast_fp16")];
            tensor<int32, [1]> var_4391_axes_0 = const()[name = string("op_4391_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_4391_cast_fp16 = expand_dims(axes = var_4391_axes_0, x = var_4390_cast_fp16)[name = string("op_4391_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_329_cast_fp16 = mul(x = state_327_cast_fp16, y = var_4391_cast_fp16)[name = string("state_329_cast_fp16")];
            tensor<int32, [4]> key_token_165_begin_0 = const()[name = string("key_token_165_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> key_token_165_end_0 = const()[name = string("key_token_165_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> key_token_165_end_mask_0 = const()[name = string("key_token_165_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_165_squeeze_mask_0 = const()[name = string("key_token_165_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_165_cast_fp16 = slice_by_index(begin = key_token_165_begin_0, end = key_token_165_end_0, end_mask = key_token_165_end_mask_0, squeeze_mask = key_token_165_squeeze_mask_0, x = key_27_cast_fp16)[name = string("key_token_165_cast_fp16")];
            tensor<int32, [4]> value_token_165_begin_0 = const()[name = string("value_token_165_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> value_token_165_end_0 = const()[name = string("value_token_165_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> value_token_165_end_mask_0 = const()[name = string("value_token_165_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_165_squeeze_mask_0 = const()[name = string("value_token_165_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_165_cast_fp16 = slice_by_index(begin = value_token_165_begin_0, end = value_token_165_end_0, end_mask = value_token_165_end_mask_0, squeeze_mask = value_token_165_squeeze_mask_0, x = value_97_cast_fp16)[name = string("value_token_165_cast_fp16")];
            tensor<int32, [1]> var_4399_axes_0 = const()[name = string("op_4399_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_4399_cast_fp16 = expand_dims(axes = var_4399_axes_0, x = key_token_165_cast_fp16)[name = string("op_4399_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4400_cast_fp16 = mul(x = state_329_cast_fp16, y = var_4399_cast_fp16)[name = string("op_4400_cast_fp16")];
            tensor<int32, [1]> memory_165_axes_0 = const()[name = string("memory_165_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_165_keep_dims_0 = const()[name = string("memory_165_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_165_cast_fp16 = reduce_sum(axes = memory_165_axes_0, keep_dims = memory_165_keep_dims_0, x = var_4400_cast_fp16)[name = string("memory_165_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_4403_cast_fp16 = sub(x = value_token_165_cast_fp16, y = memory_165_cast_fp16)[name = string("op_4403_cast_fp16")];
            tensor<int32, [3]> var_4406_begin_0 = const()[name = string("op_4406_begin_0"), val = tensor<int32, [3]>([0, 0, 2])];
            tensor<int32, [3]> var_4406_end_0 = const()[name = string("op_4406_end_0"), val = tensor<int32, [3]>([1, 16, 3])];
            tensor<bool, [3]> var_4406_end_mask_0 = const()[name = string("op_4406_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_4406_squeeze_mask_0 = const()[name = string("op_4406_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_4406_cast_fp16 = slice_by_index(begin = var_4406_begin_0, end = var_4406_end_0, end_mask = var_4406_end_mask_0, squeeze_mask = var_4406_squeeze_mask_0, x = beta_11_cast_fp16)[name = string("op_4406_cast_fp16")];
            tensor<int32, [1]> var_4407_axes_0 = const()[name = string("op_4407_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_4407_cast_fp16 = expand_dims(axes = var_4407_axes_0, x = var_4406_cast_fp16)[name = string("op_4407_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_165_cast_fp16 = mul(x = var_4403_cast_fp16, y = var_4407_cast_fp16)[name = string("delta_165_cast_fp16")];
            tensor<int32, [1]> var_4410_axes_0 = const()[name = string("op_4410_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_4410_cast_fp16 = expand_dims(axes = var_4410_axes_0, x = delta_165_cast_fp16)[name = string("op_4410_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4411_cast_fp16 = mul(x = var_4399_cast_fp16, y = var_4410_cast_fp16)[name = string("op_4411_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_331_cast_fp16 = add(x = state_329_cast_fp16, y = var_4411_cast_fp16)[name = string("state_331_cast_fp16")];
            tensor<int32, [4]> var_4415_begin_0 = const()[name = string("op_4415_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> var_4415_end_0 = const()[name = string("op_4415_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> var_4415_end_mask_0 = const()[name = string("op_4415_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_4415_squeeze_mask_0 = const()[name = string("op_4415_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_4415_cast_fp16 = slice_by_index(begin = var_4415_begin_0, end = var_4415_end_0, end_mask = var_4415_end_mask_0, squeeze_mask = var_4415_squeeze_mask_0, x = _inversed_query_27_cast_fp16)[name = string("op_4415_cast_fp16")];
            tensor<int32, [1]> var_4416_axes_0 = const()[name = string("op_4416_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_4416_cast_fp16 = expand_dims(axes = var_4416_axes_0, x = var_4415_cast_fp16)[name = string("op_4416_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4417_cast_fp16 = mul(x = state_331_cast_fp16, y = var_4416_cast_fp16)[name = string("op_4417_cast_fp16")];
            tensor<int32, [1]> var_4419_axes_0 = const()[name = string("op_4419_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_4419_keep_dims_0 = const()[name = string("op_4419_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_4419_cast_fp16 = reduce_sum(axes = var_4419_axes_0, keep_dims = var_4419_keep_dims_0, x = var_4417_cast_fp16)[name = string("op_4419_cast_fp16")];
            tensor<int32, [3]> var_4422_begin_0 = const()[name = string("op_4422_begin_0"), val = tensor<int32, [3]>([0, 0, 3])];
            tensor<int32, [3]> var_4422_end_0 = const()[name = string("op_4422_end_0"), val = tensor<int32, [3]>([1, 16, 4])];
            tensor<bool, [3]> var_4422_end_mask_0 = const()[name = string("op_4422_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_4422_squeeze_mask_0 = const()[name = string("op_4422_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_4422_cast_fp16 = slice_by_index(begin = var_4422_begin_0, end = var_4422_end_0, end_mask = var_4422_end_mask_0, squeeze_mask = var_4422_squeeze_mask_0, x = decay_11_cast_fp16)[name = string("op_4422_cast_fp16")];
            tensor<fp16, [1, 16]> var_4423_cast_fp16 = exp(x = var_4422_cast_fp16)[name = string("op_4423_cast_fp16")];
            tensor<int32, [1]> var_4424_axes_0 = const()[name = string("op_4424_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_4424_cast_fp16 = expand_dims(axes = var_4424_axes_0, x = var_4423_cast_fp16)[name = string("op_4424_cast_fp16")];
            tensor<int32, [1]> var_4425_axes_0 = const()[name = string("op_4425_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_4425_cast_fp16 = expand_dims(axes = var_4425_axes_0, x = var_4424_cast_fp16)[name = string("op_4425_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_333_cast_fp16 = mul(x = state_331_cast_fp16, y = var_4425_cast_fp16)[name = string("state_333_cast_fp16")];
            tensor<int32, [4]> key_token_167_begin_0 = const()[name = string("key_token_167_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> key_token_167_end_0 = const()[name = string("key_token_167_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> key_token_167_end_mask_0 = const()[name = string("key_token_167_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_167_squeeze_mask_0 = const()[name = string("key_token_167_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_167_cast_fp16 = slice_by_index(begin = key_token_167_begin_0, end = key_token_167_end_0, end_mask = key_token_167_end_mask_0, squeeze_mask = key_token_167_squeeze_mask_0, x = key_27_cast_fp16)[name = string("key_token_167_cast_fp16")];
            tensor<int32, [4]> value_token_167_begin_0 = const()[name = string("value_token_167_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> value_token_167_end_0 = const()[name = string("value_token_167_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> value_token_167_end_mask_0 = const()[name = string("value_token_167_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_167_squeeze_mask_0 = const()[name = string("value_token_167_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_167_cast_fp16 = slice_by_index(begin = value_token_167_begin_0, end = value_token_167_end_0, end_mask = value_token_167_end_mask_0, squeeze_mask = value_token_167_squeeze_mask_0, x = value_97_cast_fp16)[name = string("value_token_167_cast_fp16")];
            tensor<int32, [1]> var_4433_axes_0 = const()[name = string("op_4433_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_4433_cast_fp16 = expand_dims(axes = var_4433_axes_0, x = key_token_167_cast_fp16)[name = string("op_4433_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4434_cast_fp16 = mul(x = state_333_cast_fp16, y = var_4433_cast_fp16)[name = string("op_4434_cast_fp16")];
            tensor<int32, [1]> memory_167_axes_0 = const()[name = string("memory_167_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_167_keep_dims_0 = const()[name = string("memory_167_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_167_cast_fp16 = reduce_sum(axes = memory_167_axes_0, keep_dims = memory_167_keep_dims_0, x = var_4434_cast_fp16)[name = string("memory_167_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_4437_cast_fp16 = sub(x = value_token_167_cast_fp16, y = memory_167_cast_fp16)[name = string("op_4437_cast_fp16")];
            tensor<int32, [3]> var_4440_begin_0 = const()[name = string("op_4440_begin_0"), val = tensor<int32, [3]>([0, 0, 3])];
            tensor<int32, [3]> var_4440_end_0 = const()[name = string("op_4440_end_0"), val = tensor<int32, [3]>([1, 16, 4])];
            tensor<bool, [3]> var_4440_end_mask_0 = const()[name = string("op_4440_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_4440_squeeze_mask_0 = const()[name = string("op_4440_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_4440_cast_fp16 = slice_by_index(begin = var_4440_begin_0, end = var_4440_end_0, end_mask = var_4440_end_mask_0, squeeze_mask = var_4440_squeeze_mask_0, x = beta_11_cast_fp16)[name = string("op_4440_cast_fp16")];
            tensor<int32, [1]> var_4441_axes_0 = const()[name = string("op_4441_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_4441_cast_fp16 = expand_dims(axes = var_4441_axes_0, x = var_4440_cast_fp16)[name = string("op_4441_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_167_cast_fp16 = mul(x = var_4437_cast_fp16, y = var_4441_cast_fp16)[name = string("delta_167_cast_fp16")];
            tensor<int32, [1]> var_4444_axes_0 = const()[name = string("op_4444_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_4444_cast_fp16 = expand_dims(axes = var_4444_axes_0, x = delta_167_cast_fp16)[name = string("op_4444_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4445_cast_fp16 = mul(x = var_4433_cast_fp16, y = var_4444_cast_fp16)[name = string("op_4445_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_335_cast_fp16 = add(x = state_333_cast_fp16, y = var_4445_cast_fp16)[name = string("state_335_cast_fp16")];
            tensor<int32, [4]> var_4449_begin_0 = const()[name = string("op_4449_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> var_4449_end_0 = const()[name = string("op_4449_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> var_4449_end_mask_0 = const()[name = string("op_4449_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_4449_squeeze_mask_0 = const()[name = string("op_4449_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_4449_cast_fp16 = slice_by_index(begin = var_4449_begin_0, end = var_4449_end_0, end_mask = var_4449_end_mask_0, squeeze_mask = var_4449_squeeze_mask_0, x = _inversed_query_27_cast_fp16)[name = string("op_4449_cast_fp16")];
            tensor<int32, [1]> var_4450_axes_0 = const()[name = string("op_4450_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_4450_cast_fp16 = expand_dims(axes = var_4450_axes_0, x = var_4449_cast_fp16)[name = string("op_4450_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4451_cast_fp16 = mul(x = state_335_cast_fp16, y = var_4450_cast_fp16)[name = string("op_4451_cast_fp16")];
            tensor<int32, [1]> var_4453_axes_0 = const()[name = string("op_4453_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_4453_keep_dims_0 = const()[name = string("op_4453_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_4453_cast_fp16 = reduce_sum(axes = var_4453_axes_0, keep_dims = var_4453_keep_dims_0, x = var_4451_cast_fp16)[name = string("op_4453_cast_fp16")];
            tensor<int32, [3]> var_4456_begin_0 = const()[name = string("op_4456_begin_0"), val = tensor<int32, [3]>([0, 0, 4])];
            tensor<int32, [3]> var_4456_end_0 = const()[name = string("op_4456_end_0"), val = tensor<int32, [3]>([1, 16, 5])];
            tensor<bool, [3]> var_4456_end_mask_0 = const()[name = string("op_4456_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_4456_squeeze_mask_0 = const()[name = string("op_4456_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_4456_cast_fp16 = slice_by_index(begin = var_4456_begin_0, end = var_4456_end_0, end_mask = var_4456_end_mask_0, squeeze_mask = var_4456_squeeze_mask_0, x = decay_11_cast_fp16)[name = string("op_4456_cast_fp16")];
            tensor<fp16, [1, 16]> var_4457_cast_fp16 = exp(x = var_4456_cast_fp16)[name = string("op_4457_cast_fp16")];
            tensor<int32, [1]> var_4458_axes_0 = const()[name = string("op_4458_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_4458_cast_fp16 = expand_dims(axes = var_4458_axes_0, x = var_4457_cast_fp16)[name = string("op_4458_cast_fp16")];
            tensor<int32, [1]> var_4459_axes_0 = const()[name = string("op_4459_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_4459_cast_fp16 = expand_dims(axes = var_4459_axes_0, x = var_4458_cast_fp16)[name = string("op_4459_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_337_cast_fp16 = mul(x = state_335_cast_fp16, y = var_4459_cast_fp16)[name = string("state_337_cast_fp16")];
            tensor<int32, [4]> key_token_169_begin_0 = const()[name = string("key_token_169_begin_0"), val = tensor<int32, [4]>([0, 0, 4, 0])];
            tensor<int32, [4]> key_token_169_end_0 = const()[name = string("key_token_169_end_0"), val = tensor<int32, [4]>([1, 16, 5, 128])];
            tensor<bool, [4]> key_token_169_end_mask_0 = const()[name = string("key_token_169_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_169_squeeze_mask_0 = const()[name = string("key_token_169_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_169_cast_fp16 = slice_by_index(begin = key_token_169_begin_0, end = key_token_169_end_0, end_mask = key_token_169_end_mask_0, squeeze_mask = key_token_169_squeeze_mask_0, x = key_27_cast_fp16)[name = string("key_token_169_cast_fp16")];
            tensor<int32, [4]> value_token_169_begin_0 = const()[name = string("value_token_169_begin_0"), val = tensor<int32, [4]>([0, 0, 4, 0])];
            tensor<int32, [4]> value_token_169_end_0 = const()[name = string("value_token_169_end_0"), val = tensor<int32, [4]>([1, 16, 5, 128])];
            tensor<bool, [4]> value_token_169_end_mask_0 = const()[name = string("value_token_169_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_169_squeeze_mask_0 = const()[name = string("value_token_169_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_169_cast_fp16 = slice_by_index(begin = value_token_169_begin_0, end = value_token_169_end_0, end_mask = value_token_169_end_mask_0, squeeze_mask = value_token_169_squeeze_mask_0, x = value_97_cast_fp16)[name = string("value_token_169_cast_fp16")];
            tensor<int32, [1]> var_4467_axes_0 = const()[name = string("op_4467_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_4467_cast_fp16 = expand_dims(axes = var_4467_axes_0, x = key_token_169_cast_fp16)[name = string("op_4467_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4468_cast_fp16 = mul(x = state_337_cast_fp16, y = var_4467_cast_fp16)[name = string("op_4468_cast_fp16")];
            tensor<int32, [1]> memory_169_axes_0 = const()[name = string("memory_169_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_169_keep_dims_0 = const()[name = string("memory_169_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_169_cast_fp16 = reduce_sum(axes = memory_169_axes_0, keep_dims = memory_169_keep_dims_0, x = var_4468_cast_fp16)[name = string("memory_169_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_4471_cast_fp16 = sub(x = value_token_169_cast_fp16, y = memory_169_cast_fp16)[name = string("op_4471_cast_fp16")];
            tensor<int32, [3]> var_4474_begin_0 = const()[name = string("op_4474_begin_0"), val = tensor<int32, [3]>([0, 0, 4])];
            tensor<int32, [3]> var_4474_end_0 = const()[name = string("op_4474_end_0"), val = tensor<int32, [3]>([1, 16, 5])];
            tensor<bool, [3]> var_4474_end_mask_0 = const()[name = string("op_4474_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_4474_squeeze_mask_0 = const()[name = string("op_4474_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_4474_cast_fp16 = slice_by_index(begin = var_4474_begin_0, end = var_4474_end_0, end_mask = var_4474_end_mask_0, squeeze_mask = var_4474_squeeze_mask_0, x = beta_11_cast_fp16)[name = string("op_4474_cast_fp16")];
            tensor<int32, [1]> var_4475_axes_0 = const()[name = string("op_4475_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_4475_cast_fp16 = expand_dims(axes = var_4475_axes_0, x = var_4474_cast_fp16)[name = string("op_4475_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_169_cast_fp16 = mul(x = var_4471_cast_fp16, y = var_4475_cast_fp16)[name = string("delta_169_cast_fp16")];
            tensor<int32, [1]> var_4478_axes_0 = const()[name = string("op_4478_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_4478_cast_fp16 = expand_dims(axes = var_4478_axes_0, x = delta_169_cast_fp16)[name = string("op_4478_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4479_cast_fp16 = mul(x = var_4467_cast_fp16, y = var_4478_cast_fp16)[name = string("op_4479_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_339_cast_fp16 = add(x = state_337_cast_fp16, y = var_4479_cast_fp16)[name = string("state_339_cast_fp16")];
            tensor<int32, [4]> var_4483_begin_0 = const()[name = string("op_4483_begin_0"), val = tensor<int32, [4]>([0, 0, 4, 0])];
            tensor<int32, [4]> var_4483_end_0 = const()[name = string("op_4483_end_0"), val = tensor<int32, [4]>([1, 16, 5, 128])];
            tensor<bool, [4]> var_4483_end_mask_0 = const()[name = string("op_4483_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_4483_squeeze_mask_0 = const()[name = string("op_4483_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_4483_cast_fp16 = slice_by_index(begin = var_4483_begin_0, end = var_4483_end_0, end_mask = var_4483_end_mask_0, squeeze_mask = var_4483_squeeze_mask_0, x = _inversed_query_27_cast_fp16)[name = string("op_4483_cast_fp16")];
            tensor<int32, [1]> var_4484_axes_0 = const()[name = string("op_4484_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_4484_cast_fp16 = expand_dims(axes = var_4484_axes_0, x = var_4483_cast_fp16)[name = string("op_4484_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4485_cast_fp16 = mul(x = state_339_cast_fp16, y = var_4484_cast_fp16)[name = string("op_4485_cast_fp16")];
            tensor<int32, [1]> var_4487_axes_0 = const()[name = string("op_4487_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_4487_keep_dims_0 = const()[name = string("op_4487_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_4487_cast_fp16 = reduce_sum(axes = var_4487_axes_0, keep_dims = var_4487_keep_dims_0, x = var_4485_cast_fp16)[name = string("op_4487_cast_fp16")];
            tensor<int32, [3]> var_4490_begin_0 = const()[name = string("op_4490_begin_0"), val = tensor<int32, [3]>([0, 0, 5])];
            tensor<int32, [3]> var_4490_end_0 = const()[name = string("op_4490_end_0"), val = tensor<int32, [3]>([1, 16, 6])];
            tensor<bool, [3]> var_4490_end_mask_0 = const()[name = string("op_4490_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_4490_squeeze_mask_0 = const()[name = string("op_4490_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_4490_cast_fp16 = slice_by_index(begin = var_4490_begin_0, end = var_4490_end_0, end_mask = var_4490_end_mask_0, squeeze_mask = var_4490_squeeze_mask_0, x = decay_11_cast_fp16)[name = string("op_4490_cast_fp16")];
            tensor<fp16, [1, 16]> var_4491_cast_fp16 = exp(x = var_4490_cast_fp16)[name = string("op_4491_cast_fp16")];
            tensor<int32, [1]> var_4492_axes_0 = const()[name = string("op_4492_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_4492_cast_fp16 = expand_dims(axes = var_4492_axes_0, x = var_4491_cast_fp16)[name = string("op_4492_cast_fp16")];
            tensor<int32, [1]> var_4493_axes_0 = const()[name = string("op_4493_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_4493_cast_fp16 = expand_dims(axes = var_4493_axes_0, x = var_4492_cast_fp16)[name = string("op_4493_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_341_cast_fp16 = mul(x = state_339_cast_fp16, y = var_4493_cast_fp16)[name = string("state_341_cast_fp16")];
            tensor<int32, [4]> key_token_171_begin_0 = const()[name = string("key_token_171_begin_0"), val = tensor<int32, [4]>([0, 0, 5, 0])];
            tensor<int32, [4]> key_token_171_end_0 = const()[name = string("key_token_171_end_0"), val = tensor<int32, [4]>([1, 16, 6, 128])];
            tensor<bool, [4]> key_token_171_end_mask_0 = const()[name = string("key_token_171_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_171_squeeze_mask_0 = const()[name = string("key_token_171_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_171_cast_fp16 = slice_by_index(begin = key_token_171_begin_0, end = key_token_171_end_0, end_mask = key_token_171_end_mask_0, squeeze_mask = key_token_171_squeeze_mask_0, x = key_27_cast_fp16)[name = string("key_token_171_cast_fp16")];
            tensor<int32, [4]> value_token_171_begin_0 = const()[name = string("value_token_171_begin_0"), val = tensor<int32, [4]>([0, 0, 5, 0])];
            tensor<int32, [4]> value_token_171_end_0 = const()[name = string("value_token_171_end_0"), val = tensor<int32, [4]>([1, 16, 6, 128])];
            tensor<bool, [4]> value_token_171_end_mask_0 = const()[name = string("value_token_171_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_171_squeeze_mask_0 = const()[name = string("value_token_171_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_171_cast_fp16 = slice_by_index(begin = value_token_171_begin_0, end = value_token_171_end_0, end_mask = value_token_171_end_mask_0, squeeze_mask = value_token_171_squeeze_mask_0, x = value_97_cast_fp16)[name = string("value_token_171_cast_fp16")];
            tensor<int32, [1]> var_4501_axes_0 = const()[name = string("op_4501_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_4501_cast_fp16 = expand_dims(axes = var_4501_axes_0, x = key_token_171_cast_fp16)[name = string("op_4501_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4502_cast_fp16 = mul(x = state_341_cast_fp16, y = var_4501_cast_fp16)[name = string("op_4502_cast_fp16")];
            tensor<int32, [1]> memory_171_axes_0 = const()[name = string("memory_171_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_171_keep_dims_0 = const()[name = string("memory_171_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_171_cast_fp16 = reduce_sum(axes = memory_171_axes_0, keep_dims = memory_171_keep_dims_0, x = var_4502_cast_fp16)[name = string("memory_171_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_4505_cast_fp16 = sub(x = value_token_171_cast_fp16, y = memory_171_cast_fp16)[name = string("op_4505_cast_fp16")];
            tensor<int32, [3]> var_4508_begin_0 = const()[name = string("op_4508_begin_0"), val = tensor<int32, [3]>([0, 0, 5])];
            tensor<int32, [3]> var_4508_end_0 = const()[name = string("op_4508_end_0"), val = tensor<int32, [3]>([1, 16, 6])];
            tensor<bool, [3]> var_4508_end_mask_0 = const()[name = string("op_4508_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_4508_squeeze_mask_0 = const()[name = string("op_4508_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_4508_cast_fp16 = slice_by_index(begin = var_4508_begin_0, end = var_4508_end_0, end_mask = var_4508_end_mask_0, squeeze_mask = var_4508_squeeze_mask_0, x = beta_11_cast_fp16)[name = string("op_4508_cast_fp16")];
            tensor<int32, [1]> var_4509_axes_0 = const()[name = string("op_4509_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_4509_cast_fp16 = expand_dims(axes = var_4509_axes_0, x = var_4508_cast_fp16)[name = string("op_4509_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_171_cast_fp16 = mul(x = var_4505_cast_fp16, y = var_4509_cast_fp16)[name = string("delta_171_cast_fp16")];
            tensor<int32, [1]> var_4512_axes_0 = const()[name = string("op_4512_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_4512_cast_fp16 = expand_dims(axes = var_4512_axes_0, x = delta_171_cast_fp16)[name = string("op_4512_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4513_cast_fp16 = mul(x = var_4501_cast_fp16, y = var_4512_cast_fp16)[name = string("op_4513_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_343_cast_fp16 = add(x = state_341_cast_fp16, y = var_4513_cast_fp16)[name = string("state_343_cast_fp16")];
            tensor<int32, [4]> var_4517_begin_0 = const()[name = string("op_4517_begin_0"), val = tensor<int32, [4]>([0, 0, 5, 0])];
            tensor<int32, [4]> var_4517_end_0 = const()[name = string("op_4517_end_0"), val = tensor<int32, [4]>([1, 16, 6, 128])];
            tensor<bool, [4]> var_4517_end_mask_0 = const()[name = string("op_4517_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_4517_squeeze_mask_0 = const()[name = string("op_4517_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_4517_cast_fp16 = slice_by_index(begin = var_4517_begin_0, end = var_4517_end_0, end_mask = var_4517_end_mask_0, squeeze_mask = var_4517_squeeze_mask_0, x = _inversed_query_27_cast_fp16)[name = string("op_4517_cast_fp16")];
            tensor<int32, [1]> var_4518_axes_0 = const()[name = string("op_4518_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_4518_cast_fp16 = expand_dims(axes = var_4518_axes_0, x = var_4517_cast_fp16)[name = string("op_4518_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4519_cast_fp16 = mul(x = state_343_cast_fp16, y = var_4518_cast_fp16)[name = string("op_4519_cast_fp16")];
            tensor<int32, [1]> var_4521_axes_0 = const()[name = string("op_4521_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_4521_keep_dims_0 = const()[name = string("op_4521_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_4521_cast_fp16 = reduce_sum(axes = var_4521_axes_0, keep_dims = var_4521_keep_dims_0, x = var_4519_cast_fp16)[name = string("op_4521_cast_fp16")];
            tensor<int32, [3]> var_4524_begin_0 = const()[name = string("op_4524_begin_0"), val = tensor<int32, [3]>([0, 0, 6])];
            tensor<int32, [3]> var_4524_end_0 = const()[name = string("op_4524_end_0"), val = tensor<int32, [3]>([1, 16, 7])];
            tensor<bool, [3]> var_4524_end_mask_0 = const()[name = string("op_4524_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_4524_squeeze_mask_0 = const()[name = string("op_4524_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_4524_cast_fp16 = slice_by_index(begin = var_4524_begin_0, end = var_4524_end_0, end_mask = var_4524_end_mask_0, squeeze_mask = var_4524_squeeze_mask_0, x = decay_11_cast_fp16)[name = string("op_4524_cast_fp16")];
            tensor<fp16, [1, 16]> var_4525_cast_fp16 = exp(x = var_4524_cast_fp16)[name = string("op_4525_cast_fp16")];
            tensor<int32, [1]> var_4526_axes_0 = const()[name = string("op_4526_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_4526_cast_fp16 = expand_dims(axes = var_4526_axes_0, x = var_4525_cast_fp16)[name = string("op_4526_cast_fp16")];
            tensor<int32, [1]> var_4527_axes_0 = const()[name = string("op_4527_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_4527_cast_fp16 = expand_dims(axes = var_4527_axes_0, x = var_4526_cast_fp16)[name = string("op_4527_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_345_cast_fp16 = mul(x = state_343_cast_fp16, y = var_4527_cast_fp16)[name = string("state_345_cast_fp16")];
            tensor<int32, [4]> key_token_173_begin_0 = const()[name = string("key_token_173_begin_0"), val = tensor<int32, [4]>([0, 0, 6, 0])];
            tensor<int32, [4]> key_token_173_end_0 = const()[name = string("key_token_173_end_0"), val = tensor<int32, [4]>([1, 16, 7, 128])];
            tensor<bool, [4]> key_token_173_end_mask_0 = const()[name = string("key_token_173_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_173_squeeze_mask_0 = const()[name = string("key_token_173_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_173_cast_fp16 = slice_by_index(begin = key_token_173_begin_0, end = key_token_173_end_0, end_mask = key_token_173_end_mask_0, squeeze_mask = key_token_173_squeeze_mask_0, x = key_27_cast_fp16)[name = string("key_token_173_cast_fp16")];
            tensor<int32, [4]> value_token_173_begin_0 = const()[name = string("value_token_173_begin_0"), val = tensor<int32, [4]>([0, 0, 6, 0])];
            tensor<int32, [4]> value_token_173_end_0 = const()[name = string("value_token_173_end_0"), val = tensor<int32, [4]>([1, 16, 7, 128])];
            tensor<bool, [4]> value_token_173_end_mask_0 = const()[name = string("value_token_173_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_173_squeeze_mask_0 = const()[name = string("value_token_173_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_173_cast_fp16 = slice_by_index(begin = value_token_173_begin_0, end = value_token_173_end_0, end_mask = value_token_173_end_mask_0, squeeze_mask = value_token_173_squeeze_mask_0, x = value_97_cast_fp16)[name = string("value_token_173_cast_fp16")];
            tensor<int32, [1]> var_4535_axes_0 = const()[name = string("op_4535_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_4535_cast_fp16 = expand_dims(axes = var_4535_axes_0, x = key_token_173_cast_fp16)[name = string("op_4535_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4536_cast_fp16 = mul(x = state_345_cast_fp16, y = var_4535_cast_fp16)[name = string("op_4536_cast_fp16")];
            tensor<int32, [1]> memory_173_axes_0 = const()[name = string("memory_173_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_173_keep_dims_0 = const()[name = string("memory_173_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_173_cast_fp16 = reduce_sum(axes = memory_173_axes_0, keep_dims = memory_173_keep_dims_0, x = var_4536_cast_fp16)[name = string("memory_173_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_4539_cast_fp16 = sub(x = value_token_173_cast_fp16, y = memory_173_cast_fp16)[name = string("op_4539_cast_fp16")];
            tensor<int32, [3]> var_4542_begin_0 = const()[name = string("op_4542_begin_0"), val = tensor<int32, [3]>([0, 0, 6])];
            tensor<int32, [3]> var_4542_end_0 = const()[name = string("op_4542_end_0"), val = tensor<int32, [3]>([1, 16, 7])];
            tensor<bool, [3]> var_4542_end_mask_0 = const()[name = string("op_4542_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_4542_squeeze_mask_0 = const()[name = string("op_4542_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_4542_cast_fp16 = slice_by_index(begin = var_4542_begin_0, end = var_4542_end_0, end_mask = var_4542_end_mask_0, squeeze_mask = var_4542_squeeze_mask_0, x = beta_11_cast_fp16)[name = string("op_4542_cast_fp16")];
            tensor<int32, [1]> var_4543_axes_0 = const()[name = string("op_4543_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_4543_cast_fp16 = expand_dims(axes = var_4543_axes_0, x = var_4542_cast_fp16)[name = string("op_4543_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_173_cast_fp16 = mul(x = var_4539_cast_fp16, y = var_4543_cast_fp16)[name = string("delta_173_cast_fp16")];
            tensor<int32, [1]> var_4546_axes_0 = const()[name = string("op_4546_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_4546_cast_fp16 = expand_dims(axes = var_4546_axes_0, x = delta_173_cast_fp16)[name = string("op_4546_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4547_cast_fp16 = mul(x = var_4535_cast_fp16, y = var_4546_cast_fp16)[name = string("op_4547_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_347_cast_fp16 = add(x = state_345_cast_fp16, y = var_4547_cast_fp16)[name = string("state_347_cast_fp16")];
            tensor<int32, [4]> var_4551_begin_0 = const()[name = string("op_4551_begin_0"), val = tensor<int32, [4]>([0, 0, 6, 0])];
            tensor<int32, [4]> var_4551_end_0 = const()[name = string("op_4551_end_0"), val = tensor<int32, [4]>([1, 16, 7, 128])];
            tensor<bool, [4]> var_4551_end_mask_0 = const()[name = string("op_4551_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_4551_squeeze_mask_0 = const()[name = string("op_4551_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_4551_cast_fp16 = slice_by_index(begin = var_4551_begin_0, end = var_4551_end_0, end_mask = var_4551_end_mask_0, squeeze_mask = var_4551_squeeze_mask_0, x = _inversed_query_27_cast_fp16)[name = string("op_4551_cast_fp16")];
            tensor<int32, [1]> var_4552_axes_0 = const()[name = string("op_4552_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_4552_cast_fp16 = expand_dims(axes = var_4552_axes_0, x = var_4551_cast_fp16)[name = string("op_4552_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4553_cast_fp16 = mul(x = state_347_cast_fp16, y = var_4552_cast_fp16)[name = string("op_4553_cast_fp16")];
            tensor<int32, [1]> var_4555_axes_0 = const()[name = string("op_4555_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_4555_keep_dims_0 = const()[name = string("op_4555_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_4555_cast_fp16 = reduce_sum(axes = var_4555_axes_0, keep_dims = var_4555_keep_dims_0, x = var_4553_cast_fp16)[name = string("op_4555_cast_fp16")];
            tensor<int32, [3]> var_4558_begin_0 = const()[name = string("op_4558_begin_0"), val = tensor<int32, [3]>([0, 0, 7])];
            tensor<int32, [3]> var_4558_end_0 = const()[name = string("op_4558_end_0"), val = tensor<int32, [3]>([1, 16, 8])];
            tensor<bool, [3]> var_4558_end_mask_0 = const()[name = string("op_4558_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_4558_squeeze_mask_0 = const()[name = string("op_4558_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_4558_cast_fp16 = slice_by_index(begin = var_4558_begin_0, end = var_4558_end_0, end_mask = var_4558_end_mask_0, squeeze_mask = var_4558_squeeze_mask_0, x = decay_11_cast_fp16)[name = string("op_4558_cast_fp16")];
            tensor<fp16, [1, 16]> var_4559_cast_fp16 = exp(x = var_4558_cast_fp16)[name = string("op_4559_cast_fp16")];
            tensor<int32, [1]> var_4560_axes_0 = const()[name = string("op_4560_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_4560_cast_fp16 = expand_dims(axes = var_4560_axes_0, x = var_4559_cast_fp16)[name = string("op_4560_cast_fp16")];
            tensor<int32, [1]> var_4561_axes_0 = const()[name = string("op_4561_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_4561_cast_fp16 = expand_dims(axes = var_4561_axes_0, x = var_4560_cast_fp16)[name = string("op_4561_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_349_cast_fp16 = mul(x = state_347_cast_fp16, y = var_4561_cast_fp16)[name = string("state_349_cast_fp16")];
            tensor<int32, [4]> key_token_175_begin_0 = const()[name = string("key_token_175_begin_0"), val = tensor<int32, [4]>([0, 0, 7, 0])];
            tensor<int32, [4]> key_token_175_end_0 = const()[name = string("key_token_175_end_0"), val = tensor<int32, [4]>([1, 16, 8, 128])];
            tensor<bool, [4]> key_token_175_end_mask_0 = const()[name = string("key_token_175_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_175_squeeze_mask_0 = const()[name = string("key_token_175_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_175_cast_fp16 = slice_by_index(begin = key_token_175_begin_0, end = key_token_175_end_0, end_mask = key_token_175_end_mask_0, squeeze_mask = key_token_175_squeeze_mask_0, x = key_27_cast_fp16)[name = string("key_token_175_cast_fp16")];
            tensor<int32, [4]> value_token_175_begin_0 = const()[name = string("value_token_175_begin_0"), val = tensor<int32, [4]>([0, 0, 7, 0])];
            tensor<int32, [4]> value_token_175_end_0 = const()[name = string("value_token_175_end_0"), val = tensor<int32, [4]>([1, 16, 8, 128])];
            tensor<bool, [4]> value_token_175_end_mask_0 = const()[name = string("value_token_175_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_175_squeeze_mask_0 = const()[name = string("value_token_175_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_175_cast_fp16 = slice_by_index(begin = value_token_175_begin_0, end = value_token_175_end_0, end_mask = value_token_175_end_mask_0, squeeze_mask = value_token_175_squeeze_mask_0, x = value_97_cast_fp16)[name = string("value_token_175_cast_fp16")];
            tensor<int32, [1]> var_4569_axes_0 = const()[name = string("op_4569_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_4569_cast_fp16 = expand_dims(axes = var_4569_axes_0, x = key_token_175_cast_fp16)[name = string("op_4569_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4570_cast_fp16 = mul(x = state_349_cast_fp16, y = var_4569_cast_fp16)[name = string("op_4570_cast_fp16")];
            tensor<int32, [1]> memory_175_axes_0 = const()[name = string("memory_175_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_175_keep_dims_0 = const()[name = string("memory_175_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_175_cast_fp16 = reduce_sum(axes = memory_175_axes_0, keep_dims = memory_175_keep_dims_0, x = var_4570_cast_fp16)[name = string("memory_175_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_4573_cast_fp16 = sub(x = value_token_175_cast_fp16, y = memory_175_cast_fp16)[name = string("op_4573_cast_fp16")];
            tensor<int32, [3]> var_4576_begin_0 = const()[name = string("op_4576_begin_0"), val = tensor<int32, [3]>([0, 0, 7])];
            tensor<int32, [3]> var_4576_end_0 = const()[name = string("op_4576_end_0"), val = tensor<int32, [3]>([1, 16, 8])];
            tensor<bool, [3]> var_4576_end_mask_0 = const()[name = string("op_4576_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_4576_squeeze_mask_0 = const()[name = string("op_4576_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_4576_cast_fp16 = slice_by_index(begin = var_4576_begin_0, end = var_4576_end_0, end_mask = var_4576_end_mask_0, squeeze_mask = var_4576_squeeze_mask_0, x = beta_11_cast_fp16)[name = string("op_4576_cast_fp16")];
            tensor<int32, [1]> var_4577_axes_0 = const()[name = string("op_4577_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_4577_cast_fp16 = expand_dims(axes = var_4577_axes_0, x = var_4576_cast_fp16)[name = string("op_4577_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_175_cast_fp16 = mul(x = var_4573_cast_fp16, y = var_4577_cast_fp16)[name = string("delta_175_cast_fp16")];
            tensor<int32, [1]> var_4580_axes_0 = const()[name = string("op_4580_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_4580_cast_fp16 = expand_dims(axes = var_4580_axes_0, x = delta_175_cast_fp16)[name = string("op_4580_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4581_cast_fp16 = mul(x = var_4569_cast_fp16, y = var_4580_cast_fp16)[name = string("op_4581_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_351_cast_fp16 = add(x = state_349_cast_fp16, y = var_4581_cast_fp16)[name = string("state_351_cast_fp16")];
            tensor<int32, [4]> var_4585_begin_0 = const()[name = string("op_4585_begin_0"), val = tensor<int32, [4]>([0, 0, 7, 0])];
            tensor<int32, [4]> var_4585_end_0 = const()[name = string("op_4585_end_0"), val = tensor<int32, [4]>([1, 16, 8, 128])];
            tensor<bool, [4]> var_4585_end_mask_0 = const()[name = string("op_4585_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_4585_squeeze_mask_0 = const()[name = string("op_4585_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_4585_cast_fp16 = slice_by_index(begin = var_4585_begin_0, end = var_4585_end_0, end_mask = var_4585_end_mask_0, squeeze_mask = var_4585_squeeze_mask_0, x = _inversed_query_27_cast_fp16)[name = string("op_4585_cast_fp16")];
            tensor<int32, [1]> var_4586_axes_0 = const()[name = string("op_4586_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_4586_cast_fp16 = expand_dims(axes = var_4586_axes_0, x = var_4585_cast_fp16)[name = string("op_4586_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4587_cast_fp16 = mul(x = state_351_cast_fp16, y = var_4586_cast_fp16)[name = string("op_4587_cast_fp16")];
            tensor<int32, [1]> var_4589_axes_0 = const()[name = string("op_4589_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_4589_keep_dims_0 = const()[name = string("op_4589_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_4589_cast_fp16 = reduce_sum(axes = var_4589_axes_0, keep_dims = var_4589_keep_dims_0, x = var_4587_cast_fp16)[name = string("op_4589_cast_fp16")];
            tensor<int32, [3]> var_4592_begin_0 = const()[name = string("op_4592_begin_0"), val = tensor<int32, [3]>([0, 0, 8])];
            tensor<int32, [3]> var_4592_end_0 = const()[name = string("op_4592_end_0"), val = tensor<int32, [3]>([1, 16, 9])];
            tensor<bool, [3]> var_4592_end_mask_0 = const()[name = string("op_4592_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_4592_squeeze_mask_0 = const()[name = string("op_4592_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_4592_cast_fp16 = slice_by_index(begin = var_4592_begin_0, end = var_4592_end_0, end_mask = var_4592_end_mask_0, squeeze_mask = var_4592_squeeze_mask_0, x = decay_11_cast_fp16)[name = string("op_4592_cast_fp16")];
            tensor<fp16, [1, 16]> var_4593_cast_fp16 = exp(x = var_4592_cast_fp16)[name = string("op_4593_cast_fp16")];
            tensor<int32, [1]> var_4594_axes_0 = const()[name = string("op_4594_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_4594_cast_fp16 = expand_dims(axes = var_4594_axes_0, x = var_4593_cast_fp16)[name = string("op_4594_cast_fp16")];
            tensor<int32, [1]> var_4595_axes_0 = const()[name = string("op_4595_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_4595_cast_fp16 = expand_dims(axes = var_4595_axes_0, x = var_4594_cast_fp16)[name = string("op_4595_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_353_cast_fp16 = mul(x = state_351_cast_fp16, y = var_4595_cast_fp16)[name = string("state_353_cast_fp16")];
            tensor<int32, [4]> key_token_177_begin_0 = const()[name = string("key_token_177_begin_0"), val = tensor<int32, [4]>([0, 0, 8, 0])];
            tensor<int32, [4]> key_token_177_end_0 = const()[name = string("key_token_177_end_0"), val = tensor<int32, [4]>([1, 16, 9, 128])];
            tensor<bool, [4]> key_token_177_end_mask_0 = const()[name = string("key_token_177_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_177_squeeze_mask_0 = const()[name = string("key_token_177_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_177_cast_fp16 = slice_by_index(begin = key_token_177_begin_0, end = key_token_177_end_0, end_mask = key_token_177_end_mask_0, squeeze_mask = key_token_177_squeeze_mask_0, x = key_27_cast_fp16)[name = string("key_token_177_cast_fp16")];
            tensor<int32, [4]> value_token_177_begin_0 = const()[name = string("value_token_177_begin_0"), val = tensor<int32, [4]>([0, 0, 8, 0])];
            tensor<int32, [4]> value_token_177_end_0 = const()[name = string("value_token_177_end_0"), val = tensor<int32, [4]>([1, 16, 9, 128])];
            tensor<bool, [4]> value_token_177_end_mask_0 = const()[name = string("value_token_177_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_177_squeeze_mask_0 = const()[name = string("value_token_177_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_177_cast_fp16 = slice_by_index(begin = value_token_177_begin_0, end = value_token_177_end_0, end_mask = value_token_177_end_mask_0, squeeze_mask = value_token_177_squeeze_mask_0, x = value_97_cast_fp16)[name = string("value_token_177_cast_fp16")];
            tensor<int32, [1]> var_4603_axes_0 = const()[name = string("op_4603_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_4603_cast_fp16 = expand_dims(axes = var_4603_axes_0, x = key_token_177_cast_fp16)[name = string("op_4603_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4604_cast_fp16 = mul(x = state_353_cast_fp16, y = var_4603_cast_fp16)[name = string("op_4604_cast_fp16")];
            tensor<int32, [1]> memory_177_axes_0 = const()[name = string("memory_177_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_177_keep_dims_0 = const()[name = string("memory_177_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_177_cast_fp16 = reduce_sum(axes = memory_177_axes_0, keep_dims = memory_177_keep_dims_0, x = var_4604_cast_fp16)[name = string("memory_177_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_4607_cast_fp16 = sub(x = value_token_177_cast_fp16, y = memory_177_cast_fp16)[name = string("op_4607_cast_fp16")];
            tensor<int32, [3]> var_4610_begin_0 = const()[name = string("op_4610_begin_0"), val = tensor<int32, [3]>([0, 0, 8])];
            tensor<int32, [3]> var_4610_end_0 = const()[name = string("op_4610_end_0"), val = tensor<int32, [3]>([1, 16, 9])];
            tensor<bool, [3]> var_4610_end_mask_0 = const()[name = string("op_4610_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_4610_squeeze_mask_0 = const()[name = string("op_4610_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_4610_cast_fp16 = slice_by_index(begin = var_4610_begin_0, end = var_4610_end_0, end_mask = var_4610_end_mask_0, squeeze_mask = var_4610_squeeze_mask_0, x = beta_11_cast_fp16)[name = string("op_4610_cast_fp16")];
            tensor<int32, [1]> var_4611_axes_0 = const()[name = string("op_4611_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_4611_cast_fp16 = expand_dims(axes = var_4611_axes_0, x = var_4610_cast_fp16)[name = string("op_4611_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_177_cast_fp16 = mul(x = var_4607_cast_fp16, y = var_4611_cast_fp16)[name = string("delta_177_cast_fp16")];
            tensor<int32, [1]> var_4614_axes_0 = const()[name = string("op_4614_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_4614_cast_fp16 = expand_dims(axes = var_4614_axes_0, x = delta_177_cast_fp16)[name = string("op_4614_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4615_cast_fp16 = mul(x = var_4603_cast_fp16, y = var_4614_cast_fp16)[name = string("op_4615_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_355_cast_fp16 = add(x = state_353_cast_fp16, y = var_4615_cast_fp16)[name = string("state_355_cast_fp16")];
            tensor<int32, [4]> var_4619_begin_0 = const()[name = string("op_4619_begin_0"), val = tensor<int32, [4]>([0, 0, 8, 0])];
            tensor<int32, [4]> var_4619_end_0 = const()[name = string("op_4619_end_0"), val = tensor<int32, [4]>([1, 16, 9, 128])];
            tensor<bool, [4]> var_4619_end_mask_0 = const()[name = string("op_4619_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_4619_squeeze_mask_0 = const()[name = string("op_4619_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_4619_cast_fp16 = slice_by_index(begin = var_4619_begin_0, end = var_4619_end_0, end_mask = var_4619_end_mask_0, squeeze_mask = var_4619_squeeze_mask_0, x = _inversed_query_27_cast_fp16)[name = string("op_4619_cast_fp16")];
            tensor<int32, [1]> var_4620_axes_0 = const()[name = string("op_4620_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_4620_cast_fp16 = expand_dims(axes = var_4620_axes_0, x = var_4619_cast_fp16)[name = string("op_4620_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4621_cast_fp16 = mul(x = state_355_cast_fp16, y = var_4620_cast_fp16)[name = string("op_4621_cast_fp16")];
            tensor<int32, [1]> var_4623_axes_0 = const()[name = string("op_4623_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_4623_keep_dims_0 = const()[name = string("op_4623_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_4623_cast_fp16 = reduce_sum(axes = var_4623_axes_0, keep_dims = var_4623_keep_dims_0, x = var_4621_cast_fp16)[name = string("op_4623_cast_fp16")];
            tensor<int32, [3]> var_4626_begin_0 = const()[name = string("op_4626_begin_0"), val = tensor<int32, [3]>([0, 0, 9])];
            tensor<int32, [3]> var_4626_end_0 = const()[name = string("op_4626_end_0"), val = tensor<int32, [3]>([1, 16, 10])];
            tensor<bool, [3]> var_4626_end_mask_0 = const()[name = string("op_4626_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_4626_squeeze_mask_0 = const()[name = string("op_4626_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_4626_cast_fp16 = slice_by_index(begin = var_4626_begin_0, end = var_4626_end_0, end_mask = var_4626_end_mask_0, squeeze_mask = var_4626_squeeze_mask_0, x = decay_11_cast_fp16)[name = string("op_4626_cast_fp16")];
            tensor<fp16, [1, 16]> var_4627_cast_fp16 = exp(x = var_4626_cast_fp16)[name = string("op_4627_cast_fp16")];
            tensor<int32, [1]> var_4628_axes_0 = const()[name = string("op_4628_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_4628_cast_fp16 = expand_dims(axes = var_4628_axes_0, x = var_4627_cast_fp16)[name = string("op_4628_cast_fp16")];
            tensor<int32, [1]> var_4629_axes_0 = const()[name = string("op_4629_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_4629_cast_fp16 = expand_dims(axes = var_4629_axes_0, x = var_4628_cast_fp16)[name = string("op_4629_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_357_cast_fp16 = mul(x = state_355_cast_fp16, y = var_4629_cast_fp16)[name = string("state_357_cast_fp16")];
            tensor<int32, [4]> key_token_179_begin_0 = const()[name = string("key_token_179_begin_0"), val = tensor<int32, [4]>([0, 0, 9, 0])];
            tensor<int32, [4]> key_token_179_end_0 = const()[name = string("key_token_179_end_0"), val = tensor<int32, [4]>([1, 16, 10, 128])];
            tensor<bool, [4]> key_token_179_end_mask_0 = const()[name = string("key_token_179_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_179_squeeze_mask_0 = const()[name = string("key_token_179_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_179_cast_fp16 = slice_by_index(begin = key_token_179_begin_0, end = key_token_179_end_0, end_mask = key_token_179_end_mask_0, squeeze_mask = key_token_179_squeeze_mask_0, x = key_27_cast_fp16)[name = string("key_token_179_cast_fp16")];
            tensor<int32, [4]> value_token_179_begin_0 = const()[name = string("value_token_179_begin_0"), val = tensor<int32, [4]>([0, 0, 9, 0])];
            tensor<int32, [4]> value_token_179_end_0 = const()[name = string("value_token_179_end_0"), val = tensor<int32, [4]>([1, 16, 10, 128])];
            tensor<bool, [4]> value_token_179_end_mask_0 = const()[name = string("value_token_179_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_179_squeeze_mask_0 = const()[name = string("value_token_179_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_179_cast_fp16 = slice_by_index(begin = value_token_179_begin_0, end = value_token_179_end_0, end_mask = value_token_179_end_mask_0, squeeze_mask = value_token_179_squeeze_mask_0, x = value_97_cast_fp16)[name = string("value_token_179_cast_fp16")];
            tensor<int32, [1]> var_4637_axes_0 = const()[name = string("op_4637_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_4637_cast_fp16 = expand_dims(axes = var_4637_axes_0, x = key_token_179_cast_fp16)[name = string("op_4637_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4638_cast_fp16 = mul(x = state_357_cast_fp16, y = var_4637_cast_fp16)[name = string("op_4638_cast_fp16")];
            tensor<int32, [1]> memory_179_axes_0 = const()[name = string("memory_179_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_179_keep_dims_0 = const()[name = string("memory_179_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_179_cast_fp16 = reduce_sum(axes = memory_179_axes_0, keep_dims = memory_179_keep_dims_0, x = var_4638_cast_fp16)[name = string("memory_179_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_4641_cast_fp16 = sub(x = value_token_179_cast_fp16, y = memory_179_cast_fp16)[name = string("op_4641_cast_fp16")];
            tensor<int32, [3]> var_4644_begin_0 = const()[name = string("op_4644_begin_0"), val = tensor<int32, [3]>([0, 0, 9])];
            tensor<int32, [3]> var_4644_end_0 = const()[name = string("op_4644_end_0"), val = tensor<int32, [3]>([1, 16, 10])];
            tensor<bool, [3]> var_4644_end_mask_0 = const()[name = string("op_4644_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_4644_squeeze_mask_0 = const()[name = string("op_4644_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_4644_cast_fp16 = slice_by_index(begin = var_4644_begin_0, end = var_4644_end_0, end_mask = var_4644_end_mask_0, squeeze_mask = var_4644_squeeze_mask_0, x = beta_11_cast_fp16)[name = string("op_4644_cast_fp16")];
            tensor<int32, [1]> var_4645_axes_0 = const()[name = string("op_4645_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_4645_cast_fp16 = expand_dims(axes = var_4645_axes_0, x = var_4644_cast_fp16)[name = string("op_4645_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_179_cast_fp16 = mul(x = var_4641_cast_fp16, y = var_4645_cast_fp16)[name = string("delta_179_cast_fp16")];
            tensor<int32, [1]> var_4648_axes_0 = const()[name = string("op_4648_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_4648_cast_fp16 = expand_dims(axes = var_4648_axes_0, x = delta_179_cast_fp16)[name = string("op_4648_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4649_cast_fp16 = mul(x = var_4637_cast_fp16, y = var_4648_cast_fp16)[name = string("op_4649_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_359_cast_fp16 = add(x = state_357_cast_fp16, y = var_4649_cast_fp16)[name = string("state_359_cast_fp16")];
            tensor<int32, [4]> var_4653_begin_0 = const()[name = string("op_4653_begin_0"), val = tensor<int32, [4]>([0, 0, 9, 0])];
            tensor<int32, [4]> var_4653_end_0 = const()[name = string("op_4653_end_0"), val = tensor<int32, [4]>([1, 16, 10, 128])];
            tensor<bool, [4]> var_4653_end_mask_0 = const()[name = string("op_4653_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_4653_squeeze_mask_0 = const()[name = string("op_4653_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_4653_cast_fp16 = slice_by_index(begin = var_4653_begin_0, end = var_4653_end_0, end_mask = var_4653_end_mask_0, squeeze_mask = var_4653_squeeze_mask_0, x = _inversed_query_27_cast_fp16)[name = string("op_4653_cast_fp16")];
            tensor<int32, [1]> var_4654_axes_0 = const()[name = string("op_4654_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_4654_cast_fp16 = expand_dims(axes = var_4654_axes_0, x = var_4653_cast_fp16)[name = string("op_4654_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4655_cast_fp16 = mul(x = state_359_cast_fp16, y = var_4654_cast_fp16)[name = string("op_4655_cast_fp16")];
            tensor<int32, [1]> var_4657_axes_0 = const()[name = string("op_4657_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_4657_keep_dims_0 = const()[name = string("op_4657_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_4657_cast_fp16 = reduce_sum(axes = var_4657_axes_0, keep_dims = var_4657_keep_dims_0, x = var_4655_cast_fp16)[name = string("op_4657_cast_fp16")];
            tensor<int32, [3]> var_4660_begin_0 = const()[name = string("op_4660_begin_0"), val = tensor<int32, [3]>([0, 0, 10])];
            tensor<int32, [3]> var_4660_end_0 = const()[name = string("op_4660_end_0"), val = tensor<int32, [3]>([1, 16, 11])];
            tensor<bool, [3]> var_4660_end_mask_0 = const()[name = string("op_4660_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_4660_squeeze_mask_0 = const()[name = string("op_4660_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_4660_cast_fp16 = slice_by_index(begin = var_4660_begin_0, end = var_4660_end_0, end_mask = var_4660_end_mask_0, squeeze_mask = var_4660_squeeze_mask_0, x = decay_11_cast_fp16)[name = string("op_4660_cast_fp16")];
            tensor<fp16, [1, 16]> var_4661_cast_fp16 = exp(x = var_4660_cast_fp16)[name = string("op_4661_cast_fp16")];
            tensor<int32, [1]> var_4662_axes_0 = const()[name = string("op_4662_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_4662_cast_fp16 = expand_dims(axes = var_4662_axes_0, x = var_4661_cast_fp16)[name = string("op_4662_cast_fp16")];
            tensor<int32, [1]> var_4663_axes_0 = const()[name = string("op_4663_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_4663_cast_fp16 = expand_dims(axes = var_4663_axes_0, x = var_4662_cast_fp16)[name = string("op_4663_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_361_cast_fp16 = mul(x = state_359_cast_fp16, y = var_4663_cast_fp16)[name = string("state_361_cast_fp16")];
            tensor<int32, [4]> key_token_181_begin_0 = const()[name = string("key_token_181_begin_0"), val = tensor<int32, [4]>([0, 0, 10, 0])];
            tensor<int32, [4]> key_token_181_end_0 = const()[name = string("key_token_181_end_0"), val = tensor<int32, [4]>([1, 16, 11, 128])];
            tensor<bool, [4]> key_token_181_end_mask_0 = const()[name = string("key_token_181_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_181_squeeze_mask_0 = const()[name = string("key_token_181_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_181_cast_fp16 = slice_by_index(begin = key_token_181_begin_0, end = key_token_181_end_0, end_mask = key_token_181_end_mask_0, squeeze_mask = key_token_181_squeeze_mask_0, x = key_27_cast_fp16)[name = string("key_token_181_cast_fp16")];
            tensor<int32, [4]> value_token_181_begin_0 = const()[name = string("value_token_181_begin_0"), val = tensor<int32, [4]>([0, 0, 10, 0])];
            tensor<int32, [4]> value_token_181_end_0 = const()[name = string("value_token_181_end_0"), val = tensor<int32, [4]>([1, 16, 11, 128])];
            tensor<bool, [4]> value_token_181_end_mask_0 = const()[name = string("value_token_181_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_181_squeeze_mask_0 = const()[name = string("value_token_181_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_181_cast_fp16 = slice_by_index(begin = value_token_181_begin_0, end = value_token_181_end_0, end_mask = value_token_181_end_mask_0, squeeze_mask = value_token_181_squeeze_mask_0, x = value_97_cast_fp16)[name = string("value_token_181_cast_fp16")];
            tensor<int32, [1]> var_4671_axes_0 = const()[name = string("op_4671_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_4671_cast_fp16 = expand_dims(axes = var_4671_axes_0, x = key_token_181_cast_fp16)[name = string("op_4671_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4672_cast_fp16 = mul(x = state_361_cast_fp16, y = var_4671_cast_fp16)[name = string("op_4672_cast_fp16")];
            tensor<int32, [1]> memory_181_axes_0 = const()[name = string("memory_181_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_181_keep_dims_0 = const()[name = string("memory_181_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_181_cast_fp16 = reduce_sum(axes = memory_181_axes_0, keep_dims = memory_181_keep_dims_0, x = var_4672_cast_fp16)[name = string("memory_181_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_4675_cast_fp16 = sub(x = value_token_181_cast_fp16, y = memory_181_cast_fp16)[name = string("op_4675_cast_fp16")];
            tensor<int32, [3]> var_4678_begin_0 = const()[name = string("op_4678_begin_0"), val = tensor<int32, [3]>([0, 0, 10])];
            tensor<int32, [3]> var_4678_end_0 = const()[name = string("op_4678_end_0"), val = tensor<int32, [3]>([1, 16, 11])];
            tensor<bool, [3]> var_4678_end_mask_0 = const()[name = string("op_4678_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_4678_squeeze_mask_0 = const()[name = string("op_4678_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_4678_cast_fp16 = slice_by_index(begin = var_4678_begin_0, end = var_4678_end_0, end_mask = var_4678_end_mask_0, squeeze_mask = var_4678_squeeze_mask_0, x = beta_11_cast_fp16)[name = string("op_4678_cast_fp16")];
            tensor<int32, [1]> var_4679_axes_0 = const()[name = string("op_4679_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_4679_cast_fp16 = expand_dims(axes = var_4679_axes_0, x = var_4678_cast_fp16)[name = string("op_4679_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_181_cast_fp16 = mul(x = var_4675_cast_fp16, y = var_4679_cast_fp16)[name = string("delta_181_cast_fp16")];
            tensor<int32, [1]> var_4682_axes_0 = const()[name = string("op_4682_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_4682_cast_fp16 = expand_dims(axes = var_4682_axes_0, x = delta_181_cast_fp16)[name = string("op_4682_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4683_cast_fp16 = mul(x = var_4671_cast_fp16, y = var_4682_cast_fp16)[name = string("op_4683_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_363_cast_fp16 = add(x = state_361_cast_fp16, y = var_4683_cast_fp16)[name = string("state_363_cast_fp16")];
            tensor<int32, [4]> var_4687_begin_0 = const()[name = string("op_4687_begin_0"), val = tensor<int32, [4]>([0, 0, 10, 0])];
            tensor<int32, [4]> var_4687_end_0 = const()[name = string("op_4687_end_0"), val = tensor<int32, [4]>([1, 16, 11, 128])];
            tensor<bool, [4]> var_4687_end_mask_0 = const()[name = string("op_4687_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_4687_squeeze_mask_0 = const()[name = string("op_4687_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_4687_cast_fp16 = slice_by_index(begin = var_4687_begin_0, end = var_4687_end_0, end_mask = var_4687_end_mask_0, squeeze_mask = var_4687_squeeze_mask_0, x = _inversed_query_27_cast_fp16)[name = string("op_4687_cast_fp16")];
            tensor<int32, [1]> var_4688_axes_0 = const()[name = string("op_4688_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_4688_cast_fp16 = expand_dims(axes = var_4688_axes_0, x = var_4687_cast_fp16)[name = string("op_4688_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4689_cast_fp16 = mul(x = state_363_cast_fp16, y = var_4688_cast_fp16)[name = string("op_4689_cast_fp16")];
            tensor<int32, [1]> var_4691_axes_0 = const()[name = string("op_4691_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_4691_keep_dims_0 = const()[name = string("op_4691_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_4691_cast_fp16 = reduce_sum(axes = var_4691_axes_0, keep_dims = var_4691_keep_dims_0, x = var_4689_cast_fp16)[name = string("op_4691_cast_fp16")];
            tensor<int32, [3]> var_4694_begin_0 = const()[name = string("op_4694_begin_0"), val = tensor<int32, [3]>([0, 0, 11])];
            tensor<int32, [3]> var_4694_end_0 = const()[name = string("op_4694_end_0"), val = tensor<int32, [3]>([1, 16, 12])];
            tensor<bool, [3]> var_4694_end_mask_0 = const()[name = string("op_4694_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_4694_squeeze_mask_0 = const()[name = string("op_4694_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_4694_cast_fp16 = slice_by_index(begin = var_4694_begin_0, end = var_4694_end_0, end_mask = var_4694_end_mask_0, squeeze_mask = var_4694_squeeze_mask_0, x = decay_11_cast_fp16)[name = string("op_4694_cast_fp16")];
            tensor<fp16, [1, 16]> var_4695_cast_fp16 = exp(x = var_4694_cast_fp16)[name = string("op_4695_cast_fp16")];
            tensor<int32, [1]> var_4696_axes_0 = const()[name = string("op_4696_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_4696_cast_fp16 = expand_dims(axes = var_4696_axes_0, x = var_4695_cast_fp16)[name = string("op_4696_cast_fp16")];
            tensor<int32, [1]> var_4697_axes_0 = const()[name = string("op_4697_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_4697_cast_fp16 = expand_dims(axes = var_4697_axes_0, x = var_4696_cast_fp16)[name = string("op_4697_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_365_cast_fp16 = mul(x = state_363_cast_fp16, y = var_4697_cast_fp16)[name = string("state_365_cast_fp16")];
            tensor<int32, [4]> key_token_183_begin_0 = const()[name = string("key_token_183_begin_0"), val = tensor<int32, [4]>([0, 0, 11, 0])];
            tensor<int32, [4]> key_token_183_end_0 = const()[name = string("key_token_183_end_0"), val = tensor<int32, [4]>([1, 16, 12, 128])];
            tensor<bool, [4]> key_token_183_end_mask_0 = const()[name = string("key_token_183_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_183_squeeze_mask_0 = const()[name = string("key_token_183_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_183_cast_fp16 = slice_by_index(begin = key_token_183_begin_0, end = key_token_183_end_0, end_mask = key_token_183_end_mask_0, squeeze_mask = key_token_183_squeeze_mask_0, x = key_27_cast_fp16)[name = string("key_token_183_cast_fp16")];
            tensor<int32, [4]> value_token_183_begin_0 = const()[name = string("value_token_183_begin_0"), val = tensor<int32, [4]>([0, 0, 11, 0])];
            tensor<int32, [4]> value_token_183_end_0 = const()[name = string("value_token_183_end_0"), val = tensor<int32, [4]>([1, 16, 12, 128])];
            tensor<bool, [4]> value_token_183_end_mask_0 = const()[name = string("value_token_183_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_183_squeeze_mask_0 = const()[name = string("value_token_183_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_183_cast_fp16 = slice_by_index(begin = value_token_183_begin_0, end = value_token_183_end_0, end_mask = value_token_183_end_mask_0, squeeze_mask = value_token_183_squeeze_mask_0, x = value_97_cast_fp16)[name = string("value_token_183_cast_fp16")];
            tensor<int32, [1]> var_4705_axes_0 = const()[name = string("op_4705_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_4705_cast_fp16 = expand_dims(axes = var_4705_axes_0, x = key_token_183_cast_fp16)[name = string("op_4705_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4706_cast_fp16 = mul(x = state_365_cast_fp16, y = var_4705_cast_fp16)[name = string("op_4706_cast_fp16")];
            tensor<int32, [1]> memory_183_axes_0 = const()[name = string("memory_183_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_183_keep_dims_0 = const()[name = string("memory_183_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_183_cast_fp16 = reduce_sum(axes = memory_183_axes_0, keep_dims = memory_183_keep_dims_0, x = var_4706_cast_fp16)[name = string("memory_183_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_4709_cast_fp16 = sub(x = value_token_183_cast_fp16, y = memory_183_cast_fp16)[name = string("op_4709_cast_fp16")];
            tensor<int32, [3]> var_4712_begin_0 = const()[name = string("op_4712_begin_0"), val = tensor<int32, [3]>([0, 0, 11])];
            tensor<int32, [3]> var_4712_end_0 = const()[name = string("op_4712_end_0"), val = tensor<int32, [3]>([1, 16, 12])];
            tensor<bool, [3]> var_4712_end_mask_0 = const()[name = string("op_4712_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_4712_squeeze_mask_0 = const()[name = string("op_4712_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_4712_cast_fp16 = slice_by_index(begin = var_4712_begin_0, end = var_4712_end_0, end_mask = var_4712_end_mask_0, squeeze_mask = var_4712_squeeze_mask_0, x = beta_11_cast_fp16)[name = string("op_4712_cast_fp16")];
            tensor<int32, [1]> var_4713_axes_0 = const()[name = string("op_4713_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_4713_cast_fp16 = expand_dims(axes = var_4713_axes_0, x = var_4712_cast_fp16)[name = string("op_4713_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_183_cast_fp16 = mul(x = var_4709_cast_fp16, y = var_4713_cast_fp16)[name = string("delta_183_cast_fp16")];
            tensor<int32, [1]> var_4716_axes_0 = const()[name = string("op_4716_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_4716_cast_fp16 = expand_dims(axes = var_4716_axes_0, x = delta_183_cast_fp16)[name = string("op_4716_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4717_cast_fp16 = mul(x = var_4705_cast_fp16, y = var_4716_cast_fp16)[name = string("op_4717_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_367_cast_fp16 = add(x = state_365_cast_fp16, y = var_4717_cast_fp16)[name = string("state_367_cast_fp16")];
            tensor<int32, [4]> var_4721_begin_0 = const()[name = string("op_4721_begin_0"), val = tensor<int32, [4]>([0, 0, 11, 0])];
            tensor<int32, [4]> var_4721_end_0 = const()[name = string("op_4721_end_0"), val = tensor<int32, [4]>([1, 16, 12, 128])];
            tensor<bool, [4]> var_4721_end_mask_0 = const()[name = string("op_4721_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_4721_squeeze_mask_0 = const()[name = string("op_4721_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_4721_cast_fp16 = slice_by_index(begin = var_4721_begin_0, end = var_4721_end_0, end_mask = var_4721_end_mask_0, squeeze_mask = var_4721_squeeze_mask_0, x = _inversed_query_27_cast_fp16)[name = string("op_4721_cast_fp16")];
            tensor<int32, [1]> var_4722_axes_0 = const()[name = string("op_4722_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_4722_cast_fp16 = expand_dims(axes = var_4722_axes_0, x = var_4721_cast_fp16)[name = string("op_4722_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4723_cast_fp16 = mul(x = state_367_cast_fp16, y = var_4722_cast_fp16)[name = string("op_4723_cast_fp16")];
            tensor<int32, [1]> var_4725_axes_0 = const()[name = string("op_4725_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_4725_keep_dims_0 = const()[name = string("op_4725_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_4725_cast_fp16 = reduce_sum(axes = var_4725_axes_0, keep_dims = var_4725_keep_dims_0, x = var_4723_cast_fp16)[name = string("op_4725_cast_fp16")];
            tensor<int32, [3]> var_4728_begin_0 = const()[name = string("op_4728_begin_0"), val = tensor<int32, [3]>([0, 0, 12])];
            tensor<int32, [3]> var_4728_end_0 = const()[name = string("op_4728_end_0"), val = tensor<int32, [3]>([1, 16, 13])];
            tensor<bool, [3]> var_4728_end_mask_0 = const()[name = string("op_4728_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_4728_squeeze_mask_0 = const()[name = string("op_4728_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_4728_cast_fp16 = slice_by_index(begin = var_4728_begin_0, end = var_4728_end_0, end_mask = var_4728_end_mask_0, squeeze_mask = var_4728_squeeze_mask_0, x = decay_11_cast_fp16)[name = string("op_4728_cast_fp16")];
            tensor<fp16, [1, 16]> var_4729_cast_fp16 = exp(x = var_4728_cast_fp16)[name = string("op_4729_cast_fp16")];
            tensor<int32, [1]> var_4730_axes_0 = const()[name = string("op_4730_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_4730_cast_fp16 = expand_dims(axes = var_4730_axes_0, x = var_4729_cast_fp16)[name = string("op_4730_cast_fp16")];
            tensor<int32, [1]> var_4731_axes_0 = const()[name = string("op_4731_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_4731_cast_fp16 = expand_dims(axes = var_4731_axes_0, x = var_4730_cast_fp16)[name = string("op_4731_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_369_cast_fp16 = mul(x = state_367_cast_fp16, y = var_4731_cast_fp16)[name = string("state_369_cast_fp16")];
            tensor<int32, [4]> key_token_185_begin_0 = const()[name = string("key_token_185_begin_0"), val = tensor<int32, [4]>([0, 0, 12, 0])];
            tensor<int32, [4]> key_token_185_end_0 = const()[name = string("key_token_185_end_0"), val = tensor<int32, [4]>([1, 16, 13, 128])];
            tensor<bool, [4]> key_token_185_end_mask_0 = const()[name = string("key_token_185_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_185_squeeze_mask_0 = const()[name = string("key_token_185_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_185_cast_fp16 = slice_by_index(begin = key_token_185_begin_0, end = key_token_185_end_0, end_mask = key_token_185_end_mask_0, squeeze_mask = key_token_185_squeeze_mask_0, x = key_27_cast_fp16)[name = string("key_token_185_cast_fp16")];
            tensor<int32, [4]> value_token_185_begin_0 = const()[name = string("value_token_185_begin_0"), val = tensor<int32, [4]>([0, 0, 12, 0])];
            tensor<int32, [4]> value_token_185_end_0 = const()[name = string("value_token_185_end_0"), val = tensor<int32, [4]>([1, 16, 13, 128])];
            tensor<bool, [4]> value_token_185_end_mask_0 = const()[name = string("value_token_185_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_185_squeeze_mask_0 = const()[name = string("value_token_185_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_185_cast_fp16 = slice_by_index(begin = value_token_185_begin_0, end = value_token_185_end_0, end_mask = value_token_185_end_mask_0, squeeze_mask = value_token_185_squeeze_mask_0, x = value_97_cast_fp16)[name = string("value_token_185_cast_fp16")];
            tensor<int32, [1]> var_4739_axes_0 = const()[name = string("op_4739_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_4739_cast_fp16 = expand_dims(axes = var_4739_axes_0, x = key_token_185_cast_fp16)[name = string("op_4739_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4740_cast_fp16 = mul(x = state_369_cast_fp16, y = var_4739_cast_fp16)[name = string("op_4740_cast_fp16")];
            tensor<int32, [1]> memory_185_axes_0 = const()[name = string("memory_185_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_185_keep_dims_0 = const()[name = string("memory_185_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_185_cast_fp16 = reduce_sum(axes = memory_185_axes_0, keep_dims = memory_185_keep_dims_0, x = var_4740_cast_fp16)[name = string("memory_185_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_4743_cast_fp16 = sub(x = value_token_185_cast_fp16, y = memory_185_cast_fp16)[name = string("op_4743_cast_fp16")];
            tensor<int32, [3]> var_4746_begin_0 = const()[name = string("op_4746_begin_0"), val = tensor<int32, [3]>([0, 0, 12])];
            tensor<int32, [3]> var_4746_end_0 = const()[name = string("op_4746_end_0"), val = tensor<int32, [3]>([1, 16, 13])];
            tensor<bool, [3]> var_4746_end_mask_0 = const()[name = string("op_4746_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_4746_squeeze_mask_0 = const()[name = string("op_4746_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_4746_cast_fp16 = slice_by_index(begin = var_4746_begin_0, end = var_4746_end_0, end_mask = var_4746_end_mask_0, squeeze_mask = var_4746_squeeze_mask_0, x = beta_11_cast_fp16)[name = string("op_4746_cast_fp16")];
            tensor<int32, [1]> var_4747_axes_0 = const()[name = string("op_4747_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_4747_cast_fp16 = expand_dims(axes = var_4747_axes_0, x = var_4746_cast_fp16)[name = string("op_4747_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_185_cast_fp16 = mul(x = var_4743_cast_fp16, y = var_4747_cast_fp16)[name = string("delta_185_cast_fp16")];
            tensor<int32, [1]> var_4750_axes_0 = const()[name = string("op_4750_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_4750_cast_fp16 = expand_dims(axes = var_4750_axes_0, x = delta_185_cast_fp16)[name = string("op_4750_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4751_cast_fp16 = mul(x = var_4739_cast_fp16, y = var_4750_cast_fp16)[name = string("op_4751_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_371_cast_fp16 = add(x = state_369_cast_fp16, y = var_4751_cast_fp16)[name = string("state_371_cast_fp16")];
            tensor<int32, [4]> var_4755_begin_0 = const()[name = string("op_4755_begin_0"), val = tensor<int32, [4]>([0, 0, 12, 0])];
            tensor<int32, [4]> var_4755_end_0 = const()[name = string("op_4755_end_0"), val = tensor<int32, [4]>([1, 16, 13, 128])];
            tensor<bool, [4]> var_4755_end_mask_0 = const()[name = string("op_4755_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_4755_squeeze_mask_0 = const()[name = string("op_4755_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_4755_cast_fp16 = slice_by_index(begin = var_4755_begin_0, end = var_4755_end_0, end_mask = var_4755_end_mask_0, squeeze_mask = var_4755_squeeze_mask_0, x = _inversed_query_27_cast_fp16)[name = string("op_4755_cast_fp16")];
            tensor<int32, [1]> var_4756_axes_0 = const()[name = string("op_4756_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_4756_cast_fp16 = expand_dims(axes = var_4756_axes_0, x = var_4755_cast_fp16)[name = string("op_4756_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4757_cast_fp16 = mul(x = state_371_cast_fp16, y = var_4756_cast_fp16)[name = string("op_4757_cast_fp16")];
            tensor<int32, [1]> var_4759_axes_0 = const()[name = string("op_4759_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_4759_keep_dims_0 = const()[name = string("op_4759_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_4759_cast_fp16 = reduce_sum(axes = var_4759_axes_0, keep_dims = var_4759_keep_dims_0, x = var_4757_cast_fp16)[name = string("op_4759_cast_fp16")];
            tensor<int32, [3]> var_4762_begin_0 = const()[name = string("op_4762_begin_0"), val = tensor<int32, [3]>([0, 0, 13])];
            tensor<int32, [3]> var_4762_end_0 = const()[name = string("op_4762_end_0"), val = tensor<int32, [3]>([1, 16, 14])];
            tensor<bool, [3]> var_4762_end_mask_0 = const()[name = string("op_4762_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_4762_squeeze_mask_0 = const()[name = string("op_4762_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_4762_cast_fp16 = slice_by_index(begin = var_4762_begin_0, end = var_4762_end_0, end_mask = var_4762_end_mask_0, squeeze_mask = var_4762_squeeze_mask_0, x = decay_11_cast_fp16)[name = string("op_4762_cast_fp16")];
            tensor<fp16, [1, 16]> var_4763_cast_fp16 = exp(x = var_4762_cast_fp16)[name = string("op_4763_cast_fp16")];
            tensor<int32, [1]> var_4764_axes_0 = const()[name = string("op_4764_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_4764_cast_fp16 = expand_dims(axes = var_4764_axes_0, x = var_4763_cast_fp16)[name = string("op_4764_cast_fp16")];
            tensor<int32, [1]> var_4765_axes_0 = const()[name = string("op_4765_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_4765_cast_fp16 = expand_dims(axes = var_4765_axes_0, x = var_4764_cast_fp16)[name = string("op_4765_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_373_cast_fp16 = mul(x = state_371_cast_fp16, y = var_4765_cast_fp16)[name = string("state_373_cast_fp16")];
            tensor<int32, [4]> key_token_187_begin_0 = const()[name = string("key_token_187_begin_0"), val = tensor<int32, [4]>([0, 0, 13, 0])];
            tensor<int32, [4]> key_token_187_end_0 = const()[name = string("key_token_187_end_0"), val = tensor<int32, [4]>([1, 16, 14, 128])];
            tensor<bool, [4]> key_token_187_end_mask_0 = const()[name = string("key_token_187_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_187_squeeze_mask_0 = const()[name = string("key_token_187_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_187_cast_fp16 = slice_by_index(begin = key_token_187_begin_0, end = key_token_187_end_0, end_mask = key_token_187_end_mask_0, squeeze_mask = key_token_187_squeeze_mask_0, x = key_27_cast_fp16)[name = string("key_token_187_cast_fp16")];
            tensor<int32, [4]> value_token_187_begin_0 = const()[name = string("value_token_187_begin_0"), val = tensor<int32, [4]>([0, 0, 13, 0])];
            tensor<int32, [4]> value_token_187_end_0 = const()[name = string("value_token_187_end_0"), val = tensor<int32, [4]>([1, 16, 14, 128])];
            tensor<bool, [4]> value_token_187_end_mask_0 = const()[name = string("value_token_187_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_187_squeeze_mask_0 = const()[name = string("value_token_187_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_187_cast_fp16 = slice_by_index(begin = value_token_187_begin_0, end = value_token_187_end_0, end_mask = value_token_187_end_mask_0, squeeze_mask = value_token_187_squeeze_mask_0, x = value_97_cast_fp16)[name = string("value_token_187_cast_fp16")];
            tensor<int32, [1]> var_4773_axes_0 = const()[name = string("op_4773_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_4773_cast_fp16 = expand_dims(axes = var_4773_axes_0, x = key_token_187_cast_fp16)[name = string("op_4773_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4774_cast_fp16 = mul(x = state_373_cast_fp16, y = var_4773_cast_fp16)[name = string("op_4774_cast_fp16")];
            tensor<int32, [1]> memory_187_axes_0 = const()[name = string("memory_187_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_187_keep_dims_0 = const()[name = string("memory_187_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_187_cast_fp16 = reduce_sum(axes = memory_187_axes_0, keep_dims = memory_187_keep_dims_0, x = var_4774_cast_fp16)[name = string("memory_187_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_4777_cast_fp16 = sub(x = value_token_187_cast_fp16, y = memory_187_cast_fp16)[name = string("op_4777_cast_fp16")];
            tensor<int32, [3]> var_4780_begin_0 = const()[name = string("op_4780_begin_0"), val = tensor<int32, [3]>([0, 0, 13])];
            tensor<int32, [3]> var_4780_end_0 = const()[name = string("op_4780_end_0"), val = tensor<int32, [3]>([1, 16, 14])];
            tensor<bool, [3]> var_4780_end_mask_0 = const()[name = string("op_4780_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_4780_squeeze_mask_0 = const()[name = string("op_4780_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_4780_cast_fp16 = slice_by_index(begin = var_4780_begin_0, end = var_4780_end_0, end_mask = var_4780_end_mask_0, squeeze_mask = var_4780_squeeze_mask_0, x = beta_11_cast_fp16)[name = string("op_4780_cast_fp16")];
            tensor<int32, [1]> var_4781_axes_0 = const()[name = string("op_4781_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_4781_cast_fp16 = expand_dims(axes = var_4781_axes_0, x = var_4780_cast_fp16)[name = string("op_4781_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_187_cast_fp16 = mul(x = var_4777_cast_fp16, y = var_4781_cast_fp16)[name = string("delta_187_cast_fp16")];
            tensor<int32, [1]> var_4784_axes_0 = const()[name = string("op_4784_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_4784_cast_fp16 = expand_dims(axes = var_4784_axes_0, x = delta_187_cast_fp16)[name = string("op_4784_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4785_cast_fp16 = mul(x = var_4773_cast_fp16, y = var_4784_cast_fp16)[name = string("op_4785_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_375_cast_fp16 = add(x = state_373_cast_fp16, y = var_4785_cast_fp16)[name = string("state_375_cast_fp16")];
            tensor<int32, [4]> var_4789_begin_0 = const()[name = string("op_4789_begin_0"), val = tensor<int32, [4]>([0, 0, 13, 0])];
            tensor<int32, [4]> var_4789_end_0 = const()[name = string("op_4789_end_0"), val = tensor<int32, [4]>([1, 16, 14, 128])];
            tensor<bool, [4]> var_4789_end_mask_0 = const()[name = string("op_4789_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_4789_squeeze_mask_0 = const()[name = string("op_4789_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_4789_cast_fp16 = slice_by_index(begin = var_4789_begin_0, end = var_4789_end_0, end_mask = var_4789_end_mask_0, squeeze_mask = var_4789_squeeze_mask_0, x = _inversed_query_27_cast_fp16)[name = string("op_4789_cast_fp16")];
            tensor<int32, [1]> var_4790_axes_0 = const()[name = string("op_4790_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_4790_cast_fp16 = expand_dims(axes = var_4790_axes_0, x = var_4789_cast_fp16)[name = string("op_4790_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4791_cast_fp16 = mul(x = state_375_cast_fp16, y = var_4790_cast_fp16)[name = string("op_4791_cast_fp16")];
            tensor<int32, [1]> var_4793_axes_0 = const()[name = string("op_4793_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_4793_keep_dims_0 = const()[name = string("op_4793_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_4793_cast_fp16 = reduce_sum(axes = var_4793_axes_0, keep_dims = var_4793_keep_dims_0, x = var_4791_cast_fp16)[name = string("op_4793_cast_fp16")];
            tensor<int32, [3]> var_4796_begin_0 = const()[name = string("op_4796_begin_0"), val = tensor<int32, [3]>([0, 0, 14])];
            tensor<int32, [3]> var_4796_end_0 = const()[name = string("op_4796_end_0"), val = tensor<int32, [3]>([1, 16, 15])];
            tensor<bool, [3]> var_4796_end_mask_0 = const()[name = string("op_4796_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_4796_squeeze_mask_0 = const()[name = string("op_4796_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_4796_cast_fp16 = slice_by_index(begin = var_4796_begin_0, end = var_4796_end_0, end_mask = var_4796_end_mask_0, squeeze_mask = var_4796_squeeze_mask_0, x = decay_11_cast_fp16)[name = string("op_4796_cast_fp16")];
            tensor<fp16, [1, 16]> var_4797_cast_fp16 = exp(x = var_4796_cast_fp16)[name = string("op_4797_cast_fp16")];
            tensor<int32, [1]> var_4798_axes_0 = const()[name = string("op_4798_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_4798_cast_fp16 = expand_dims(axes = var_4798_axes_0, x = var_4797_cast_fp16)[name = string("op_4798_cast_fp16")];
            tensor<int32, [1]> var_4799_axes_0 = const()[name = string("op_4799_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_4799_cast_fp16 = expand_dims(axes = var_4799_axes_0, x = var_4798_cast_fp16)[name = string("op_4799_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_377_cast_fp16 = mul(x = state_375_cast_fp16, y = var_4799_cast_fp16)[name = string("state_377_cast_fp16")];
            tensor<int32, [4]> key_token_189_begin_0 = const()[name = string("key_token_189_begin_0"), val = tensor<int32, [4]>([0, 0, 14, 0])];
            tensor<int32, [4]> key_token_189_end_0 = const()[name = string("key_token_189_end_0"), val = tensor<int32, [4]>([1, 16, 15, 128])];
            tensor<bool, [4]> key_token_189_end_mask_0 = const()[name = string("key_token_189_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_189_squeeze_mask_0 = const()[name = string("key_token_189_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_189_cast_fp16 = slice_by_index(begin = key_token_189_begin_0, end = key_token_189_end_0, end_mask = key_token_189_end_mask_0, squeeze_mask = key_token_189_squeeze_mask_0, x = key_27_cast_fp16)[name = string("key_token_189_cast_fp16")];
            tensor<int32, [4]> value_token_189_begin_0 = const()[name = string("value_token_189_begin_0"), val = tensor<int32, [4]>([0, 0, 14, 0])];
            tensor<int32, [4]> value_token_189_end_0 = const()[name = string("value_token_189_end_0"), val = tensor<int32, [4]>([1, 16, 15, 128])];
            tensor<bool, [4]> value_token_189_end_mask_0 = const()[name = string("value_token_189_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_189_squeeze_mask_0 = const()[name = string("value_token_189_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_189_cast_fp16 = slice_by_index(begin = value_token_189_begin_0, end = value_token_189_end_0, end_mask = value_token_189_end_mask_0, squeeze_mask = value_token_189_squeeze_mask_0, x = value_97_cast_fp16)[name = string("value_token_189_cast_fp16")];
            tensor<int32, [1]> var_4807_axes_0 = const()[name = string("op_4807_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_4807_cast_fp16 = expand_dims(axes = var_4807_axes_0, x = key_token_189_cast_fp16)[name = string("op_4807_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4808_cast_fp16 = mul(x = state_377_cast_fp16, y = var_4807_cast_fp16)[name = string("op_4808_cast_fp16")];
            tensor<int32, [1]> memory_189_axes_0 = const()[name = string("memory_189_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_189_keep_dims_0 = const()[name = string("memory_189_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_189_cast_fp16 = reduce_sum(axes = memory_189_axes_0, keep_dims = memory_189_keep_dims_0, x = var_4808_cast_fp16)[name = string("memory_189_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_4811_cast_fp16 = sub(x = value_token_189_cast_fp16, y = memory_189_cast_fp16)[name = string("op_4811_cast_fp16")];
            tensor<int32, [3]> var_4814_begin_0 = const()[name = string("op_4814_begin_0"), val = tensor<int32, [3]>([0, 0, 14])];
            tensor<int32, [3]> var_4814_end_0 = const()[name = string("op_4814_end_0"), val = tensor<int32, [3]>([1, 16, 15])];
            tensor<bool, [3]> var_4814_end_mask_0 = const()[name = string("op_4814_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_4814_squeeze_mask_0 = const()[name = string("op_4814_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_4814_cast_fp16 = slice_by_index(begin = var_4814_begin_0, end = var_4814_end_0, end_mask = var_4814_end_mask_0, squeeze_mask = var_4814_squeeze_mask_0, x = beta_11_cast_fp16)[name = string("op_4814_cast_fp16")];
            tensor<int32, [1]> var_4815_axes_0 = const()[name = string("op_4815_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_4815_cast_fp16 = expand_dims(axes = var_4815_axes_0, x = var_4814_cast_fp16)[name = string("op_4815_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_189_cast_fp16 = mul(x = var_4811_cast_fp16, y = var_4815_cast_fp16)[name = string("delta_189_cast_fp16")];
            tensor<int32, [1]> var_4818_axes_0 = const()[name = string("op_4818_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_4818_cast_fp16 = expand_dims(axes = var_4818_axes_0, x = delta_189_cast_fp16)[name = string("op_4818_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4819_cast_fp16 = mul(x = var_4807_cast_fp16, y = var_4818_cast_fp16)[name = string("op_4819_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_379_cast_fp16 = add(x = state_377_cast_fp16, y = var_4819_cast_fp16)[name = string("state_379_cast_fp16")];
            tensor<int32, [4]> var_4823_begin_0 = const()[name = string("op_4823_begin_0"), val = tensor<int32, [4]>([0, 0, 14, 0])];
            tensor<int32, [4]> var_4823_end_0 = const()[name = string("op_4823_end_0"), val = tensor<int32, [4]>([1, 16, 15, 128])];
            tensor<bool, [4]> var_4823_end_mask_0 = const()[name = string("op_4823_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_4823_squeeze_mask_0 = const()[name = string("op_4823_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_4823_cast_fp16 = slice_by_index(begin = var_4823_begin_0, end = var_4823_end_0, end_mask = var_4823_end_mask_0, squeeze_mask = var_4823_squeeze_mask_0, x = _inversed_query_27_cast_fp16)[name = string("op_4823_cast_fp16")];
            tensor<int32, [1]> var_4824_axes_0 = const()[name = string("op_4824_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_4824_cast_fp16 = expand_dims(axes = var_4824_axes_0, x = var_4823_cast_fp16)[name = string("op_4824_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4825_cast_fp16 = mul(x = state_379_cast_fp16, y = var_4824_cast_fp16)[name = string("op_4825_cast_fp16")];
            tensor<int32, [1]> var_4827_axes_0 = const()[name = string("op_4827_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_4827_keep_dims_0 = const()[name = string("op_4827_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_4827_cast_fp16 = reduce_sum(axes = var_4827_axes_0, keep_dims = var_4827_keep_dims_0, x = var_4825_cast_fp16)[name = string("op_4827_cast_fp16")];
            tensor<int32, [3]> var_4830_begin_0 = const()[name = string("op_4830_begin_0"), val = tensor<int32, [3]>([0, 0, 15])];
            tensor<int32, [3]> var_4830_end_0 = const()[name = string("op_4830_end_0"), val = tensor<int32, [3]>([1, 16, 16])];
            tensor<bool, [3]> var_4830_end_mask_0 = const()[name = string("op_4830_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_4830_squeeze_mask_0 = const()[name = string("op_4830_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_4830_cast_fp16 = slice_by_index(begin = var_4830_begin_0, end = var_4830_end_0, end_mask = var_4830_end_mask_0, squeeze_mask = var_4830_squeeze_mask_0, x = decay_11_cast_fp16)[name = string("op_4830_cast_fp16")];
            tensor<fp16, [1, 16]> var_4831_cast_fp16 = exp(x = var_4830_cast_fp16)[name = string("op_4831_cast_fp16")];
            tensor<int32, [1]> var_4832_axes_0 = const()[name = string("op_4832_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_4832_cast_fp16 = expand_dims(axes = var_4832_axes_0, x = var_4831_cast_fp16)[name = string("op_4832_cast_fp16")];
            tensor<int32, [1]> var_4833_axes_0 = const()[name = string("op_4833_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_4833_cast_fp16 = expand_dims(axes = var_4833_axes_0, x = var_4832_cast_fp16)[name = string("op_4833_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_381_cast_fp16 = mul(x = state_379_cast_fp16, y = var_4833_cast_fp16)[name = string("state_381_cast_fp16")];
            tensor<int32, [4]> key_token_191_begin_0 = const()[name = string("key_token_191_begin_0"), val = tensor<int32, [4]>([0, 0, 15, 0])];
            tensor<int32, [4]> key_token_191_end_0 = const()[name = string("key_token_191_end_0"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<bool, [4]> key_token_191_end_mask_0 = const()[name = string("key_token_191_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_191_squeeze_mask_0 = const()[name = string("key_token_191_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_191_cast_fp16 = slice_by_index(begin = key_token_191_begin_0, end = key_token_191_end_0, end_mask = key_token_191_end_mask_0, squeeze_mask = key_token_191_squeeze_mask_0, x = key_27_cast_fp16)[name = string("key_token_191_cast_fp16")];
            tensor<int32, [4]> value_token_191_begin_0 = const()[name = string("value_token_191_begin_0"), val = tensor<int32, [4]>([0, 0, 15, 0])];
            tensor<int32, [4]> value_token_191_end_0 = const()[name = string("value_token_191_end_0"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<bool, [4]> value_token_191_end_mask_0 = const()[name = string("value_token_191_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_191_squeeze_mask_0 = const()[name = string("value_token_191_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_191_cast_fp16 = slice_by_index(begin = value_token_191_begin_0, end = value_token_191_end_0, end_mask = value_token_191_end_mask_0, squeeze_mask = value_token_191_squeeze_mask_0, x = value_97_cast_fp16)[name = string("value_token_191_cast_fp16")];
            tensor<int32, [1]> var_4841_axes_0 = const()[name = string("op_4841_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_4841_cast_fp16 = expand_dims(axes = var_4841_axes_0, x = key_token_191_cast_fp16)[name = string("op_4841_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4842_cast_fp16 = mul(x = state_381_cast_fp16, y = var_4841_cast_fp16)[name = string("op_4842_cast_fp16")];
            tensor<int32, [1]> memory_191_axes_0 = const()[name = string("memory_191_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_191_keep_dims_0 = const()[name = string("memory_191_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_191_cast_fp16 = reduce_sum(axes = memory_191_axes_0, keep_dims = memory_191_keep_dims_0, x = var_4842_cast_fp16)[name = string("memory_191_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_4845_cast_fp16 = sub(x = value_token_191_cast_fp16, y = memory_191_cast_fp16)[name = string("op_4845_cast_fp16")];
            tensor<int32, [3]> var_4848_begin_0 = const()[name = string("op_4848_begin_0"), val = tensor<int32, [3]>([0, 0, 15])];
            tensor<int32, [3]> var_4848_end_0 = const()[name = string("op_4848_end_0"), val = tensor<int32, [3]>([1, 16, 16])];
            tensor<bool, [3]> var_4848_end_mask_0 = const()[name = string("op_4848_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_4848_squeeze_mask_0 = const()[name = string("op_4848_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_4848_cast_fp16 = slice_by_index(begin = var_4848_begin_0, end = var_4848_end_0, end_mask = var_4848_end_mask_0, squeeze_mask = var_4848_squeeze_mask_0, x = beta_11_cast_fp16)[name = string("op_4848_cast_fp16")];
            tensor<int32, [1]> var_4849_axes_0 = const()[name = string("op_4849_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_4849_cast_fp16 = expand_dims(axes = var_4849_axes_0, x = var_4848_cast_fp16)[name = string("op_4849_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_191_cast_fp16 = mul(x = var_4845_cast_fp16, y = var_4849_cast_fp16)[name = string("delta_191_cast_fp16")];
            tensor<int32, [1]> var_4852_axes_0 = const()[name = string("op_4852_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_4852_cast_fp16 = expand_dims(axes = var_4852_axes_0, x = delta_191_cast_fp16)[name = string("op_4852_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4853_cast_fp16 = mul(x = var_4841_cast_fp16, y = var_4852_cast_fp16)[name = string("op_4853_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> rec6_out = add(x = state_381_cast_fp16, y = var_4853_cast_fp16)[name = string("state_383_cast_fp16")];
            tensor<int32, [4]> var_4857_begin_0 = const()[name = string("op_4857_begin_0"), val = tensor<int32, [4]>([0, 0, 15, 0])];
            tensor<int32, [4]> var_4857_end_0 = const()[name = string("op_4857_end_0"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<bool, [4]> var_4857_end_mask_0 = const()[name = string("op_4857_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_4857_squeeze_mask_0 = const()[name = string("op_4857_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_4857_cast_fp16 = slice_by_index(begin = var_4857_begin_0, end = var_4857_end_0, end_mask = var_4857_end_mask_0, squeeze_mask = var_4857_squeeze_mask_0, x = _inversed_query_27_cast_fp16)[name = string("op_4857_cast_fp16")];
            tensor<int32, [1]> var_4858_axes_0 = const()[name = string("op_4858_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_4858_cast_fp16 = expand_dims(axes = var_4858_axes_0, x = var_4857_cast_fp16)[name = string("op_4858_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4859_cast_fp16 = mul(x = rec6_out, y = var_4858_cast_fp16)[name = string("op_4859_cast_fp16")];
            tensor<int32, [1]> var_4861_axes_0 = const()[name = string("op_4861_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_4861_keep_dims_0 = const()[name = string("op_4861_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_4861_cast_fp16 = reduce_sum(axes = var_4861_axes_0, keep_dims = var_4861_keep_dims_0, x = var_4859_cast_fp16)[name = string("op_4861_cast_fp16")];
            int32 attention_51_axis_0 = const()[name = string("attention_51_axis_0"), val = int32(1)];
            tensor<fp16, [1, 16, 16, 128]> attention_51_cast_fp16 = stack(axis = attention_51_axis_0, values = (var_4351_cast_fp16, var_4385_cast_fp16, var_4419_cast_fp16, var_4453_cast_fp16, var_4487_cast_fp16, var_4521_cast_fp16, var_4555_cast_fp16, var_4589_cast_fp16, var_4623_cast_fp16, var_4657_cast_fp16, var_4691_cast_fp16, var_4725_cast_fp16, var_4759_cast_fp16, var_4793_cast_fp16, var_4827_cast_fp16, var_4861_cast_fp16))[name = string("attention_51_cast_fp16")];
            tensor<int32, [2]> var_4864 = const()[name = string("op_4864"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [256, 128]> attention_53_cast_fp16 = reshape(shape = var_4864, x = attention_51_cast_fp16)[name = string("attention_53_cast_fp16")];
            tensor<int32, [2]> var_4866 = const()[name = string("op_4866"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [256, 128]> input_85_cast_fp16 = reshape(shape = var_4866, x = linear_50_cast_fp16)[name = string("input_85_cast_fp16")];
            tensor<fp16, [256, 128]> var_4868_cast_fp16 = mul(x = attention_53_cast_fp16, y = attention_53_cast_fp16)[name = string("op_4868_cast_fp16")];
            tensor<int32, [1]> variance_41_axes_0 = const()[name = string("variance_41_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_41_keep_dims_0 = const()[name = string("variance_41_keep_dims_0"), val = bool(true)];
            tensor<fp16, [256, 1]> variance_41_cast_fp16 = reduce_mean(axes = variance_41_axes_0, keep_dims = variance_41_keep_dims_0, x = var_4868_cast_fp16)[name = string("variance_41_cast_fp16")];
            fp16 var_4871_to_fp16 = const()[name = string("op_4871_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [256, 1]> var_4872_cast_fp16 = add(x = variance_41_cast_fp16, y = var_4871_to_fp16)[name = string("op_4872_cast_fp16")];
            fp32 var_4873_epsilon_0 = const()[name = string("op_4873_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [256, 1]> var_4873_cast_fp16 = rsqrt(epsilon = var_4873_epsilon_0, x = var_4872_cast_fp16)[name = string("op_4873_cast_fp16")];
            tensor<fp16, [256, 128]> attention_55_cast_fp16 = mul(x = attention_53_cast_fp16, y = var_4873_cast_fp16)[name = string("attention_55_cast_fp16")];
            tensor<fp16, [128]> groups_1_2_gated_norm_promoted_to_fp16 = const()[name = string("groups_1_2_gated_norm_promoted_to_fp16"), val = tensor<fp16, [128]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(102168768)))];
            tensor<fp16, [256, 128]> attention_57_cast_fp16 = mul(x = attention_55_cast_fp16, y = groups_1_2_gated_norm_promoted_to_fp16)[name = string("attention_57_cast_fp16")];
            tensor<fp16, [256, 128]> var_4876_cast_fp16 = silu(x = input_85_cast_fp16)[name = string("op_4876_cast_fp16")];
            tensor<fp16, [256, 128]> attention_59_cast_fp16 = mul(x = attention_57_cast_fp16, y = var_4876_cast_fp16)[name = string("attention_59_cast_fp16")];
            tensor<int32, [2]> var_4878 = const()[name = string("op_4878"), val = tensor<int32, [2]>([16, 2048])];
            tensor<fp16, [16, 2048]> input_87_cast_fp16 = reshape(shape = var_4878, x = attention_59_cast_fp16)[name = string("input_87_cast_fp16")];
            tensor<fp16, [1024, 2048]> groups_1_2_out_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 2048]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(102169088))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(103742016))))[name = string("groups_1_2_out_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 1024]> linear_51_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = groups_1_2_out_proj_weight_promoted_to_fp16_palettized, x = input_87_cast_fp16)[name = string("linear_51_cast_fp16")];
            tensor<fp16, [16, 1024]> value_99_cast_fp16 = add(x = value_87_cast_fp16, y = linear_51_cast_fp16)[name = string("value_99_cast_fp16")];
            tensor<fp16, [16, 1024]> var_4883_cast_fp16 = mul(x = value_99_cast_fp16, y = value_99_cast_fp16)[name = string("op_4883_cast_fp16")];
            tensor<int32, [1]> variance_43_axes_0 = const()[name = string("variance_43_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_43_keep_dims_0 = const()[name = string("variance_43_keep_dims_0"), val = bool(true)];
            tensor<fp16, [16, 1]> variance_43_cast_fp16 = reduce_mean(axes = variance_43_axes_0, keep_dims = variance_43_keep_dims_0, x = var_4883_cast_fp16)[name = string("variance_43_cast_fp16")];
            fp16 var_4886_to_fp16 = const()[name = string("op_4886_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [16, 1]> var_4887_cast_fp16 = add(x = variance_43_cast_fp16, y = var_4886_to_fp16)[name = string("op_4887_cast_fp16")];
            fp32 var_4888_epsilon_0 = const()[name = string("op_4888_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [16, 1]> var_4888_cast_fp16 = rsqrt(epsilon = var_4888_epsilon_0, x = var_4887_cast_fp16)[name = string("op_4888_cast_fp16")];
            tensor<fp16, [16, 1024]> var_4889_cast_fp16 = mul(x = value_99_cast_fp16, y = var_4888_cast_fp16)[name = string("op_4889_cast_fp16")];
            tensor<fp16, [1024]> var_4891_to_fp16 = const()[name = string("op_4891_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(103750272)))];
            tensor<fp16, [16, 1024]> input_89_cast_fp16 = mul(x = var_4889_cast_fp16, y = var_4891_to_fp16)[name = string("input_89_cast_fp16")];
            tensor<fp16, [3584, 1024]> groups_1_2_gate_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(103752384))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(106504960))))[name = string("groups_1_2_gate_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 3584]> linear_52_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = groups_1_2_gate_proj_weight_promoted_to_fp16_palettized, x = input_89_cast_fp16)[name = string("linear_52_cast_fp16")];
            tensor<fp16, [16, 3584]> var_4895_cast_fp16 = silu(x = linear_52_cast_fp16)[name = string("op_4895_cast_fp16")];
            tensor<fp16, [3584, 1024]> groups_1_2_up_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(106533696))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(109286272))))[name = string("groups_1_2_up_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 3584]> linear_53_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = groups_1_2_up_proj_weight_promoted_to_fp16_palettized, x = input_89_cast_fp16)[name = string("linear_53_cast_fp16")];
            tensor<fp16, [16, 3584]> input_93_cast_fp16 = mul(x = var_4895_cast_fp16, y = linear_53_cast_fp16)[name = string("input_93_cast_fp16")];
            tensor<fp16, [1024, 3584]> groups_1_2_down_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 3584]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(109315008))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(112067584))))[name = string("groups_1_2_down_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 1024]> linear_54_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = groups_1_2_down_proj_weight_promoted_to_fp16_palettized, x = input_93_cast_fp16)[name = string("linear_54_cast_fp16")];
            tensor<fp16, [16, 1024]> value_101_cast_fp16 = add(x = value_99_cast_fp16, y = linear_54_cast_fp16)[name = string("value_101_cast_fp16")];
            int32 var_4920 = const()[name = string("op_4920"), val = int32(-1)];
            tensor<fp16, [16, 1024]> var_4927_cast_fp16 = mul(x = value_101_cast_fp16, y = value_101_cast_fp16)[name = string("op_4927_cast_fp16")];
            tensor<int32, [1]> variance_45_axes_0 = const()[name = string("variance_45_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_45_keep_dims_0 = const()[name = string("variance_45_keep_dims_0"), val = bool(true)];
            tensor<fp16, [16, 1]> variance_45_cast_fp16 = reduce_mean(axes = variance_45_axes_0, keep_dims = variance_45_keep_dims_0, x = var_4927_cast_fp16)[name = string("variance_45_cast_fp16")];
            fp16 var_4930_to_fp16 = const()[name = string("op_4930_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [16, 1]> var_4931_cast_fp16 = add(x = variance_45_cast_fp16, y = var_4930_to_fp16)[name = string("op_4931_cast_fp16")];
            fp32 var_4932_epsilon_0 = const()[name = string("op_4932_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [16, 1]> var_4932_cast_fp16 = rsqrt(epsilon = var_4932_epsilon_0, x = var_4931_cast_fp16)[name = string("op_4932_cast_fp16")];
            tensor<fp16, [16, 1024]> var_4933_cast_fp16 = mul(x = value_101_cast_fp16, y = var_4932_cast_fp16)[name = string("op_4933_cast_fp16")];
            tensor<fp16, [1024]> var_4935_to_fp16 = const()[name = string("op_4935_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(112075840)))];
            tensor<fp16, [16, 1024]> input_95_cast_fp16 = mul(x = var_4933_cast_fp16, y = var_4935_to_fp16)[name = string("input_95_cast_fp16")];
            tensor<fp16, [4096, 1024]> projections_1_q_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [4096, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(112077952))), lut = tensor<fp16, [256, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(115223744))))[name = string("projections_1_q_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 4096]> linear_55_cast_fp16 = linear(bias = linear_24_bias_0_to_fp16, weight = projections_1_q_proj_weight_promoted_to_fp16_palettized, x = input_95_cast_fp16)[name = string("linear_55_cast_fp16")];
            tensor<int32, [3]> var_4939 = const()[name = string("op_4939"), val = tensor<int32, [3]>([16, 8, 512])];
            tensor<fp16, [16, 8, 512]> query_and_gate_3_cast_fp16 = reshape(shape = var_4939, x = linear_55_cast_fp16)[name = string("query_and_gate_3_cast_fp16")];
            tensor<int32, [2]> var_4941_split_sizes_0 = const()[name = string("op_4941_split_sizes_0"), val = tensor<int32, [2]>([256, 256])];
            int32 var_4941_axis_0 = const()[name = string("op_4941_axis_0"), val = int32(-1)];
            tensor<fp16, [16, 8, 256]> var_4941_cast_fp16_0, tensor<fp16, [16, 8, 256]> var_4941_cast_fp16_1 = split(axis = var_4941_axis_0, split_sizes = var_4941_split_sizes_0, x = query_and_gate_3_cast_fp16)[name = string("op_4941_cast_fp16")];
            tensor<fp16, [512, 1024]> projections_1_k_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [512, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(115256576))), lut = tensor<fp16, [32, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(115649856))))[name = string("projections_1_k_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 512]> linear_56_cast_fp16 = linear(bias = linear_25_bias_0_to_fp16, weight = projections_1_k_proj_weight_promoted_to_fp16_palettized, x = input_95_cast_fp16)[name = string("linear_56_cast_fp16")];
            tensor<int32, [3]> var_4945 = const()[name = string("op_4945"), val = tensor<int32, [3]>([16, 2, 256])];
            tensor<fp16, [16, 2, 256]> value_105_cast_fp16 = reshape(shape = var_4945, x = linear_56_cast_fp16)[name = string("value_105_cast_fp16")];
            tensor<fp16, [512, 1024]> projections_1_v_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [512, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(115654016))), lut = tensor<fp16, [32, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(116047296))))[name = string("projections_1_v_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 512]> linear_57_cast_fp16 = linear(bias = linear_25_bias_0_to_fp16, weight = projections_1_v_proj_weight_promoted_to_fp16_palettized, x = input_95_cast_fp16)[name = string("linear_57_cast_fp16")];
            tensor<int32, [3]> var_4949 = const()[name = string("op_4949"), val = tensor<int32, [3]>([16, 2, 256])];
            tensor<fp16, [16, 2, 256]> value_111_cast_fp16 = reshape(shape = var_4949, x = linear_57_cast_fp16)[name = string("value_111_cast_fp16")];
            tensor<fp16, [16, 8, 256]> var_4951_cast_fp16 = mul(x = var_4941_cast_fp16_0, y = var_4941_cast_fp16_0)[name = string("op_4951_cast_fp16")];
            tensor<int32, [1]> variance_47_axes_0 = const()[name = string("variance_47_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_47_keep_dims_0 = const()[name = string("variance_47_keep_dims_0"), val = bool(true)];
            tensor<fp16, [16, 8, 1]> variance_47_cast_fp16 = reduce_mean(axes = variance_47_axes_0, keep_dims = variance_47_keep_dims_0, x = var_4951_cast_fp16)[name = string("variance_47_cast_fp16")];
            fp16 var_4954_to_fp16 = const()[name = string("op_4954_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [16, 8, 1]> var_4955_cast_fp16 = add(x = variance_47_cast_fp16, y = var_4954_to_fp16)[name = string("op_4955_cast_fp16")];
            fp32 var_4956_epsilon_0 = const()[name = string("op_4956_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [16, 8, 1]> var_4956_cast_fp16 = rsqrt(epsilon = var_4956_epsilon_0, x = var_4955_cast_fp16)[name = string("op_4956_cast_fp16")];
            tensor<fp16, [16, 8, 256]> var_4957_cast_fp16 = mul(x = var_4941_cast_fp16_0, y = var_4956_cast_fp16)[name = string("op_4957_cast_fp16")];
            tensor<fp16, [256]> var_4959_to_fp16 = const()[name = string("op_4959_to_fp16"), val = tensor<fp16, [256]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(116051456)))];
            tensor<fp16, [16, 8, 256]> var_4960_cast_fp16 = mul(x = var_4957_cast_fp16, y = var_4959_to_fp16)[name = string("op_4960_cast_fp16")];
            tensor<int32, [3]> var_4961 = const()[name = string("op_4961"), val = tensor<int32, [3]>([1, 0, 2])];
            tensor<int32, [1]> value_107_axes_0 = const()[name = string("value_107_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [8, 16, 256]> var_4962_cast_fp16 = transpose(perm = var_4961, x = var_4960_cast_fp16)[name = string("transpose_90")];
            tensor<fp16, [1, 8, 16, 256]> value_107_cast_fp16 = expand_dims(axes = value_107_axes_0, x = var_4962_cast_fp16)[name = string("value_107_cast_fp16")];
            tensor<fp16, [16, 2, 256]> var_4964_cast_fp16 = mul(x = value_105_cast_fp16, y = value_105_cast_fp16)[name = string("op_4964_cast_fp16")];
            tensor<int32, [1]> variance_49_axes_0 = const()[name = string("variance_49_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_49_keep_dims_0 = const()[name = string("variance_49_keep_dims_0"), val = bool(true)];
            tensor<fp16, [16, 2, 1]> variance_49_cast_fp16 = reduce_mean(axes = variance_49_axes_0, keep_dims = variance_49_keep_dims_0, x = var_4964_cast_fp16)[name = string("variance_49_cast_fp16")];
            fp16 var_4967_to_fp16 = const()[name = string("op_4967_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [16, 2, 1]> var_4968_cast_fp16 = add(x = variance_49_cast_fp16, y = var_4967_to_fp16)[name = string("op_4968_cast_fp16")];
            fp32 var_4969_epsilon_0 = const()[name = string("op_4969_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [16, 2, 1]> var_4969_cast_fp16 = rsqrt(epsilon = var_4969_epsilon_0, x = var_4968_cast_fp16)[name = string("op_4969_cast_fp16")];
            tensor<fp16, [16, 2, 256]> var_4970_cast_fp16 = mul(x = value_105_cast_fp16, y = var_4969_cast_fp16)[name = string("op_4970_cast_fp16")];
            tensor<fp16, [256]> var_4972_to_fp16 = const()[name = string("op_4972_to_fp16"), val = tensor<fp16, [256]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(116052032)))];
            tensor<fp16, [16, 2, 256]> var_4973_cast_fp16 = mul(x = var_4970_cast_fp16, y = var_4972_to_fp16)[name = string("op_4973_cast_fp16")];
            tensor<int32, [3]> var_4974 = const()[name = string("op_4974"), val = tensor<int32, [3]>([1, 0, 2])];
            tensor<int32, [1]> value_109_axes_0 = const()[name = string("value_109_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [2, 16, 256]> var_4975_cast_fp16 = transpose(perm = var_4974, x = var_4973_cast_fp16)[name = string("transpose_89")];
            tensor<fp16, [1, 2, 16, 256]> value_109_cast_fp16 = expand_dims(axes = value_109_axes_0, x = var_4975_cast_fp16)[name = string("value_109_cast_fp16")];
            tensor<int32, [4]> rotated_5_begin_0 = const()[name = string("rotated_5_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> rotated_5_end_0 = const()[name = string("rotated_5_end_0"), val = tensor<int32, [4]>([1, 8, 16, 64])];
            tensor<bool, [4]> rotated_5_end_mask_0 = const()[name = string("rotated_5_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
            tensor<fp16, [1, 8, 16, 64]> rotated_5_cast_fp16 = slice_by_index(begin = rotated_5_begin_0, end = rotated_5_end_0, end_mask = rotated_5_end_mask_0, x = value_107_cast_fp16)[name = string("rotated_5_cast_fp16")];
            tensor<int32, [4]> passthrough_5_begin_0 = const()[name = string("passthrough_5_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 64])];
            tensor<int32, [4]> passthrough_5_end_0 = const()[name = string("passthrough_5_end_0"), val = tensor<int32, [4]>([1, 8, 16, 256])];
            tensor<bool, [4]> passthrough_5_end_mask_0 = const()[name = string("passthrough_5_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
            tensor<fp16, [1, 8, 16, 192]> passthrough_5_cast_fp16 = slice_by_index(begin = passthrough_5_begin_0, end = passthrough_5_end_0, end_mask = passthrough_5_end_mask_0, x = value_107_cast_fp16)[name = string("passthrough_5_cast_fp16")];
            tensor<int32, [2]> var_4979_split_sizes_0 = const()[name = string("op_4979_split_sizes_0"), val = tensor<int32, [2]>([32, 32])];
            int32 var_4979_axis_0 = const()[name = string("op_4979_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 8, 16, 32]> var_4979_cast_fp16_0, tensor<fp16, [1, 8, 16, 32]> var_4979_cast_fp16_1 = split(axis = var_4979_axis_0, split_sizes = var_4979_split_sizes_0, x = rotated_5_cast_fp16)[name = string("op_4979_cast_fp16")];
            fp16 const_46_promoted_to_fp16 = const()[name = string("const_46_promoted_to_fp16"), val = fp16(-0x1p+0)];
            tensor<fp16, [1, 8, 16, 32]> var_4981_cast_fp16 = mul(x = var_4979_cast_fp16_1, y = const_46_promoted_to_fp16)[name = string("op_4981_cast_fp16")];
            bool rotated_half_5_interleave_0 = const()[name = string("rotated_half_5_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 8, 16, 64]> rotated_half_5_cast_fp16 = concat(axis = var_4920, interleave = rotated_half_5_interleave_0, values = (var_4981_cast_fp16, var_4979_cast_fp16_0))[name = string("rotated_half_5_cast_fp16")];
            tensor<fp16, [1, 8, 16, 64]> var_4984_cast_fp16 = mul(x = rotated_5_cast_fp16, y = cos)[name = string("op_4984_cast_fp16")];
            tensor<fp16, [1, 8, 16, 64]> var_4985_cast_fp16 = mul(x = rotated_half_5_cast_fp16, y = sin)[name = string("op_4985_cast_fp16")];
            tensor<fp16, [1, 8, 16, 64]> var_4986_cast_fp16 = add(x = var_4984_cast_fp16, y = var_4985_cast_fp16)[name = string("op_4986_cast_fp16")];
            bool query_29_interleave_0 = const()[name = string("query_29_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 8, 16, 256]> query_29_cast_fp16 = concat(axis = var_4920, interleave = query_29_interleave_0, values = (var_4986_cast_fp16, passthrough_5_cast_fp16))[name = string("query_29_cast_fp16")];
            tensor<int32, [4]> rotated_7_begin_0 = const()[name = string("rotated_7_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> rotated_7_end_0 = const()[name = string("rotated_7_end_0"), val = tensor<int32, [4]>([1, 2, 16, 64])];
            tensor<bool, [4]> rotated_7_end_mask_0 = const()[name = string("rotated_7_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
            tensor<fp16, [1, 2, 16, 64]> rotated_7_cast_fp16 = slice_by_index(begin = rotated_7_begin_0, end = rotated_7_end_0, end_mask = rotated_7_end_mask_0, x = value_109_cast_fp16)[name = string("rotated_7_cast_fp16")];
            tensor<int32, [4]> passthrough_7_begin_0 = const()[name = string("passthrough_7_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 64])];
            tensor<int32, [4]> passthrough_7_end_0 = const()[name = string("passthrough_7_end_0"), val = tensor<int32, [4]>([1, 2, 16, 256])];
            tensor<bool, [4]> passthrough_7_end_mask_0 = const()[name = string("passthrough_7_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
            tensor<fp16, [1, 2, 16, 192]> passthrough_7_cast_fp16 = slice_by_index(begin = passthrough_7_begin_0, end = passthrough_7_end_0, end_mask = passthrough_7_end_mask_0, x = value_109_cast_fp16)[name = string("passthrough_7_cast_fp16")];
            tensor<int32, [2]> var_4991_split_sizes_0 = const()[name = string("op_4991_split_sizes_0"), val = tensor<int32, [2]>([32, 32])];
            int32 var_4991_axis_0 = const()[name = string("op_4991_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 2, 16, 32]> var_4991_cast_fp16_0, tensor<fp16, [1, 2, 16, 32]> var_4991_cast_fp16_1 = split(axis = var_4991_axis_0, split_sizes = var_4991_split_sizes_0, x = rotated_7_cast_fp16)[name = string("op_4991_cast_fp16")];
            fp16 const_47_promoted_to_fp16 = const()[name = string("const_47_promoted_to_fp16"), val = fp16(-0x1p+0)];
            tensor<fp16, [1, 2, 16, 32]> var_4993_cast_fp16 = mul(x = var_4991_cast_fp16_1, y = const_47_promoted_to_fp16)[name = string("op_4993_cast_fp16")];
            bool rotated_half_7_interleave_0 = const()[name = string("rotated_half_7_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 2, 16, 64]> rotated_half_7_cast_fp16 = concat(axis = var_4920, interleave = rotated_half_7_interleave_0, values = (var_4993_cast_fp16, var_4991_cast_fp16_0))[name = string("rotated_half_7_cast_fp16")];
            tensor<fp16, [1, 2, 16, 64]> var_4996_cast_fp16 = mul(x = rotated_7_cast_fp16, y = cos)[name = string("op_4996_cast_fp16")];
            tensor<fp16, [1, 2, 16, 64]> var_4997_cast_fp16 = mul(x = rotated_half_7_cast_fp16, y = sin)[name = string("op_4997_cast_fp16")];
            tensor<fp16, [1, 2, 16, 64]> var_4998_cast_fp16 = add(x = var_4996_cast_fp16, y = var_4997_cast_fp16)[name = string("op_4998_cast_fp16")];
            bool key_29_interleave_0 = const()[name = string("key_29_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 2, 16, 256]> key_29_cast_fp16 = concat(axis = var_4920, interleave = key_29_interleave_0, values = (var_4998_cast_fp16, passthrough_7_cast_fp16))[name = string("key_29_cast_fp16")];
            tensor<int32, [4]> var_5001 = const()[name = string("op_5001"), val = tensor<int32, [4]>([2, 4, 16, 256])];
            tensor<fp16, [2, 4, 16, 256]> var_5002_cast_fp16 = reshape(shape = var_5001, x = query_29_cast_fp16)[name = string("op_5002_cast_fp16")];
            tensor<int32, [4]> transpose_2_perm_0 = const()[name = string("transpose_2_perm_0"), val = tensor<int32, [4]>([0, 1, 3, 2])];
            tensor<int32, [3]> var_5005 = const()[name = string("op_5005"), val = tensor<int32, [3]>([2, 16, 256])];
            tensor<fp16, [2, 16, 256]> key_31_cast_fp16 = reshape(shape = var_5005, x = key_29_cast_fp16)[name = string("key_31_cast_fp16")];
            tensor<int32, [3]> var_5007 = const()[name = string("op_5007"), val = tensor<int32, [3]>([1, 0, 2])];
            tensor<int32, [2]> var_5009 = const()[name = string("op_5009"), val = tensor<int32, [2]>([16, 2048])];
            tensor<fp16, [16, 2048]> gate_7_cast_fp16 = reshape(shape = var_5009, x = var_4941_cast_fp16_1)[name = string("gate_7_cast_fp16")];
            tensor<int32, [1]> var_5017_axes_0 = const()[name = string("op_5017_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [1, 2, 16, 256]> var_5017_cast_fp16 = expand_dims(axes = var_5017_axes_0, x = key_31_cast_fp16)[name = string("op_5017_cast_fp16")];
            tensor<int32, [1]> var_5019_axes_0 = const()[name = string("op_5019_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [1, 1, 2, 16, 256]> var_5019_cast_fp16 = expand_dims(axes = var_5019_axes_0, x = var_5017_cast_fp16)[name = string("op_5019_cast_fp16")];
            tensor<int32, [1]> expand_dims_16 = const()[name = string("expand_dims_16"), val = tensor<int32, [1]>([1])];
            tensor<int32, [1]> expand_dims_17 = const()[name = string("expand_dims_17"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_18 = const()[name = string("expand_dims_18"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_20 = const()[name = string("expand_dims_20"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_21 = const()[name = string("expand_dims_21"), val = tensor<int32, [1]>([2])];
            tensor<int32, [1]> expand_dims_22 = const()[name = string("expand_dims_22"), val = tensor<int32, [1]>([1])];
            int32 concat_10_axis_0 = const()[name = string("concat_10_axis_0"), val = int32(0)];
            bool concat_10_interleave_0 = const()[name = string("concat_10_interleave_0"), val = bool(false)];
            tensor<int32, [5]> concat_10 = concat(axis = concat_10_axis_0, interleave = concat_10_interleave_0, values = (expand_dims_16, expand_dims_17, expand_dims_18, current_pos, expand_dims_20))[name = string("concat_10")];
            tensor<int32, [1]> concat_11_values2_0 = const()[name = string("concat_11_values2_0"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> concat_11_values4_0 = const()[name = string("concat_11_values4_0"), val = tensor<int32, [1]>([0])];
            int32 concat_11_axis_0 = const()[name = string("concat_11_axis_0"), val = int32(0)];
            bool concat_11_interleave_0 = const()[name = string("concat_11_interleave_0"), val = bool(false)];
            tensor<int32, [5]> concat_11 = concat(axis = concat_11_axis_0, interleave = concat_11_interleave_0, values = (expand_dims_21, expand_dims_22, concat_11_values2_0, var_2332, concat_11_values4_0))[name = string("concat_11")];
            tensor<int32, [5]> kv_cache_internal_tensor_assign_3_stride_0 = const()[name = string("kv_cache_internal_tensor_assign_3_stride_0"), val = tensor<int32, [5]>([1, 1, 1, 1, 1])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_3_begin_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_3_begin_mask_0"), val = tensor<bool, [5]>([false, false, false, false, false])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_3_end_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_3_end_mask_0"), val = tensor<bool, [5]>([false, false, true, false, true])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_3_squeeze_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_3_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, false, false, false])];
            tensor<fp16, [4, 2, 2, 2048, 256]> kv_cache_internal_tensor_assign_3_cast_fp16 = slice_update(begin = concat_10, begin_mask = kv_cache_internal_tensor_assign_3_begin_mask_0, end = concat_11, end_mask = kv_cache_internal_tensor_assign_3_end_mask_0, squeeze_mask = kv_cache_internal_tensor_assign_3_squeeze_mask_0, stride = kv_cache_internal_tensor_assign_3_stride_0, update = var_5019_cast_fp16, x = coreml_update_state_9)[name = string("kv_cache_internal_tensor_assign_3_cast_fp16")];
            write_state(data = kv_cache_internal_tensor_assign_3_cast_fp16, input = kv_cache)[name = string("coreml_update_state_10_write_state")];
            tensor<fp16, [4, 2, 2, 2048, 256]> coreml_update_state_10 = read_state(input = kv_cache)[name = string("coreml_update_state_10")];
            tensor<int32, [1]> var_5050_axes_0 = const()[name = string("op_5050_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [2, 16, 256]> value_113_cast_fp16 = transpose(perm = var_5007, x = value_111_cast_fp16)[name = string("transpose_88")];
            tensor<fp16, [1, 2, 16, 256]> var_5050_cast_fp16 = expand_dims(axes = var_5050_axes_0, x = value_113_cast_fp16)[name = string("op_5050_cast_fp16")];
            tensor<int32, [1]> var_5052_axes_0 = const()[name = string("op_5052_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [1, 1, 2, 16, 256]> var_5052_cast_fp16 = expand_dims(axes = var_5052_axes_0, x = var_5050_cast_fp16)[name = string("op_5052_cast_fp16")];
            tensor<int32, [1]> expand_dims_24 = const()[name = string("expand_dims_24"), val = tensor<int32, [1]>([1])];
            tensor<int32, [1]> expand_dims_25 = const()[name = string("expand_dims_25"), val = tensor<int32, [1]>([1])];
            tensor<int32, [1]> expand_dims_26 = const()[name = string("expand_dims_26"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_28 = const()[name = string("expand_dims_28"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_29 = const()[name = string("expand_dims_29"), val = tensor<int32, [1]>([2])];
            tensor<int32, [1]> expand_dims_30 = const()[name = string("expand_dims_30"), val = tensor<int32, [1]>([2])];
            int32 concat_14_axis_0 = const()[name = string("concat_14_axis_0"), val = int32(0)];
            bool concat_14_interleave_0 = const()[name = string("concat_14_interleave_0"), val = bool(false)];
            tensor<int32, [5]> concat_14 = concat(axis = concat_14_axis_0, interleave = concat_14_interleave_0, values = (expand_dims_24, expand_dims_25, expand_dims_26, current_pos, expand_dims_28))[name = string("concat_14")];
            tensor<int32, [1]> concat_15_values2_0 = const()[name = string("concat_15_values2_0"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> concat_15_values4_0 = const()[name = string("concat_15_values4_0"), val = tensor<int32, [1]>([0])];
            int32 concat_15_axis_0 = const()[name = string("concat_15_axis_0"), val = int32(0)];
            bool concat_15_interleave_0 = const()[name = string("concat_15_interleave_0"), val = bool(false)];
            tensor<int32, [5]> concat_15 = concat(axis = concat_15_axis_0, interleave = concat_15_interleave_0, values = (expand_dims_29, expand_dims_30, concat_15_values2_0, var_2332, concat_15_values4_0))[name = string("concat_15")];
            tensor<int32, [5]> kv_cache_internal_tensor_assign_4_stride_0 = const()[name = string("kv_cache_internal_tensor_assign_4_stride_0"), val = tensor<int32, [5]>([1, 1, 1, 1, 1])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_4_begin_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_4_begin_mask_0"), val = tensor<bool, [5]>([false, false, false, false, false])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_4_end_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_4_end_mask_0"), val = tensor<bool, [5]>([false, false, true, false, true])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_4_squeeze_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_4_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, false, false, false])];
            tensor<fp16, [4, 2, 2, 2048, 256]> kv_cache_internal_tensor_assign_4_cast_fp16 = slice_update(begin = concat_14, begin_mask = kv_cache_internal_tensor_assign_4_begin_mask_0, end = concat_15, end_mask = kv_cache_internal_tensor_assign_4_end_mask_0, squeeze_mask = kv_cache_internal_tensor_assign_4_squeeze_mask_0, stride = kv_cache_internal_tensor_assign_4_stride_0, update = var_5052_cast_fp16, x = coreml_update_state_10)[name = string("kv_cache_internal_tensor_assign_4_cast_fp16")];
            write_state(data = kv_cache_internal_tensor_assign_4_cast_fp16, input = kv_cache)[name = string("coreml_update_state_11_write_state")];
            tensor<fp16, [4, 2, 2, 2048, 256]> coreml_update_state_11 = read_state(input = kv_cache)[name = string("coreml_update_state_11")];
            tensor<int32, [5]> var_5084_begin_0 = const()[name = string("op_5084_begin_0"), val = tensor<int32, [5]>([1, 0, 0, 0, 0])];
            tensor<int32, [5]> var_5084_end_0 = const()[name = string("op_5084_end_0"), val = tensor<int32, [5]>([2, 2, 2, 2048, 256])];
            tensor<bool, [5]> var_5084_end_mask_0 = const()[name = string("op_5084_end_mask_0"), val = tensor<bool, [5]>([false, true, true, true, true])];
            tensor<bool, [5]> var_5084_squeeze_mask_0 = const()[name = string("op_5084_squeeze_mask_0"), val = tensor<bool, [5]>([true, false, false, false, false])];
            tensor<fp16, [2, 2, 2048, 256]> var_5084_cast_fp16 = slice_by_index(begin = var_5084_begin_0, end = var_5084_end_0, end_mask = var_5084_end_mask_0, squeeze_mask = var_5084_squeeze_mask_0, x = coreml_update_state_11)[name = string("op_5084_cast_fp16")];
            tensor<int32, [4]> keys_3_begin_0 = const()[name = string("keys_3_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> keys_3_end_0 = const()[name = string("keys_3_end_0"), val = tensor<int32, [4]>([1, 2, 2048, 256])];
            tensor<bool, [4]> keys_3_end_mask_0 = const()[name = string("keys_3_end_mask_0"), val = tensor<bool, [4]>([false, true, true, true])];
            tensor<bool, [4]> keys_3_squeeze_mask_0 = const()[name = string("keys_3_squeeze_mask_0"), val = tensor<bool, [4]>([true, false, false, false])];
            tensor<fp16, [2, 2048, 256]> keys_3_cast_fp16 = slice_by_index(begin = keys_3_begin_0, end = keys_3_end_0, end_mask = keys_3_end_mask_0, squeeze_mask = keys_3_squeeze_mask_0, x = var_5084_cast_fp16)[name = string("keys_3_cast_fp16")];
            tensor<int32, [4]> values_3_begin_0 = const()[name = string("values_3_begin_0"), val = tensor<int32, [4]>([1, 0, 0, 0])];
            tensor<int32, [4]> values_3_end_0 = const()[name = string("values_3_end_0"), val = tensor<int32, [4]>([2, 2, 2048, 256])];
            tensor<bool, [4]> values_3_end_mask_0 = const()[name = string("values_3_end_mask_0"), val = tensor<bool, [4]>([false, true, true, true])];
            tensor<bool, [4]> values_3_squeeze_mask_0 = const()[name = string("values_3_squeeze_mask_0"), val = tensor<bool, [4]>([true, false, false, false])];
            tensor<fp16, [2, 2048, 256]> values_3_cast_fp16 = slice_by_index(begin = values_3_begin_0, end = values_3_end_0, end_mask = values_3_end_mask_0, squeeze_mask = values_3_squeeze_mask_0, x = var_5084_cast_fp16)[name = string("values_3_cast_fp16")];
            int32 var_5102 = const()[name = string("op_5102"), val = int32(1)];
            tensor<int32, [4]> var_5109 = const()[name = string("op_5109"), val = tensor<int32, [4]>([1, 2048, 1, 16])];
            tensor<fp16, [2, 4, 256, 16]> transpose_2_cast_fp16 = transpose(perm = transpose_2_perm_0, x = var_5002_cast_fp16)[name = string("transpose_87")];
            tensor<fp16, [1, 2048, 1, 16]> var_5110_cast_fp16 = reshape(shape = var_5109, x = transpose_2_cast_fp16)[name = string("op_5110_cast_fp16")];
            tensor<int32, [3]> var_5111 = const()[name = string("op_5111"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [4]> var_5114 = const()[name = string("op_5114"), val = tensor<int32, [4]>([1, 512, 1, 2048])];
            tensor<fp16, [2, 256, 2048]> var_5112_cast_fp16 = transpose(perm = var_5111, x = keys_3_cast_fp16)[name = string("transpose_86")];
            tensor<fp16, [1, 512, 1, 2048]> key_native_3_cast_fp16 = reshape(shape = var_5114, x = var_5112_cast_fp16)[name = string("key_native_3_cast_fp16")];
            tensor<int32, [3]> var_5116 = const()[name = string("op_5116"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [4]> var_5119 = const()[name = string("op_5119"), val = tensor<int32, [4]>([1, 512, 1, 2048])];
            tensor<fp16, [2, 256, 2048]> var_5117_cast_fp16 = transpose(perm = var_5116, x = values_3_cast_fp16)[name = string("transpose_85")];
            tensor<fp16, [1, 512, 1, 2048]> var_5120_cast_fp16 = reshape(shape = var_5119, x = var_5117_cast_fp16)[name = string("op_5120_cast_fp16")];
            tensor<int32, [8]> tile_19 = const()[name = string("tile_19"), val = tensor<int32, [8]>([256, 256, 256, 256, 256, 256, 256, 256])];
            int32 var_5124_axis_0 = const()[name = string("op_5124_axis_0"), val = int32(1)];
            tensor<fp16, [1, 256, 1, 16]> var_5124_cast_fp16_0, tensor<fp16, [1, 256, 1, 16]> var_5124_cast_fp16_1, tensor<fp16, [1, 256, 1, 16]> var_5124_cast_fp16_2, tensor<fp16, [1, 256, 1, 16]> var_5124_cast_fp16_3, tensor<fp16, [1, 256, 1, 16]> var_5124_cast_fp16_4, tensor<fp16, [1, 256, 1, 16]> var_5124_cast_fp16_5, tensor<fp16, [1, 256, 1, 16]> var_5124_cast_fp16_6, tensor<fp16, [1, 256, 1, 16]> var_5124_cast_fp16_7 = split(axis = var_5124_axis_0, split_sizes = tile_19, x = var_5110_cast_fp16)[name = string("op_5124_cast_fp16")];
            tensor<int32, [4]> var_5133_perm_0 = const()[name = string("op_5133_perm_0"), val = tensor<int32, [4]>([0, 3, 2, 1])];
            tensor<int32, [2]> tile_20 = const()[name = string("tile_20"), val = tensor<int32, [2]>([256, 256])];
            int32 var_5134_axis_0 = const()[name = string("op_5134_axis_0"), val = int32(3)];
            tensor<fp16, [1, 2048, 1, 512]> var_5133_cast_fp16 = transpose(perm = var_5133_perm_0, x = key_native_3_cast_fp16)[name = string("transpose_84")];
            tensor<fp16, [1, 2048, 1, 256]> var_5134_cast_fp16_0, tensor<fp16, [1, 2048, 1, 256]> var_5134_cast_fp16_1 = split(axis = var_5134_axis_0, split_sizes = tile_20, x = var_5133_cast_fp16)[name = string("op_5134_cast_fp16")];
            tensor<int32, [2]> tile_21 = const()[name = string("tile_21"), val = tensor<int32, [2]>([256, 256])];
            int32 var_5137_axis_0 = const()[name = string("op_5137_axis_0"), val = int32(1)];
            tensor<fp16, [1, 256, 1, 2048]> var_5137_cast_fp16_0, tensor<fp16, [1, 256, 1, 2048]> var_5137_cast_fp16_1 = split(axis = var_5137_axis_0, split_sizes = tile_21, x = var_5120_cast_fp16)[name = string("op_5137_cast_fp16")];
            string scores_17_equation_0 = const()[name = string("scores_17_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 16]> scores_17_cast_fp16 = einsum(equation = scores_17_equation_0, values = (var_5134_cast_fp16_0, var_5124_cast_fp16_0))[name = string("scores_17_cast_fp16")];
            fp16 var_5142_to_fp16 = const()[name = string("op_5142_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 16]> var_5143_cast_fp16 = mul(x = scores_17_cast_fp16, y = var_5142_to_fp16)[name = string("op_5143_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_5144_cast_fp16 = add(x = var_5143_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_5144_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_5145_cast_fp16 = softmax(axis = var_5102, x = var_5144_cast_fp16)[name = string("op_5145_cast_fp16")];
            tensor<int32, [4]> tile_22 = const()[name = string("tile_22"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_5146_axis_0 = const()[name = string("op_5146_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 16]> var_5146_cast_fp16_0, tensor<fp16, [1, 512, 1, 16]> var_5146_cast_fp16_1, tensor<fp16, [1, 512, 1, 16]> var_5146_cast_fp16_2, tensor<fp16, [1, 512, 1, 16]> var_5146_cast_fp16_3 = split(axis = var_5146_axis_0, split_sizes = tile_22, x = var_5145_cast_fp16)[name = string("op_5146_cast_fp16")];
            tensor<int32, [4]> tile_23 = const()[name = string("tile_23"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_5151_axis_0 = const()[name = string("op_5151_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_5151_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_5151_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_5151_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_5151_cast_fp16_3 = split(axis = var_5151_axis_0, split_sizes = tile_23, x = var_5137_cast_fp16_0)[name = string("op_5151_cast_fp16")];
            fp16 var_5156_to_fp16 = const()[name = string("op_5156_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_5157_cast_fp16 = mul(x = var_5146_cast_fp16_0, y = var_5156_to_fp16)[name = string("op_5157_cast_fp16")];
            string var_5159_equation_0 = const()[name = string("op_5159_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_5159_cast_fp16 = einsum(equation = var_5159_equation_0, values = (var_5151_cast_fp16_0, var_5157_cast_fp16))[name = string("op_5159_cast_fp16")];
            fp16 var_5160_to_fp16 = const()[name = string("op_5160_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_5161_cast_fp16 = mul(x = var_5146_cast_fp16_1, y = var_5160_to_fp16)[name = string("op_5161_cast_fp16")];
            string var_5163_equation_0 = const()[name = string("op_5163_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_5163_cast_fp16 = einsum(equation = var_5163_equation_0, values = (var_5151_cast_fp16_1, var_5161_cast_fp16))[name = string("op_5163_cast_fp16")];
            fp16 var_5164_to_fp16 = const()[name = string("op_5164_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_5165_cast_fp16 = mul(x = var_5146_cast_fp16_2, y = var_5164_to_fp16)[name = string("op_5165_cast_fp16")];
            string var_5167_equation_0 = const()[name = string("op_5167_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_5167_cast_fp16 = einsum(equation = var_5167_equation_0, values = (var_5151_cast_fp16_2, var_5165_cast_fp16))[name = string("op_5167_cast_fp16")];
            fp16 var_5168_to_fp16 = const()[name = string("op_5168_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_5169_cast_fp16 = mul(x = var_5146_cast_fp16_3, y = var_5168_to_fp16)[name = string("op_5169_cast_fp16")];
            string var_5171_equation_0 = const()[name = string("op_5171_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_5171_cast_fp16 = einsum(equation = var_5171_equation_0, values = (var_5151_cast_fp16_3, var_5169_cast_fp16))[name = string("op_5171_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_5172_cast_fp16 = add(x = var_5159_cast_fp16, y = var_5163_cast_fp16)[name = string("op_5172_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_5173_cast_fp16 = add(x = var_5167_cast_fp16, y = var_5171_cast_fp16)[name = string("op_5173_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_5174_cast_fp16 = add(x = var_5172_cast_fp16, y = var_5173_cast_fp16)[name = string("op_5174_cast_fp16")];
            fp16 _inversed_5176_y_0_to_fp16 = const()[name = string("_inversed_5176_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 16]> _inversed_5176_cast_fp16 = mul(x = var_5174_cast_fp16, y = _inversed_5176_y_0_to_fp16)[name = string("_inversed_5176_cast_fp16")];
            string scores_19_equation_0 = const()[name = string("scores_19_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 16]> scores_19_cast_fp16 = einsum(equation = scores_19_equation_0, values = (var_5134_cast_fp16_0, var_5124_cast_fp16_1))[name = string("scores_19_cast_fp16")];
            fp16 var_5179_to_fp16 = const()[name = string("op_5179_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 16]> var_5180_cast_fp16 = mul(x = scores_19_cast_fp16, y = var_5179_to_fp16)[name = string("op_5180_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_5181_cast_fp16 = add(x = var_5180_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_5181_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_5182_cast_fp16 = softmax(axis = var_5102, x = var_5181_cast_fp16)[name = string("op_5182_cast_fp16")];
            tensor<int32, [4]> tile_24 = const()[name = string("tile_24"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_5183_axis_0 = const()[name = string("op_5183_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 16]> var_5183_cast_fp16_0, tensor<fp16, [1, 512, 1, 16]> var_5183_cast_fp16_1, tensor<fp16, [1, 512, 1, 16]> var_5183_cast_fp16_2, tensor<fp16, [1, 512, 1, 16]> var_5183_cast_fp16_3 = split(axis = var_5183_axis_0, split_sizes = tile_24, x = var_5182_cast_fp16)[name = string("op_5183_cast_fp16")];
            tensor<int32, [4]> tile_25 = const()[name = string("tile_25"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_5188_axis_0 = const()[name = string("op_5188_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_5188_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_5188_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_5188_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_5188_cast_fp16_3 = split(axis = var_5188_axis_0, split_sizes = tile_25, x = var_5137_cast_fp16_0)[name = string("op_5188_cast_fp16")];
            fp16 var_5193_to_fp16 = const()[name = string("op_5193_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_5194_cast_fp16 = mul(x = var_5183_cast_fp16_0, y = var_5193_to_fp16)[name = string("op_5194_cast_fp16")];
            string var_5196_equation_0 = const()[name = string("op_5196_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_5196_cast_fp16 = einsum(equation = var_5196_equation_0, values = (var_5188_cast_fp16_0, var_5194_cast_fp16))[name = string("op_5196_cast_fp16")];
            fp16 var_5197_to_fp16 = const()[name = string("op_5197_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_5198_cast_fp16 = mul(x = var_5183_cast_fp16_1, y = var_5197_to_fp16)[name = string("op_5198_cast_fp16")];
            string var_5200_equation_0 = const()[name = string("op_5200_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_5200_cast_fp16 = einsum(equation = var_5200_equation_0, values = (var_5188_cast_fp16_1, var_5198_cast_fp16))[name = string("op_5200_cast_fp16")];
            fp16 var_5201_to_fp16 = const()[name = string("op_5201_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_5202_cast_fp16 = mul(x = var_5183_cast_fp16_2, y = var_5201_to_fp16)[name = string("op_5202_cast_fp16")];
            string var_5204_equation_0 = const()[name = string("op_5204_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_5204_cast_fp16 = einsum(equation = var_5204_equation_0, values = (var_5188_cast_fp16_2, var_5202_cast_fp16))[name = string("op_5204_cast_fp16")];
            fp16 var_5205_to_fp16 = const()[name = string("op_5205_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_5206_cast_fp16 = mul(x = var_5183_cast_fp16_3, y = var_5205_to_fp16)[name = string("op_5206_cast_fp16")];
            string var_5208_equation_0 = const()[name = string("op_5208_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_5208_cast_fp16 = einsum(equation = var_5208_equation_0, values = (var_5188_cast_fp16_3, var_5206_cast_fp16))[name = string("op_5208_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_5209_cast_fp16 = add(x = var_5196_cast_fp16, y = var_5200_cast_fp16)[name = string("op_5209_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_5210_cast_fp16 = add(x = var_5204_cast_fp16, y = var_5208_cast_fp16)[name = string("op_5210_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_5211_cast_fp16 = add(x = var_5209_cast_fp16, y = var_5210_cast_fp16)[name = string("op_5211_cast_fp16")];
            fp16 _inversed_5213_y_0_to_fp16 = const()[name = string("_inversed_5213_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 16]> _inversed_5213_cast_fp16 = mul(x = var_5211_cast_fp16, y = _inversed_5213_y_0_to_fp16)[name = string("_inversed_5213_cast_fp16")];
            string scores_21_equation_0 = const()[name = string("scores_21_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 16]> scores_21_cast_fp16 = einsum(equation = scores_21_equation_0, values = (var_5134_cast_fp16_0, var_5124_cast_fp16_2))[name = string("scores_21_cast_fp16")];
            fp16 var_5216_to_fp16 = const()[name = string("op_5216_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 16]> var_5217_cast_fp16 = mul(x = scores_21_cast_fp16, y = var_5216_to_fp16)[name = string("op_5217_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_5218_cast_fp16 = add(x = var_5217_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_5218_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_5219_cast_fp16 = softmax(axis = var_5102, x = var_5218_cast_fp16)[name = string("op_5219_cast_fp16")];
            tensor<int32, [4]> tile_26 = const()[name = string("tile_26"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_5220_axis_0 = const()[name = string("op_5220_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 16]> var_5220_cast_fp16_0, tensor<fp16, [1, 512, 1, 16]> var_5220_cast_fp16_1, tensor<fp16, [1, 512, 1, 16]> var_5220_cast_fp16_2, tensor<fp16, [1, 512, 1, 16]> var_5220_cast_fp16_3 = split(axis = var_5220_axis_0, split_sizes = tile_26, x = var_5219_cast_fp16)[name = string("op_5220_cast_fp16")];
            tensor<int32, [4]> tile_27 = const()[name = string("tile_27"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_5225_axis_0 = const()[name = string("op_5225_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_5225_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_5225_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_5225_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_5225_cast_fp16_3 = split(axis = var_5225_axis_0, split_sizes = tile_27, x = var_5137_cast_fp16_0)[name = string("op_5225_cast_fp16")];
            fp16 var_5230_to_fp16 = const()[name = string("op_5230_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_5231_cast_fp16 = mul(x = var_5220_cast_fp16_0, y = var_5230_to_fp16)[name = string("op_5231_cast_fp16")];
            string var_5233_equation_0 = const()[name = string("op_5233_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_5233_cast_fp16 = einsum(equation = var_5233_equation_0, values = (var_5225_cast_fp16_0, var_5231_cast_fp16))[name = string("op_5233_cast_fp16")];
            fp16 var_5234_to_fp16 = const()[name = string("op_5234_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_5235_cast_fp16 = mul(x = var_5220_cast_fp16_1, y = var_5234_to_fp16)[name = string("op_5235_cast_fp16")];
            string var_5237_equation_0 = const()[name = string("op_5237_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_5237_cast_fp16 = einsum(equation = var_5237_equation_0, values = (var_5225_cast_fp16_1, var_5235_cast_fp16))[name = string("op_5237_cast_fp16")];
            fp16 var_5238_to_fp16 = const()[name = string("op_5238_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_5239_cast_fp16 = mul(x = var_5220_cast_fp16_2, y = var_5238_to_fp16)[name = string("op_5239_cast_fp16")];
            string var_5241_equation_0 = const()[name = string("op_5241_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_5241_cast_fp16 = einsum(equation = var_5241_equation_0, values = (var_5225_cast_fp16_2, var_5239_cast_fp16))[name = string("op_5241_cast_fp16")];
            fp16 var_5242_to_fp16 = const()[name = string("op_5242_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_5243_cast_fp16 = mul(x = var_5220_cast_fp16_3, y = var_5242_to_fp16)[name = string("op_5243_cast_fp16")];
            string var_5245_equation_0 = const()[name = string("op_5245_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_5245_cast_fp16 = einsum(equation = var_5245_equation_0, values = (var_5225_cast_fp16_3, var_5243_cast_fp16))[name = string("op_5245_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_5246_cast_fp16 = add(x = var_5233_cast_fp16, y = var_5237_cast_fp16)[name = string("op_5246_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_5247_cast_fp16 = add(x = var_5241_cast_fp16, y = var_5245_cast_fp16)[name = string("op_5247_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_5248_cast_fp16 = add(x = var_5246_cast_fp16, y = var_5247_cast_fp16)[name = string("op_5248_cast_fp16")];
            fp16 _inversed_5250_y_0_to_fp16 = const()[name = string("_inversed_5250_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 16]> _inversed_5250_cast_fp16 = mul(x = var_5248_cast_fp16, y = _inversed_5250_y_0_to_fp16)[name = string("_inversed_5250_cast_fp16")];
            string scores_23_equation_0 = const()[name = string("scores_23_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 16]> scores_23_cast_fp16 = einsum(equation = scores_23_equation_0, values = (var_5134_cast_fp16_0, var_5124_cast_fp16_3))[name = string("scores_23_cast_fp16")];
            fp16 var_5253_to_fp16 = const()[name = string("op_5253_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 16]> var_5254_cast_fp16 = mul(x = scores_23_cast_fp16, y = var_5253_to_fp16)[name = string("op_5254_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_5255_cast_fp16 = add(x = var_5254_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_5255_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_5256_cast_fp16 = softmax(axis = var_5102, x = var_5255_cast_fp16)[name = string("op_5256_cast_fp16")];
            tensor<int32, [4]> tile_28 = const()[name = string("tile_28"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_5257_axis_0 = const()[name = string("op_5257_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 16]> var_5257_cast_fp16_0, tensor<fp16, [1, 512, 1, 16]> var_5257_cast_fp16_1, tensor<fp16, [1, 512, 1, 16]> var_5257_cast_fp16_2, tensor<fp16, [1, 512, 1, 16]> var_5257_cast_fp16_3 = split(axis = var_5257_axis_0, split_sizes = tile_28, x = var_5256_cast_fp16)[name = string("op_5257_cast_fp16")];
            tensor<int32, [4]> tile_29 = const()[name = string("tile_29"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_5262_axis_0 = const()[name = string("op_5262_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_5262_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_5262_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_5262_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_5262_cast_fp16_3 = split(axis = var_5262_axis_0, split_sizes = tile_29, x = var_5137_cast_fp16_0)[name = string("op_5262_cast_fp16")];
            fp16 var_5267_to_fp16 = const()[name = string("op_5267_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_5268_cast_fp16 = mul(x = var_5257_cast_fp16_0, y = var_5267_to_fp16)[name = string("op_5268_cast_fp16")];
            string var_5270_equation_0 = const()[name = string("op_5270_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_5270_cast_fp16 = einsum(equation = var_5270_equation_0, values = (var_5262_cast_fp16_0, var_5268_cast_fp16))[name = string("op_5270_cast_fp16")];
            fp16 var_5271_to_fp16 = const()[name = string("op_5271_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_5272_cast_fp16 = mul(x = var_5257_cast_fp16_1, y = var_5271_to_fp16)[name = string("op_5272_cast_fp16")];
            string var_5274_equation_0 = const()[name = string("op_5274_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_5274_cast_fp16 = einsum(equation = var_5274_equation_0, values = (var_5262_cast_fp16_1, var_5272_cast_fp16))[name = string("op_5274_cast_fp16")];
            fp16 var_5275_to_fp16 = const()[name = string("op_5275_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_5276_cast_fp16 = mul(x = var_5257_cast_fp16_2, y = var_5275_to_fp16)[name = string("op_5276_cast_fp16")];
            string var_5278_equation_0 = const()[name = string("op_5278_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_5278_cast_fp16 = einsum(equation = var_5278_equation_0, values = (var_5262_cast_fp16_2, var_5276_cast_fp16))[name = string("op_5278_cast_fp16")];
            fp16 var_5279_to_fp16 = const()[name = string("op_5279_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_5280_cast_fp16 = mul(x = var_5257_cast_fp16_3, y = var_5279_to_fp16)[name = string("op_5280_cast_fp16")];
            string var_5282_equation_0 = const()[name = string("op_5282_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_5282_cast_fp16 = einsum(equation = var_5282_equation_0, values = (var_5262_cast_fp16_3, var_5280_cast_fp16))[name = string("op_5282_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_5283_cast_fp16 = add(x = var_5270_cast_fp16, y = var_5274_cast_fp16)[name = string("op_5283_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_5284_cast_fp16 = add(x = var_5278_cast_fp16, y = var_5282_cast_fp16)[name = string("op_5284_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_5285_cast_fp16 = add(x = var_5283_cast_fp16, y = var_5284_cast_fp16)[name = string("op_5285_cast_fp16")];
            fp16 _inversed_5287_y_0_to_fp16 = const()[name = string("_inversed_5287_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 16]> _inversed_5287_cast_fp16 = mul(x = var_5285_cast_fp16, y = _inversed_5287_y_0_to_fp16)[name = string("_inversed_5287_cast_fp16")];
            string scores_25_equation_0 = const()[name = string("scores_25_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 16]> scores_25_cast_fp16 = einsum(equation = scores_25_equation_0, values = (var_5134_cast_fp16_1, var_5124_cast_fp16_4))[name = string("scores_25_cast_fp16")];
            fp16 var_5290_to_fp16 = const()[name = string("op_5290_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 16]> var_5291_cast_fp16 = mul(x = scores_25_cast_fp16, y = var_5290_to_fp16)[name = string("op_5291_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_5292_cast_fp16 = add(x = var_5291_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_5292_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_5293_cast_fp16 = softmax(axis = var_5102, x = var_5292_cast_fp16)[name = string("op_5293_cast_fp16")];
            tensor<int32, [4]> tile_30 = const()[name = string("tile_30"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_5294_axis_0 = const()[name = string("op_5294_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 16]> var_5294_cast_fp16_0, tensor<fp16, [1, 512, 1, 16]> var_5294_cast_fp16_1, tensor<fp16, [1, 512, 1, 16]> var_5294_cast_fp16_2, tensor<fp16, [1, 512, 1, 16]> var_5294_cast_fp16_3 = split(axis = var_5294_axis_0, split_sizes = tile_30, x = var_5293_cast_fp16)[name = string("op_5294_cast_fp16")];
            tensor<int32, [4]> tile_31 = const()[name = string("tile_31"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_5299_axis_0 = const()[name = string("op_5299_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_5299_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_5299_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_5299_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_5299_cast_fp16_3 = split(axis = var_5299_axis_0, split_sizes = tile_31, x = var_5137_cast_fp16_1)[name = string("op_5299_cast_fp16")];
            fp16 var_5304_to_fp16 = const()[name = string("op_5304_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_5305_cast_fp16 = mul(x = var_5294_cast_fp16_0, y = var_5304_to_fp16)[name = string("op_5305_cast_fp16")];
            string var_5307_equation_0 = const()[name = string("op_5307_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_5307_cast_fp16 = einsum(equation = var_5307_equation_0, values = (var_5299_cast_fp16_0, var_5305_cast_fp16))[name = string("op_5307_cast_fp16")];
            fp16 var_5308_to_fp16 = const()[name = string("op_5308_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_5309_cast_fp16 = mul(x = var_5294_cast_fp16_1, y = var_5308_to_fp16)[name = string("op_5309_cast_fp16")];
            string var_5311_equation_0 = const()[name = string("op_5311_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_5311_cast_fp16 = einsum(equation = var_5311_equation_0, values = (var_5299_cast_fp16_1, var_5309_cast_fp16))[name = string("op_5311_cast_fp16")];
            fp16 var_5312_to_fp16 = const()[name = string("op_5312_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_5313_cast_fp16 = mul(x = var_5294_cast_fp16_2, y = var_5312_to_fp16)[name = string("op_5313_cast_fp16")];
            string var_5315_equation_0 = const()[name = string("op_5315_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_5315_cast_fp16 = einsum(equation = var_5315_equation_0, values = (var_5299_cast_fp16_2, var_5313_cast_fp16))[name = string("op_5315_cast_fp16")];
            fp16 var_5316_to_fp16 = const()[name = string("op_5316_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_5317_cast_fp16 = mul(x = var_5294_cast_fp16_3, y = var_5316_to_fp16)[name = string("op_5317_cast_fp16")];
            string var_5319_equation_0 = const()[name = string("op_5319_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_5319_cast_fp16 = einsum(equation = var_5319_equation_0, values = (var_5299_cast_fp16_3, var_5317_cast_fp16))[name = string("op_5319_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_5320_cast_fp16 = add(x = var_5307_cast_fp16, y = var_5311_cast_fp16)[name = string("op_5320_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_5321_cast_fp16 = add(x = var_5315_cast_fp16, y = var_5319_cast_fp16)[name = string("op_5321_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_5322_cast_fp16 = add(x = var_5320_cast_fp16, y = var_5321_cast_fp16)[name = string("op_5322_cast_fp16")];
            fp16 _inversed_5324_y_0_to_fp16 = const()[name = string("_inversed_5324_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 16]> _inversed_5324_cast_fp16 = mul(x = var_5322_cast_fp16, y = _inversed_5324_y_0_to_fp16)[name = string("_inversed_5324_cast_fp16")];
            string scores_27_equation_0 = const()[name = string("scores_27_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 16]> scores_27_cast_fp16 = einsum(equation = scores_27_equation_0, values = (var_5134_cast_fp16_1, var_5124_cast_fp16_5))[name = string("scores_27_cast_fp16")];
            fp16 var_5327_to_fp16 = const()[name = string("op_5327_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 16]> var_5328_cast_fp16 = mul(x = scores_27_cast_fp16, y = var_5327_to_fp16)[name = string("op_5328_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_5329_cast_fp16 = add(x = var_5328_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_5329_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_5330_cast_fp16 = softmax(axis = var_5102, x = var_5329_cast_fp16)[name = string("op_5330_cast_fp16")];
            tensor<int32, [4]> tile_32 = const()[name = string("tile_32"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_5331_axis_0 = const()[name = string("op_5331_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 16]> var_5331_cast_fp16_0, tensor<fp16, [1, 512, 1, 16]> var_5331_cast_fp16_1, tensor<fp16, [1, 512, 1, 16]> var_5331_cast_fp16_2, tensor<fp16, [1, 512, 1, 16]> var_5331_cast_fp16_3 = split(axis = var_5331_axis_0, split_sizes = tile_32, x = var_5330_cast_fp16)[name = string("op_5331_cast_fp16")];
            tensor<int32, [4]> tile_33 = const()[name = string("tile_33"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_5336_axis_0 = const()[name = string("op_5336_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_5336_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_5336_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_5336_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_5336_cast_fp16_3 = split(axis = var_5336_axis_0, split_sizes = tile_33, x = var_5137_cast_fp16_1)[name = string("op_5336_cast_fp16")];
            fp16 var_5341_to_fp16 = const()[name = string("op_5341_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_5342_cast_fp16 = mul(x = var_5331_cast_fp16_0, y = var_5341_to_fp16)[name = string("op_5342_cast_fp16")];
            string var_5344_equation_0 = const()[name = string("op_5344_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_5344_cast_fp16 = einsum(equation = var_5344_equation_0, values = (var_5336_cast_fp16_0, var_5342_cast_fp16))[name = string("op_5344_cast_fp16")];
            fp16 var_5345_to_fp16 = const()[name = string("op_5345_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_5346_cast_fp16 = mul(x = var_5331_cast_fp16_1, y = var_5345_to_fp16)[name = string("op_5346_cast_fp16")];
            string var_5348_equation_0 = const()[name = string("op_5348_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_5348_cast_fp16 = einsum(equation = var_5348_equation_0, values = (var_5336_cast_fp16_1, var_5346_cast_fp16))[name = string("op_5348_cast_fp16")];
            fp16 var_5349_to_fp16 = const()[name = string("op_5349_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_5350_cast_fp16 = mul(x = var_5331_cast_fp16_2, y = var_5349_to_fp16)[name = string("op_5350_cast_fp16")];
            string var_5352_equation_0 = const()[name = string("op_5352_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_5352_cast_fp16 = einsum(equation = var_5352_equation_0, values = (var_5336_cast_fp16_2, var_5350_cast_fp16))[name = string("op_5352_cast_fp16")];
            fp16 var_5353_to_fp16 = const()[name = string("op_5353_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_5354_cast_fp16 = mul(x = var_5331_cast_fp16_3, y = var_5353_to_fp16)[name = string("op_5354_cast_fp16")];
            string var_5356_equation_0 = const()[name = string("op_5356_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_5356_cast_fp16 = einsum(equation = var_5356_equation_0, values = (var_5336_cast_fp16_3, var_5354_cast_fp16))[name = string("op_5356_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_5357_cast_fp16 = add(x = var_5344_cast_fp16, y = var_5348_cast_fp16)[name = string("op_5357_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_5358_cast_fp16 = add(x = var_5352_cast_fp16, y = var_5356_cast_fp16)[name = string("op_5358_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_5359_cast_fp16 = add(x = var_5357_cast_fp16, y = var_5358_cast_fp16)[name = string("op_5359_cast_fp16")];
            fp16 _inversed_5361_y_0_to_fp16 = const()[name = string("_inversed_5361_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 16]> _inversed_5361_cast_fp16 = mul(x = var_5359_cast_fp16, y = _inversed_5361_y_0_to_fp16)[name = string("_inversed_5361_cast_fp16")];
            string scores_29_equation_0 = const()[name = string("scores_29_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 16]> scores_29_cast_fp16 = einsum(equation = scores_29_equation_0, values = (var_5134_cast_fp16_1, var_5124_cast_fp16_6))[name = string("scores_29_cast_fp16")];
            fp16 var_5364_to_fp16 = const()[name = string("op_5364_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 16]> var_5365_cast_fp16 = mul(x = scores_29_cast_fp16, y = var_5364_to_fp16)[name = string("op_5365_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_5366_cast_fp16 = add(x = var_5365_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_5366_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_5367_cast_fp16 = softmax(axis = var_5102, x = var_5366_cast_fp16)[name = string("op_5367_cast_fp16")];
            tensor<int32, [4]> tile_34 = const()[name = string("tile_34"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_5368_axis_0 = const()[name = string("op_5368_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 16]> var_5368_cast_fp16_0, tensor<fp16, [1, 512, 1, 16]> var_5368_cast_fp16_1, tensor<fp16, [1, 512, 1, 16]> var_5368_cast_fp16_2, tensor<fp16, [1, 512, 1, 16]> var_5368_cast_fp16_3 = split(axis = var_5368_axis_0, split_sizes = tile_34, x = var_5367_cast_fp16)[name = string("op_5368_cast_fp16")];
            tensor<int32, [4]> tile_35 = const()[name = string("tile_35"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_5373_axis_0 = const()[name = string("op_5373_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_5373_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_5373_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_5373_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_5373_cast_fp16_3 = split(axis = var_5373_axis_0, split_sizes = tile_35, x = var_5137_cast_fp16_1)[name = string("op_5373_cast_fp16")];
            fp16 var_5378_to_fp16 = const()[name = string("op_5378_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_5379_cast_fp16 = mul(x = var_5368_cast_fp16_0, y = var_5378_to_fp16)[name = string("op_5379_cast_fp16")];
            string var_5381_equation_0 = const()[name = string("op_5381_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_5381_cast_fp16 = einsum(equation = var_5381_equation_0, values = (var_5373_cast_fp16_0, var_5379_cast_fp16))[name = string("op_5381_cast_fp16")];
            fp16 var_5382_to_fp16 = const()[name = string("op_5382_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_5383_cast_fp16 = mul(x = var_5368_cast_fp16_1, y = var_5382_to_fp16)[name = string("op_5383_cast_fp16")];
            string var_5385_equation_0 = const()[name = string("op_5385_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_5385_cast_fp16 = einsum(equation = var_5385_equation_0, values = (var_5373_cast_fp16_1, var_5383_cast_fp16))[name = string("op_5385_cast_fp16")];
            fp16 var_5386_to_fp16 = const()[name = string("op_5386_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_5387_cast_fp16 = mul(x = var_5368_cast_fp16_2, y = var_5386_to_fp16)[name = string("op_5387_cast_fp16")];
            string var_5389_equation_0 = const()[name = string("op_5389_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_5389_cast_fp16 = einsum(equation = var_5389_equation_0, values = (var_5373_cast_fp16_2, var_5387_cast_fp16))[name = string("op_5389_cast_fp16")];
            fp16 var_5390_to_fp16 = const()[name = string("op_5390_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_5391_cast_fp16 = mul(x = var_5368_cast_fp16_3, y = var_5390_to_fp16)[name = string("op_5391_cast_fp16")];
            string var_5393_equation_0 = const()[name = string("op_5393_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_5393_cast_fp16 = einsum(equation = var_5393_equation_0, values = (var_5373_cast_fp16_3, var_5391_cast_fp16))[name = string("op_5393_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_5394_cast_fp16 = add(x = var_5381_cast_fp16, y = var_5385_cast_fp16)[name = string("op_5394_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_5395_cast_fp16 = add(x = var_5389_cast_fp16, y = var_5393_cast_fp16)[name = string("op_5395_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_5396_cast_fp16 = add(x = var_5394_cast_fp16, y = var_5395_cast_fp16)[name = string("op_5396_cast_fp16")];
            fp16 _inversed_5398_y_0_to_fp16 = const()[name = string("_inversed_5398_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 16]> _inversed_5398_cast_fp16 = mul(x = var_5396_cast_fp16, y = _inversed_5398_y_0_to_fp16)[name = string("_inversed_5398_cast_fp16")];
            string scores_31_equation_0 = const()[name = string("scores_31_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 16]> scores_31_cast_fp16 = einsum(equation = scores_31_equation_0, values = (var_5134_cast_fp16_1, var_5124_cast_fp16_7))[name = string("scores_31_cast_fp16")];
            fp16 var_5401_to_fp16 = const()[name = string("op_5401_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 16]> var_5402_cast_fp16 = mul(x = scores_31_cast_fp16, y = var_5401_to_fp16)[name = string("op_5402_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_5403_cast_fp16 = add(x = var_5402_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_5403_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_5404_cast_fp16 = softmax(axis = var_5102, x = var_5403_cast_fp16)[name = string("op_5404_cast_fp16")];
            tensor<int32, [4]> tile_36 = const()[name = string("tile_36"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_5405_axis_0 = const()[name = string("op_5405_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 16]> var_5405_cast_fp16_0, tensor<fp16, [1, 512, 1, 16]> var_5405_cast_fp16_1, tensor<fp16, [1, 512, 1, 16]> var_5405_cast_fp16_2, tensor<fp16, [1, 512, 1, 16]> var_5405_cast_fp16_3 = split(axis = var_5405_axis_0, split_sizes = tile_36, x = var_5404_cast_fp16)[name = string("op_5405_cast_fp16")];
            tensor<int32, [4]> tile_37 = const()[name = string("tile_37"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_5410_axis_0 = const()[name = string("op_5410_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_5410_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_5410_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_5410_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_5410_cast_fp16_3 = split(axis = var_5410_axis_0, split_sizes = tile_37, x = var_5137_cast_fp16_1)[name = string("op_5410_cast_fp16")];
            fp16 var_5415_to_fp16 = const()[name = string("op_5415_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_5416_cast_fp16 = mul(x = var_5405_cast_fp16_0, y = var_5415_to_fp16)[name = string("op_5416_cast_fp16")];
            string var_5418_equation_0 = const()[name = string("op_5418_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_5418_cast_fp16 = einsum(equation = var_5418_equation_0, values = (var_5410_cast_fp16_0, var_5416_cast_fp16))[name = string("op_5418_cast_fp16")];
            fp16 var_5419_to_fp16 = const()[name = string("op_5419_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_5420_cast_fp16 = mul(x = var_5405_cast_fp16_1, y = var_5419_to_fp16)[name = string("op_5420_cast_fp16")];
            string var_5422_equation_0 = const()[name = string("op_5422_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_5422_cast_fp16 = einsum(equation = var_5422_equation_0, values = (var_5410_cast_fp16_1, var_5420_cast_fp16))[name = string("op_5422_cast_fp16")];
            fp16 var_5423_to_fp16 = const()[name = string("op_5423_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_5424_cast_fp16 = mul(x = var_5405_cast_fp16_2, y = var_5423_to_fp16)[name = string("op_5424_cast_fp16")];
            string var_5426_equation_0 = const()[name = string("op_5426_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_5426_cast_fp16 = einsum(equation = var_5426_equation_0, values = (var_5410_cast_fp16_2, var_5424_cast_fp16))[name = string("op_5426_cast_fp16")];
            fp16 var_5427_to_fp16 = const()[name = string("op_5427_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_5428_cast_fp16 = mul(x = var_5405_cast_fp16_3, y = var_5427_to_fp16)[name = string("op_5428_cast_fp16")];
            string var_5430_equation_0 = const()[name = string("op_5430_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_5430_cast_fp16 = einsum(equation = var_5430_equation_0, values = (var_5410_cast_fp16_3, var_5428_cast_fp16))[name = string("op_5430_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_5431_cast_fp16 = add(x = var_5418_cast_fp16, y = var_5422_cast_fp16)[name = string("op_5431_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_5432_cast_fp16 = add(x = var_5426_cast_fp16, y = var_5430_cast_fp16)[name = string("op_5432_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_5433_cast_fp16 = add(x = var_5431_cast_fp16, y = var_5432_cast_fp16)[name = string("op_5433_cast_fp16")];
            fp16 _inversed_5435_y_0_to_fp16 = const()[name = string("_inversed_5435_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 16]> _inversed_5435_cast_fp16 = mul(x = var_5433_cast_fp16, y = _inversed_5435_y_0_to_fp16)[name = string("_inversed_5435_cast_fp16")];
            bool var_5437_interleave_0 = const()[name = string("op_5437_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 2048, 1, 16]> var_5437_cast_fp16 = concat(axis = var_5102, interleave = var_5437_interleave_0, values = (_inversed_5176_cast_fp16, _inversed_5213_cast_fp16, _inversed_5250_cast_fp16, _inversed_5287_cast_fp16, _inversed_5324_cast_fp16, _inversed_5361_cast_fp16, _inversed_5398_cast_fp16, _inversed_5435_cast_fp16))[name = string("op_5437_cast_fp16")];
            tensor<int32, [4]> var_5438 = const()[name = string("op_5438"), val = tensor<int32, [4]>([2, 4, 256, 16])];
            tensor<fp16, [2, 4, 256, 16]> var_5439_cast_fp16 = reshape(shape = var_5438, x = var_5437_cast_fp16)[name = string("op_5439_cast_fp16")];
            tensor<int32, [4]> transpose_3_perm_0 = const()[name = string("transpose_3_perm_0"), val = tensor<int32, [4]>([3, 0, 1, 2])];
            tensor<int32, [2]> var_5458 = const()[name = string("op_5458"), val = tensor<int32, [2]>([16, 2048])];
            tensor<fp16, [16, 2, 4, 256]> transpose_3_cast_fp16 = transpose(perm = transpose_3_perm_0, x = var_5439_cast_fp16)[name = string("transpose_83")];
            tensor<fp16, [16, 2048]> attention_output_7_cast_fp16 = reshape(shape = var_5458, x = transpose_3_cast_fp16)[name = string("attention_output_7_cast_fp16")];
            tensor<fp16, [16, 2048]> var_5460_cast_fp16 = sigmoid(x = gate_7_cast_fp16)[name = string("op_5460_cast_fp16")];
            tensor<fp16, [16, 2048]> input_97_cast_fp16 = mul(x = attention_output_7_cast_fp16, y = var_5460_cast_fp16)[name = string("input_97_cast_fp16")];
            tensor<fp16, [1024, 2048]> completions_1_o_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 2048]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(116052608))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(117625536))))[name = string("completions_1_o_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 1024]> linear_58_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = completions_1_o_proj_weight_promoted_to_fp16_palettized, x = input_97_cast_fp16)[name = string("linear_58_cast_fp16")];
            tensor<fp16, [16, 1024]> value_115_cast_fp16 = add(x = value_101_cast_fp16, y = linear_58_cast_fp16)[name = string("value_115_cast_fp16")];
            tensor<fp16, [16, 1024]> var_5465_cast_fp16 = mul(x = value_115_cast_fp16, y = value_115_cast_fp16)[name = string("op_5465_cast_fp16")];
            tensor<int32, [1]> variance_51_axes_0 = const()[name = string("variance_51_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_51_keep_dims_0 = const()[name = string("variance_51_keep_dims_0"), val = bool(true)];
            tensor<fp16, [16, 1]> variance_51_cast_fp16 = reduce_mean(axes = variance_51_axes_0, keep_dims = variance_51_keep_dims_0, x = var_5465_cast_fp16)[name = string("variance_51_cast_fp16")];
            fp16 var_5468_to_fp16 = const()[name = string("op_5468_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [16, 1]> var_5469_cast_fp16 = add(x = variance_51_cast_fp16, y = var_5468_to_fp16)[name = string("op_5469_cast_fp16")];
            fp32 var_5470_epsilon_0 = const()[name = string("op_5470_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [16, 1]> var_5470_cast_fp16 = rsqrt(epsilon = var_5470_epsilon_0, x = var_5469_cast_fp16)[name = string("op_5470_cast_fp16")];
            tensor<fp16, [16, 1024]> var_5471_cast_fp16 = mul(x = value_115_cast_fp16, y = var_5470_cast_fp16)[name = string("op_5471_cast_fp16")];
            tensor<fp16, [1024]> var_5473_to_fp16 = const()[name = string("op_5473_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(117633792)))];
            tensor<fp16, [16, 1024]> input_99_cast_fp16 = mul(x = var_5471_cast_fp16, y = var_5473_to_fp16)[name = string("input_99_cast_fp16")];
            tensor<fp16, [3584, 1024]> completions_1_gate_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(117635904))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(120388480))))[name = string("completions_1_gate_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 3584]> linear_59_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = completions_1_gate_proj_weight_promoted_to_fp16_palettized, x = input_99_cast_fp16)[name = string("linear_59_cast_fp16")];
            tensor<fp16, [16, 3584]> var_5477_cast_fp16 = silu(x = linear_59_cast_fp16)[name = string("op_5477_cast_fp16")];
            tensor<fp16, [3584, 1024]> completions_1_up_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(120417216))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(123169792))))[name = string("completions_1_up_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 3584]> linear_60_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = completions_1_up_proj_weight_promoted_to_fp16_palettized, x = input_99_cast_fp16)[name = string("linear_60_cast_fp16")];
            tensor<fp16, [16, 3584]> input_103_cast_fp16 = mul(x = var_5477_cast_fp16, y = linear_60_cast_fp16)[name = string("input_103_cast_fp16")];
            tensor<fp16, [1024, 3584]> completions_1_down_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 3584]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(123198528))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(125951104))))[name = string("completions_1_down_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 1024]> linear_61_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = completions_1_down_proj_weight_promoted_to_fp16_palettized, x = input_103_cast_fp16)[name = string("linear_61_cast_fp16")];
            tensor<fp16, [16, 1024]> value_117_cast_fp16 = add(x = value_115_cast_fp16, y = linear_61_cast_fp16)[name = string("value_117_cast_fp16")];
            int32 var_5511 = const()[name = string("op_5511"), val = int32(-1)];
            tensor<fp16, [16, 1024]> var_5526_cast_fp16 = mul(x = value_117_cast_fp16, y = value_117_cast_fp16)[name = string("op_5526_cast_fp16")];
            tensor<int32, [1]> variance_53_axes_0 = const()[name = string("variance_53_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_53_keep_dims_0 = const()[name = string("variance_53_keep_dims_0"), val = bool(true)];
            tensor<fp16, [16, 1]> variance_53_cast_fp16 = reduce_mean(axes = variance_53_axes_0, keep_dims = variance_53_keep_dims_0, x = var_5526_cast_fp16)[name = string("variance_53_cast_fp16")];
            fp16 var_5529_to_fp16 = const()[name = string("op_5529_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [16, 1]> var_5530_cast_fp16 = add(x = variance_53_cast_fp16, y = var_5529_to_fp16)[name = string("op_5530_cast_fp16")];
            fp32 var_5531_epsilon_0 = const()[name = string("op_5531_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [16, 1]> var_5531_cast_fp16 = rsqrt(epsilon = var_5531_epsilon_0, x = var_5530_cast_fp16)[name = string("op_5531_cast_fp16")];
            tensor<fp16, [16, 1024]> var_5532_cast_fp16 = mul(x = value_117_cast_fp16, y = var_5531_cast_fp16)[name = string("op_5532_cast_fp16")];
            tensor<fp16, [1024]> var_5534_to_fp16 = const()[name = string("op_5534_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(125959360)))];
            tensor<fp16, [16, 1024]> input_105_cast_fp16 = mul(x = var_5532_cast_fp16, y = var_5534_to_fp16)[name = string("input_105_cast_fp16")];
            tensor<fp16, [6144, 1024]> groups_2_0_in_proj_qkv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(125961472))), lut = tensor<fp16, [384, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(130680128))))[name = string("groups_2_0_in_proj_qkv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 6144]> linear_62_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = groups_2_0_in_proj_qkv_weight_promoted_to_fp16_palettized, x = input_105_cast_fp16)[name = string("linear_62_cast_fp16")];
            tensor<int32, [2]> var_5538_perm_0 = const()[name = string("op_5538_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> mixed_13_axes_0 = const()[name = string("mixed_13_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [6144, 16]> var_5538_cast_fp16 = transpose(perm = var_5538_perm_0, x = linear_62_cast_fp16)[name = string("transpose_82")];
            tensor<fp16, [1, 6144, 16]> mixed_13_cast_fp16 = expand_dims(axes = mixed_13_axes_0, x = var_5538_cast_fp16)[name = string("mixed_13_cast_fp16")];
            bool convolution_input_13_interleave_0 = const()[name = string("convolution_input_13_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 6144, 19]> convolution_input_13_cast_fp16 = concat(axis = var_5511, interleave = convolution_input_13_interleave_0, values = (conv8, mixed_13_cast_fp16))[name = string("convolution_input_13_cast_fp16")];
            string input_107_pad_type_0 = const()[name = string("input_107_pad_type_0"), val = string("valid")];
            int32 input_107_groups_0 = const()[name = string("input_107_groups_0"), val = int32(6144)];
            tensor<int32, [1]> input_107_strides_0 = const()[name = string("input_107_strides_0"), val = tensor<int32, [1]>([1])];
            tensor<int32, [2]> input_107_pad_0 = const()[name = string("input_107_pad_0"), val = tensor<int32, [2]>([0, 0])];
            tensor<int32, [1]> input_107_dilations_0 = const()[name = string("input_107_dilations_0"), val = tensor<int32, [1]>([1])];
            tensor<fp16, [6144, 1, 4]> groups_2_0_conv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1, 4]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(130729344))), lut = tensor<fp16, [384, 1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(130747840))))[name = string("groups_2_0_conv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 6144, 16]> input_107_cast_fp16 = conv(dilations = input_107_dilations_0, groups = input_107_groups_0, pad = input_107_pad_0, pad_type = input_107_pad_type_0, strides = input_107_strides_0, weight = groups_2_0_conv_weight_promoted_to_fp16_palettized, x = convolution_input_13_cast_fp16)[name = string("input_107_cast_fp16")];
            tensor<fp16, [1, 6144, 16]> var_5547_cast_fp16 = silu(x = input_107_cast_fp16)[name = string("op_5547_cast_fp16")];
            tensor<int32, [3]> var_5548_perm_0 = const()[name = string("op_5548_perm_0"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [3]> var_5551_begin_0 = const()[name = string("op_5551_begin_0"), val = tensor<int32, [3]>([0, 0, 16])];
            tensor<int32, [3]> var_5551_end_0 = const()[name = string("op_5551_end_0"), val = tensor<int32, [3]>([1, 6144, 19])];
            tensor<bool, [3]> var_5551_end_mask_0 = const()[name = string("op_5551_end_mask_0"), val = tensor<bool, [3]>([true, true, true])];
            tensor<fp16, [1, 6144, 3]> conv8_out = slice_by_index(begin = var_5551_begin_0, end = var_5551_end_0, end_mask = var_5551_end_mask_0, x = convolution_input_13_cast_fp16)[name = string("op_5551_cast_fp16")];
            tensor<int32, [3]> var_5552 = const()[name = string("op_5552"), val = tensor<int32, [3]>([2048, 2048, 2048])];
            int32 var_5553_axis_0 = const()[name = string("op_5553_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 16, 6144]> var_5548_cast_fp16 = transpose(perm = var_5548_perm_0, x = var_5547_cast_fp16)[name = string("transpose_81")];
            tensor<fp16, [1, 16, 2048]> var_5553_cast_fp16_0, tensor<fp16, [1, 16, 2048]> var_5553_cast_fp16_1, tensor<fp16, [1, 16, 2048]> var_5553_cast_fp16_2 = split(axis = var_5553_axis_0, split_sizes = var_5552, x = var_5548_cast_fp16)[name = string("op_5553_cast_fp16")];
            tensor<int32, [4]> var_5557 = const()[name = string("op_5557"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<fp16, [1, 16, 16, 128]> value_121_cast_fp16 = reshape(shape = var_5557, x = var_5553_cast_fp16_0)[name = string("value_121_cast_fp16")];
            tensor<int32, [4]> var_5559 = const()[name = string("op_5559"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<fp16, [1, 16, 16, 128]> value_123_cast_fp16 = reshape(shape = var_5559, x = var_5553_cast_fp16_1)[name = string("value_123_cast_fp16")];
            tensor<int32, [4]> var_5561 = const()[name = string("op_5561"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<fp16, [1, 16, 16, 128]> value_125_cast_fp16 = reshape(shape = var_5561, x = var_5553_cast_fp16_2)[name = string("value_125_cast_fp16")];
            tensor<fp16, [1, 16, 16, 128]> var_5563_cast_fp16 = mul(x = value_121_cast_fp16, y = value_121_cast_fp16)[name = string("op_5563_cast_fp16")];
            tensor<int32, [1]> var_5565_axes_0 = const()[name = string("op_5565_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_5565_keep_dims_0 = const()[name = string("op_5565_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 16, 16, 1]> var_5565_cast_fp16 = reduce_sum(axes = var_5565_axes_0, keep_dims = var_5565_keep_dims_0, x = var_5563_cast_fp16)[name = string("op_5565_cast_fp16")];
            fp16 var_5566_to_fp16 = const()[name = string("op_5566_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 16, 16, 1]> var_5567_cast_fp16 = add(x = var_5565_cast_fp16, y = var_5566_to_fp16)[name = string("op_5567_cast_fp16")];
            fp32 var_5568_epsilon_0 = const()[name = string("op_5568_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 16, 16, 1]> var_5568_cast_fp16 = rsqrt(epsilon = var_5568_epsilon_0, x = var_5567_cast_fp16)[name = string("op_5568_cast_fp16")];
            tensor<fp16, [1, 16, 16, 128]> var_5569_cast_fp16 = mul(x = value_121_cast_fp16, y = var_5568_cast_fp16)[name = string("op_5569_cast_fp16")];
            tensor<int32, [4]> var_5570_perm_0 = const()[name = string("op_5570_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            fp16 _inversed_query_35_y_0_to_fp16 = const()[name = string("_inversed_query_35_y_0_to_fp16"), val = fp16(0x1.6ap-4)];
            tensor<fp16, [1, 16, 16, 128]> var_5570_cast_fp16 = transpose(perm = var_5570_perm_0, x = var_5569_cast_fp16)[name = string("transpose_80")];
            tensor<fp16, [1, 16, 16, 128]> _inversed_query_35_cast_fp16 = mul(x = var_5570_cast_fp16, y = _inversed_query_35_y_0_to_fp16)[name = string("_inversed_query_35_cast_fp16")];
            tensor<fp16, [1, 16, 16, 128]> var_5573_cast_fp16 = mul(x = value_123_cast_fp16, y = value_123_cast_fp16)[name = string("op_5573_cast_fp16")];
            tensor<int32, [1]> var_5575_axes_0 = const()[name = string("op_5575_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_5575_keep_dims_0 = const()[name = string("op_5575_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 16, 16, 1]> var_5575_cast_fp16 = reduce_sum(axes = var_5575_axes_0, keep_dims = var_5575_keep_dims_0, x = var_5573_cast_fp16)[name = string("op_5575_cast_fp16")];
            fp16 var_5576_to_fp16 = const()[name = string("op_5576_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 16, 16, 1]> var_5577_cast_fp16 = add(x = var_5575_cast_fp16, y = var_5576_to_fp16)[name = string("op_5577_cast_fp16")];
            fp32 var_5578_epsilon_0 = const()[name = string("op_5578_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 16, 16, 1]> var_5578_cast_fp16 = rsqrt(epsilon = var_5578_epsilon_0, x = var_5577_cast_fp16)[name = string("op_5578_cast_fp16")];
            tensor<fp16, [1, 16, 16, 128]> var_5579_cast_fp16 = mul(x = value_123_cast_fp16, y = var_5578_cast_fp16)[name = string("op_5579_cast_fp16")];
            tensor<int32, [4]> key_35_perm_0 = const()[name = string("key_35_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<int32, [4]> value_127_perm_0 = const()[name = string("value_127_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<fp16, [16, 1024]> groups_2_0_in_proj_b_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(130797056))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(130809408))))[name = string("groups_2_0_in_proj_b_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16]> linear_63_bias_0_to_fp16 = const()[name = string("linear_63_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(4862656)))];
            tensor<fp16, [16, 16]> linear_63_cast_fp16 = linear(bias = linear_63_bias_0_to_fp16, weight = groups_2_0_in_proj_b_weight_promoted_to_fp16_palettized, x = input_105_cast_fp16)[name = string("linear_63_cast_fp16")];
            tensor<fp16, [16, 16]> var_5584_cast_fp16 = sigmoid(x = linear_63_cast_fp16)[name = string("op_5584_cast_fp16")];
            tensor<int32, [2]> var_5585_perm_0 = const()[name = string("op_5585_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> beta_13_axes_0 = const()[name = string("beta_13_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 16]> var_5585_cast_fp16 = transpose(perm = var_5585_perm_0, x = var_5584_cast_fp16)[name = string("transpose_79")];
            tensor<fp16, [1, 16, 16]> beta_13_cast_fp16 = expand_dims(axes = beta_13_axes_0, x = var_5585_cast_fp16)[name = string("beta_13_cast_fp16")];
            tensor<fp16, [16, 1024]> op_5591_weight_0_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(130809600))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(130821952))))[name = string("op_5591_weight_0_to_fp16_palettized")];
            tensor<fp16, [16]> var_5591_bias_0_to_fp16 = const()[name = string("op_5591_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(130822144)))];
            tensor<fp16, [16, 16]> var_5591_cast_fp16 = linear(bias = var_5591_bias_0_to_fp16, weight = op_5591_weight_0_to_fp16_palettized, x = input_105_cast_fp16)[name = string("op_5591_cast_fp16")];
            tensor<fp16, [16, 16]> var_5592_cast_fp16 = softplus(x = var_5591_cast_fp16)[name = string("op_5592_cast_fp16")];
            tensor<fp16, [16]> var_5588_promoted_to_fp16 = const()[name = string("op_5588_promoted_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(130822272)))];
            tensor<fp16, [16, 16]> var_5593_cast_fp16 = mul(x = var_5588_promoted_to_fp16, y = var_5592_cast_fp16)[name = string("op_5593_cast_fp16")];
            tensor<int32, [2]> var_5594_perm_0 = const()[name = string("op_5594_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> decay_13_axes_0 = const()[name = string("decay_13_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 16]> var_5594_cast_fp16 = transpose(perm = var_5594_perm_0, x = var_5593_cast_fp16)[name = string("transpose_78")];
            tensor<fp16, [1, 16, 16]> decay_13_cast_fp16 = expand_dims(axes = decay_13_axes_0, x = var_5594_cast_fp16)[name = string("decay_13_cast_fp16")];
            tensor<fp16, [2048, 1024]> groups_2_0_in_proj_z_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [2048, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(130822400))), lut = tensor<fp16, [128, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(132395328))))[name = string("groups_2_0_in_proj_z_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 2048]> linear_65_cast_fp16 = linear(bias = linear_3_bias_0_to_fp16, weight = groups_2_0_in_proj_z_weight_promoted_to_fp16_palettized, x = input_105_cast_fp16)[name = string("linear_65_cast_fp16")];
            tensor<int32, [3]> var_5602_begin_0 = const()[name = string("op_5602_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_5602_end_0 = const()[name = string("op_5602_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_5602_end_mask_0 = const()[name = string("op_5602_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_5602_squeeze_mask_0 = const()[name = string("op_5602_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_5602_cast_fp16 = slice_by_index(begin = var_5602_begin_0, end = var_5602_end_0, end_mask = var_5602_end_mask_0, squeeze_mask = var_5602_squeeze_mask_0, x = decay_13_cast_fp16)[name = string("op_5602_cast_fp16")];
            tensor<fp16, [1, 16]> var_5603_cast_fp16 = exp(x = var_5602_cast_fp16)[name = string("op_5603_cast_fp16")];
            tensor<int32, [1]> var_5604_axes_0 = const()[name = string("op_5604_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_5604_cast_fp16 = expand_dims(axes = var_5604_axes_0, x = var_5603_cast_fp16)[name = string("op_5604_cast_fp16")];
            tensor<int32, [1]> var_5605_axes_0 = const()[name = string("op_5605_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_5605_cast_fp16 = expand_dims(axes = var_5605_axes_0, x = var_5604_cast_fp16)[name = string("op_5605_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_385_cast_fp16 = mul(x = rec8, y = var_5605_cast_fp16)[name = string("state_385_cast_fp16")];
            tensor<int32, [4]> key_token_193_begin_0 = const()[name = string("key_token_193_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> key_token_193_end_0 = const()[name = string("key_token_193_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> key_token_193_end_mask_0 = const()[name = string("key_token_193_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_193_squeeze_mask_0 = const()[name = string("key_token_193_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 16, 128]> key_35_cast_fp16 = transpose(perm = key_35_perm_0, x = var_5579_cast_fp16)[name = string("transpose_77")];
            tensor<fp16, [1, 16, 128]> key_token_193_cast_fp16 = slice_by_index(begin = key_token_193_begin_0, end = key_token_193_end_0, end_mask = key_token_193_end_mask_0, squeeze_mask = key_token_193_squeeze_mask_0, x = key_35_cast_fp16)[name = string("key_token_193_cast_fp16")];
            tensor<int32, [4]> value_token_193_begin_0 = const()[name = string("value_token_193_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> value_token_193_end_0 = const()[name = string("value_token_193_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> value_token_193_end_mask_0 = const()[name = string("value_token_193_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_193_squeeze_mask_0 = const()[name = string("value_token_193_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 16, 128]> value_127_cast_fp16 = transpose(perm = value_127_perm_0, x = value_125_cast_fp16)[name = string("transpose_76")];
            tensor<fp16, [1, 16, 128]> value_token_193_cast_fp16 = slice_by_index(begin = value_token_193_begin_0, end = value_token_193_end_0, end_mask = value_token_193_end_mask_0, squeeze_mask = value_token_193_squeeze_mask_0, x = value_127_cast_fp16)[name = string("value_token_193_cast_fp16")];
            tensor<int32, [1]> var_5613_axes_0 = const()[name = string("op_5613_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_5613_cast_fp16 = expand_dims(axes = var_5613_axes_0, x = key_token_193_cast_fp16)[name = string("op_5613_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_5614_cast_fp16 = mul(x = state_385_cast_fp16, y = var_5613_cast_fp16)[name = string("op_5614_cast_fp16")];
            tensor<int32, [1]> memory_193_axes_0 = const()[name = string("memory_193_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_193_keep_dims_0 = const()[name = string("memory_193_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_193_cast_fp16 = reduce_sum(axes = memory_193_axes_0, keep_dims = memory_193_keep_dims_0, x = var_5614_cast_fp16)[name = string("memory_193_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_5617_cast_fp16 = sub(x = value_token_193_cast_fp16, y = memory_193_cast_fp16)[name = string("op_5617_cast_fp16")];
            tensor<int32, [3]> var_5620_begin_0 = const()[name = string("op_5620_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_5620_end_0 = const()[name = string("op_5620_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_5620_end_mask_0 = const()[name = string("op_5620_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_5620_squeeze_mask_0 = const()[name = string("op_5620_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_5620_cast_fp16 = slice_by_index(begin = var_5620_begin_0, end = var_5620_end_0, end_mask = var_5620_end_mask_0, squeeze_mask = var_5620_squeeze_mask_0, x = beta_13_cast_fp16)[name = string("op_5620_cast_fp16")];
            tensor<int32, [1]> var_5621_axes_0 = const()[name = string("op_5621_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_5621_cast_fp16 = expand_dims(axes = var_5621_axes_0, x = var_5620_cast_fp16)[name = string("op_5621_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_193_cast_fp16 = mul(x = var_5617_cast_fp16, y = var_5621_cast_fp16)[name = string("delta_193_cast_fp16")];
            tensor<int32, [1]> var_5624_axes_0 = const()[name = string("op_5624_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_5624_cast_fp16 = expand_dims(axes = var_5624_axes_0, x = delta_193_cast_fp16)[name = string("op_5624_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_5625_cast_fp16 = mul(x = var_5613_cast_fp16, y = var_5624_cast_fp16)[name = string("op_5625_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_387_cast_fp16 = add(x = state_385_cast_fp16, y = var_5625_cast_fp16)[name = string("state_387_cast_fp16")];
            tensor<int32, [4]> var_5629_begin_0 = const()[name = string("op_5629_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> var_5629_end_0 = const()[name = string("op_5629_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> var_5629_end_mask_0 = const()[name = string("op_5629_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_5629_squeeze_mask_0 = const()[name = string("op_5629_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_5629_cast_fp16 = slice_by_index(begin = var_5629_begin_0, end = var_5629_end_0, end_mask = var_5629_end_mask_0, squeeze_mask = var_5629_squeeze_mask_0, x = _inversed_query_35_cast_fp16)[name = string("op_5629_cast_fp16")];
            tensor<int32, [1]> var_5630_axes_0 = const()[name = string("op_5630_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_5630_cast_fp16 = expand_dims(axes = var_5630_axes_0, x = var_5629_cast_fp16)[name = string("op_5630_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_5631_cast_fp16 = mul(x = state_387_cast_fp16, y = var_5630_cast_fp16)[name = string("op_5631_cast_fp16")];
            tensor<int32, [1]> var_5633_axes_0 = const()[name = string("op_5633_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_5633_keep_dims_0 = const()[name = string("op_5633_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_5633_cast_fp16 = reduce_sum(axes = var_5633_axes_0, keep_dims = var_5633_keep_dims_0, x = var_5631_cast_fp16)[name = string("op_5633_cast_fp16")];
            tensor<int32, [3]> var_5636_begin_0 = const()[name = string("op_5636_begin_0"), val = tensor<int32, [3]>([0, 0, 1])];
            tensor<int32, [3]> var_5636_end_0 = const()[name = string("op_5636_end_0"), val = tensor<int32, [3]>([1, 16, 2])];
            tensor<bool, [3]> var_5636_end_mask_0 = const()[name = string("op_5636_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_5636_squeeze_mask_0 = const()[name = string("op_5636_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_5636_cast_fp16 = slice_by_index(begin = var_5636_begin_0, end = var_5636_end_0, end_mask = var_5636_end_mask_0, squeeze_mask = var_5636_squeeze_mask_0, x = decay_13_cast_fp16)[name = string("op_5636_cast_fp16")];
            tensor<fp16, [1, 16]> var_5637_cast_fp16 = exp(x = var_5636_cast_fp16)[name = string("op_5637_cast_fp16")];
            tensor<int32, [1]> var_5638_axes_0 = const()[name = string("op_5638_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_5638_cast_fp16 = expand_dims(axes = var_5638_axes_0, x = var_5637_cast_fp16)[name = string("op_5638_cast_fp16")];
            tensor<int32, [1]> var_5639_axes_0 = const()[name = string("op_5639_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_5639_cast_fp16 = expand_dims(axes = var_5639_axes_0, x = var_5638_cast_fp16)[name = string("op_5639_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_389_cast_fp16 = mul(x = state_387_cast_fp16, y = var_5639_cast_fp16)[name = string("state_389_cast_fp16")];
            tensor<int32, [4]> key_token_195_begin_0 = const()[name = string("key_token_195_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> key_token_195_end_0 = const()[name = string("key_token_195_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> key_token_195_end_mask_0 = const()[name = string("key_token_195_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_195_squeeze_mask_0 = const()[name = string("key_token_195_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_195_cast_fp16 = slice_by_index(begin = key_token_195_begin_0, end = key_token_195_end_0, end_mask = key_token_195_end_mask_0, squeeze_mask = key_token_195_squeeze_mask_0, x = key_35_cast_fp16)[name = string("key_token_195_cast_fp16")];
            tensor<int32, [4]> value_token_195_begin_0 = const()[name = string("value_token_195_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> value_token_195_end_0 = const()[name = string("value_token_195_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> value_token_195_end_mask_0 = const()[name = string("value_token_195_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_195_squeeze_mask_0 = const()[name = string("value_token_195_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_195_cast_fp16 = slice_by_index(begin = value_token_195_begin_0, end = value_token_195_end_0, end_mask = value_token_195_end_mask_0, squeeze_mask = value_token_195_squeeze_mask_0, x = value_127_cast_fp16)[name = string("value_token_195_cast_fp16")];
            tensor<int32, [1]> var_5647_axes_0 = const()[name = string("op_5647_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_5647_cast_fp16 = expand_dims(axes = var_5647_axes_0, x = key_token_195_cast_fp16)[name = string("op_5647_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_5648_cast_fp16 = mul(x = state_389_cast_fp16, y = var_5647_cast_fp16)[name = string("op_5648_cast_fp16")];
            tensor<int32, [1]> memory_195_axes_0 = const()[name = string("memory_195_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_195_keep_dims_0 = const()[name = string("memory_195_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_195_cast_fp16 = reduce_sum(axes = memory_195_axes_0, keep_dims = memory_195_keep_dims_0, x = var_5648_cast_fp16)[name = string("memory_195_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_5651_cast_fp16 = sub(x = value_token_195_cast_fp16, y = memory_195_cast_fp16)[name = string("op_5651_cast_fp16")];
            tensor<int32, [3]> var_5654_begin_0 = const()[name = string("op_5654_begin_0"), val = tensor<int32, [3]>([0, 0, 1])];
            tensor<int32, [3]> var_5654_end_0 = const()[name = string("op_5654_end_0"), val = tensor<int32, [3]>([1, 16, 2])];
            tensor<bool, [3]> var_5654_end_mask_0 = const()[name = string("op_5654_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_5654_squeeze_mask_0 = const()[name = string("op_5654_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_5654_cast_fp16 = slice_by_index(begin = var_5654_begin_0, end = var_5654_end_0, end_mask = var_5654_end_mask_0, squeeze_mask = var_5654_squeeze_mask_0, x = beta_13_cast_fp16)[name = string("op_5654_cast_fp16")];
            tensor<int32, [1]> var_5655_axes_0 = const()[name = string("op_5655_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_5655_cast_fp16 = expand_dims(axes = var_5655_axes_0, x = var_5654_cast_fp16)[name = string("op_5655_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_195_cast_fp16 = mul(x = var_5651_cast_fp16, y = var_5655_cast_fp16)[name = string("delta_195_cast_fp16")];
            tensor<int32, [1]> var_5658_axes_0 = const()[name = string("op_5658_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_5658_cast_fp16 = expand_dims(axes = var_5658_axes_0, x = delta_195_cast_fp16)[name = string("op_5658_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_5659_cast_fp16 = mul(x = var_5647_cast_fp16, y = var_5658_cast_fp16)[name = string("op_5659_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_391_cast_fp16 = add(x = state_389_cast_fp16, y = var_5659_cast_fp16)[name = string("state_391_cast_fp16")];
            tensor<int32, [4]> var_5663_begin_0 = const()[name = string("op_5663_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> var_5663_end_0 = const()[name = string("op_5663_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> var_5663_end_mask_0 = const()[name = string("op_5663_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_5663_squeeze_mask_0 = const()[name = string("op_5663_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_5663_cast_fp16 = slice_by_index(begin = var_5663_begin_0, end = var_5663_end_0, end_mask = var_5663_end_mask_0, squeeze_mask = var_5663_squeeze_mask_0, x = _inversed_query_35_cast_fp16)[name = string("op_5663_cast_fp16")];
            tensor<int32, [1]> var_5664_axes_0 = const()[name = string("op_5664_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_5664_cast_fp16 = expand_dims(axes = var_5664_axes_0, x = var_5663_cast_fp16)[name = string("op_5664_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_5665_cast_fp16 = mul(x = state_391_cast_fp16, y = var_5664_cast_fp16)[name = string("op_5665_cast_fp16")];
            tensor<int32, [1]> var_5667_axes_0 = const()[name = string("op_5667_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_5667_keep_dims_0 = const()[name = string("op_5667_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_5667_cast_fp16 = reduce_sum(axes = var_5667_axes_0, keep_dims = var_5667_keep_dims_0, x = var_5665_cast_fp16)[name = string("op_5667_cast_fp16")];
            tensor<int32, [3]> var_5670_begin_0 = const()[name = string("op_5670_begin_0"), val = tensor<int32, [3]>([0, 0, 2])];
            tensor<int32, [3]> var_5670_end_0 = const()[name = string("op_5670_end_0"), val = tensor<int32, [3]>([1, 16, 3])];
            tensor<bool, [3]> var_5670_end_mask_0 = const()[name = string("op_5670_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_5670_squeeze_mask_0 = const()[name = string("op_5670_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_5670_cast_fp16 = slice_by_index(begin = var_5670_begin_0, end = var_5670_end_0, end_mask = var_5670_end_mask_0, squeeze_mask = var_5670_squeeze_mask_0, x = decay_13_cast_fp16)[name = string("op_5670_cast_fp16")];
            tensor<fp16, [1, 16]> var_5671_cast_fp16 = exp(x = var_5670_cast_fp16)[name = string("op_5671_cast_fp16")];
            tensor<int32, [1]> var_5672_axes_0 = const()[name = string("op_5672_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_5672_cast_fp16 = expand_dims(axes = var_5672_axes_0, x = var_5671_cast_fp16)[name = string("op_5672_cast_fp16")];
            tensor<int32, [1]> var_5673_axes_0 = const()[name = string("op_5673_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_5673_cast_fp16 = expand_dims(axes = var_5673_axes_0, x = var_5672_cast_fp16)[name = string("op_5673_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_393_cast_fp16 = mul(x = state_391_cast_fp16, y = var_5673_cast_fp16)[name = string("state_393_cast_fp16")];
            tensor<int32, [4]> key_token_197_begin_0 = const()[name = string("key_token_197_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> key_token_197_end_0 = const()[name = string("key_token_197_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> key_token_197_end_mask_0 = const()[name = string("key_token_197_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_197_squeeze_mask_0 = const()[name = string("key_token_197_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_197_cast_fp16 = slice_by_index(begin = key_token_197_begin_0, end = key_token_197_end_0, end_mask = key_token_197_end_mask_0, squeeze_mask = key_token_197_squeeze_mask_0, x = key_35_cast_fp16)[name = string("key_token_197_cast_fp16")];
            tensor<int32, [4]> value_token_197_begin_0 = const()[name = string("value_token_197_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> value_token_197_end_0 = const()[name = string("value_token_197_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> value_token_197_end_mask_0 = const()[name = string("value_token_197_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_197_squeeze_mask_0 = const()[name = string("value_token_197_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_197_cast_fp16 = slice_by_index(begin = value_token_197_begin_0, end = value_token_197_end_0, end_mask = value_token_197_end_mask_0, squeeze_mask = value_token_197_squeeze_mask_0, x = value_127_cast_fp16)[name = string("value_token_197_cast_fp16")];
            tensor<int32, [1]> var_5681_axes_0 = const()[name = string("op_5681_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_5681_cast_fp16 = expand_dims(axes = var_5681_axes_0, x = key_token_197_cast_fp16)[name = string("op_5681_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_5682_cast_fp16 = mul(x = state_393_cast_fp16, y = var_5681_cast_fp16)[name = string("op_5682_cast_fp16")];
            tensor<int32, [1]> memory_197_axes_0 = const()[name = string("memory_197_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_197_keep_dims_0 = const()[name = string("memory_197_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_197_cast_fp16 = reduce_sum(axes = memory_197_axes_0, keep_dims = memory_197_keep_dims_0, x = var_5682_cast_fp16)[name = string("memory_197_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_5685_cast_fp16 = sub(x = value_token_197_cast_fp16, y = memory_197_cast_fp16)[name = string("op_5685_cast_fp16")];
            tensor<int32, [3]> var_5688_begin_0 = const()[name = string("op_5688_begin_0"), val = tensor<int32, [3]>([0, 0, 2])];
            tensor<int32, [3]> var_5688_end_0 = const()[name = string("op_5688_end_0"), val = tensor<int32, [3]>([1, 16, 3])];
            tensor<bool, [3]> var_5688_end_mask_0 = const()[name = string("op_5688_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_5688_squeeze_mask_0 = const()[name = string("op_5688_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_5688_cast_fp16 = slice_by_index(begin = var_5688_begin_0, end = var_5688_end_0, end_mask = var_5688_end_mask_0, squeeze_mask = var_5688_squeeze_mask_0, x = beta_13_cast_fp16)[name = string("op_5688_cast_fp16")];
            tensor<int32, [1]> var_5689_axes_0 = const()[name = string("op_5689_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_5689_cast_fp16 = expand_dims(axes = var_5689_axes_0, x = var_5688_cast_fp16)[name = string("op_5689_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_197_cast_fp16 = mul(x = var_5685_cast_fp16, y = var_5689_cast_fp16)[name = string("delta_197_cast_fp16")];
            tensor<int32, [1]> var_5692_axes_0 = const()[name = string("op_5692_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_5692_cast_fp16 = expand_dims(axes = var_5692_axes_0, x = delta_197_cast_fp16)[name = string("op_5692_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_5693_cast_fp16 = mul(x = var_5681_cast_fp16, y = var_5692_cast_fp16)[name = string("op_5693_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_395_cast_fp16 = add(x = state_393_cast_fp16, y = var_5693_cast_fp16)[name = string("state_395_cast_fp16")];
            tensor<int32, [4]> var_5697_begin_0 = const()[name = string("op_5697_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> var_5697_end_0 = const()[name = string("op_5697_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> var_5697_end_mask_0 = const()[name = string("op_5697_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_5697_squeeze_mask_0 = const()[name = string("op_5697_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_5697_cast_fp16 = slice_by_index(begin = var_5697_begin_0, end = var_5697_end_0, end_mask = var_5697_end_mask_0, squeeze_mask = var_5697_squeeze_mask_0, x = _inversed_query_35_cast_fp16)[name = string("op_5697_cast_fp16")];
            tensor<int32, [1]> var_5698_axes_0 = const()[name = string("op_5698_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_5698_cast_fp16 = expand_dims(axes = var_5698_axes_0, x = var_5697_cast_fp16)[name = string("op_5698_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_5699_cast_fp16 = mul(x = state_395_cast_fp16, y = var_5698_cast_fp16)[name = string("op_5699_cast_fp16")];
            tensor<int32, [1]> var_5701_axes_0 = const()[name = string("op_5701_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_5701_keep_dims_0 = const()[name = string("op_5701_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_5701_cast_fp16 = reduce_sum(axes = var_5701_axes_0, keep_dims = var_5701_keep_dims_0, x = var_5699_cast_fp16)[name = string("op_5701_cast_fp16")];
            tensor<int32, [3]> var_5704_begin_0 = const()[name = string("op_5704_begin_0"), val = tensor<int32, [3]>([0, 0, 3])];
            tensor<int32, [3]> var_5704_end_0 = const()[name = string("op_5704_end_0"), val = tensor<int32, [3]>([1, 16, 4])];
            tensor<bool, [3]> var_5704_end_mask_0 = const()[name = string("op_5704_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_5704_squeeze_mask_0 = const()[name = string("op_5704_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_5704_cast_fp16 = slice_by_index(begin = var_5704_begin_0, end = var_5704_end_0, end_mask = var_5704_end_mask_0, squeeze_mask = var_5704_squeeze_mask_0, x = decay_13_cast_fp16)[name = string("op_5704_cast_fp16")];
            tensor<fp16, [1, 16]> var_5705_cast_fp16 = exp(x = var_5704_cast_fp16)[name = string("op_5705_cast_fp16")];
            tensor<int32, [1]> var_5706_axes_0 = const()[name = string("op_5706_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_5706_cast_fp16 = expand_dims(axes = var_5706_axes_0, x = var_5705_cast_fp16)[name = string("op_5706_cast_fp16")];
            tensor<int32, [1]> var_5707_axes_0 = const()[name = string("op_5707_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_5707_cast_fp16 = expand_dims(axes = var_5707_axes_0, x = var_5706_cast_fp16)[name = string("op_5707_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_397_cast_fp16 = mul(x = state_395_cast_fp16, y = var_5707_cast_fp16)[name = string("state_397_cast_fp16")];
            tensor<int32, [4]> key_token_199_begin_0 = const()[name = string("key_token_199_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> key_token_199_end_0 = const()[name = string("key_token_199_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> key_token_199_end_mask_0 = const()[name = string("key_token_199_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_199_squeeze_mask_0 = const()[name = string("key_token_199_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_199_cast_fp16 = slice_by_index(begin = key_token_199_begin_0, end = key_token_199_end_0, end_mask = key_token_199_end_mask_0, squeeze_mask = key_token_199_squeeze_mask_0, x = key_35_cast_fp16)[name = string("key_token_199_cast_fp16")];
            tensor<int32, [4]> value_token_199_begin_0 = const()[name = string("value_token_199_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> value_token_199_end_0 = const()[name = string("value_token_199_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> value_token_199_end_mask_0 = const()[name = string("value_token_199_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_199_squeeze_mask_0 = const()[name = string("value_token_199_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_199_cast_fp16 = slice_by_index(begin = value_token_199_begin_0, end = value_token_199_end_0, end_mask = value_token_199_end_mask_0, squeeze_mask = value_token_199_squeeze_mask_0, x = value_127_cast_fp16)[name = string("value_token_199_cast_fp16")];
            tensor<int32, [1]> var_5715_axes_0 = const()[name = string("op_5715_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_5715_cast_fp16 = expand_dims(axes = var_5715_axes_0, x = key_token_199_cast_fp16)[name = string("op_5715_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_5716_cast_fp16 = mul(x = state_397_cast_fp16, y = var_5715_cast_fp16)[name = string("op_5716_cast_fp16")];
            tensor<int32, [1]> memory_199_axes_0 = const()[name = string("memory_199_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_199_keep_dims_0 = const()[name = string("memory_199_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_199_cast_fp16 = reduce_sum(axes = memory_199_axes_0, keep_dims = memory_199_keep_dims_0, x = var_5716_cast_fp16)[name = string("memory_199_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_5719_cast_fp16 = sub(x = value_token_199_cast_fp16, y = memory_199_cast_fp16)[name = string("op_5719_cast_fp16")];
            tensor<int32, [3]> var_5722_begin_0 = const()[name = string("op_5722_begin_0"), val = tensor<int32, [3]>([0, 0, 3])];
            tensor<int32, [3]> var_5722_end_0 = const()[name = string("op_5722_end_0"), val = tensor<int32, [3]>([1, 16, 4])];
            tensor<bool, [3]> var_5722_end_mask_0 = const()[name = string("op_5722_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_5722_squeeze_mask_0 = const()[name = string("op_5722_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_5722_cast_fp16 = slice_by_index(begin = var_5722_begin_0, end = var_5722_end_0, end_mask = var_5722_end_mask_0, squeeze_mask = var_5722_squeeze_mask_0, x = beta_13_cast_fp16)[name = string("op_5722_cast_fp16")];
            tensor<int32, [1]> var_5723_axes_0 = const()[name = string("op_5723_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_5723_cast_fp16 = expand_dims(axes = var_5723_axes_0, x = var_5722_cast_fp16)[name = string("op_5723_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_199_cast_fp16 = mul(x = var_5719_cast_fp16, y = var_5723_cast_fp16)[name = string("delta_199_cast_fp16")];
            tensor<int32, [1]> var_5726_axes_0 = const()[name = string("op_5726_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_5726_cast_fp16 = expand_dims(axes = var_5726_axes_0, x = delta_199_cast_fp16)[name = string("op_5726_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_5727_cast_fp16 = mul(x = var_5715_cast_fp16, y = var_5726_cast_fp16)[name = string("op_5727_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_399_cast_fp16 = add(x = state_397_cast_fp16, y = var_5727_cast_fp16)[name = string("state_399_cast_fp16")];
            tensor<int32, [4]> var_5731_begin_0 = const()[name = string("op_5731_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> var_5731_end_0 = const()[name = string("op_5731_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> var_5731_end_mask_0 = const()[name = string("op_5731_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_5731_squeeze_mask_0 = const()[name = string("op_5731_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_5731_cast_fp16 = slice_by_index(begin = var_5731_begin_0, end = var_5731_end_0, end_mask = var_5731_end_mask_0, squeeze_mask = var_5731_squeeze_mask_0, x = _inversed_query_35_cast_fp16)[name = string("op_5731_cast_fp16")];
            tensor<int32, [1]> var_5732_axes_0 = const()[name = string("op_5732_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_5732_cast_fp16 = expand_dims(axes = var_5732_axes_0, x = var_5731_cast_fp16)[name = string("op_5732_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_5733_cast_fp16 = mul(x = state_399_cast_fp16, y = var_5732_cast_fp16)[name = string("op_5733_cast_fp16")];
            tensor<int32, [1]> var_5735_axes_0 = const()[name = string("op_5735_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_5735_keep_dims_0 = const()[name = string("op_5735_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_5735_cast_fp16 = reduce_sum(axes = var_5735_axes_0, keep_dims = var_5735_keep_dims_0, x = var_5733_cast_fp16)[name = string("op_5735_cast_fp16")];
            tensor<int32, [3]> var_5738_begin_0 = const()[name = string("op_5738_begin_0"), val = tensor<int32, [3]>([0, 0, 4])];
            tensor<int32, [3]> var_5738_end_0 = const()[name = string("op_5738_end_0"), val = tensor<int32, [3]>([1, 16, 5])];
            tensor<bool, [3]> var_5738_end_mask_0 = const()[name = string("op_5738_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_5738_squeeze_mask_0 = const()[name = string("op_5738_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_5738_cast_fp16 = slice_by_index(begin = var_5738_begin_0, end = var_5738_end_0, end_mask = var_5738_end_mask_0, squeeze_mask = var_5738_squeeze_mask_0, x = decay_13_cast_fp16)[name = string("op_5738_cast_fp16")];
            tensor<fp16, [1, 16]> var_5739_cast_fp16 = exp(x = var_5738_cast_fp16)[name = string("op_5739_cast_fp16")];
            tensor<int32, [1]> var_5740_axes_0 = const()[name = string("op_5740_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_5740_cast_fp16 = expand_dims(axes = var_5740_axes_0, x = var_5739_cast_fp16)[name = string("op_5740_cast_fp16")];
            tensor<int32, [1]> var_5741_axes_0 = const()[name = string("op_5741_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_5741_cast_fp16 = expand_dims(axes = var_5741_axes_0, x = var_5740_cast_fp16)[name = string("op_5741_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_401_cast_fp16 = mul(x = state_399_cast_fp16, y = var_5741_cast_fp16)[name = string("state_401_cast_fp16")];
            tensor<int32, [4]> key_token_201_begin_0 = const()[name = string("key_token_201_begin_0"), val = tensor<int32, [4]>([0, 0, 4, 0])];
            tensor<int32, [4]> key_token_201_end_0 = const()[name = string("key_token_201_end_0"), val = tensor<int32, [4]>([1, 16, 5, 128])];
            tensor<bool, [4]> key_token_201_end_mask_0 = const()[name = string("key_token_201_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_201_squeeze_mask_0 = const()[name = string("key_token_201_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_201_cast_fp16 = slice_by_index(begin = key_token_201_begin_0, end = key_token_201_end_0, end_mask = key_token_201_end_mask_0, squeeze_mask = key_token_201_squeeze_mask_0, x = key_35_cast_fp16)[name = string("key_token_201_cast_fp16")];
            tensor<int32, [4]> value_token_201_begin_0 = const()[name = string("value_token_201_begin_0"), val = tensor<int32, [4]>([0, 0, 4, 0])];
            tensor<int32, [4]> value_token_201_end_0 = const()[name = string("value_token_201_end_0"), val = tensor<int32, [4]>([1, 16, 5, 128])];
            tensor<bool, [4]> value_token_201_end_mask_0 = const()[name = string("value_token_201_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_201_squeeze_mask_0 = const()[name = string("value_token_201_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_201_cast_fp16 = slice_by_index(begin = value_token_201_begin_0, end = value_token_201_end_0, end_mask = value_token_201_end_mask_0, squeeze_mask = value_token_201_squeeze_mask_0, x = value_127_cast_fp16)[name = string("value_token_201_cast_fp16")];
            tensor<int32, [1]> var_5749_axes_0 = const()[name = string("op_5749_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_5749_cast_fp16 = expand_dims(axes = var_5749_axes_0, x = key_token_201_cast_fp16)[name = string("op_5749_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_5750_cast_fp16 = mul(x = state_401_cast_fp16, y = var_5749_cast_fp16)[name = string("op_5750_cast_fp16")];
            tensor<int32, [1]> memory_201_axes_0 = const()[name = string("memory_201_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_201_keep_dims_0 = const()[name = string("memory_201_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_201_cast_fp16 = reduce_sum(axes = memory_201_axes_0, keep_dims = memory_201_keep_dims_0, x = var_5750_cast_fp16)[name = string("memory_201_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_5753_cast_fp16 = sub(x = value_token_201_cast_fp16, y = memory_201_cast_fp16)[name = string("op_5753_cast_fp16")];
            tensor<int32, [3]> var_5756_begin_0 = const()[name = string("op_5756_begin_0"), val = tensor<int32, [3]>([0, 0, 4])];
            tensor<int32, [3]> var_5756_end_0 = const()[name = string("op_5756_end_0"), val = tensor<int32, [3]>([1, 16, 5])];
            tensor<bool, [3]> var_5756_end_mask_0 = const()[name = string("op_5756_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_5756_squeeze_mask_0 = const()[name = string("op_5756_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_5756_cast_fp16 = slice_by_index(begin = var_5756_begin_0, end = var_5756_end_0, end_mask = var_5756_end_mask_0, squeeze_mask = var_5756_squeeze_mask_0, x = beta_13_cast_fp16)[name = string("op_5756_cast_fp16")];
            tensor<int32, [1]> var_5757_axes_0 = const()[name = string("op_5757_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_5757_cast_fp16 = expand_dims(axes = var_5757_axes_0, x = var_5756_cast_fp16)[name = string("op_5757_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_201_cast_fp16 = mul(x = var_5753_cast_fp16, y = var_5757_cast_fp16)[name = string("delta_201_cast_fp16")];
            tensor<int32, [1]> var_5760_axes_0 = const()[name = string("op_5760_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_5760_cast_fp16 = expand_dims(axes = var_5760_axes_0, x = delta_201_cast_fp16)[name = string("op_5760_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_5761_cast_fp16 = mul(x = var_5749_cast_fp16, y = var_5760_cast_fp16)[name = string("op_5761_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_403_cast_fp16 = add(x = state_401_cast_fp16, y = var_5761_cast_fp16)[name = string("state_403_cast_fp16")];
            tensor<int32, [4]> var_5765_begin_0 = const()[name = string("op_5765_begin_0"), val = tensor<int32, [4]>([0, 0, 4, 0])];
            tensor<int32, [4]> var_5765_end_0 = const()[name = string("op_5765_end_0"), val = tensor<int32, [4]>([1, 16, 5, 128])];
            tensor<bool, [4]> var_5765_end_mask_0 = const()[name = string("op_5765_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_5765_squeeze_mask_0 = const()[name = string("op_5765_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_5765_cast_fp16 = slice_by_index(begin = var_5765_begin_0, end = var_5765_end_0, end_mask = var_5765_end_mask_0, squeeze_mask = var_5765_squeeze_mask_0, x = _inversed_query_35_cast_fp16)[name = string("op_5765_cast_fp16")];
            tensor<int32, [1]> var_5766_axes_0 = const()[name = string("op_5766_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_5766_cast_fp16 = expand_dims(axes = var_5766_axes_0, x = var_5765_cast_fp16)[name = string("op_5766_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_5767_cast_fp16 = mul(x = state_403_cast_fp16, y = var_5766_cast_fp16)[name = string("op_5767_cast_fp16")];
            tensor<int32, [1]> var_5769_axes_0 = const()[name = string("op_5769_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_5769_keep_dims_0 = const()[name = string("op_5769_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_5769_cast_fp16 = reduce_sum(axes = var_5769_axes_0, keep_dims = var_5769_keep_dims_0, x = var_5767_cast_fp16)[name = string("op_5769_cast_fp16")];
            tensor<int32, [3]> var_5772_begin_0 = const()[name = string("op_5772_begin_0"), val = tensor<int32, [3]>([0, 0, 5])];
            tensor<int32, [3]> var_5772_end_0 = const()[name = string("op_5772_end_0"), val = tensor<int32, [3]>([1, 16, 6])];
            tensor<bool, [3]> var_5772_end_mask_0 = const()[name = string("op_5772_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_5772_squeeze_mask_0 = const()[name = string("op_5772_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_5772_cast_fp16 = slice_by_index(begin = var_5772_begin_0, end = var_5772_end_0, end_mask = var_5772_end_mask_0, squeeze_mask = var_5772_squeeze_mask_0, x = decay_13_cast_fp16)[name = string("op_5772_cast_fp16")];
            tensor<fp16, [1, 16]> var_5773_cast_fp16 = exp(x = var_5772_cast_fp16)[name = string("op_5773_cast_fp16")];
            tensor<int32, [1]> var_5774_axes_0 = const()[name = string("op_5774_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_5774_cast_fp16 = expand_dims(axes = var_5774_axes_0, x = var_5773_cast_fp16)[name = string("op_5774_cast_fp16")];
            tensor<int32, [1]> var_5775_axes_0 = const()[name = string("op_5775_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_5775_cast_fp16 = expand_dims(axes = var_5775_axes_0, x = var_5774_cast_fp16)[name = string("op_5775_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_405_cast_fp16 = mul(x = state_403_cast_fp16, y = var_5775_cast_fp16)[name = string("state_405_cast_fp16")];
            tensor<int32, [4]> key_token_203_begin_0 = const()[name = string("key_token_203_begin_0"), val = tensor<int32, [4]>([0, 0, 5, 0])];
            tensor<int32, [4]> key_token_203_end_0 = const()[name = string("key_token_203_end_0"), val = tensor<int32, [4]>([1, 16, 6, 128])];
            tensor<bool, [4]> key_token_203_end_mask_0 = const()[name = string("key_token_203_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_203_squeeze_mask_0 = const()[name = string("key_token_203_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_203_cast_fp16 = slice_by_index(begin = key_token_203_begin_0, end = key_token_203_end_0, end_mask = key_token_203_end_mask_0, squeeze_mask = key_token_203_squeeze_mask_0, x = key_35_cast_fp16)[name = string("key_token_203_cast_fp16")];
            tensor<int32, [4]> value_token_203_begin_0 = const()[name = string("value_token_203_begin_0"), val = tensor<int32, [4]>([0, 0, 5, 0])];
            tensor<int32, [4]> value_token_203_end_0 = const()[name = string("value_token_203_end_0"), val = tensor<int32, [4]>([1, 16, 6, 128])];
            tensor<bool, [4]> value_token_203_end_mask_0 = const()[name = string("value_token_203_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_203_squeeze_mask_0 = const()[name = string("value_token_203_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_203_cast_fp16 = slice_by_index(begin = value_token_203_begin_0, end = value_token_203_end_0, end_mask = value_token_203_end_mask_0, squeeze_mask = value_token_203_squeeze_mask_0, x = value_127_cast_fp16)[name = string("value_token_203_cast_fp16")];
            tensor<int32, [1]> var_5783_axes_0 = const()[name = string("op_5783_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_5783_cast_fp16 = expand_dims(axes = var_5783_axes_0, x = key_token_203_cast_fp16)[name = string("op_5783_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_5784_cast_fp16 = mul(x = state_405_cast_fp16, y = var_5783_cast_fp16)[name = string("op_5784_cast_fp16")];
            tensor<int32, [1]> memory_203_axes_0 = const()[name = string("memory_203_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_203_keep_dims_0 = const()[name = string("memory_203_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_203_cast_fp16 = reduce_sum(axes = memory_203_axes_0, keep_dims = memory_203_keep_dims_0, x = var_5784_cast_fp16)[name = string("memory_203_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_5787_cast_fp16 = sub(x = value_token_203_cast_fp16, y = memory_203_cast_fp16)[name = string("op_5787_cast_fp16")];
            tensor<int32, [3]> var_5790_begin_0 = const()[name = string("op_5790_begin_0"), val = tensor<int32, [3]>([0, 0, 5])];
            tensor<int32, [3]> var_5790_end_0 = const()[name = string("op_5790_end_0"), val = tensor<int32, [3]>([1, 16, 6])];
            tensor<bool, [3]> var_5790_end_mask_0 = const()[name = string("op_5790_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_5790_squeeze_mask_0 = const()[name = string("op_5790_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_5790_cast_fp16 = slice_by_index(begin = var_5790_begin_0, end = var_5790_end_0, end_mask = var_5790_end_mask_0, squeeze_mask = var_5790_squeeze_mask_0, x = beta_13_cast_fp16)[name = string("op_5790_cast_fp16")];
            tensor<int32, [1]> var_5791_axes_0 = const()[name = string("op_5791_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_5791_cast_fp16 = expand_dims(axes = var_5791_axes_0, x = var_5790_cast_fp16)[name = string("op_5791_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_203_cast_fp16 = mul(x = var_5787_cast_fp16, y = var_5791_cast_fp16)[name = string("delta_203_cast_fp16")];
            tensor<int32, [1]> var_5794_axes_0 = const()[name = string("op_5794_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_5794_cast_fp16 = expand_dims(axes = var_5794_axes_0, x = delta_203_cast_fp16)[name = string("op_5794_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_5795_cast_fp16 = mul(x = var_5783_cast_fp16, y = var_5794_cast_fp16)[name = string("op_5795_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_407_cast_fp16 = add(x = state_405_cast_fp16, y = var_5795_cast_fp16)[name = string("state_407_cast_fp16")];
            tensor<int32, [4]> var_5799_begin_0 = const()[name = string("op_5799_begin_0"), val = tensor<int32, [4]>([0, 0, 5, 0])];
            tensor<int32, [4]> var_5799_end_0 = const()[name = string("op_5799_end_0"), val = tensor<int32, [4]>([1, 16, 6, 128])];
            tensor<bool, [4]> var_5799_end_mask_0 = const()[name = string("op_5799_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_5799_squeeze_mask_0 = const()[name = string("op_5799_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_5799_cast_fp16 = slice_by_index(begin = var_5799_begin_0, end = var_5799_end_0, end_mask = var_5799_end_mask_0, squeeze_mask = var_5799_squeeze_mask_0, x = _inversed_query_35_cast_fp16)[name = string("op_5799_cast_fp16")];
            tensor<int32, [1]> var_5800_axes_0 = const()[name = string("op_5800_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_5800_cast_fp16 = expand_dims(axes = var_5800_axes_0, x = var_5799_cast_fp16)[name = string("op_5800_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_5801_cast_fp16 = mul(x = state_407_cast_fp16, y = var_5800_cast_fp16)[name = string("op_5801_cast_fp16")];
            tensor<int32, [1]> var_5803_axes_0 = const()[name = string("op_5803_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_5803_keep_dims_0 = const()[name = string("op_5803_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_5803_cast_fp16 = reduce_sum(axes = var_5803_axes_0, keep_dims = var_5803_keep_dims_0, x = var_5801_cast_fp16)[name = string("op_5803_cast_fp16")];
            tensor<int32, [3]> var_5806_begin_0 = const()[name = string("op_5806_begin_0"), val = tensor<int32, [3]>([0, 0, 6])];
            tensor<int32, [3]> var_5806_end_0 = const()[name = string("op_5806_end_0"), val = tensor<int32, [3]>([1, 16, 7])];
            tensor<bool, [3]> var_5806_end_mask_0 = const()[name = string("op_5806_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_5806_squeeze_mask_0 = const()[name = string("op_5806_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_5806_cast_fp16 = slice_by_index(begin = var_5806_begin_0, end = var_5806_end_0, end_mask = var_5806_end_mask_0, squeeze_mask = var_5806_squeeze_mask_0, x = decay_13_cast_fp16)[name = string("op_5806_cast_fp16")];
            tensor<fp16, [1, 16]> var_5807_cast_fp16 = exp(x = var_5806_cast_fp16)[name = string("op_5807_cast_fp16")];
            tensor<int32, [1]> var_5808_axes_0 = const()[name = string("op_5808_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_5808_cast_fp16 = expand_dims(axes = var_5808_axes_0, x = var_5807_cast_fp16)[name = string("op_5808_cast_fp16")];
            tensor<int32, [1]> var_5809_axes_0 = const()[name = string("op_5809_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_5809_cast_fp16 = expand_dims(axes = var_5809_axes_0, x = var_5808_cast_fp16)[name = string("op_5809_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_409_cast_fp16 = mul(x = state_407_cast_fp16, y = var_5809_cast_fp16)[name = string("state_409_cast_fp16")];
            tensor<int32, [4]> key_token_205_begin_0 = const()[name = string("key_token_205_begin_0"), val = tensor<int32, [4]>([0, 0, 6, 0])];
            tensor<int32, [4]> key_token_205_end_0 = const()[name = string("key_token_205_end_0"), val = tensor<int32, [4]>([1, 16, 7, 128])];
            tensor<bool, [4]> key_token_205_end_mask_0 = const()[name = string("key_token_205_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_205_squeeze_mask_0 = const()[name = string("key_token_205_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_205_cast_fp16 = slice_by_index(begin = key_token_205_begin_0, end = key_token_205_end_0, end_mask = key_token_205_end_mask_0, squeeze_mask = key_token_205_squeeze_mask_0, x = key_35_cast_fp16)[name = string("key_token_205_cast_fp16")];
            tensor<int32, [4]> value_token_205_begin_0 = const()[name = string("value_token_205_begin_0"), val = tensor<int32, [4]>([0, 0, 6, 0])];
            tensor<int32, [4]> value_token_205_end_0 = const()[name = string("value_token_205_end_0"), val = tensor<int32, [4]>([1, 16, 7, 128])];
            tensor<bool, [4]> value_token_205_end_mask_0 = const()[name = string("value_token_205_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_205_squeeze_mask_0 = const()[name = string("value_token_205_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_205_cast_fp16 = slice_by_index(begin = value_token_205_begin_0, end = value_token_205_end_0, end_mask = value_token_205_end_mask_0, squeeze_mask = value_token_205_squeeze_mask_0, x = value_127_cast_fp16)[name = string("value_token_205_cast_fp16")];
            tensor<int32, [1]> var_5817_axes_0 = const()[name = string("op_5817_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_5817_cast_fp16 = expand_dims(axes = var_5817_axes_0, x = key_token_205_cast_fp16)[name = string("op_5817_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_5818_cast_fp16 = mul(x = state_409_cast_fp16, y = var_5817_cast_fp16)[name = string("op_5818_cast_fp16")];
            tensor<int32, [1]> memory_205_axes_0 = const()[name = string("memory_205_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_205_keep_dims_0 = const()[name = string("memory_205_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_205_cast_fp16 = reduce_sum(axes = memory_205_axes_0, keep_dims = memory_205_keep_dims_0, x = var_5818_cast_fp16)[name = string("memory_205_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_5821_cast_fp16 = sub(x = value_token_205_cast_fp16, y = memory_205_cast_fp16)[name = string("op_5821_cast_fp16")];
            tensor<int32, [3]> var_5824_begin_0 = const()[name = string("op_5824_begin_0"), val = tensor<int32, [3]>([0, 0, 6])];
            tensor<int32, [3]> var_5824_end_0 = const()[name = string("op_5824_end_0"), val = tensor<int32, [3]>([1, 16, 7])];
            tensor<bool, [3]> var_5824_end_mask_0 = const()[name = string("op_5824_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_5824_squeeze_mask_0 = const()[name = string("op_5824_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_5824_cast_fp16 = slice_by_index(begin = var_5824_begin_0, end = var_5824_end_0, end_mask = var_5824_end_mask_0, squeeze_mask = var_5824_squeeze_mask_0, x = beta_13_cast_fp16)[name = string("op_5824_cast_fp16")];
            tensor<int32, [1]> var_5825_axes_0 = const()[name = string("op_5825_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_5825_cast_fp16 = expand_dims(axes = var_5825_axes_0, x = var_5824_cast_fp16)[name = string("op_5825_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_205_cast_fp16 = mul(x = var_5821_cast_fp16, y = var_5825_cast_fp16)[name = string("delta_205_cast_fp16")];
            tensor<int32, [1]> var_5828_axes_0 = const()[name = string("op_5828_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_5828_cast_fp16 = expand_dims(axes = var_5828_axes_0, x = delta_205_cast_fp16)[name = string("op_5828_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_5829_cast_fp16 = mul(x = var_5817_cast_fp16, y = var_5828_cast_fp16)[name = string("op_5829_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_411_cast_fp16 = add(x = state_409_cast_fp16, y = var_5829_cast_fp16)[name = string("state_411_cast_fp16")];
            tensor<int32, [4]> var_5833_begin_0 = const()[name = string("op_5833_begin_0"), val = tensor<int32, [4]>([0, 0, 6, 0])];
            tensor<int32, [4]> var_5833_end_0 = const()[name = string("op_5833_end_0"), val = tensor<int32, [4]>([1, 16, 7, 128])];
            tensor<bool, [4]> var_5833_end_mask_0 = const()[name = string("op_5833_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_5833_squeeze_mask_0 = const()[name = string("op_5833_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_5833_cast_fp16 = slice_by_index(begin = var_5833_begin_0, end = var_5833_end_0, end_mask = var_5833_end_mask_0, squeeze_mask = var_5833_squeeze_mask_0, x = _inversed_query_35_cast_fp16)[name = string("op_5833_cast_fp16")];
            tensor<int32, [1]> var_5834_axes_0 = const()[name = string("op_5834_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_5834_cast_fp16 = expand_dims(axes = var_5834_axes_0, x = var_5833_cast_fp16)[name = string("op_5834_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_5835_cast_fp16 = mul(x = state_411_cast_fp16, y = var_5834_cast_fp16)[name = string("op_5835_cast_fp16")];
            tensor<int32, [1]> var_5837_axes_0 = const()[name = string("op_5837_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_5837_keep_dims_0 = const()[name = string("op_5837_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_5837_cast_fp16 = reduce_sum(axes = var_5837_axes_0, keep_dims = var_5837_keep_dims_0, x = var_5835_cast_fp16)[name = string("op_5837_cast_fp16")];
            tensor<int32, [3]> var_5840_begin_0 = const()[name = string("op_5840_begin_0"), val = tensor<int32, [3]>([0, 0, 7])];
            tensor<int32, [3]> var_5840_end_0 = const()[name = string("op_5840_end_0"), val = tensor<int32, [3]>([1, 16, 8])];
            tensor<bool, [3]> var_5840_end_mask_0 = const()[name = string("op_5840_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_5840_squeeze_mask_0 = const()[name = string("op_5840_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_5840_cast_fp16 = slice_by_index(begin = var_5840_begin_0, end = var_5840_end_0, end_mask = var_5840_end_mask_0, squeeze_mask = var_5840_squeeze_mask_0, x = decay_13_cast_fp16)[name = string("op_5840_cast_fp16")];
            tensor<fp16, [1, 16]> var_5841_cast_fp16 = exp(x = var_5840_cast_fp16)[name = string("op_5841_cast_fp16")];
            tensor<int32, [1]> var_5842_axes_0 = const()[name = string("op_5842_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_5842_cast_fp16 = expand_dims(axes = var_5842_axes_0, x = var_5841_cast_fp16)[name = string("op_5842_cast_fp16")];
            tensor<int32, [1]> var_5843_axes_0 = const()[name = string("op_5843_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_5843_cast_fp16 = expand_dims(axes = var_5843_axes_0, x = var_5842_cast_fp16)[name = string("op_5843_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_413_cast_fp16 = mul(x = state_411_cast_fp16, y = var_5843_cast_fp16)[name = string("state_413_cast_fp16")];
            tensor<int32, [4]> key_token_207_begin_0 = const()[name = string("key_token_207_begin_0"), val = tensor<int32, [4]>([0, 0, 7, 0])];
            tensor<int32, [4]> key_token_207_end_0 = const()[name = string("key_token_207_end_0"), val = tensor<int32, [4]>([1, 16, 8, 128])];
            tensor<bool, [4]> key_token_207_end_mask_0 = const()[name = string("key_token_207_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_207_squeeze_mask_0 = const()[name = string("key_token_207_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_207_cast_fp16 = slice_by_index(begin = key_token_207_begin_0, end = key_token_207_end_0, end_mask = key_token_207_end_mask_0, squeeze_mask = key_token_207_squeeze_mask_0, x = key_35_cast_fp16)[name = string("key_token_207_cast_fp16")];
            tensor<int32, [4]> value_token_207_begin_0 = const()[name = string("value_token_207_begin_0"), val = tensor<int32, [4]>([0, 0, 7, 0])];
            tensor<int32, [4]> value_token_207_end_0 = const()[name = string("value_token_207_end_0"), val = tensor<int32, [4]>([1, 16, 8, 128])];
            tensor<bool, [4]> value_token_207_end_mask_0 = const()[name = string("value_token_207_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_207_squeeze_mask_0 = const()[name = string("value_token_207_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_207_cast_fp16 = slice_by_index(begin = value_token_207_begin_0, end = value_token_207_end_0, end_mask = value_token_207_end_mask_0, squeeze_mask = value_token_207_squeeze_mask_0, x = value_127_cast_fp16)[name = string("value_token_207_cast_fp16")];
            tensor<int32, [1]> var_5851_axes_0 = const()[name = string("op_5851_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_5851_cast_fp16 = expand_dims(axes = var_5851_axes_0, x = key_token_207_cast_fp16)[name = string("op_5851_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_5852_cast_fp16 = mul(x = state_413_cast_fp16, y = var_5851_cast_fp16)[name = string("op_5852_cast_fp16")];
            tensor<int32, [1]> memory_207_axes_0 = const()[name = string("memory_207_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_207_keep_dims_0 = const()[name = string("memory_207_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_207_cast_fp16 = reduce_sum(axes = memory_207_axes_0, keep_dims = memory_207_keep_dims_0, x = var_5852_cast_fp16)[name = string("memory_207_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_5855_cast_fp16 = sub(x = value_token_207_cast_fp16, y = memory_207_cast_fp16)[name = string("op_5855_cast_fp16")];
            tensor<int32, [3]> var_5858_begin_0 = const()[name = string("op_5858_begin_0"), val = tensor<int32, [3]>([0, 0, 7])];
            tensor<int32, [3]> var_5858_end_0 = const()[name = string("op_5858_end_0"), val = tensor<int32, [3]>([1, 16, 8])];
            tensor<bool, [3]> var_5858_end_mask_0 = const()[name = string("op_5858_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_5858_squeeze_mask_0 = const()[name = string("op_5858_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_5858_cast_fp16 = slice_by_index(begin = var_5858_begin_0, end = var_5858_end_0, end_mask = var_5858_end_mask_0, squeeze_mask = var_5858_squeeze_mask_0, x = beta_13_cast_fp16)[name = string("op_5858_cast_fp16")];
            tensor<int32, [1]> var_5859_axes_0 = const()[name = string("op_5859_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_5859_cast_fp16 = expand_dims(axes = var_5859_axes_0, x = var_5858_cast_fp16)[name = string("op_5859_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_207_cast_fp16 = mul(x = var_5855_cast_fp16, y = var_5859_cast_fp16)[name = string("delta_207_cast_fp16")];
            tensor<int32, [1]> var_5862_axes_0 = const()[name = string("op_5862_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_5862_cast_fp16 = expand_dims(axes = var_5862_axes_0, x = delta_207_cast_fp16)[name = string("op_5862_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_5863_cast_fp16 = mul(x = var_5851_cast_fp16, y = var_5862_cast_fp16)[name = string("op_5863_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_415_cast_fp16 = add(x = state_413_cast_fp16, y = var_5863_cast_fp16)[name = string("state_415_cast_fp16")];
            tensor<int32, [4]> var_5867_begin_0 = const()[name = string("op_5867_begin_0"), val = tensor<int32, [4]>([0, 0, 7, 0])];
            tensor<int32, [4]> var_5867_end_0 = const()[name = string("op_5867_end_0"), val = tensor<int32, [4]>([1, 16, 8, 128])];
            tensor<bool, [4]> var_5867_end_mask_0 = const()[name = string("op_5867_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_5867_squeeze_mask_0 = const()[name = string("op_5867_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_5867_cast_fp16 = slice_by_index(begin = var_5867_begin_0, end = var_5867_end_0, end_mask = var_5867_end_mask_0, squeeze_mask = var_5867_squeeze_mask_0, x = _inversed_query_35_cast_fp16)[name = string("op_5867_cast_fp16")];
            tensor<int32, [1]> var_5868_axes_0 = const()[name = string("op_5868_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_5868_cast_fp16 = expand_dims(axes = var_5868_axes_0, x = var_5867_cast_fp16)[name = string("op_5868_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_5869_cast_fp16 = mul(x = state_415_cast_fp16, y = var_5868_cast_fp16)[name = string("op_5869_cast_fp16")];
            tensor<int32, [1]> var_5871_axes_0 = const()[name = string("op_5871_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_5871_keep_dims_0 = const()[name = string("op_5871_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_5871_cast_fp16 = reduce_sum(axes = var_5871_axes_0, keep_dims = var_5871_keep_dims_0, x = var_5869_cast_fp16)[name = string("op_5871_cast_fp16")];
            tensor<int32, [3]> var_5874_begin_0 = const()[name = string("op_5874_begin_0"), val = tensor<int32, [3]>([0, 0, 8])];
            tensor<int32, [3]> var_5874_end_0 = const()[name = string("op_5874_end_0"), val = tensor<int32, [3]>([1, 16, 9])];
            tensor<bool, [3]> var_5874_end_mask_0 = const()[name = string("op_5874_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_5874_squeeze_mask_0 = const()[name = string("op_5874_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_5874_cast_fp16 = slice_by_index(begin = var_5874_begin_0, end = var_5874_end_0, end_mask = var_5874_end_mask_0, squeeze_mask = var_5874_squeeze_mask_0, x = decay_13_cast_fp16)[name = string("op_5874_cast_fp16")];
            tensor<fp16, [1, 16]> var_5875_cast_fp16 = exp(x = var_5874_cast_fp16)[name = string("op_5875_cast_fp16")];
            tensor<int32, [1]> var_5876_axes_0 = const()[name = string("op_5876_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_5876_cast_fp16 = expand_dims(axes = var_5876_axes_0, x = var_5875_cast_fp16)[name = string("op_5876_cast_fp16")];
            tensor<int32, [1]> var_5877_axes_0 = const()[name = string("op_5877_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_5877_cast_fp16 = expand_dims(axes = var_5877_axes_0, x = var_5876_cast_fp16)[name = string("op_5877_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_417_cast_fp16 = mul(x = state_415_cast_fp16, y = var_5877_cast_fp16)[name = string("state_417_cast_fp16")];
            tensor<int32, [4]> key_token_209_begin_0 = const()[name = string("key_token_209_begin_0"), val = tensor<int32, [4]>([0, 0, 8, 0])];
            tensor<int32, [4]> key_token_209_end_0 = const()[name = string("key_token_209_end_0"), val = tensor<int32, [4]>([1, 16, 9, 128])];
            tensor<bool, [4]> key_token_209_end_mask_0 = const()[name = string("key_token_209_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_209_squeeze_mask_0 = const()[name = string("key_token_209_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_209_cast_fp16 = slice_by_index(begin = key_token_209_begin_0, end = key_token_209_end_0, end_mask = key_token_209_end_mask_0, squeeze_mask = key_token_209_squeeze_mask_0, x = key_35_cast_fp16)[name = string("key_token_209_cast_fp16")];
            tensor<int32, [4]> value_token_209_begin_0 = const()[name = string("value_token_209_begin_0"), val = tensor<int32, [4]>([0, 0, 8, 0])];
            tensor<int32, [4]> value_token_209_end_0 = const()[name = string("value_token_209_end_0"), val = tensor<int32, [4]>([1, 16, 9, 128])];
            tensor<bool, [4]> value_token_209_end_mask_0 = const()[name = string("value_token_209_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_209_squeeze_mask_0 = const()[name = string("value_token_209_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_209_cast_fp16 = slice_by_index(begin = value_token_209_begin_0, end = value_token_209_end_0, end_mask = value_token_209_end_mask_0, squeeze_mask = value_token_209_squeeze_mask_0, x = value_127_cast_fp16)[name = string("value_token_209_cast_fp16")];
            tensor<int32, [1]> var_5885_axes_0 = const()[name = string("op_5885_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_5885_cast_fp16 = expand_dims(axes = var_5885_axes_0, x = key_token_209_cast_fp16)[name = string("op_5885_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_5886_cast_fp16 = mul(x = state_417_cast_fp16, y = var_5885_cast_fp16)[name = string("op_5886_cast_fp16")];
            tensor<int32, [1]> memory_209_axes_0 = const()[name = string("memory_209_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_209_keep_dims_0 = const()[name = string("memory_209_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_209_cast_fp16 = reduce_sum(axes = memory_209_axes_0, keep_dims = memory_209_keep_dims_0, x = var_5886_cast_fp16)[name = string("memory_209_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_5889_cast_fp16 = sub(x = value_token_209_cast_fp16, y = memory_209_cast_fp16)[name = string("op_5889_cast_fp16")];
            tensor<int32, [3]> var_5892_begin_0 = const()[name = string("op_5892_begin_0"), val = tensor<int32, [3]>([0, 0, 8])];
            tensor<int32, [3]> var_5892_end_0 = const()[name = string("op_5892_end_0"), val = tensor<int32, [3]>([1, 16, 9])];
            tensor<bool, [3]> var_5892_end_mask_0 = const()[name = string("op_5892_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_5892_squeeze_mask_0 = const()[name = string("op_5892_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_5892_cast_fp16 = slice_by_index(begin = var_5892_begin_0, end = var_5892_end_0, end_mask = var_5892_end_mask_0, squeeze_mask = var_5892_squeeze_mask_0, x = beta_13_cast_fp16)[name = string("op_5892_cast_fp16")];
            tensor<int32, [1]> var_5893_axes_0 = const()[name = string("op_5893_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_5893_cast_fp16 = expand_dims(axes = var_5893_axes_0, x = var_5892_cast_fp16)[name = string("op_5893_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_209_cast_fp16 = mul(x = var_5889_cast_fp16, y = var_5893_cast_fp16)[name = string("delta_209_cast_fp16")];
            tensor<int32, [1]> var_5896_axes_0 = const()[name = string("op_5896_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_5896_cast_fp16 = expand_dims(axes = var_5896_axes_0, x = delta_209_cast_fp16)[name = string("op_5896_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_5897_cast_fp16 = mul(x = var_5885_cast_fp16, y = var_5896_cast_fp16)[name = string("op_5897_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_419_cast_fp16 = add(x = state_417_cast_fp16, y = var_5897_cast_fp16)[name = string("state_419_cast_fp16")];
            tensor<int32, [4]> var_5901_begin_0 = const()[name = string("op_5901_begin_0"), val = tensor<int32, [4]>([0, 0, 8, 0])];
            tensor<int32, [4]> var_5901_end_0 = const()[name = string("op_5901_end_0"), val = tensor<int32, [4]>([1, 16, 9, 128])];
            tensor<bool, [4]> var_5901_end_mask_0 = const()[name = string("op_5901_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_5901_squeeze_mask_0 = const()[name = string("op_5901_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_5901_cast_fp16 = slice_by_index(begin = var_5901_begin_0, end = var_5901_end_0, end_mask = var_5901_end_mask_0, squeeze_mask = var_5901_squeeze_mask_0, x = _inversed_query_35_cast_fp16)[name = string("op_5901_cast_fp16")];
            tensor<int32, [1]> var_5902_axes_0 = const()[name = string("op_5902_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_5902_cast_fp16 = expand_dims(axes = var_5902_axes_0, x = var_5901_cast_fp16)[name = string("op_5902_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_5903_cast_fp16 = mul(x = state_419_cast_fp16, y = var_5902_cast_fp16)[name = string("op_5903_cast_fp16")];
            tensor<int32, [1]> var_5905_axes_0 = const()[name = string("op_5905_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_5905_keep_dims_0 = const()[name = string("op_5905_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_5905_cast_fp16 = reduce_sum(axes = var_5905_axes_0, keep_dims = var_5905_keep_dims_0, x = var_5903_cast_fp16)[name = string("op_5905_cast_fp16")];
            tensor<int32, [3]> var_5908_begin_0 = const()[name = string("op_5908_begin_0"), val = tensor<int32, [3]>([0, 0, 9])];
            tensor<int32, [3]> var_5908_end_0 = const()[name = string("op_5908_end_0"), val = tensor<int32, [3]>([1, 16, 10])];
            tensor<bool, [3]> var_5908_end_mask_0 = const()[name = string("op_5908_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_5908_squeeze_mask_0 = const()[name = string("op_5908_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_5908_cast_fp16 = slice_by_index(begin = var_5908_begin_0, end = var_5908_end_0, end_mask = var_5908_end_mask_0, squeeze_mask = var_5908_squeeze_mask_0, x = decay_13_cast_fp16)[name = string("op_5908_cast_fp16")];
            tensor<fp16, [1, 16]> var_5909_cast_fp16 = exp(x = var_5908_cast_fp16)[name = string("op_5909_cast_fp16")];
            tensor<int32, [1]> var_5910_axes_0 = const()[name = string("op_5910_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_5910_cast_fp16 = expand_dims(axes = var_5910_axes_0, x = var_5909_cast_fp16)[name = string("op_5910_cast_fp16")];
            tensor<int32, [1]> var_5911_axes_0 = const()[name = string("op_5911_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_5911_cast_fp16 = expand_dims(axes = var_5911_axes_0, x = var_5910_cast_fp16)[name = string("op_5911_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_421_cast_fp16 = mul(x = state_419_cast_fp16, y = var_5911_cast_fp16)[name = string("state_421_cast_fp16")];
            tensor<int32, [4]> key_token_211_begin_0 = const()[name = string("key_token_211_begin_0"), val = tensor<int32, [4]>([0, 0, 9, 0])];
            tensor<int32, [4]> key_token_211_end_0 = const()[name = string("key_token_211_end_0"), val = tensor<int32, [4]>([1, 16, 10, 128])];
            tensor<bool, [4]> key_token_211_end_mask_0 = const()[name = string("key_token_211_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_211_squeeze_mask_0 = const()[name = string("key_token_211_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_211_cast_fp16 = slice_by_index(begin = key_token_211_begin_0, end = key_token_211_end_0, end_mask = key_token_211_end_mask_0, squeeze_mask = key_token_211_squeeze_mask_0, x = key_35_cast_fp16)[name = string("key_token_211_cast_fp16")];
            tensor<int32, [4]> value_token_211_begin_0 = const()[name = string("value_token_211_begin_0"), val = tensor<int32, [4]>([0, 0, 9, 0])];
            tensor<int32, [4]> value_token_211_end_0 = const()[name = string("value_token_211_end_0"), val = tensor<int32, [4]>([1, 16, 10, 128])];
            tensor<bool, [4]> value_token_211_end_mask_0 = const()[name = string("value_token_211_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_211_squeeze_mask_0 = const()[name = string("value_token_211_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_211_cast_fp16 = slice_by_index(begin = value_token_211_begin_0, end = value_token_211_end_0, end_mask = value_token_211_end_mask_0, squeeze_mask = value_token_211_squeeze_mask_0, x = value_127_cast_fp16)[name = string("value_token_211_cast_fp16")];
            tensor<int32, [1]> var_5919_axes_0 = const()[name = string("op_5919_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_5919_cast_fp16 = expand_dims(axes = var_5919_axes_0, x = key_token_211_cast_fp16)[name = string("op_5919_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_5920_cast_fp16 = mul(x = state_421_cast_fp16, y = var_5919_cast_fp16)[name = string("op_5920_cast_fp16")];
            tensor<int32, [1]> memory_211_axes_0 = const()[name = string("memory_211_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_211_keep_dims_0 = const()[name = string("memory_211_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_211_cast_fp16 = reduce_sum(axes = memory_211_axes_0, keep_dims = memory_211_keep_dims_0, x = var_5920_cast_fp16)[name = string("memory_211_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_5923_cast_fp16 = sub(x = value_token_211_cast_fp16, y = memory_211_cast_fp16)[name = string("op_5923_cast_fp16")];
            tensor<int32, [3]> var_5926_begin_0 = const()[name = string("op_5926_begin_0"), val = tensor<int32, [3]>([0, 0, 9])];
            tensor<int32, [3]> var_5926_end_0 = const()[name = string("op_5926_end_0"), val = tensor<int32, [3]>([1, 16, 10])];
            tensor<bool, [3]> var_5926_end_mask_0 = const()[name = string("op_5926_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_5926_squeeze_mask_0 = const()[name = string("op_5926_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_5926_cast_fp16 = slice_by_index(begin = var_5926_begin_0, end = var_5926_end_0, end_mask = var_5926_end_mask_0, squeeze_mask = var_5926_squeeze_mask_0, x = beta_13_cast_fp16)[name = string("op_5926_cast_fp16")];
            tensor<int32, [1]> var_5927_axes_0 = const()[name = string("op_5927_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_5927_cast_fp16 = expand_dims(axes = var_5927_axes_0, x = var_5926_cast_fp16)[name = string("op_5927_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_211_cast_fp16 = mul(x = var_5923_cast_fp16, y = var_5927_cast_fp16)[name = string("delta_211_cast_fp16")];
            tensor<int32, [1]> var_5930_axes_0 = const()[name = string("op_5930_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_5930_cast_fp16 = expand_dims(axes = var_5930_axes_0, x = delta_211_cast_fp16)[name = string("op_5930_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_5931_cast_fp16 = mul(x = var_5919_cast_fp16, y = var_5930_cast_fp16)[name = string("op_5931_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_423_cast_fp16 = add(x = state_421_cast_fp16, y = var_5931_cast_fp16)[name = string("state_423_cast_fp16")];
            tensor<int32, [4]> var_5935_begin_0 = const()[name = string("op_5935_begin_0"), val = tensor<int32, [4]>([0, 0, 9, 0])];
            tensor<int32, [4]> var_5935_end_0 = const()[name = string("op_5935_end_0"), val = tensor<int32, [4]>([1, 16, 10, 128])];
            tensor<bool, [4]> var_5935_end_mask_0 = const()[name = string("op_5935_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_5935_squeeze_mask_0 = const()[name = string("op_5935_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_5935_cast_fp16 = slice_by_index(begin = var_5935_begin_0, end = var_5935_end_0, end_mask = var_5935_end_mask_0, squeeze_mask = var_5935_squeeze_mask_0, x = _inversed_query_35_cast_fp16)[name = string("op_5935_cast_fp16")];
            tensor<int32, [1]> var_5936_axes_0 = const()[name = string("op_5936_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_5936_cast_fp16 = expand_dims(axes = var_5936_axes_0, x = var_5935_cast_fp16)[name = string("op_5936_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_5937_cast_fp16 = mul(x = state_423_cast_fp16, y = var_5936_cast_fp16)[name = string("op_5937_cast_fp16")];
            tensor<int32, [1]> var_5939_axes_0 = const()[name = string("op_5939_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_5939_keep_dims_0 = const()[name = string("op_5939_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_5939_cast_fp16 = reduce_sum(axes = var_5939_axes_0, keep_dims = var_5939_keep_dims_0, x = var_5937_cast_fp16)[name = string("op_5939_cast_fp16")];
            tensor<int32, [3]> var_5942_begin_0 = const()[name = string("op_5942_begin_0"), val = tensor<int32, [3]>([0, 0, 10])];
            tensor<int32, [3]> var_5942_end_0 = const()[name = string("op_5942_end_0"), val = tensor<int32, [3]>([1, 16, 11])];
            tensor<bool, [3]> var_5942_end_mask_0 = const()[name = string("op_5942_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_5942_squeeze_mask_0 = const()[name = string("op_5942_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_5942_cast_fp16 = slice_by_index(begin = var_5942_begin_0, end = var_5942_end_0, end_mask = var_5942_end_mask_0, squeeze_mask = var_5942_squeeze_mask_0, x = decay_13_cast_fp16)[name = string("op_5942_cast_fp16")];
            tensor<fp16, [1, 16]> var_5943_cast_fp16 = exp(x = var_5942_cast_fp16)[name = string("op_5943_cast_fp16")];
            tensor<int32, [1]> var_5944_axes_0 = const()[name = string("op_5944_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_5944_cast_fp16 = expand_dims(axes = var_5944_axes_0, x = var_5943_cast_fp16)[name = string("op_5944_cast_fp16")];
            tensor<int32, [1]> var_5945_axes_0 = const()[name = string("op_5945_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_5945_cast_fp16 = expand_dims(axes = var_5945_axes_0, x = var_5944_cast_fp16)[name = string("op_5945_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_425_cast_fp16 = mul(x = state_423_cast_fp16, y = var_5945_cast_fp16)[name = string("state_425_cast_fp16")];
            tensor<int32, [4]> key_token_213_begin_0 = const()[name = string("key_token_213_begin_0"), val = tensor<int32, [4]>([0, 0, 10, 0])];
            tensor<int32, [4]> key_token_213_end_0 = const()[name = string("key_token_213_end_0"), val = tensor<int32, [4]>([1, 16, 11, 128])];
            tensor<bool, [4]> key_token_213_end_mask_0 = const()[name = string("key_token_213_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_213_squeeze_mask_0 = const()[name = string("key_token_213_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_213_cast_fp16 = slice_by_index(begin = key_token_213_begin_0, end = key_token_213_end_0, end_mask = key_token_213_end_mask_0, squeeze_mask = key_token_213_squeeze_mask_0, x = key_35_cast_fp16)[name = string("key_token_213_cast_fp16")];
            tensor<int32, [4]> value_token_213_begin_0 = const()[name = string("value_token_213_begin_0"), val = tensor<int32, [4]>([0, 0, 10, 0])];
            tensor<int32, [4]> value_token_213_end_0 = const()[name = string("value_token_213_end_0"), val = tensor<int32, [4]>([1, 16, 11, 128])];
            tensor<bool, [4]> value_token_213_end_mask_0 = const()[name = string("value_token_213_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_213_squeeze_mask_0 = const()[name = string("value_token_213_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_213_cast_fp16 = slice_by_index(begin = value_token_213_begin_0, end = value_token_213_end_0, end_mask = value_token_213_end_mask_0, squeeze_mask = value_token_213_squeeze_mask_0, x = value_127_cast_fp16)[name = string("value_token_213_cast_fp16")];
            tensor<int32, [1]> var_5953_axes_0 = const()[name = string("op_5953_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_5953_cast_fp16 = expand_dims(axes = var_5953_axes_0, x = key_token_213_cast_fp16)[name = string("op_5953_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_5954_cast_fp16 = mul(x = state_425_cast_fp16, y = var_5953_cast_fp16)[name = string("op_5954_cast_fp16")];
            tensor<int32, [1]> memory_213_axes_0 = const()[name = string("memory_213_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_213_keep_dims_0 = const()[name = string("memory_213_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_213_cast_fp16 = reduce_sum(axes = memory_213_axes_0, keep_dims = memory_213_keep_dims_0, x = var_5954_cast_fp16)[name = string("memory_213_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_5957_cast_fp16 = sub(x = value_token_213_cast_fp16, y = memory_213_cast_fp16)[name = string("op_5957_cast_fp16")];
            tensor<int32, [3]> var_5960_begin_0 = const()[name = string("op_5960_begin_0"), val = tensor<int32, [3]>([0, 0, 10])];
            tensor<int32, [3]> var_5960_end_0 = const()[name = string("op_5960_end_0"), val = tensor<int32, [3]>([1, 16, 11])];
            tensor<bool, [3]> var_5960_end_mask_0 = const()[name = string("op_5960_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_5960_squeeze_mask_0 = const()[name = string("op_5960_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_5960_cast_fp16 = slice_by_index(begin = var_5960_begin_0, end = var_5960_end_0, end_mask = var_5960_end_mask_0, squeeze_mask = var_5960_squeeze_mask_0, x = beta_13_cast_fp16)[name = string("op_5960_cast_fp16")];
            tensor<int32, [1]> var_5961_axes_0 = const()[name = string("op_5961_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_5961_cast_fp16 = expand_dims(axes = var_5961_axes_0, x = var_5960_cast_fp16)[name = string("op_5961_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_213_cast_fp16 = mul(x = var_5957_cast_fp16, y = var_5961_cast_fp16)[name = string("delta_213_cast_fp16")];
            tensor<int32, [1]> var_5964_axes_0 = const()[name = string("op_5964_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_5964_cast_fp16 = expand_dims(axes = var_5964_axes_0, x = delta_213_cast_fp16)[name = string("op_5964_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_5965_cast_fp16 = mul(x = var_5953_cast_fp16, y = var_5964_cast_fp16)[name = string("op_5965_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_427_cast_fp16 = add(x = state_425_cast_fp16, y = var_5965_cast_fp16)[name = string("state_427_cast_fp16")];
            tensor<int32, [4]> var_5969_begin_0 = const()[name = string("op_5969_begin_0"), val = tensor<int32, [4]>([0, 0, 10, 0])];
            tensor<int32, [4]> var_5969_end_0 = const()[name = string("op_5969_end_0"), val = tensor<int32, [4]>([1, 16, 11, 128])];
            tensor<bool, [4]> var_5969_end_mask_0 = const()[name = string("op_5969_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_5969_squeeze_mask_0 = const()[name = string("op_5969_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_5969_cast_fp16 = slice_by_index(begin = var_5969_begin_0, end = var_5969_end_0, end_mask = var_5969_end_mask_0, squeeze_mask = var_5969_squeeze_mask_0, x = _inversed_query_35_cast_fp16)[name = string("op_5969_cast_fp16")];
            tensor<int32, [1]> var_5970_axes_0 = const()[name = string("op_5970_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_5970_cast_fp16 = expand_dims(axes = var_5970_axes_0, x = var_5969_cast_fp16)[name = string("op_5970_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_5971_cast_fp16 = mul(x = state_427_cast_fp16, y = var_5970_cast_fp16)[name = string("op_5971_cast_fp16")];
            tensor<int32, [1]> var_5973_axes_0 = const()[name = string("op_5973_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_5973_keep_dims_0 = const()[name = string("op_5973_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_5973_cast_fp16 = reduce_sum(axes = var_5973_axes_0, keep_dims = var_5973_keep_dims_0, x = var_5971_cast_fp16)[name = string("op_5973_cast_fp16")];
            tensor<int32, [3]> var_5976_begin_0 = const()[name = string("op_5976_begin_0"), val = tensor<int32, [3]>([0, 0, 11])];
            tensor<int32, [3]> var_5976_end_0 = const()[name = string("op_5976_end_0"), val = tensor<int32, [3]>([1, 16, 12])];
            tensor<bool, [3]> var_5976_end_mask_0 = const()[name = string("op_5976_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_5976_squeeze_mask_0 = const()[name = string("op_5976_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_5976_cast_fp16 = slice_by_index(begin = var_5976_begin_0, end = var_5976_end_0, end_mask = var_5976_end_mask_0, squeeze_mask = var_5976_squeeze_mask_0, x = decay_13_cast_fp16)[name = string("op_5976_cast_fp16")];
            tensor<fp16, [1, 16]> var_5977_cast_fp16 = exp(x = var_5976_cast_fp16)[name = string("op_5977_cast_fp16")];
            tensor<int32, [1]> var_5978_axes_0 = const()[name = string("op_5978_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_5978_cast_fp16 = expand_dims(axes = var_5978_axes_0, x = var_5977_cast_fp16)[name = string("op_5978_cast_fp16")];
            tensor<int32, [1]> var_5979_axes_0 = const()[name = string("op_5979_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_5979_cast_fp16 = expand_dims(axes = var_5979_axes_0, x = var_5978_cast_fp16)[name = string("op_5979_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_429_cast_fp16 = mul(x = state_427_cast_fp16, y = var_5979_cast_fp16)[name = string("state_429_cast_fp16")];
            tensor<int32, [4]> key_token_215_begin_0 = const()[name = string("key_token_215_begin_0"), val = tensor<int32, [4]>([0, 0, 11, 0])];
            tensor<int32, [4]> key_token_215_end_0 = const()[name = string("key_token_215_end_0"), val = tensor<int32, [4]>([1, 16, 12, 128])];
            tensor<bool, [4]> key_token_215_end_mask_0 = const()[name = string("key_token_215_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_215_squeeze_mask_0 = const()[name = string("key_token_215_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_215_cast_fp16 = slice_by_index(begin = key_token_215_begin_0, end = key_token_215_end_0, end_mask = key_token_215_end_mask_0, squeeze_mask = key_token_215_squeeze_mask_0, x = key_35_cast_fp16)[name = string("key_token_215_cast_fp16")];
            tensor<int32, [4]> value_token_215_begin_0 = const()[name = string("value_token_215_begin_0"), val = tensor<int32, [4]>([0, 0, 11, 0])];
            tensor<int32, [4]> value_token_215_end_0 = const()[name = string("value_token_215_end_0"), val = tensor<int32, [4]>([1, 16, 12, 128])];
            tensor<bool, [4]> value_token_215_end_mask_0 = const()[name = string("value_token_215_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_215_squeeze_mask_0 = const()[name = string("value_token_215_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_215_cast_fp16 = slice_by_index(begin = value_token_215_begin_0, end = value_token_215_end_0, end_mask = value_token_215_end_mask_0, squeeze_mask = value_token_215_squeeze_mask_0, x = value_127_cast_fp16)[name = string("value_token_215_cast_fp16")];
            tensor<int32, [1]> var_5987_axes_0 = const()[name = string("op_5987_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_5987_cast_fp16 = expand_dims(axes = var_5987_axes_0, x = key_token_215_cast_fp16)[name = string("op_5987_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_5988_cast_fp16 = mul(x = state_429_cast_fp16, y = var_5987_cast_fp16)[name = string("op_5988_cast_fp16")];
            tensor<int32, [1]> memory_215_axes_0 = const()[name = string("memory_215_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_215_keep_dims_0 = const()[name = string("memory_215_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_215_cast_fp16 = reduce_sum(axes = memory_215_axes_0, keep_dims = memory_215_keep_dims_0, x = var_5988_cast_fp16)[name = string("memory_215_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_5991_cast_fp16 = sub(x = value_token_215_cast_fp16, y = memory_215_cast_fp16)[name = string("op_5991_cast_fp16")];
            tensor<int32, [3]> var_5994_begin_0 = const()[name = string("op_5994_begin_0"), val = tensor<int32, [3]>([0, 0, 11])];
            tensor<int32, [3]> var_5994_end_0 = const()[name = string("op_5994_end_0"), val = tensor<int32, [3]>([1, 16, 12])];
            tensor<bool, [3]> var_5994_end_mask_0 = const()[name = string("op_5994_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_5994_squeeze_mask_0 = const()[name = string("op_5994_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_5994_cast_fp16 = slice_by_index(begin = var_5994_begin_0, end = var_5994_end_0, end_mask = var_5994_end_mask_0, squeeze_mask = var_5994_squeeze_mask_0, x = beta_13_cast_fp16)[name = string("op_5994_cast_fp16")];
            tensor<int32, [1]> var_5995_axes_0 = const()[name = string("op_5995_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_5995_cast_fp16 = expand_dims(axes = var_5995_axes_0, x = var_5994_cast_fp16)[name = string("op_5995_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_215_cast_fp16 = mul(x = var_5991_cast_fp16, y = var_5995_cast_fp16)[name = string("delta_215_cast_fp16")];
            tensor<int32, [1]> var_5998_axes_0 = const()[name = string("op_5998_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_5998_cast_fp16 = expand_dims(axes = var_5998_axes_0, x = delta_215_cast_fp16)[name = string("op_5998_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_5999_cast_fp16 = mul(x = var_5987_cast_fp16, y = var_5998_cast_fp16)[name = string("op_5999_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_431_cast_fp16 = add(x = state_429_cast_fp16, y = var_5999_cast_fp16)[name = string("state_431_cast_fp16")];
            tensor<int32, [4]> var_6003_begin_0 = const()[name = string("op_6003_begin_0"), val = tensor<int32, [4]>([0, 0, 11, 0])];
            tensor<int32, [4]> var_6003_end_0 = const()[name = string("op_6003_end_0"), val = tensor<int32, [4]>([1, 16, 12, 128])];
            tensor<bool, [4]> var_6003_end_mask_0 = const()[name = string("op_6003_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_6003_squeeze_mask_0 = const()[name = string("op_6003_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_6003_cast_fp16 = slice_by_index(begin = var_6003_begin_0, end = var_6003_end_0, end_mask = var_6003_end_mask_0, squeeze_mask = var_6003_squeeze_mask_0, x = _inversed_query_35_cast_fp16)[name = string("op_6003_cast_fp16")];
            tensor<int32, [1]> var_6004_axes_0 = const()[name = string("op_6004_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_6004_cast_fp16 = expand_dims(axes = var_6004_axes_0, x = var_6003_cast_fp16)[name = string("op_6004_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6005_cast_fp16 = mul(x = state_431_cast_fp16, y = var_6004_cast_fp16)[name = string("op_6005_cast_fp16")];
            tensor<int32, [1]> var_6007_axes_0 = const()[name = string("op_6007_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_6007_keep_dims_0 = const()[name = string("op_6007_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_6007_cast_fp16 = reduce_sum(axes = var_6007_axes_0, keep_dims = var_6007_keep_dims_0, x = var_6005_cast_fp16)[name = string("op_6007_cast_fp16")];
            tensor<int32, [3]> var_6010_begin_0 = const()[name = string("op_6010_begin_0"), val = tensor<int32, [3]>([0, 0, 12])];
            tensor<int32, [3]> var_6010_end_0 = const()[name = string("op_6010_end_0"), val = tensor<int32, [3]>([1, 16, 13])];
            tensor<bool, [3]> var_6010_end_mask_0 = const()[name = string("op_6010_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_6010_squeeze_mask_0 = const()[name = string("op_6010_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_6010_cast_fp16 = slice_by_index(begin = var_6010_begin_0, end = var_6010_end_0, end_mask = var_6010_end_mask_0, squeeze_mask = var_6010_squeeze_mask_0, x = decay_13_cast_fp16)[name = string("op_6010_cast_fp16")];
            tensor<fp16, [1, 16]> var_6011_cast_fp16 = exp(x = var_6010_cast_fp16)[name = string("op_6011_cast_fp16")];
            tensor<int32, [1]> var_6012_axes_0 = const()[name = string("op_6012_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_6012_cast_fp16 = expand_dims(axes = var_6012_axes_0, x = var_6011_cast_fp16)[name = string("op_6012_cast_fp16")];
            tensor<int32, [1]> var_6013_axes_0 = const()[name = string("op_6013_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_6013_cast_fp16 = expand_dims(axes = var_6013_axes_0, x = var_6012_cast_fp16)[name = string("op_6013_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_433_cast_fp16 = mul(x = state_431_cast_fp16, y = var_6013_cast_fp16)[name = string("state_433_cast_fp16")];
            tensor<int32, [4]> key_token_217_begin_0 = const()[name = string("key_token_217_begin_0"), val = tensor<int32, [4]>([0, 0, 12, 0])];
            tensor<int32, [4]> key_token_217_end_0 = const()[name = string("key_token_217_end_0"), val = tensor<int32, [4]>([1, 16, 13, 128])];
            tensor<bool, [4]> key_token_217_end_mask_0 = const()[name = string("key_token_217_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_217_squeeze_mask_0 = const()[name = string("key_token_217_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_217_cast_fp16 = slice_by_index(begin = key_token_217_begin_0, end = key_token_217_end_0, end_mask = key_token_217_end_mask_0, squeeze_mask = key_token_217_squeeze_mask_0, x = key_35_cast_fp16)[name = string("key_token_217_cast_fp16")];
            tensor<int32, [4]> value_token_217_begin_0 = const()[name = string("value_token_217_begin_0"), val = tensor<int32, [4]>([0, 0, 12, 0])];
            tensor<int32, [4]> value_token_217_end_0 = const()[name = string("value_token_217_end_0"), val = tensor<int32, [4]>([1, 16, 13, 128])];
            tensor<bool, [4]> value_token_217_end_mask_0 = const()[name = string("value_token_217_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_217_squeeze_mask_0 = const()[name = string("value_token_217_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_217_cast_fp16 = slice_by_index(begin = value_token_217_begin_0, end = value_token_217_end_0, end_mask = value_token_217_end_mask_0, squeeze_mask = value_token_217_squeeze_mask_0, x = value_127_cast_fp16)[name = string("value_token_217_cast_fp16")];
            tensor<int32, [1]> var_6021_axes_0 = const()[name = string("op_6021_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_6021_cast_fp16 = expand_dims(axes = var_6021_axes_0, x = key_token_217_cast_fp16)[name = string("op_6021_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6022_cast_fp16 = mul(x = state_433_cast_fp16, y = var_6021_cast_fp16)[name = string("op_6022_cast_fp16")];
            tensor<int32, [1]> memory_217_axes_0 = const()[name = string("memory_217_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_217_keep_dims_0 = const()[name = string("memory_217_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_217_cast_fp16 = reduce_sum(axes = memory_217_axes_0, keep_dims = memory_217_keep_dims_0, x = var_6022_cast_fp16)[name = string("memory_217_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_6025_cast_fp16 = sub(x = value_token_217_cast_fp16, y = memory_217_cast_fp16)[name = string("op_6025_cast_fp16")];
            tensor<int32, [3]> var_6028_begin_0 = const()[name = string("op_6028_begin_0"), val = tensor<int32, [3]>([0, 0, 12])];
            tensor<int32, [3]> var_6028_end_0 = const()[name = string("op_6028_end_0"), val = tensor<int32, [3]>([1, 16, 13])];
            tensor<bool, [3]> var_6028_end_mask_0 = const()[name = string("op_6028_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_6028_squeeze_mask_0 = const()[name = string("op_6028_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_6028_cast_fp16 = slice_by_index(begin = var_6028_begin_0, end = var_6028_end_0, end_mask = var_6028_end_mask_0, squeeze_mask = var_6028_squeeze_mask_0, x = beta_13_cast_fp16)[name = string("op_6028_cast_fp16")];
            tensor<int32, [1]> var_6029_axes_0 = const()[name = string("op_6029_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_6029_cast_fp16 = expand_dims(axes = var_6029_axes_0, x = var_6028_cast_fp16)[name = string("op_6029_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_217_cast_fp16 = mul(x = var_6025_cast_fp16, y = var_6029_cast_fp16)[name = string("delta_217_cast_fp16")];
            tensor<int32, [1]> var_6032_axes_0 = const()[name = string("op_6032_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_6032_cast_fp16 = expand_dims(axes = var_6032_axes_0, x = delta_217_cast_fp16)[name = string("op_6032_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6033_cast_fp16 = mul(x = var_6021_cast_fp16, y = var_6032_cast_fp16)[name = string("op_6033_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_435_cast_fp16 = add(x = state_433_cast_fp16, y = var_6033_cast_fp16)[name = string("state_435_cast_fp16")];
            tensor<int32, [4]> var_6037_begin_0 = const()[name = string("op_6037_begin_0"), val = tensor<int32, [4]>([0, 0, 12, 0])];
            tensor<int32, [4]> var_6037_end_0 = const()[name = string("op_6037_end_0"), val = tensor<int32, [4]>([1, 16, 13, 128])];
            tensor<bool, [4]> var_6037_end_mask_0 = const()[name = string("op_6037_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_6037_squeeze_mask_0 = const()[name = string("op_6037_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_6037_cast_fp16 = slice_by_index(begin = var_6037_begin_0, end = var_6037_end_0, end_mask = var_6037_end_mask_0, squeeze_mask = var_6037_squeeze_mask_0, x = _inversed_query_35_cast_fp16)[name = string("op_6037_cast_fp16")];
            tensor<int32, [1]> var_6038_axes_0 = const()[name = string("op_6038_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_6038_cast_fp16 = expand_dims(axes = var_6038_axes_0, x = var_6037_cast_fp16)[name = string("op_6038_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6039_cast_fp16 = mul(x = state_435_cast_fp16, y = var_6038_cast_fp16)[name = string("op_6039_cast_fp16")];
            tensor<int32, [1]> var_6041_axes_0 = const()[name = string("op_6041_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_6041_keep_dims_0 = const()[name = string("op_6041_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_6041_cast_fp16 = reduce_sum(axes = var_6041_axes_0, keep_dims = var_6041_keep_dims_0, x = var_6039_cast_fp16)[name = string("op_6041_cast_fp16")];
            tensor<int32, [3]> var_6044_begin_0 = const()[name = string("op_6044_begin_0"), val = tensor<int32, [3]>([0, 0, 13])];
            tensor<int32, [3]> var_6044_end_0 = const()[name = string("op_6044_end_0"), val = tensor<int32, [3]>([1, 16, 14])];
            tensor<bool, [3]> var_6044_end_mask_0 = const()[name = string("op_6044_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_6044_squeeze_mask_0 = const()[name = string("op_6044_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_6044_cast_fp16 = slice_by_index(begin = var_6044_begin_0, end = var_6044_end_0, end_mask = var_6044_end_mask_0, squeeze_mask = var_6044_squeeze_mask_0, x = decay_13_cast_fp16)[name = string("op_6044_cast_fp16")];
            tensor<fp16, [1, 16]> var_6045_cast_fp16 = exp(x = var_6044_cast_fp16)[name = string("op_6045_cast_fp16")];
            tensor<int32, [1]> var_6046_axes_0 = const()[name = string("op_6046_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_6046_cast_fp16 = expand_dims(axes = var_6046_axes_0, x = var_6045_cast_fp16)[name = string("op_6046_cast_fp16")];
            tensor<int32, [1]> var_6047_axes_0 = const()[name = string("op_6047_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_6047_cast_fp16 = expand_dims(axes = var_6047_axes_0, x = var_6046_cast_fp16)[name = string("op_6047_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_437_cast_fp16 = mul(x = state_435_cast_fp16, y = var_6047_cast_fp16)[name = string("state_437_cast_fp16")];
            tensor<int32, [4]> key_token_219_begin_0 = const()[name = string("key_token_219_begin_0"), val = tensor<int32, [4]>([0, 0, 13, 0])];
            tensor<int32, [4]> key_token_219_end_0 = const()[name = string("key_token_219_end_0"), val = tensor<int32, [4]>([1, 16, 14, 128])];
            tensor<bool, [4]> key_token_219_end_mask_0 = const()[name = string("key_token_219_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_219_squeeze_mask_0 = const()[name = string("key_token_219_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_219_cast_fp16 = slice_by_index(begin = key_token_219_begin_0, end = key_token_219_end_0, end_mask = key_token_219_end_mask_0, squeeze_mask = key_token_219_squeeze_mask_0, x = key_35_cast_fp16)[name = string("key_token_219_cast_fp16")];
            tensor<int32, [4]> value_token_219_begin_0 = const()[name = string("value_token_219_begin_0"), val = tensor<int32, [4]>([0, 0, 13, 0])];
            tensor<int32, [4]> value_token_219_end_0 = const()[name = string("value_token_219_end_0"), val = tensor<int32, [4]>([1, 16, 14, 128])];
            tensor<bool, [4]> value_token_219_end_mask_0 = const()[name = string("value_token_219_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_219_squeeze_mask_0 = const()[name = string("value_token_219_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_219_cast_fp16 = slice_by_index(begin = value_token_219_begin_0, end = value_token_219_end_0, end_mask = value_token_219_end_mask_0, squeeze_mask = value_token_219_squeeze_mask_0, x = value_127_cast_fp16)[name = string("value_token_219_cast_fp16")];
            tensor<int32, [1]> var_6055_axes_0 = const()[name = string("op_6055_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_6055_cast_fp16 = expand_dims(axes = var_6055_axes_0, x = key_token_219_cast_fp16)[name = string("op_6055_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6056_cast_fp16 = mul(x = state_437_cast_fp16, y = var_6055_cast_fp16)[name = string("op_6056_cast_fp16")];
            tensor<int32, [1]> memory_219_axes_0 = const()[name = string("memory_219_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_219_keep_dims_0 = const()[name = string("memory_219_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_219_cast_fp16 = reduce_sum(axes = memory_219_axes_0, keep_dims = memory_219_keep_dims_0, x = var_6056_cast_fp16)[name = string("memory_219_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_6059_cast_fp16 = sub(x = value_token_219_cast_fp16, y = memory_219_cast_fp16)[name = string("op_6059_cast_fp16")];
            tensor<int32, [3]> var_6062_begin_0 = const()[name = string("op_6062_begin_0"), val = tensor<int32, [3]>([0, 0, 13])];
            tensor<int32, [3]> var_6062_end_0 = const()[name = string("op_6062_end_0"), val = tensor<int32, [3]>([1, 16, 14])];
            tensor<bool, [3]> var_6062_end_mask_0 = const()[name = string("op_6062_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_6062_squeeze_mask_0 = const()[name = string("op_6062_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_6062_cast_fp16 = slice_by_index(begin = var_6062_begin_0, end = var_6062_end_0, end_mask = var_6062_end_mask_0, squeeze_mask = var_6062_squeeze_mask_0, x = beta_13_cast_fp16)[name = string("op_6062_cast_fp16")];
            tensor<int32, [1]> var_6063_axes_0 = const()[name = string("op_6063_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_6063_cast_fp16 = expand_dims(axes = var_6063_axes_0, x = var_6062_cast_fp16)[name = string("op_6063_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_219_cast_fp16 = mul(x = var_6059_cast_fp16, y = var_6063_cast_fp16)[name = string("delta_219_cast_fp16")];
            tensor<int32, [1]> var_6066_axes_0 = const()[name = string("op_6066_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_6066_cast_fp16 = expand_dims(axes = var_6066_axes_0, x = delta_219_cast_fp16)[name = string("op_6066_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6067_cast_fp16 = mul(x = var_6055_cast_fp16, y = var_6066_cast_fp16)[name = string("op_6067_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_439_cast_fp16 = add(x = state_437_cast_fp16, y = var_6067_cast_fp16)[name = string("state_439_cast_fp16")];
            tensor<int32, [4]> var_6071_begin_0 = const()[name = string("op_6071_begin_0"), val = tensor<int32, [4]>([0, 0, 13, 0])];
            tensor<int32, [4]> var_6071_end_0 = const()[name = string("op_6071_end_0"), val = tensor<int32, [4]>([1, 16, 14, 128])];
            tensor<bool, [4]> var_6071_end_mask_0 = const()[name = string("op_6071_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_6071_squeeze_mask_0 = const()[name = string("op_6071_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_6071_cast_fp16 = slice_by_index(begin = var_6071_begin_0, end = var_6071_end_0, end_mask = var_6071_end_mask_0, squeeze_mask = var_6071_squeeze_mask_0, x = _inversed_query_35_cast_fp16)[name = string("op_6071_cast_fp16")];
            tensor<int32, [1]> var_6072_axes_0 = const()[name = string("op_6072_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_6072_cast_fp16 = expand_dims(axes = var_6072_axes_0, x = var_6071_cast_fp16)[name = string("op_6072_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6073_cast_fp16 = mul(x = state_439_cast_fp16, y = var_6072_cast_fp16)[name = string("op_6073_cast_fp16")];
            tensor<int32, [1]> var_6075_axes_0 = const()[name = string("op_6075_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_6075_keep_dims_0 = const()[name = string("op_6075_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_6075_cast_fp16 = reduce_sum(axes = var_6075_axes_0, keep_dims = var_6075_keep_dims_0, x = var_6073_cast_fp16)[name = string("op_6075_cast_fp16")];
            tensor<int32, [3]> var_6078_begin_0 = const()[name = string("op_6078_begin_0"), val = tensor<int32, [3]>([0, 0, 14])];
            tensor<int32, [3]> var_6078_end_0 = const()[name = string("op_6078_end_0"), val = tensor<int32, [3]>([1, 16, 15])];
            tensor<bool, [3]> var_6078_end_mask_0 = const()[name = string("op_6078_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_6078_squeeze_mask_0 = const()[name = string("op_6078_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_6078_cast_fp16 = slice_by_index(begin = var_6078_begin_0, end = var_6078_end_0, end_mask = var_6078_end_mask_0, squeeze_mask = var_6078_squeeze_mask_0, x = decay_13_cast_fp16)[name = string("op_6078_cast_fp16")];
            tensor<fp16, [1, 16]> var_6079_cast_fp16 = exp(x = var_6078_cast_fp16)[name = string("op_6079_cast_fp16")];
            tensor<int32, [1]> var_6080_axes_0 = const()[name = string("op_6080_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_6080_cast_fp16 = expand_dims(axes = var_6080_axes_0, x = var_6079_cast_fp16)[name = string("op_6080_cast_fp16")];
            tensor<int32, [1]> var_6081_axes_0 = const()[name = string("op_6081_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_6081_cast_fp16 = expand_dims(axes = var_6081_axes_0, x = var_6080_cast_fp16)[name = string("op_6081_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_441_cast_fp16 = mul(x = state_439_cast_fp16, y = var_6081_cast_fp16)[name = string("state_441_cast_fp16")];
            tensor<int32, [4]> key_token_221_begin_0 = const()[name = string("key_token_221_begin_0"), val = tensor<int32, [4]>([0, 0, 14, 0])];
            tensor<int32, [4]> key_token_221_end_0 = const()[name = string("key_token_221_end_0"), val = tensor<int32, [4]>([1, 16, 15, 128])];
            tensor<bool, [4]> key_token_221_end_mask_0 = const()[name = string("key_token_221_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_221_squeeze_mask_0 = const()[name = string("key_token_221_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_221_cast_fp16 = slice_by_index(begin = key_token_221_begin_0, end = key_token_221_end_0, end_mask = key_token_221_end_mask_0, squeeze_mask = key_token_221_squeeze_mask_0, x = key_35_cast_fp16)[name = string("key_token_221_cast_fp16")];
            tensor<int32, [4]> value_token_221_begin_0 = const()[name = string("value_token_221_begin_0"), val = tensor<int32, [4]>([0, 0, 14, 0])];
            tensor<int32, [4]> value_token_221_end_0 = const()[name = string("value_token_221_end_0"), val = tensor<int32, [4]>([1, 16, 15, 128])];
            tensor<bool, [4]> value_token_221_end_mask_0 = const()[name = string("value_token_221_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_221_squeeze_mask_0 = const()[name = string("value_token_221_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_221_cast_fp16 = slice_by_index(begin = value_token_221_begin_0, end = value_token_221_end_0, end_mask = value_token_221_end_mask_0, squeeze_mask = value_token_221_squeeze_mask_0, x = value_127_cast_fp16)[name = string("value_token_221_cast_fp16")];
            tensor<int32, [1]> var_6089_axes_0 = const()[name = string("op_6089_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_6089_cast_fp16 = expand_dims(axes = var_6089_axes_0, x = key_token_221_cast_fp16)[name = string("op_6089_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6090_cast_fp16 = mul(x = state_441_cast_fp16, y = var_6089_cast_fp16)[name = string("op_6090_cast_fp16")];
            tensor<int32, [1]> memory_221_axes_0 = const()[name = string("memory_221_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_221_keep_dims_0 = const()[name = string("memory_221_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_221_cast_fp16 = reduce_sum(axes = memory_221_axes_0, keep_dims = memory_221_keep_dims_0, x = var_6090_cast_fp16)[name = string("memory_221_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_6093_cast_fp16 = sub(x = value_token_221_cast_fp16, y = memory_221_cast_fp16)[name = string("op_6093_cast_fp16")];
            tensor<int32, [3]> var_6096_begin_0 = const()[name = string("op_6096_begin_0"), val = tensor<int32, [3]>([0, 0, 14])];
            tensor<int32, [3]> var_6096_end_0 = const()[name = string("op_6096_end_0"), val = tensor<int32, [3]>([1, 16, 15])];
            tensor<bool, [3]> var_6096_end_mask_0 = const()[name = string("op_6096_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_6096_squeeze_mask_0 = const()[name = string("op_6096_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_6096_cast_fp16 = slice_by_index(begin = var_6096_begin_0, end = var_6096_end_0, end_mask = var_6096_end_mask_0, squeeze_mask = var_6096_squeeze_mask_0, x = beta_13_cast_fp16)[name = string("op_6096_cast_fp16")];
            tensor<int32, [1]> var_6097_axes_0 = const()[name = string("op_6097_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_6097_cast_fp16 = expand_dims(axes = var_6097_axes_0, x = var_6096_cast_fp16)[name = string("op_6097_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_221_cast_fp16 = mul(x = var_6093_cast_fp16, y = var_6097_cast_fp16)[name = string("delta_221_cast_fp16")];
            tensor<int32, [1]> var_6100_axes_0 = const()[name = string("op_6100_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_6100_cast_fp16 = expand_dims(axes = var_6100_axes_0, x = delta_221_cast_fp16)[name = string("op_6100_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6101_cast_fp16 = mul(x = var_6089_cast_fp16, y = var_6100_cast_fp16)[name = string("op_6101_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_443_cast_fp16 = add(x = state_441_cast_fp16, y = var_6101_cast_fp16)[name = string("state_443_cast_fp16")];
            tensor<int32, [4]> var_6105_begin_0 = const()[name = string("op_6105_begin_0"), val = tensor<int32, [4]>([0, 0, 14, 0])];
            tensor<int32, [4]> var_6105_end_0 = const()[name = string("op_6105_end_0"), val = tensor<int32, [4]>([1, 16, 15, 128])];
            tensor<bool, [4]> var_6105_end_mask_0 = const()[name = string("op_6105_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_6105_squeeze_mask_0 = const()[name = string("op_6105_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_6105_cast_fp16 = slice_by_index(begin = var_6105_begin_0, end = var_6105_end_0, end_mask = var_6105_end_mask_0, squeeze_mask = var_6105_squeeze_mask_0, x = _inversed_query_35_cast_fp16)[name = string("op_6105_cast_fp16")];
            tensor<int32, [1]> var_6106_axes_0 = const()[name = string("op_6106_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_6106_cast_fp16 = expand_dims(axes = var_6106_axes_0, x = var_6105_cast_fp16)[name = string("op_6106_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6107_cast_fp16 = mul(x = state_443_cast_fp16, y = var_6106_cast_fp16)[name = string("op_6107_cast_fp16")];
            tensor<int32, [1]> var_6109_axes_0 = const()[name = string("op_6109_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_6109_keep_dims_0 = const()[name = string("op_6109_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_6109_cast_fp16 = reduce_sum(axes = var_6109_axes_0, keep_dims = var_6109_keep_dims_0, x = var_6107_cast_fp16)[name = string("op_6109_cast_fp16")];
            tensor<int32, [3]> var_6112_begin_0 = const()[name = string("op_6112_begin_0"), val = tensor<int32, [3]>([0, 0, 15])];
            tensor<int32, [3]> var_6112_end_0 = const()[name = string("op_6112_end_0"), val = tensor<int32, [3]>([1, 16, 16])];
            tensor<bool, [3]> var_6112_end_mask_0 = const()[name = string("op_6112_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_6112_squeeze_mask_0 = const()[name = string("op_6112_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_6112_cast_fp16 = slice_by_index(begin = var_6112_begin_0, end = var_6112_end_0, end_mask = var_6112_end_mask_0, squeeze_mask = var_6112_squeeze_mask_0, x = decay_13_cast_fp16)[name = string("op_6112_cast_fp16")];
            tensor<fp16, [1, 16]> var_6113_cast_fp16 = exp(x = var_6112_cast_fp16)[name = string("op_6113_cast_fp16")];
            tensor<int32, [1]> var_6114_axes_0 = const()[name = string("op_6114_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_6114_cast_fp16 = expand_dims(axes = var_6114_axes_0, x = var_6113_cast_fp16)[name = string("op_6114_cast_fp16")];
            tensor<int32, [1]> var_6115_axes_0 = const()[name = string("op_6115_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_6115_cast_fp16 = expand_dims(axes = var_6115_axes_0, x = var_6114_cast_fp16)[name = string("op_6115_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_445_cast_fp16 = mul(x = state_443_cast_fp16, y = var_6115_cast_fp16)[name = string("state_445_cast_fp16")];
            tensor<int32, [4]> key_token_223_begin_0 = const()[name = string("key_token_223_begin_0"), val = tensor<int32, [4]>([0, 0, 15, 0])];
            tensor<int32, [4]> key_token_223_end_0 = const()[name = string("key_token_223_end_0"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<bool, [4]> key_token_223_end_mask_0 = const()[name = string("key_token_223_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_223_squeeze_mask_0 = const()[name = string("key_token_223_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_223_cast_fp16 = slice_by_index(begin = key_token_223_begin_0, end = key_token_223_end_0, end_mask = key_token_223_end_mask_0, squeeze_mask = key_token_223_squeeze_mask_0, x = key_35_cast_fp16)[name = string("key_token_223_cast_fp16")];
            tensor<int32, [4]> value_token_223_begin_0 = const()[name = string("value_token_223_begin_0"), val = tensor<int32, [4]>([0, 0, 15, 0])];
            tensor<int32, [4]> value_token_223_end_0 = const()[name = string("value_token_223_end_0"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<bool, [4]> value_token_223_end_mask_0 = const()[name = string("value_token_223_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_223_squeeze_mask_0 = const()[name = string("value_token_223_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_223_cast_fp16 = slice_by_index(begin = value_token_223_begin_0, end = value_token_223_end_0, end_mask = value_token_223_end_mask_0, squeeze_mask = value_token_223_squeeze_mask_0, x = value_127_cast_fp16)[name = string("value_token_223_cast_fp16")];
            tensor<int32, [1]> var_6123_axes_0 = const()[name = string("op_6123_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_6123_cast_fp16 = expand_dims(axes = var_6123_axes_0, x = key_token_223_cast_fp16)[name = string("op_6123_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6124_cast_fp16 = mul(x = state_445_cast_fp16, y = var_6123_cast_fp16)[name = string("op_6124_cast_fp16")];
            tensor<int32, [1]> memory_223_axes_0 = const()[name = string("memory_223_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_223_keep_dims_0 = const()[name = string("memory_223_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_223_cast_fp16 = reduce_sum(axes = memory_223_axes_0, keep_dims = memory_223_keep_dims_0, x = var_6124_cast_fp16)[name = string("memory_223_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_6127_cast_fp16 = sub(x = value_token_223_cast_fp16, y = memory_223_cast_fp16)[name = string("op_6127_cast_fp16")];
            tensor<int32, [3]> var_6130_begin_0 = const()[name = string("op_6130_begin_0"), val = tensor<int32, [3]>([0, 0, 15])];
            tensor<int32, [3]> var_6130_end_0 = const()[name = string("op_6130_end_0"), val = tensor<int32, [3]>([1, 16, 16])];
            tensor<bool, [3]> var_6130_end_mask_0 = const()[name = string("op_6130_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_6130_squeeze_mask_0 = const()[name = string("op_6130_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_6130_cast_fp16 = slice_by_index(begin = var_6130_begin_0, end = var_6130_end_0, end_mask = var_6130_end_mask_0, squeeze_mask = var_6130_squeeze_mask_0, x = beta_13_cast_fp16)[name = string("op_6130_cast_fp16")];
            tensor<int32, [1]> var_6131_axes_0 = const()[name = string("op_6131_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_6131_cast_fp16 = expand_dims(axes = var_6131_axes_0, x = var_6130_cast_fp16)[name = string("op_6131_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_223_cast_fp16 = mul(x = var_6127_cast_fp16, y = var_6131_cast_fp16)[name = string("delta_223_cast_fp16")];
            tensor<int32, [1]> var_6134_axes_0 = const()[name = string("op_6134_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_6134_cast_fp16 = expand_dims(axes = var_6134_axes_0, x = delta_223_cast_fp16)[name = string("op_6134_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6135_cast_fp16 = mul(x = var_6123_cast_fp16, y = var_6134_cast_fp16)[name = string("op_6135_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> rec8_out = add(x = state_445_cast_fp16, y = var_6135_cast_fp16)[name = string("state_447_cast_fp16")];
            tensor<int32, [4]> var_6139_begin_0 = const()[name = string("op_6139_begin_0"), val = tensor<int32, [4]>([0, 0, 15, 0])];
            tensor<int32, [4]> var_6139_end_0 = const()[name = string("op_6139_end_0"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<bool, [4]> var_6139_end_mask_0 = const()[name = string("op_6139_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_6139_squeeze_mask_0 = const()[name = string("op_6139_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_6139_cast_fp16 = slice_by_index(begin = var_6139_begin_0, end = var_6139_end_0, end_mask = var_6139_end_mask_0, squeeze_mask = var_6139_squeeze_mask_0, x = _inversed_query_35_cast_fp16)[name = string("op_6139_cast_fp16")];
            tensor<int32, [1]> var_6140_axes_0 = const()[name = string("op_6140_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_6140_cast_fp16 = expand_dims(axes = var_6140_axes_0, x = var_6139_cast_fp16)[name = string("op_6140_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6141_cast_fp16 = mul(x = rec8_out, y = var_6140_cast_fp16)[name = string("op_6141_cast_fp16")];
            tensor<int32, [1]> var_6143_axes_0 = const()[name = string("op_6143_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_6143_keep_dims_0 = const()[name = string("op_6143_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_6143_cast_fp16 = reduce_sum(axes = var_6143_axes_0, keep_dims = var_6143_keep_dims_0, x = var_6141_cast_fp16)[name = string("op_6143_cast_fp16")];
            int32 attention_61_axis_0 = const()[name = string("attention_61_axis_0"), val = int32(1)];
            tensor<fp16, [1, 16, 16, 128]> attention_61_cast_fp16 = stack(axis = attention_61_axis_0, values = (var_5633_cast_fp16, var_5667_cast_fp16, var_5701_cast_fp16, var_5735_cast_fp16, var_5769_cast_fp16, var_5803_cast_fp16, var_5837_cast_fp16, var_5871_cast_fp16, var_5905_cast_fp16, var_5939_cast_fp16, var_5973_cast_fp16, var_6007_cast_fp16, var_6041_cast_fp16, var_6075_cast_fp16, var_6109_cast_fp16, var_6143_cast_fp16))[name = string("attention_61_cast_fp16")];
            tensor<int32, [2]> var_6146 = const()[name = string("op_6146"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [256, 128]> attention_63_cast_fp16 = reshape(shape = var_6146, x = attention_61_cast_fp16)[name = string("attention_63_cast_fp16")];
            tensor<int32, [2]> var_6148 = const()[name = string("op_6148"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [256, 128]> input_109_cast_fp16 = reshape(shape = var_6148, x = linear_65_cast_fp16)[name = string("input_109_cast_fp16")];
            tensor<fp16, [256, 128]> var_6150_cast_fp16 = mul(x = attention_63_cast_fp16, y = attention_63_cast_fp16)[name = string("op_6150_cast_fp16")];
            tensor<int32, [1]> variance_55_axes_0 = const()[name = string("variance_55_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_55_keep_dims_0 = const()[name = string("variance_55_keep_dims_0"), val = bool(true)];
            tensor<fp16, [256, 1]> variance_55_cast_fp16 = reduce_mean(axes = variance_55_axes_0, keep_dims = variance_55_keep_dims_0, x = var_6150_cast_fp16)[name = string("variance_55_cast_fp16")];
            fp16 var_6153_to_fp16 = const()[name = string("op_6153_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [256, 1]> var_6154_cast_fp16 = add(x = variance_55_cast_fp16, y = var_6153_to_fp16)[name = string("op_6154_cast_fp16")];
            fp32 var_6155_epsilon_0 = const()[name = string("op_6155_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [256, 1]> var_6155_cast_fp16 = rsqrt(epsilon = var_6155_epsilon_0, x = var_6154_cast_fp16)[name = string("op_6155_cast_fp16")];
            tensor<fp16, [256, 128]> attention_65_cast_fp16 = mul(x = attention_63_cast_fp16, y = var_6155_cast_fp16)[name = string("attention_65_cast_fp16")];
            tensor<fp16, [128]> groups_2_0_gated_norm_promoted_to_fp16 = const()[name = string("groups_2_0_gated_norm_promoted_to_fp16"), val = tensor<fp16, [128]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(132411776)))];
            tensor<fp16, [256, 128]> attention_67_cast_fp16 = mul(x = attention_65_cast_fp16, y = groups_2_0_gated_norm_promoted_to_fp16)[name = string("attention_67_cast_fp16")];
            tensor<fp16, [256, 128]> var_6158_cast_fp16 = silu(x = input_109_cast_fp16)[name = string("op_6158_cast_fp16")];
            tensor<fp16, [256, 128]> attention_69_cast_fp16 = mul(x = attention_67_cast_fp16, y = var_6158_cast_fp16)[name = string("attention_69_cast_fp16")];
            tensor<int32, [2]> var_6160 = const()[name = string("op_6160"), val = tensor<int32, [2]>([16, 2048])];
            tensor<fp16, [16, 2048]> input_111_cast_fp16 = reshape(shape = var_6160, x = attention_69_cast_fp16)[name = string("input_111_cast_fp16")];
            tensor<fp16, [1024, 2048]> groups_2_0_out_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 2048]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(132412096))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(133985024))))[name = string("groups_2_0_out_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 1024]> linear_66_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = groups_2_0_out_proj_weight_promoted_to_fp16_palettized, x = input_111_cast_fp16)[name = string("linear_66_cast_fp16")];
            tensor<fp16, [16, 1024]> value_129_cast_fp16 = add(x = value_117_cast_fp16, y = linear_66_cast_fp16)[name = string("value_129_cast_fp16")];
            tensor<fp16, [16, 1024]> var_6165_cast_fp16 = mul(x = value_129_cast_fp16, y = value_129_cast_fp16)[name = string("op_6165_cast_fp16")];
            tensor<int32, [1]> variance_57_axes_0 = const()[name = string("variance_57_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_57_keep_dims_0 = const()[name = string("variance_57_keep_dims_0"), val = bool(true)];
            tensor<fp16, [16, 1]> variance_57_cast_fp16 = reduce_mean(axes = variance_57_axes_0, keep_dims = variance_57_keep_dims_0, x = var_6165_cast_fp16)[name = string("variance_57_cast_fp16")];
            fp16 var_6168_to_fp16 = const()[name = string("op_6168_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [16, 1]> var_6169_cast_fp16 = add(x = variance_57_cast_fp16, y = var_6168_to_fp16)[name = string("op_6169_cast_fp16")];
            fp32 var_6170_epsilon_0 = const()[name = string("op_6170_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [16, 1]> var_6170_cast_fp16 = rsqrt(epsilon = var_6170_epsilon_0, x = var_6169_cast_fp16)[name = string("op_6170_cast_fp16")];
            tensor<fp16, [16, 1024]> var_6171_cast_fp16 = mul(x = value_129_cast_fp16, y = var_6170_cast_fp16)[name = string("op_6171_cast_fp16")];
            tensor<fp16, [1024]> var_6173_to_fp16 = const()[name = string("op_6173_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(133993280)))];
            tensor<fp16, [16, 1024]> input_113_cast_fp16 = mul(x = var_6171_cast_fp16, y = var_6173_to_fp16)[name = string("input_113_cast_fp16")];
            tensor<fp16, [3584, 1024]> groups_2_0_gate_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(133995392))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(136747968))))[name = string("groups_2_0_gate_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 3584]> linear_67_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = groups_2_0_gate_proj_weight_promoted_to_fp16_palettized, x = input_113_cast_fp16)[name = string("linear_67_cast_fp16")];
            tensor<fp16, [16, 3584]> var_6177_cast_fp16 = silu(x = linear_67_cast_fp16)[name = string("op_6177_cast_fp16")];
            tensor<fp16, [3584, 1024]> groups_2_0_up_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(136776704))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(139529280))))[name = string("groups_2_0_up_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 3584]> linear_68_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = groups_2_0_up_proj_weight_promoted_to_fp16_palettized, x = input_113_cast_fp16)[name = string("linear_68_cast_fp16")];
            tensor<fp16, [16, 3584]> input_117_cast_fp16 = mul(x = var_6177_cast_fp16, y = linear_68_cast_fp16)[name = string("input_117_cast_fp16")];
            tensor<fp16, [1024, 3584]> groups_2_0_down_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 3584]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(139558016))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(142310592))))[name = string("groups_2_0_down_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 1024]> linear_69_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = groups_2_0_down_proj_weight_promoted_to_fp16_palettized, x = input_117_cast_fp16)[name = string("linear_69_cast_fp16")];
            tensor<fp16, [16, 1024]> value_131_cast_fp16 = add(x = value_129_cast_fp16, y = linear_69_cast_fp16)[name = string("value_131_cast_fp16")];
            int32 var_6215 = const()[name = string("op_6215"), val = int32(-1)];
            tensor<fp16, [16, 1024]> var_6230_cast_fp16 = mul(x = value_131_cast_fp16, y = value_131_cast_fp16)[name = string("op_6230_cast_fp16")];
            tensor<int32, [1]> variance_59_axes_0 = const()[name = string("variance_59_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_59_keep_dims_0 = const()[name = string("variance_59_keep_dims_0"), val = bool(true)];
            tensor<fp16, [16, 1]> variance_59_cast_fp16 = reduce_mean(axes = variance_59_axes_0, keep_dims = variance_59_keep_dims_0, x = var_6230_cast_fp16)[name = string("variance_59_cast_fp16")];
            fp16 var_6233_to_fp16 = const()[name = string("op_6233_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [16, 1]> var_6234_cast_fp16 = add(x = variance_59_cast_fp16, y = var_6233_to_fp16)[name = string("op_6234_cast_fp16")];
            fp32 var_6235_epsilon_0 = const()[name = string("op_6235_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [16, 1]> var_6235_cast_fp16 = rsqrt(epsilon = var_6235_epsilon_0, x = var_6234_cast_fp16)[name = string("op_6235_cast_fp16")];
            tensor<fp16, [16, 1024]> var_6236_cast_fp16 = mul(x = value_131_cast_fp16, y = var_6235_cast_fp16)[name = string("op_6236_cast_fp16")];
            tensor<fp16, [1024]> var_6238_to_fp16 = const()[name = string("op_6238_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(142318848)))];
            tensor<fp16, [16, 1024]> input_119_cast_fp16 = mul(x = var_6236_cast_fp16, y = var_6238_to_fp16)[name = string("input_119_cast_fp16")];
            tensor<fp16, [6144, 1024]> groups_2_1_in_proj_qkv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(142320960))), lut = tensor<fp16, [384, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(147039616))))[name = string("groups_2_1_in_proj_qkv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 6144]> linear_70_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = groups_2_1_in_proj_qkv_weight_promoted_to_fp16_palettized, x = input_119_cast_fp16)[name = string("linear_70_cast_fp16")];
            tensor<int32, [2]> var_6242_perm_0 = const()[name = string("op_6242_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> mixed_15_axes_0 = const()[name = string("mixed_15_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [6144, 16]> var_6242_cast_fp16 = transpose(perm = var_6242_perm_0, x = linear_70_cast_fp16)[name = string("transpose_75")];
            tensor<fp16, [1, 6144, 16]> mixed_15_cast_fp16 = expand_dims(axes = mixed_15_axes_0, x = var_6242_cast_fp16)[name = string("mixed_15_cast_fp16")];
            bool convolution_input_15_interleave_0 = const()[name = string("convolution_input_15_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 6144, 19]> convolution_input_15_cast_fp16 = concat(axis = var_6215, interleave = convolution_input_15_interleave_0, values = (conv9, mixed_15_cast_fp16))[name = string("convolution_input_15_cast_fp16")];
            string input_121_pad_type_0 = const()[name = string("input_121_pad_type_0"), val = string("valid")];
            int32 input_121_groups_0 = const()[name = string("input_121_groups_0"), val = int32(6144)];
            tensor<int32, [1]> input_121_strides_0 = const()[name = string("input_121_strides_0"), val = tensor<int32, [1]>([1])];
            tensor<int32, [2]> input_121_pad_0 = const()[name = string("input_121_pad_0"), val = tensor<int32, [2]>([0, 0])];
            tensor<int32, [1]> input_121_dilations_0 = const()[name = string("input_121_dilations_0"), val = tensor<int32, [1]>([1])];
            tensor<fp16, [6144, 1, 4]> groups_2_1_conv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1, 4]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(147088832))), lut = tensor<fp16, [384, 1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(147107328))))[name = string("groups_2_1_conv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 6144, 16]> input_121_cast_fp16 = conv(dilations = input_121_dilations_0, groups = input_121_groups_0, pad = input_121_pad_0, pad_type = input_121_pad_type_0, strides = input_121_strides_0, weight = groups_2_1_conv_weight_promoted_to_fp16_palettized, x = convolution_input_15_cast_fp16)[name = string("input_121_cast_fp16")];
            tensor<fp16, [1, 6144, 16]> var_6251_cast_fp16 = silu(x = input_121_cast_fp16)[name = string("op_6251_cast_fp16")];
            tensor<int32, [3]> var_6252_perm_0 = const()[name = string("op_6252_perm_0"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [3]> var_6255_begin_0 = const()[name = string("op_6255_begin_0"), val = tensor<int32, [3]>([0, 0, 16])];
            tensor<int32, [3]> var_6255_end_0 = const()[name = string("op_6255_end_0"), val = tensor<int32, [3]>([1, 6144, 19])];
            tensor<bool, [3]> var_6255_end_mask_0 = const()[name = string("op_6255_end_mask_0"), val = tensor<bool, [3]>([true, true, true])];
            tensor<fp16, [1, 6144, 3]> conv9_out = slice_by_index(begin = var_6255_begin_0, end = var_6255_end_0, end_mask = var_6255_end_mask_0, x = convolution_input_15_cast_fp16)[name = string("op_6255_cast_fp16")];
            tensor<int32, [3]> var_6256 = const()[name = string("op_6256"), val = tensor<int32, [3]>([2048, 2048, 2048])];
            int32 var_6257_axis_0 = const()[name = string("op_6257_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 16, 6144]> var_6252_cast_fp16 = transpose(perm = var_6252_perm_0, x = var_6251_cast_fp16)[name = string("transpose_74")];
            tensor<fp16, [1, 16, 2048]> var_6257_cast_fp16_0, tensor<fp16, [1, 16, 2048]> var_6257_cast_fp16_1, tensor<fp16, [1, 16, 2048]> var_6257_cast_fp16_2 = split(axis = var_6257_axis_0, split_sizes = var_6256, x = var_6252_cast_fp16)[name = string("op_6257_cast_fp16")];
            tensor<int32, [4]> var_6261 = const()[name = string("op_6261"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<fp16, [1, 16, 16, 128]> value_135_cast_fp16 = reshape(shape = var_6261, x = var_6257_cast_fp16_0)[name = string("value_135_cast_fp16")];
            tensor<int32, [4]> var_6263 = const()[name = string("op_6263"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<fp16, [1, 16, 16, 128]> value_137_cast_fp16 = reshape(shape = var_6263, x = var_6257_cast_fp16_1)[name = string("value_137_cast_fp16")];
            tensor<int32, [4]> var_6265 = const()[name = string("op_6265"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<fp16, [1, 16, 16, 128]> value_139_cast_fp16 = reshape(shape = var_6265, x = var_6257_cast_fp16_2)[name = string("value_139_cast_fp16")];
            tensor<fp16, [1, 16, 16, 128]> var_6267_cast_fp16 = mul(x = value_135_cast_fp16, y = value_135_cast_fp16)[name = string("op_6267_cast_fp16")];
            tensor<int32, [1]> var_6269_axes_0 = const()[name = string("op_6269_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_6269_keep_dims_0 = const()[name = string("op_6269_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 16, 16, 1]> var_6269_cast_fp16 = reduce_sum(axes = var_6269_axes_0, keep_dims = var_6269_keep_dims_0, x = var_6267_cast_fp16)[name = string("op_6269_cast_fp16")];
            fp16 var_6270_to_fp16 = const()[name = string("op_6270_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 16, 16, 1]> var_6271_cast_fp16 = add(x = var_6269_cast_fp16, y = var_6270_to_fp16)[name = string("op_6271_cast_fp16")];
            fp32 var_6272_epsilon_0 = const()[name = string("op_6272_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 16, 16, 1]> var_6272_cast_fp16 = rsqrt(epsilon = var_6272_epsilon_0, x = var_6271_cast_fp16)[name = string("op_6272_cast_fp16")];
            tensor<fp16, [1, 16, 16, 128]> var_6273_cast_fp16 = mul(x = value_135_cast_fp16, y = var_6272_cast_fp16)[name = string("op_6273_cast_fp16")];
            tensor<int32, [4]> var_6274_perm_0 = const()[name = string("op_6274_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            fp16 _inversed_query_39_y_0_to_fp16 = const()[name = string("_inversed_query_39_y_0_to_fp16"), val = fp16(0x1.6ap-4)];
            tensor<fp16, [1, 16, 16, 128]> var_6274_cast_fp16 = transpose(perm = var_6274_perm_0, x = var_6273_cast_fp16)[name = string("transpose_73")];
            tensor<fp16, [1, 16, 16, 128]> _inversed_query_39_cast_fp16 = mul(x = var_6274_cast_fp16, y = _inversed_query_39_y_0_to_fp16)[name = string("_inversed_query_39_cast_fp16")];
            tensor<fp16, [1, 16, 16, 128]> var_6277_cast_fp16 = mul(x = value_137_cast_fp16, y = value_137_cast_fp16)[name = string("op_6277_cast_fp16")];
            tensor<int32, [1]> var_6279_axes_0 = const()[name = string("op_6279_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_6279_keep_dims_0 = const()[name = string("op_6279_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 16, 16, 1]> var_6279_cast_fp16 = reduce_sum(axes = var_6279_axes_0, keep_dims = var_6279_keep_dims_0, x = var_6277_cast_fp16)[name = string("op_6279_cast_fp16")];
            fp16 var_6280_to_fp16 = const()[name = string("op_6280_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 16, 16, 1]> var_6281_cast_fp16 = add(x = var_6279_cast_fp16, y = var_6280_to_fp16)[name = string("op_6281_cast_fp16")];
            fp32 var_6282_epsilon_0 = const()[name = string("op_6282_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 16, 16, 1]> var_6282_cast_fp16 = rsqrt(epsilon = var_6282_epsilon_0, x = var_6281_cast_fp16)[name = string("op_6282_cast_fp16")];
            tensor<fp16, [1, 16, 16, 128]> var_6283_cast_fp16 = mul(x = value_137_cast_fp16, y = var_6282_cast_fp16)[name = string("op_6283_cast_fp16")];
            tensor<int32, [4]> key_39_perm_0 = const()[name = string("key_39_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<int32, [4]> value_141_perm_0 = const()[name = string("value_141_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<fp16, [16, 1024]> groups_2_1_in_proj_b_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(147156544))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(147168896))))[name = string("groups_2_1_in_proj_b_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16]> linear_71_bias_0_to_fp16 = const()[name = string("linear_71_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(4862656)))];
            tensor<fp16, [16, 16]> linear_71_cast_fp16 = linear(bias = linear_71_bias_0_to_fp16, weight = groups_2_1_in_proj_b_weight_promoted_to_fp16_palettized, x = input_119_cast_fp16)[name = string("linear_71_cast_fp16")];
            tensor<fp16, [16, 16]> var_6288_cast_fp16 = sigmoid(x = linear_71_cast_fp16)[name = string("op_6288_cast_fp16")];
            tensor<int32, [2]> var_6289_perm_0 = const()[name = string("op_6289_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> beta_15_axes_0 = const()[name = string("beta_15_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 16]> var_6289_cast_fp16 = transpose(perm = var_6289_perm_0, x = var_6288_cast_fp16)[name = string("transpose_72")];
            tensor<fp16, [1, 16, 16]> beta_15_cast_fp16 = expand_dims(axes = beta_15_axes_0, x = var_6289_cast_fp16)[name = string("beta_15_cast_fp16")];
            tensor<fp16, [16, 1024]> op_6295_weight_0_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(147169088))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(147181440))))[name = string("op_6295_weight_0_to_fp16_palettized")];
            tensor<fp16, [16]> var_6295_bias_0_to_fp16 = const()[name = string("op_6295_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(147181632)))];
            tensor<fp16, [16, 16]> var_6295_cast_fp16 = linear(bias = var_6295_bias_0_to_fp16, weight = op_6295_weight_0_to_fp16_palettized, x = input_119_cast_fp16)[name = string("op_6295_cast_fp16")];
            tensor<fp16, [16, 16]> var_6296_cast_fp16 = softplus(x = var_6295_cast_fp16)[name = string("op_6296_cast_fp16")];
            tensor<fp16, [16]> var_6292_promoted_to_fp16 = const()[name = string("op_6292_promoted_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(147181760)))];
            tensor<fp16, [16, 16]> var_6297_cast_fp16 = mul(x = var_6292_promoted_to_fp16, y = var_6296_cast_fp16)[name = string("op_6297_cast_fp16")];
            tensor<int32, [2]> var_6298_perm_0 = const()[name = string("op_6298_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> decay_15_axes_0 = const()[name = string("decay_15_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 16]> var_6298_cast_fp16 = transpose(perm = var_6298_perm_0, x = var_6297_cast_fp16)[name = string("transpose_71")];
            tensor<fp16, [1, 16, 16]> decay_15_cast_fp16 = expand_dims(axes = decay_15_axes_0, x = var_6298_cast_fp16)[name = string("decay_15_cast_fp16")];
            tensor<fp16, [2048, 1024]> groups_2_1_in_proj_z_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [2048, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(147181888))), lut = tensor<fp16, [128, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(148754816))))[name = string("groups_2_1_in_proj_z_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 2048]> linear_73_cast_fp16 = linear(bias = linear_3_bias_0_to_fp16, weight = groups_2_1_in_proj_z_weight_promoted_to_fp16_palettized, x = input_119_cast_fp16)[name = string("linear_73_cast_fp16")];
            tensor<int32, [3]> var_6306_begin_0 = const()[name = string("op_6306_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_6306_end_0 = const()[name = string("op_6306_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_6306_end_mask_0 = const()[name = string("op_6306_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_6306_squeeze_mask_0 = const()[name = string("op_6306_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_6306_cast_fp16 = slice_by_index(begin = var_6306_begin_0, end = var_6306_end_0, end_mask = var_6306_end_mask_0, squeeze_mask = var_6306_squeeze_mask_0, x = decay_15_cast_fp16)[name = string("op_6306_cast_fp16")];
            tensor<fp16, [1, 16]> var_6307_cast_fp16 = exp(x = var_6306_cast_fp16)[name = string("op_6307_cast_fp16")];
            tensor<int32, [1]> var_6308_axes_0 = const()[name = string("op_6308_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_6308_cast_fp16 = expand_dims(axes = var_6308_axes_0, x = var_6307_cast_fp16)[name = string("op_6308_cast_fp16")];
            tensor<int32, [1]> var_6309_axes_0 = const()[name = string("op_6309_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_6309_cast_fp16 = expand_dims(axes = var_6309_axes_0, x = var_6308_cast_fp16)[name = string("op_6309_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_449_cast_fp16 = mul(x = rec9, y = var_6309_cast_fp16)[name = string("state_449_cast_fp16")];
            tensor<int32, [4]> key_token_225_begin_0 = const()[name = string("key_token_225_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> key_token_225_end_0 = const()[name = string("key_token_225_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> key_token_225_end_mask_0 = const()[name = string("key_token_225_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_225_squeeze_mask_0 = const()[name = string("key_token_225_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 16, 128]> key_39_cast_fp16 = transpose(perm = key_39_perm_0, x = var_6283_cast_fp16)[name = string("transpose_70")];
            tensor<fp16, [1, 16, 128]> key_token_225_cast_fp16 = slice_by_index(begin = key_token_225_begin_0, end = key_token_225_end_0, end_mask = key_token_225_end_mask_0, squeeze_mask = key_token_225_squeeze_mask_0, x = key_39_cast_fp16)[name = string("key_token_225_cast_fp16")];
            tensor<int32, [4]> value_token_225_begin_0 = const()[name = string("value_token_225_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> value_token_225_end_0 = const()[name = string("value_token_225_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> value_token_225_end_mask_0 = const()[name = string("value_token_225_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_225_squeeze_mask_0 = const()[name = string("value_token_225_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 16, 128]> value_141_cast_fp16 = transpose(perm = value_141_perm_0, x = value_139_cast_fp16)[name = string("transpose_69")];
            tensor<fp16, [1, 16, 128]> value_token_225_cast_fp16 = slice_by_index(begin = value_token_225_begin_0, end = value_token_225_end_0, end_mask = value_token_225_end_mask_0, squeeze_mask = value_token_225_squeeze_mask_0, x = value_141_cast_fp16)[name = string("value_token_225_cast_fp16")];
            tensor<int32, [1]> var_6317_axes_0 = const()[name = string("op_6317_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_6317_cast_fp16 = expand_dims(axes = var_6317_axes_0, x = key_token_225_cast_fp16)[name = string("op_6317_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6318_cast_fp16 = mul(x = state_449_cast_fp16, y = var_6317_cast_fp16)[name = string("op_6318_cast_fp16")];
            tensor<int32, [1]> memory_225_axes_0 = const()[name = string("memory_225_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_225_keep_dims_0 = const()[name = string("memory_225_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_225_cast_fp16 = reduce_sum(axes = memory_225_axes_0, keep_dims = memory_225_keep_dims_0, x = var_6318_cast_fp16)[name = string("memory_225_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_6321_cast_fp16 = sub(x = value_token_225_cast_fp16, y = memory_225_cast_fp16)[name = string("op_6321_cast_fp16")];
            tensor<int32, [3]> var_6324_begin_0 = const()[name = string("op_6324_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_6324_end_0 = const()[name = string("op_6324_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_6324_end_mask_0 = const()[name = string("op_6324_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_6324_squeeze_mask_0 = const()[name = string("op_6324_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_6324_cast_fp16 = slice_by_index(begin = var_6324_begin_0, end = var_6324_end_0, end_mask = var_6324_end_mask_0, squeeze_mask = var_6324_squeeze_mask_0, x = beta_15_cast_fp16)[name = string("op_6324_cast_fp16")];
            tensor<int32, [1]> var_6325_axes_0 = const()[name = string("op_6325_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_6325_cast_fp16 = expand_dims(axes = var_6325_axes_0, x = var_6324_cast_fp16)[name = string("op_6325_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_225_cast_fp16 = mul(x = var_6321_cast_fp16, y = var_6325_cast_fp16)[name = string("delta_225_cast_fp16")];
            tensor<int32, [1]> var_6328_axes_0 = const()[name = string("op_6328_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_6328_cast_fp16 = expand_dims(axes = var_6328_axes_0, x = delta_225_cast_fp16)[name = string("op_6328_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6329_cast_fp16 = mul(x = var_6317_cast_fp16, y = var_6328_cast_fp16)[name = string("op_6329_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_451_cast_fp16 = add(x = state_449_cast_fp16, y = var_6329_cast_fp16)[name = string("state_451_cast_fp16")];
            tensor<int32, [4]> var_6333_begin_0 = const()[name = string("op_6333_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> var_6333_end_0 = const()[name = string("op_6333_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> var_6333_end_mask_0 = const()[name = string("op_6333_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_6333_squeeze_mask_0 = const()[name = string("op_6333_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_6333_cast_fp16 = slice_by_index(begin = var_6333_begin_0, end = var_6333_end_0, end_mask = var_6333_end_mask_0, squeeze_mask = var_6333_squeeze_mask_0, x = _inversed_query_39_cast_fp16)[name = string("op_6333_cast_fp16")];
            tensor<int32, [1]> var_6334_axes_0 = const()[name = string("op_6334_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_6334_cast_fp16 = expand_dims(axes = var_6334_axes_0, x = var_6333_cast_fp16)[name = string("op_6334_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6335_cast_fp16 = mul(x = state_451_cast_fp16, y = var_6334_cast_fp16)[name = string("op_6335_cast_fp16")];
            tensor<int32, [1]> var_6337_axes_0 = const()[name = string("op_6337_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_6337_keep_dims_0 = const()[name = string("op_6337_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_6337_cast_fp16 = reduce_sum(axes = var_6337_axes_0, keep_dims = var_6337_keep_dims_0, x = var_6335_cast_fp16)[name = string("op_6337_cast_fp16")];
            tensor<int32, [3]> var_6340_begin_0 = const()[name = string("op_6340_begin_0"), val = tensor<int32, [3]>([0, 0, 1])];
            tensor<int32, [3]> var_6340_end_0 = const()[name = string("op_6340_end_0"), val = tensor<int32, [3]>([1, 16, 2])];
            tensor<bool, [3]> var_6340_end_mask_0 = const()[name = string("op_6340_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_6340_squeeze_mask_0 = const()[name = string("op_6340_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_6340_cast_fp16 = slice_by_index(begin = var_6340_begin_0, end = var_6340_end_0, end_mask = var_6340_end_mask_0, squeeze_mask = var_6340_squeeze_mask_0, x = decay_15_cast_fp16)[name = string("op_6340_cast_fp16")];
            tensor<fp16, [1, 16]> var_6341_cast_fp16 = exp(x = var_6340_cast_fp16)[name = string("op_6341_cast_fp16")];
            tensor<int32, [1]> var_6342_axes_0 = const()[name = string("op_6342_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_6342_cast_fp16 = expand_dims(axes = var_6342_axes_0, x = var_6341_cast_fp16)[name = string("op_6342_cast_fp16")];
            tensor<int32, [1]> var_6343_axes_0 = const()[name = string("op_6343_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_6343_cast_fp16 = expand_dims(axes = var_6343_axes_0, x = var_6342_cast_fp16)[name = string("op_6343_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_453_cast_fp16 = mul(x = state_451_cast_fp16, y = var_6343_cast_fp16)[name = string("state_453_cast_fp16")];
            tensor<int32, [4]> key_token_227_begin_0 = const()[name = string("key_token_227_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> key_token_227_end_0 = const()[name = string("key_token_227_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> key_token_227_end_mask_0 = const()[name = string("key_token_227_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_227_squeeze_mask_0 = const()[name = string("key_token_227_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_227_cast_fp16 = slice_by_index(begin = key_token_227_begin_0, end = key_token_227_end_0, end_mask = key_token_227_end_mask_0, squeeze_mask = key_token_227_squeeze_mask_0, x = key_39_cast_fp16)[name = string("key_token_227_cast_fp16")];
            tensor<int32, [4]> value_token_227_begin_0 = const()[name = string("value_token_227_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> value_token_227_end_0 = const()[name = string("value_token_227_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> value_token_227_end_mask_0 = const()[name = string("value_token_227_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_227_squeeze_mask_0 = const()[name = string("value_token_227_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_227_cast_fp16 = slice_by_index(begin = value_token_227_begin_0, end = value_token_227_end_0, end_mask = value_token_227_end_mask_0, squeeze_mask = value_token_227_squeeze_mask_0, x = value_141_cast_fp16)[name = string("value_token_227_cast_fp16")];
            tensor<int32, [1]> var_6351_axes_0 = const()[name = string("op_6351_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_6351_cast_fp16 = expand_dims(axes = var_6351_axes_0, x = key_token_227_cast_fp16)[name = string("op_6351_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6352_cast_fp16 = mul(x = state_453_cast_fp16, y = var_6351_cast_fp16)[name = string("op_6352_cast_fp16")];
            tensor<int32, [1]> memory_227_axes_0 = const()[name = string("memory_227_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_227_keep_dims_0 = const()[name = string("memory_227_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_227_cast_fp16 = reduce_sum(axes = memory_227_axes_0, keep_dims = memory_227_keep_dims_0, x = var_6352_cast_fp16)[name = string("memory_227_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_6355_cast_fp16 = sub(x = value_token_227_cast_fp16, y = memory_227_cast_fp16)[name = string("op_6355_cast_fp16")];
            tensor<int32, [3]> var_6358_begin_0 = const()[name = string("op_6358_begin_0"), val = tensor<int32, [3]>([0, 0, 1])];
            tensor<int32, [3]> var_6358_end_0 = const()[name = string("op_6358_end_0"), val = tensor<int32, [3]>([1, 16, 2])];
            tensor<bool, [3]> var_6358_end_mask_0 = const()[name = string("op_6358_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_6358_squeeze_mask_0 = const()[name = string("op_6358_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_6358_cast_fp16 = slice_by_index(begin = var_6358_begin_0, end = var_6358_end_0, end_mask = var_6358_end_mask_0, squeeze_mask = var_6358_squeeze_mask_0, x = beta_15_cast_fp16)[name = string("op_6358_cast_fp16")];
            tensor<int32, [1]> var_6359_axes_0 = const()[name = string("op_6359_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_6359_cast_fp16 = expand_dims(axes = var_6359_axes_0, x = var_6358_cast_fp16)[name = string("op_6359_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_227_cast_fp16 = mul(x = var_6355_cast_fp16, y = var_6359_cast_fp16)[name = string("delta_227_cast_fp16")];
            tensor<int32, [1]> var_6362_axes_0 = const()[name = string("op_6362_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_6362_cast_fp16 = expand_dims(axes = var_6362_axes_0, x = delta_227_cast_fp16)[name = string("op_6362_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6363_cast_fp16 = mul(x = var_6351_cast_fp16, y = var_6362_cast_fp16)[name = string("op_6363_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_455_cast_fp16 = add(x = state_453_cast_fp16, y = var_6363_cast_fp16)[name = string("state_455_cast_fp16")];
            tensor<int32, [4]> var_6367_begin_0 = const()[name = string("op_6367_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> var_6367_end_0 = const()[name = string("op_6367_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> var_6367_end_mask_0 = const()[name = string("op_6367_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_6367_squeeze_mask_0 = const()[name = string("op_6367_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_6367_cast_fp16 = slice_by_index(begin = var_6367_begin_0, end = var_6367_end_0, end_mask = var_6367_end_mask_0, squeeze_mask = var_6367_squeeze_mask_0, x = _inversed_query_39_cast_fp16)[name = string("op_6367_cast_fp16")];
            tensor<int32, [1]> var_6368_axes_0 = const()[name = string("op_6368_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_6368_cast_fp16 = expand_dims(axes = var_6368_axes_0, x = var_6367_cast_fp16)[name = string("op_6368_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6369_cast_fp16 = mul(x = state_455_cast_fp16, y = var_6368_cast_fp16)[name = string("op_6369_cast_fp16")];
            tensor<int32, [1]> var_6371_axes_0 = const()[name = string("op_6371_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_6371_keep_dims_0 = const()[name = string("op_6371_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_6371_cast_fp16 = reduce_sum(axes = var_6371_axes_0, keep_dims = var_6371_keep_dims_0, x = var_6369_cast_fp16)[name = string("op_6371_cast_fp16")];
            tensor<int32, [3]> var_6374_begin_0 = const()[name = string("op_6374_begin_0"), val = tensor<int32, [3]>([0, 0, 2])];
            tensor<int32, [3]> var_6374_end_0 = const()[name = string("op_6374_end_0"), val = tensor<int32, [3]>([1, 16, 3])];
            tensor<bool, [3]> var_6374_end_mask_0 = const()[name = string("op_6374_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_6374_squeeze_mask_0 = const()[name = string("op_6374_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_6374_cast_fp16 = slice_by_index(begin = var_6374_begin_0, end = var_6374_end_0, end_mask = var_6374_end_mask_0, squeeze_mask = var_6374_squeeze_mask_0, x = decay_15_cast_fp16)[name = string("op_6374_cast_fp16")];
            tensor<fp16, [1, 16]> var_6375_cast_fp16 = exp(x = var_6374_cast_fp16)[name = string("op_6375_cast_fp16")];
            tensor<int32, [1]> var_6376_axes_0 = const()[name = string("op_6376_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_6376_cast_fp16 = expand_dims(axes = var_6376_axes_0, x = var_6375_cast_fp16)[name = string("op_6376_cast_fp16")];
            tensor<int32, [1]> var_6377_axes_0 = const()[name = string("op_6377_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_6377_cast_fp16 = expand_dims(axes = var_6377_axes_0, x = var_6376_cast_fp16)[name = string("op_6377_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_457_cast_fp16 = mul(x = state_455_cast_fp16, y = var_6377_cast_fp16)[name = string("state_457_cast_fp16")];
            tensor<int32, [4]> key_token_229_begin_0 = const()[name = string("key_token_229_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> key_token_229_end_0 = const()[name = string("key_token_229_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> key_token_229_end_mask_0 = const()[name = string("key_token_229_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_229_squeeze_mask_0 = const()[name = string("key_token_229_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_229_cast_fp16 = slice_by_index(begin = key_token_229_begin_0, end = key_token_229_end_0, end_mask = key_token_229_end_mask_0, squeeze_mask = key_token_229_squeeze_mask_0, x = key_39_cast_fp16)[name = string("key_token_229_cast_fp16")];
            tensor<int32, [4]> value_token_229_begin_0 = const()[name = string("value_token_229_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> value_token_229_end_0 = const()[name = string("value_token_229_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> value_token_229_end_mask_0 = const()[name = string("value_token_229_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_229_squeeze_mask_0 = const()[name = string("value_token_229_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_229_cast_fp16 = slice_by_index(begin = value_token_229_begin_0, end = value_token_229_end_0, end_mask = value_token_229_end_mask_0, squeeze_mask = value_token_229_squeeze_mask_0, x = value_141_cast_fp16)[name = string("value_token_229_cast_fp16")];
            tensor<int32, [1]> var_6385_axes_0 = const()[name = string("op_6385_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_6385_cast_fp16 = expand_dims(axes = var_6385_axes_0, x = key_token_229_cast_fp16)[name = string("op_6385_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6386_cast_fp16 = mul(x = state_457_cast_fp16, y = var_6385_cast_fp16)[name = string("op_6386_cast_fp16")];
            tensor<int32, [1]> memory_229_axes_0 = const()[name = string("memory_229_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_229_keep_dims_0 = const()[name = string("memory_229_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_229_cast_fp16 = reduce_sum(axes = memory_229_axes_0, keep_dims = memory_229_keep_dims_0, x = var_6386_cast_fp16)[name = string("memory_229_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_6389_cast_fp16 = sub(x = value_token_229_cast_fp16, y = memory_229_cast_fp16)[name = string("op_6389_cast_fp16")];
            tensor<int32, [3]> var_6392_begin_0 = const()[name = string("op_6392_begin_0"), val = tensor<int32, [3]>([0, 0, 2])];
            tensor<int32, [3]> var_6392_end_0 = const()[name = string("op_6392_end_0"), val = tensor<int32, [3]>([1, 16, 3])];
            tensor<bool, [3]> var_6392_end_mask_0 = const()[name = string("op_6392_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_6392_squeeze_mask_0 = const()[name = string("op_6392_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_6392_cast_fp16 = slice_by_index(begin = var_6392_begin_0, end = var_6392_end_0, end_mask = var_6392_end_mask_0, squeeze_mask = var_6392_squeeze_mask_0, x = beta_15_cast_fp16)[name = string("op_6392_cast_fp16")];
            tensor<int32, [1]> var_6393_axes_0 = const()[name = string("op_6393_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_6393_cast_fp16 = expand_dims(axes = var_6393_axes_0, x = var_6392_cast_fp16)[name = string("op_6393_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_229_cast_fp16 = mul(x = var_6389_cast_fp16, y = var_6393_cast_fp16)[name = string("delta_229_cast_fp16")];
            tensor<int32, [1]> var_6396_axes_0 = const()[name = string("op_6396_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_6396_cast_fp16 = expand_dims(axes = var_6396_axes_0, x = delta_229_cast_fp16)[name = string("op_6396_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6397_cast_fp16 = mul(x = var_6385_cast_fp16, y = var_6396_cast_fp16)[name = string("op_6397_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_459_cast_fp16 = add(x = state_457_cast_fp16, y = var_6397_cast_fp16)[name = string("state_459_cast_fp16")];
            tensor<int32, [4]> var_6401_begin_0 = const()[name = string("op_6401_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> var_6401_end_0 = const()[name = string("op_6401_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> var_6401_end_mask_0 = const()[name = string("op_6401_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_6401_squeeze_mask_0 = const()[name = string("op_6401_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_6401_cast_fp16 = slice_by_index(begin = var_6401_begin_0, end = var_6401_end_0, end_mask = var_6401_end_mask_0, squeeze_mask = var_6401_squeeze_mask_0, x = _inversed_query_39_cast_fp16)[name = string("op_6401_cast_fp16")];
            tensor<int32, [1]> var_6402_axes_0 = const()[name = string("op_6402_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_6402_cast_fp16 = expand_dims(axes = var_6402_axes_0, x = var_6401_cast_fp16)[name = string("op_6402_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6403_cast_fp16 = mul(x = state_459_cast_fp16, y = var_6402_cast_fp16)[name = string("op_6403_cast_fp16")];
            tensor<int32, [1]> var_6405_axes_0 = const()[name = string("op_6405_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_6405_keep_dims_0 = const()[name = string("op_6405_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_6405_cast_fp16 = reduce_sum(axes = var_6405_axes_0, keep_dims = var_6405_keep_dims_0, x = var_6403_cast_fp16)[name = string("op_6405_cast_fp16")];
            tensor<int32, [3]> var_6408_begin_0 = const()[name = string("op_6408_begin_0"), val = tensor<int32, [3]>([0, 0, 3])];
            tensor<int32, [3]> var_6408_end_0 = const()[name = string("op_6408_end_0"), val = tensor<int32, [3]>([1, 16, 4])];
            tensor<bool, [3]> var_6408_end_mask_0 = const()[name = string("op_6408_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_6408_squeeze_mask_0 = const()[name = string("op_6408_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_6408_cast_fp16 = slice_by_index(begin = var_6408_begin_0, end = var_6408_end_0, end_mask = var_6408_end_mask_0, squeeze_mask = var_6408_squeeze_mask_0, x = decay_15_cast_fp16)[name = string("op_6408_cast_fp16")];
            tensor<fp16, [1, 16]> var_6409_cast_fp16 = exp(x = var_6408_cast_fp16)[name = string("op_6409_cast_fp16")];
            tensor<int32, [1]> var_6410_axes_0 = const()[name = string("op_6410_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_6410_cast_fp16 = expand_dims(axes = var_6410_axes_0, x = var_6409_cast_fp16)[name = string("op_6410_cast_fp16")];
            tensor<int32, [1]> var_6411_axes_0 = const()[name = string("op_6411_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_6411_cast_fp16 = expand_dims(axes = var_6411_axes_0, x = var_6410_cast_fp16)[name = string("op_6411_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_461_cast_fp16 = mul(x = state_459_cast_fp16, y = var_6411_cast_fp16)[name = string("state_461_cast_fp16")];
            tensor<int32, [4]> key_token_231_begin_0 = const()[name = string("key_token_231_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> key_token_231_end_0 = const()[name = string("key_token_231_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> key_token_231_end_mask_0 = const()[name = string("key_token_231_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_231_squeeze_mask_0 = const()[name = string("key_token_231_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_231_cast_fp16 = slice_by_index(begin = key_token_231_begin_0, end = key_token_231_end_0, end_mask = key_token_231_end_mask_0, squeeze_mask = key_token_231_squeeze_mask_0, x = key_39_cast_fp16)[name = string("key_token_231_cast_fp16")];
            tensor<int32, [4]> value_token_231_begin_0 = const()[name = string("value_token_231_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> value_token_231_end_0 = const()[name = string("value_token_231_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> value_token_231_end_mask_0 = const()[name = string("value_token_231_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_231_squeeze_mask_0 = const()[name = string("value_token_231_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_231_cast_fp16 = slice_by_index(begin = value_token_231_begin_0, end = value_token_231_end_0, end_mask = value_token_231_end_mask_0, squeeze_mask = value_token_231_squeeze_mask_0, x = value_141_cast_fp16)[name = string("value_token_231_cast_fp16")];
            tensor<int32, [1]> var_6419_axes_0 = const()[name = string("op_6419_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_6419_cast_fp16 = expand_dims(axes = var_6419_axes_0, x = key_token_231_cast_fp16)[name = string("op_6419_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6420_cast_fp16 = mul(x = state_461_cast_fp16, y = var_6419_cast_fp16)[name = string("op_6420_cast_fp16")];
            tensor<int32, [1]> memory_231_axes_0 = const()[name = string("memory_231_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_231_keep_dims_0 = const()[name = string("memory_231_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_231_cast_fp16 = reduce_sum(axes = memory_231_axes_0, keep_dims = memory_231_keep_dims_0, x = var_6420_cast_fp16)[name = string("memory_231_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_6423_cast_fp16 = sub(x = value_token_231_cast_fp16, y = memory_231_cast_fp16)[name = string("op_6423_cast_fp16")];
            tensor<int32, [3]> var_6426_begin_0 = const()[name = string("op_6426_begin_0"), val = tensor<int32, [3]>([0, 0, 3])];
            tensor<int32, [3]> var_6426_end_0 = const()[name = string("op_6426_end_0"), val = tensor<int32, [3]>([1, 16, 4])];
            tensor<bool, [3]> var_6426_end_mask_0 = const()[name = string("op_6426_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_6426_squeeze_mask_0 = const()[name = string("op_6426_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_6426_cast_fp16 = slice_by_index(begin = var_6426_begin_0, end = var_6426_end_0, end_mask = var_6426_end_mask_0, squeeze_mask = var_6426_squeeze_mask_0, x = beta_15_cast_fp16)[name = string("op_6426_cast_fp16")];
            tensor<int32, [1]> var_6427_axes_0 = const()[name = string("op_6427_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_6427_cast_fp16 = expand_dims(axes = var_6427_axes_0, x = var_6426_cast_fp16)[name = string("op_6427_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_231_cast_fp16 = mul(x = var_6423_cast_fp16, y = var_6427_cast_fp16)[name = string("delta_231_cast_fp16")];
            tensor<int32, [1]> var_6430_axes_0 = const()[name = string("op_6430_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_6430_cast_fp16 = expand_dims(axes = var_6430_axes_0, x = delta_231_cast_fp16)[name = string("op_6430_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6431_cast_fp16 = mul(x = var_6419_cast_fp16, y = var_6430_cast_fp16)[name = string("op_6431_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_463_cast_fp16 = add(x = state_461_cast_fp16, y = var_6431_cast_fp16)[name = string("state_463_cast_fp16")];
            tensor<int32, [4]> var_6435_begin_0 = const()[name = string("op_6435_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> var_6435_end_0 = const()[name = string("op_6435_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> var_6435_end_mask_0 = const()[name = string("op_6435_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_6435_squeeze_mask_0 = const()[name = string("op_6435_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_6435_cast_fp16 = slice_by_index(begin = var_6435_begin_0, end = var_6435_end_0, end_mask = var_6435_end_mask_0, squeeze_mask = var_6435_squeeze_mask_0, x = _inversed_query_39_cast_fp16)[name = string("op_6435_cast_fp16")];
            tensor<int32, [1]> var_6436_axes_0 = const()[name = string("op_6436_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_6436_cast_fp16 = expand_dims(axes = var_6436_axes_0, x = var_6435_cast_fp16)[name = string("op_6436_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6437_cast_fp16 = mul(x = state_463_cast_fp16, y = var_6436_cast_fp16)[name = string("op_6437_cast_fp16")];
            tensor<int32, [1]> var_6439_axes_0 = const()[name = string("op_6439_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_6439_keep_dims_0 = const()[name = string("op_6439_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_6439_cast_fp16 = reduce_sum(axes = var_6439_axes_0, keep_dims = var_6439_keep_dims_0, x = var_6437_cast_fp16)[name = string("op_6439_cast_fp16")];
            tensor<int32, [3]> var_6442_begin_0 = const()[name = string("op_6442_begin_0"), val = tensor<int32, [3]>([0, 0, 4])];
            tensor<int32, [3]> var_6442_end_0 = const()[name = string("op_6442_end_0"), val = tensor<int32, [3]>([1, 16, 5])];
            tensor<bool, [3]> var_6442_end_mask_0 = const()[name = string("op_6442_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_6442_squeeze_mask_0 = const()[name = string("op_6442_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_6442_cast_fp16 = slice_by_index(begin = var_6442_begin_0, end = var_6442_end_0, end_mask = var_6442_end_mask_0, squeeze_mask = var_6442_squeeze_mask_0, x = decay_15_cast_fp16)[name = string("op_6442_cast_fp16")];
            tensor<fp16, [1, 16]> var_6443_cast_fp16 = exp(x = var_6442_cast_fp16)[name = string("op_6443_cast_fp16")];
            tensor<int32, [1]> var_6444_axes_0 = const()[name = string("op_6444_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_6444_cast_fp16 = expand_dims(axes = var_6444_axes_0, x = var_6443_cast_fp16)[name = string("op_6444_cast_fp16")];
            tensor<int32, [1]> var_6445_axes_0 = const()[name = string("op_6445_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_6445_cast_fp16 = expand_dims(axes = var_6445_axes_0, x = var_6444_cast_fp16)[name = string("op_6445_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_465_cast_fp16 = mul(x = state_463_cast_fp16, y = var_6445_cast_fp16)[name = string("state_465_cast_fp16")];
            tensor<int32, [4]> key_token_233_begin_0 = const()[name = string("key_token_233_begin_0"), val = tensor<int32, [4]>([0, 0, 4, 0])];
            tensor<int32, [4]> key_token_233_end_0 = const()[name = string("key_token_233_end_0"), val = tensor<int32, [4]>([1, 16, 5, 128])];
            tensor<bool, [4]> key_token_233_end_mask_0 = const()[name = string("key_token_233_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_233_squeeze_mask_0 = const()[name = string("key_token_233_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_233_cast_fp16 = slice_by_index(begin = key_token_233_begin_0, end = key_token_233_end_0, end_mask = key_token_233_end_mask_0, squeeze_mask = key_token_233_squeeze_mask_0, x = key_39_cast_fp16)[name = string("key_token_233_cast_fp16")];
            tensor<int32, [4]> value_token_233_begin_0 = const()[name = string("value_token_233_begin_0"), val = tensor<int32, [4]>([0, 0, 4, 0])];
            tensor<int32, [4]> value_token_233_end_0 = const()[name = string("value_token_233_end_0"), val = tensor<int32, [4]>([1, 16, 5, 128])];
            tensor<bool, [4]> value_token_233_end_mask_0 = const()[name = string("value_token_233_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_233_squeeze_mask_0 = const()[name = string("value_token_233_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_233_cast_fp16 = slice_by_index(begin = value_token_233_begin_0, end = value_token_233_end_0, end_mask = value_token_233_end_mask_0, squeeze_mask = value_token_233_squeeze_mask_0, x = value_141_cast_fp16)[name = string("value_token_233_cast_fp16")];
            tensor<int32, [1]> var_6453_axes_0 = const()[name = string("op_6453_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_6453_cast_fp16 = expand_dims(axes = var_6453_axes_0, x = key_token_233_cast_fp16)[name = string("op_6453_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6454_cast_fp16 = mul(x = state_465_cast_fp16, y = var_6453_cast_fp16)[name = string("op_6454_cast_fp16")];
            tensor<int32, [1]> memory_233_axes_0 = const()[name = string("memory_233_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_233_keep_dims_0 = const()[name = string("memory_233_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_233_cast_fp16 = reduce_sum(axes = memory_233_axes_0, keep_dims = memory_233_keep_dims_0, x = var_6454_cast_fp16)[name = string("memory_233_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_6457_cast_fp16 = sub(x = value_token_233_cast_fp16, y = memory_233_cast_fp16)[name = string("op_6457_cast_fp16")];
            tensor<int32, [3]> var_6460_begin_0 = const()[name = string("op_6460_begin_0"), val = tensor<int32, [3]>([0, 0, 4])];
            tensor<int32, [3]> var_6460_end_0 = const()[name = string("op_6460_end_0"), val = tensor<int32, [3]>([1, 16, 5])];
            tensor<bool, [3]> var_6460_end_mask_0 = const()[name = string("op_6460_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_6460_squeeze_mask_0 = const()[name = string("op_6460_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_6460_cast_fp16 = slice_by_index(begin = var_6460_begin_0, end = var_6460_end_0, end_mask = var_6460_end_mask_0, squeeze_mask = var_6460_squeeze_mask_0, x = beta_15_cast_fp16)[name = string("op_6460_cast_fp16")];
            tensor<int32, [1]> var_6461_axes_0 = const()[name = string("op_6461_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_6461_cast_fp16 = expand_dims(axes = var_6461_axes_0, x = var_6460_cast_fp16)[name = string("op_6461_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_233_cast_fp16 = mul(x = var_6457_cast_fp16, y = var_6461_cast_fp16)[name = string("delta_233_cast_fp16")];
            tensor<int32, [1]> var_6464_axes_0 = const()[name = string("op_6464_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_6464_cast_fp16 = expand_dims(axes = var_6464_axes_0, x = delta_233_cast_fp16)[name = string("op_6464_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6465_cast_fp16 = mul(x = var_6453_cast_fp16, y = var_6464_cast_fp16)[name = string("op_6465_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_467_cast_fp16 = add(x = state_465_cast_fp16, y = var_6465_cast_fp16)[name = string("state_467_cast_fp16")];
            tensor<int32, [4]> var_6469_begin_0 = const()[name = string("op_6469_begin_0"), val = tensor<int32, [4]>([0, 0, 4, 0])];
            tensor<int32, [4]> var_6469_end_0 = const()[name = string("op_6469_end_0"), val = tensor<int32, [4]>([1, 16, 5, 128])];
            tensor<bool, [4]> var_6469_end_mask_0 = const()[name = string("op_6469_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_6469_squeeze_mask_0 = const()[name = string("op_6469_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_6469_cast_fp16 = slice_by_index(begin = var_6469_begin_0, end = var_6469_end_0, end_mask = var_6469_end_mask_0, squeeze_mask = var_6469_squeeze_mask_0, x = _inversed_query_39_cast_fp16)[name = string("op_6469_cast_fp16")];
            tensor<int32, [1]> var_6470_axes_0 = const()[name = string("op_6470_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_6470_cast_fp16 = expand_dims(axes = var_6470_axes_0, x = var_6469_cast_fp16)[name = string("op_6470_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6471_cast_fp16 = mul(x = state_467_cast_fp16, y = var_6470_cast_fp16)[name = string("op_6471_cast_fp16")];
            tensor<int32, [1]> var_6473_axes_0 = const()[name = string("op_6473_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_6473_keep_dims_0 = const()[name = string("op_6473_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_6473_cast_fp16 = reduce_sum(axes = var_6473_axes_0, keep_dims = var_6473_keep_dims_0, x = var_6471_cast_fp16)[name = string("op_6473_cast_fp16")];
            tensor<int32, [3]> var_6476_begin_0 = const()[name = string("op_6476_begin_0"), val = tensor<int32, [3]>([0, 0, 5])];
            tensor<int32, [3]> var_6476_end_0 = const()[name = string("op_6476_end_0"), val = tensor<int32, [3]>([1, 16, 6])];
            tensor<bool, [3]> var_6476_end_mask_0 = const()[name = string("op_6476_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_6476_squeeze_mask_0 = const()[name = string("op_6476_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_6476_cast_fp16 = slice_by_index(begin = var_6476_begin_0, end = var_6476_end_0, end_mask = var_6476_end_mask_0, squeeze_mask = var_6476_squeeze_mask_0, x = decay_15_cast_fp16)[name = string("op_6476_cast_fp16")];
            tensor<fp16, [1, 16]> var_6477_cast_fp16 = exp(x = var_6476_cast_fp16)[name = string("op_6477_cast_fp16")];
            tensor<int32, [1]> var_6478_axes_0 = const()[name = string("op_6478_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_6478_cast_fp16 = expand_dims(axes = var_6478_axes_0, x = var_6477_cast_fp16)[name = string("op_6478_cast_fp16")];
            tensor<int32, [1]> var_6479_axes_0 = const()[name = string("op_6479_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_6479_cast_fp16 = expand_dims(axes = var_6479_axes_0, x = var_6478_cast_fp16)[name = string("op_6479_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_469_cast_fp16 = mul(x = state_467_cast_fp16, y = var_6479_cast_fp16)[name = string("state_469_cast_fp16")];
            tensor<int32, [4]> key_token_235_begin_0 = const()[name = string("key_token_235_begin_0"), val = tensor<int32, [4]>([0, 0, 5, 0])];
            tensor<int32, [4]> key_token_235_end_0 = const()[name = string("key_token_235_end_0"), val = tensor<int32, [4]>([1, 16, 6, 128])];
            tensor<bool, [4]> key_token_235_end_mask_0 = const()[name = string("key_token_235_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_235_squeeze_mask_0 = const()[name = string("key_token_235_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_235_cast_fp16 = slice_by_index(begin = key_token_235_begin_0, end = key_token_235_end_0, end_mask = key_token_235_end_mask_0, squeeze_mask = key_token_235_squeeze_mask_0, x = key_39_cast_fp16)[name = string("key_token_235_cast_fp16")];
            tensor<int32, [4]> value_token_235_begin_0 = const()[name = string("value_token_235_begin_0"), val = tensor<int32, [4]>([0, 0, 5, 0])];
            tensor<int32, [4]> value_token_235_end_0 = const()[name = string("value_token_235_end_0"), val = tensor<int32, [4]>([1, 16, 6, 128])];
            tensor<bool, [4]> value_token_235_end_mask_0 = const()[name = string("value_token_235_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_235_squeeze_mask_0 = const()[name = string("value_token_235_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_235_cast_fp16 = slice_by_index(begin = value_token_235_begin_0, end = value_token_235_end_0, end_mask = value_token_235_end_mask_0, squeeze_mask = value_token_235_squeeze_mask_0, x = value_141_cast_fp16)[name = string("value_token_235_cast_fp16")];
            tensor<int32, [1]> var_6487_axes_0 = const()[name = string("op_6487_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_6487_cast_fp16 = expand_dims(axes = var_6487_axes_0, x = key_token_235_cast_fp16)[name = string("op_6487_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6488_cast_fp16 = mul(x = state_469_cast_fp16, y = var_6487_cast_fp16)[name = string("op_6488_cast_fp16")];
            tensor<int32, [1]> memory_235_axes_0 = const()[name = string("memory_235_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_235_keep_dims_0 = const()[name = string("memory_235_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_235_cast_fp16 = reduce_sum(axes = memory_235_axes_0, keep_dims = memory_235_keep_dims_0, x = var_6488_cast_fp16)[name = string("memory_235_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_6491_cast_fp16 = sub(x = value_token_235_cast_fp16, y = memory_235_cast_fp16)[name = string("op_6491_cast_fp16")];
            tensor<int32, [3]> var_6494_begin_0 = const()[name = string("op_6494_begin_0"), val = tensor<int32, [3]>([0, 0, 5])];
            tensor<int32, [3]> var_6494_end_0 = const()[name = string("op_6494_end_0"), val = tensor<int32, [3]>([1, 16, 6])];
            tensor<bool, [3]> var_6494_end_mask_0 = const()[name = string("op_6494_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_6494_squeeze_mask_0 = const()[name = string("op_6494_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_6494_cast_fp16 = slice_by_index(begin = var_6494_begin_0, end = var_6494_end_0, end_mask = var_6494_end_mask_0, squeeze_mask = var_6494_squeeze_mask_0, x = beta_15_cast_fp16)[name = string("op_6494_cast_fp16")];
            tensor<int32, [1]> var_6495_axes_0 = const()[name = string("op_6495_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_6495_cast_fp16 = expand_dims(axes = var_6495_axes_0, x = var_6494_cast_fp16)[name = string("op_6495_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_235_cast_fp16 = mul(x = var_6491_cast_fp16, y = var_6495_cast_fp16)[name = string("delta_235_cast_fp16")];
            tensor<int32, [1]> var_6498_axes_0 = const()[name = string("op_6498_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_6498_cast_fp16 = expand_dims(axes = var_6498_axes_0, x = delta_235_cast_fp16)[name = string("op_6498_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6499_cast_fp16 = mul(x = var_6487_cast_fp16, y = var_6498_cast_fp16)[name = string("op_6499_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_471_cast_fp16 = add(x = state_469_cast_fp16, y = var_6499_cast_fp16)[name = string("state_471_cast_fp16")];
            tensor<int32, [4]> var_6503_begin_0 = const()[name = string("op_6503_begin_0"), val = tensor<int32, [4]>([0, 0, 5, 0])];
            tensor<int32, [4]> var_6503_end_0 = const()[name = string("op_6503_end_0"), val = tensor<int32, [4]>([1, 16, 6, 128])];
            tensor<bool, [4]> var_6503_end_mask_0 = const()[name = string("op_6503_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_6503_squeeze_mask_0 = const()[name = string("op_6503_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_6503_cast_fp16 = slice_by_index(begin = var_6503_begin_0, end = var_6503_end_0, end_mask = var_6503_end_mask_0, squeeze_mask = var_6503_squeeze_mask_0, x = _inversed_query_39_cast_fp16)[name = string("op_6503_cast_fp16")];
            tensor<int32, [1]> var_6504_axes_0 = const()[name = string("op_6504_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_6504_cast_fp16 = expand_dims(axes = var_6504_axes_0, x = var_6503_cast_fp16)[name = string("op_6504_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6505_cast_fp16 = mul(x = state_471_cast_fp16, y = var_6504_cast_fp16)[name = string("op_6505_cast_fp16")];
            tensor<int32, [1]> var_6507_axes_0 = const()[name = string("op_6507_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_6507_keep_dims_0 = const()[name = string("op_6507_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_6507_cast_fp16 = reduce_sum(axes = var_6507_axes_0, keep_dims = var_6507_keep_dims_0, x = var_6505_cast_fp16)[name = string("op_6507_cast_fp16")];
            tensor<int32, [3]> var_6510_begin_0 = const()[name = string("op_6510_begin_0"), val = tensor<int32, [3]>([0, 0, 6])];
            tensor<int32, [3]> var_6510_end_0 = const()[name = string("op_6510_end_0"), val = tensor<int32, [3]>([1, 16, 7])];
            tensor<bool, [3]> var_6510_end_mask_0 = const()[name = string("op_6510_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_6510_squeeze_mask_0 = const()[name = string("op_6510_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_6510_cast_fp16 = slice_by_index(begin = var_6510_begin_0, end = var_6510_end_0, end_mask = var_6510_end_mask_0, squeeze_mask = var_6510_squeeze_mask_0, x = decay_15_cast_fp16)[name = string("op_6510_cast_fp16")];
            tensor<fp16, [1, 16]> var_6511_cast_fp16 = exp(x = var_6510_cast_fp16)[name = string("op_6511_cast_fp16")];
            tensor<int32, [1]> var_6512_axes_0 = const()[name = string("op_6512_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_6512_cast_fp16 = expand_dims(axes = var_6512_axes_0, x = var_6511_cast_fp16)[name = string("op_6512_cast_fp16")];
            tensor<int32, [1]> var_6513_axes_0 = const()[name = string("op_6513_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_6513_cast_fp16 = expand_dims(axes = var_6513_axes_0, x = var_6512_cast_fp16)[name = string("op_6513_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_473_cast_fp16 = mul(x = state_471_cast_fp16, y = var_6513_cast_fp16)[name = string("state_473_cast_fp16")];
            tensor<int32, [4]> key_token_237_begin_0 = const()[name = string("key_token_237_begin_0"), val = tensor<int32, [4]>([0, 0, 6, 0])];
            tensor<int32, [4]> key_token_237_end_0 = const()[name = string("key_token_237_end_0"), val = tensor<int32, [4]>([1, 16, 7, 128])];
            tensor<bool, [4]> key_token_237_end_mask_0 = const()[name = string("key_token_237_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_237_squeeze_mask_0 = const()[name = string("key_token_237_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_237_cast_fp16 = slice_by_index(begin = key_token_237_begin_0, end = key_token_237_end_0, end_mask = key_token_237_end_mask_0, squeeze_mask = key_token_237_squeeze_mask_0, x = key_39_cast_fp16)[name = string("key_token_237_cast_fp16")];
            tensor<int32, [4]> value_token_237_begin_0 = const()[name = string("value_token_237_begin_0"), val = tensor<int32, [4]>([0, 0, 6, 0])];
            tensor<int32, [4]> value_token_237_end_0 = const()[name = string("value_token_237_end_0"), val = tensor<int32, [4]>([1, 16, 7, 128])];
            tensor<bool, [4]> value_token_237_end_mask_0 = const()[name = string("value_token_237_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_237_squeeze_mask_0 = const()[name = string("value_token_237_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_237_cast_fp16 = slice_by_index(begin = value_token_237_begin_0, end = value_token_237_end_0, end_mask = value_token_237_end_mask_0, squeeze_mask = value_token_237_squeeze_mask_0, x = value_141_cast_fp16)[name = string("value_token_237_cast_fp16")];
            tensor<int32, [1]> var_6521_axes_0 = const()[name = string("op_6521_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_6521_cast_fp16 = expand_dims(axes = var_6521_axes_0, x = key_token_237_cast_fp16)[name = string("op_6521_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6522_cast_fp16 = mul(x = state_473_cast_fp16, y = var_6521_cast_fp16)[name = string("op_6522_cast_fp16")];
            tensor<int32, [1]> memory_237_axes_0 = const()[name = string("memory_237_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_237_keep_dims_0 = const()[name = string("memory_237_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_237_cast_fp16 = reduce_sum(axes = memory_237_axes_0, keep_dims = memory_237_keep_dims_0, x = var_6522_cast_fp16)[name = string("memory_237_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_6525_cast_fp16 = sub(x = value_token_237_cast_fp16, y = memory_237_cast_fp16)[name = string("op_6525_cast_fp16")];
            tensor<int32, [3]> var_6528_begin_0 = const()[name = string("op_6528_begin_0"), val = tensor<int32, [3]>([0, 0, 6])];
            tensor<int32, [3]> var_6528_end_0 = const()[name = string("op_6528_end_0"), val = tensor<int32, [3]>([1, 16, 7])];
            tensor<bool, [3]> var_6528_end_mask_0 = const()[name = string("op_6528_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_6528_squeeze_mask_0 = const()[name = string("op_6528_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_6528_cast_fp16 = slice_by_index(begin = var_6528_begin_0, end = var_6528_end_0, end_mask = var_6528_end_mask_0, squeeze_mask = var_6528_squeeze_mask_0, x = beta_15_cast_fp16)[name = string("op_6528_cast_fp16")];
            tensor<int32, [1]> var_6529_axes_0 = const()[name = string("op_6529_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_6529_cast_fp16 = expand_dims(axes = var_6529_axes_0, x = var_6528_cast_fp16)[name = string("op_6529_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_237_cast_fp16 = mul(x = var_6525_cast_fp16, y = var_6529_cast_fp16)[name = string("delta_237_cast_fp16")];
            tensor<int32, [1]> var_6532_axes_0 = const()[name = string("op_6532_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_6532_cast_fp16 = expand_dims(axes = var_6532_axes_0, x = delta_237_cast_fp16)[name = string("op_6532_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6533_cast_fp16 = mul(x = var_6521_cast_fp16, y = var_6532_cast_fp16)[name = string("op_6533_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_475_cast_fp16 = add(x = state_473_cast_fp16, y = var_6533_cast_fp16)[name = string("state_475_cast_fp16")];
            tensor<int32, [4]> var_6537_begin_0 = const()[name = string("op_6537_begin_0"), val = tensor<int32, [4]>([0, 0, 6, 0])];
            tensor<int32, [4]> var_6537_end_0 = const()[name = string("op_6537_end_0"), val = tensor<int32, [4]>([1, 16, 7, 128])];
            tensor<bool, [4]> var_6537_end_mask_0 = const()[name = string("op_6537_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_6537_squeeze_mask_0 = const()[name = string("op_6537_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_6537_cast_fp16 = slice_by_index(begin = var_6537_begin_0, end = var_6537_end_0, end_mask = var_6537_end_mask_0, squeeze_mask = var_6537_squeeze_mask_0, x = _inversed_query_39_cast_fp16)[name = string("op_6537_cast_fp16")];
            tensor<int32, [1]> var_6538_axes_0 = const()[name = string("op_6538_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_6538_cast_fp16 = expand_dims(axes = var_6538_axes_0, x = var_6537_cast_fp16)[name = string("op_6538_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6539_cast_fp16 = mul(x = state_475_cast_fp16, y = var_6538_cast_fp16)[name = string("op_6539_cast_fp16")];
            tensor<int32, [1]> var_6541_axes_0 = const()[name = string("op_6541_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_6541_keep_dims_0 = const()[name = string("op_6541_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_6541_cast_fp16 = reduce_sum(axes = var_6541_axes_0, keep_dims = var_6541_keep_dims_0, x = var_6539_cast_fp16)[name = string("op_6541_cast_fp16")];
            tensor<int32, [3]> var_6544_begin_0 = const()[name = string("op_6544_begin_0"), val = tensor<int32, [3]>([0, 0, 7])];
            tensor<int32, [3]> var_6544_end_0 = const()[name = string("op_6544_end_0"), val = tensor<int32, [3]>([1, 16, 8])];
            tensor<bool, [3]> var_6544_end_mask_0 = const()[name = string("op_6544_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_6544_squeeze_mask_0 = const()[name = string("op_6544_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_6544_cast_fp16 = slice_by_index(begin = var_6544_begin_0, end = var_6544_end_0, end_mask = var_6544_end_mask_0, squeeze_mask = var_6544_squeeze_mask_0, x = decay_15_cast_fp16)[name = string("op_6544_cast_fp16")];
            tensor<fp16, [1, 16]> var_6545_cast_fp16 = exp(x = var_6544_cast_fp16)[name = string("op_6545_cast_fp16")];
            tensor<int32, [1]> var_6546_axes_0 = const()[name = string("op_6546_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_6546_cast_fp16 = expand_dims(axes = var_6546_axes_0, x = var_6545_cast_fp16)[name = string("op_6546_cast_fp16")];
            tensor<int32, [1]> var_6547_axes_0 = const()[name = string("op_6547_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_6547_cast_fp16 = expand_dims(axes = var_6547_axes_0, x = var_6546_cast_fp16)[name = string("op_6547_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_477_cast_fp16 = mul(x = state_475_cast_fp16, y = var_6547_cast_fp16)[name = string("state_477_cast_fp16")];
            tensor<int32, [4]> key_token_239_begin_0 = const()[name = string("key_token_239_begin_0"), val = tensor<int32, [4]>([0, 0, 7, 0])];
            tensor<int32, [4]> key_token_239_end_0 = const()[name = string("key_token_239_end_0"), val = tensor<int32, [4]>([1, 16, 8, 128])];
            tensor<bool, [4]> key_token_239_end_mask_0 = const()[name = string("key_token_239_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_239_squeeze_mask_0 = const()[name = string("key_token_239_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_239_cast_fp16 = slice_by_index(begin = key_token_239_begin_0, end = key_token_239_end_0, end_mask = key_token_239_end_mask_0, squeeze_mask = key_token_239_squeeze_mask_0, x = key_39_cast_fp16)[name = string("key_token_239_cast_fp16")];
            tensor<int32, [4]> value_token_239_begin_0 = const()[name = string("value_token_239_begin_0"), val = tensor<int32, [4]>([0, 0, 7, 0])];
            tensor<int32, [4]> value_token_239_end_0 = const()[name = string("value_token_239_end_0"), val = tensor<int32, [4]>([1, 16, 8, 128])];
            tensor<bool, [4]> value_token_239_end_mask_0 = const()[name = string("value_token_239_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_239_squeeze_mask_0 = const()[name = string("value_token_239_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_239_cast_fp16 = slice_by_index(begin = value_token_239_begin_0, end = value_token_239_end_0, end_mask = value_token_239_end_mask_0, squeeze_mask = value_token_239_squeeze_mask_0, x = value_141_cast_fp16)[name = string("value_token_239_cast_fp16")];
            tensor<int32, [1]> var_6555_axes_0 = const()[name = string("op_6555_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_6555_cast_fp16 = expand_dims(axes = var_6555_axes_0, x = key_token_239_cast_fp16)[name = string("op_6555_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6556_cast_fp16 = mul(x = state_477_cast_fp16, y = var_6555_cast_fp16)[name = string("op_6556_cast_fp16")];
            tensor<int32, [1]> memory_239_axes_0 = const()[name = string("memory_239_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_239_keep_dims_0 = const()[name = string("memory_239_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_239_cast_fp16 = reduce_sum(axes = memory_239_axes_0, keep_dims = memory_239_keep_dims_0, x = var_6556_cast_fp16)[name = string("memory_239_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_6559_cast_fp16 = sub(x = value_token_239_cast_fp16, y = memory_239_cast_fp16)[name = string("op_6559_cast_fp16")];
            tensor<int32, [3]> var_6562_begin_0 = const()[name = string("op_6562_begin_0"), val = tensor<int32, [3]>([0, 0, 7])];
            tensor<int32, [3]> var_6562_end_0 = const()[name = string("op_6562_end_0"), val = tensor<int32, [3]>([1, 16, 8])];
            tensor<bool, [3]> var_6562_end_mask_0 = const()[name = string("op_6562_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_6562_squeeze_mask_0 = const()[name = string("op_6562_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_6562_cast_fp16 = slice_by_index(begin = var_6562_begin_0, end = var_6562_end_0, end_mask = var_6562_end_mask_0, squeeze_mask = var_6562_squeeze_mask_0, x = beta_15_cast_fp16)[name = string("op_6562_cast_fp16")];
            tensor<int32, [1]> var_6563_axes_0 = const()[name = string("op_6563_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_6563_cast_fp16 = expand_dims(axes = var_6563_axes_0, x = var_6562_cast_fp16)[name = string("op_6563_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_239_cast_fp16 = mul(x = var_6559_cast_fp16, y = var_6563_cast_fp16)[name = string("delta_239_cast_fp16")];
            tensor<int32, [1]> var_6566_axes_0 = const()[name = string("op_6566_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_6566_cast_fp16 = expand_dims(axes = var_6566_axes_0, x = delta_239_cast_fp16)[name = string("op_6566_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6567_cast_fp16 = mul(x = var_6555_cast_fp16, y = var_6566_cast_fp16)[name = string("op_6567_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_479_cast_fp16 = add(x = state_477_cast_fp16, y = var_6567_cast_fp16)[name = string("state_479_cast_fp16")];
            tensor<int32, [4]> var_6571_begin_0 = const()[name = string("op_6571_begin_0"), val = tensor<int32, [4]>([0, 0, 7, 0])];
            tensor<int32, [4]> var_6571_end_0 = const()[name = string("op_6571_end_0"), val = tensor<int32, [4]>([1, 16, 8, 128])];
            tensor<bool, [4]> var_6571_end_mask_0 = const()[name = string("op_6571_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_6571_squeeze_mask_0 = const()[name = string("op_6571_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_6571_cast_fp16 = slice_by_index(begin = var_6571_begin_0, end = var_6571_end_0, end_mask = var_6571_end_mask_0, squeeze_mask = var_6571_squeeze_mask_0, x = _inversed_query_39_cast_fp16)[name = string("op_6571_cast_fp16")];
            tensor<int32, [1]> var_6572_axes_0 = const()[name = string("op_6572_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_6572_cast_fp16 = expand_dims(axes = var_6572_axes_0, x = var_6571_cast_fp16)[name = string("op_6572_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6573_cast_fp16 = mul(x = state_479_cast_fp16, y = var_6572_cast_fp16)[name = string("op_6573_cast_fp16")];
            tensor<int32, [1]> var_6575_axes_0 = const()[name = string("op_6575_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_6575_keep_dims_0 = const()[name = string("op_6575_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_6575_cast_fp16 = reduce_sum(axes = var_6575_axes_0, keep_dims = var_6575_keep_dims_0, x = var_6573_cast_fp16)[name = string("op_6575_cast_fp16")];
            tensor<int32, [3]> var_6578_begin_0 = const()[name = string("op_6578_begin_0"), val = tensor<int32, [3]>([0, 0, 8])];
            tensor<int32, [3]> var_6578_end_0 = const()[name = string("op_6578_end_0"), val = tensor<int32, [3]>([1, 16, 9])];
            tensor<bool, [3]> var_6578_end_mask_0 = const()[name = string("op_6578_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_6578_squeeze_mask_0 = const()[name = string("op_6578_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_6578_cast_fp16 = slice_by_index(begin = var_6578_begin_0, end = var_6578_end_0, end_mask = var_6578_end_mask_0, squeeze_mask = var_6578_squeeze_mask_0, x = decay_15_cast_fp16)[name = string("op_6578_cast_fp16")];
            tensor<fp16, [1, 16]> var_6579_cast_fp16 = exp(x = var_6578_cast_fp16)[name = string("op_6579_cast_fp16")];
            tensor<int32, [1]> var_6580_axes_0 = const()[name = string("op_6580_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_6580_cast_fp16 = expand_dims(axes = var_6580_axes_0, x = var_6579_cast_fp16)[name = string("op_6580_cast_fp16")];
            tensor<int32, [1]> var_6581_axes_0 = const()[name = string("op_6581_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_6581_cast_fp16 = expand_dims(axes = var_6581_axes_0, x = var_6580_cast_fp16)[name = string("op_6581_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_481_cast_fp16 = mul(x = state_479_cast_fp16, y = var_6581_cast_fp16)[name = string("state_481_cast_fp16")];
            tensor<int32, [4]> key_token_241_begin_0 = const()[name = string("key_token_241_begin_0"), val = tensor<int32, [4]>([0, 0, 8, 0])];
            tensor<int32, [4]> key_token_241_end_0 = const()[name = string("key_token_241_end_0"), val = tensor<int32, [4]>([1, 16, 9, 128])];
            tensor<bool, [4]> key_token_241_end_mask_0 = const()[name = string("key_token_241_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_241_squeeze_mask_0 = const()[name = string("key_token_241_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_241_cast_fp16 = slice_by_index(begin = key_token_241_begin_0, end = key_token_241_end_0, end_mask = key_token_241_end_mask_0, squeeze_mask = key_token_241_squeeze_mask_0, x = key_39_cast_fp16)[name = string("key_token_241_cast_fp16")];
            tensor<int32, [4]> value_token_241_begin_0 = const()[name = string("value_token_241_begin_0"), val = tensor<int32, [4]>([0, 0, 8, 0])];
            tensor<int32, [4]> value_token_241_end_0 = const()[name = string("value_token_241_end_0"), val = tensor<int32, [4]>([1, 16, 9, 128])];
            tensor<bool, [4]> value_token_241_end_mask_0 = const()[name = string("value_token_241_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_241_squeeze_mask_0 = const()[name = string("value_token_241_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_241_cast_fp16 = slice_by_index(begin = value_token_241_begin_0, end = value_token_241_end_0, end_mask = value_token_241_end_mask_0, squeeze_mask = value_token_241_squeeze_mask_0, x = value_141_cast_fp16)[name = string("value_token_241_cast_fp16")];
            tensor<int32, [1]> var_6589_axes_0 = const()[name = string("op_6589_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_6589_cast_fp16 = expand_dims(axes = var_6589_axes_0, x = key_token_241_cast_fp16)[name = string("op_6589_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6590_cast_fp16 = mul(x = state_481_cast_fp16, y = var_6589_cast_fp16)[name = string("op_6590_cast_fp16")];
            tensor<int32, [1]> memory_241_axes_0 = const()[name = string("memory_241_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_241_keep_dims_0 = const()[name = string("memory_241_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_241_cast_fp16 = reduce_sum(axes = memory_241_axes_0, keep_dims = memory_241_keep_dims_0, x = var_6590_cast_fp16)[name = string("memory_241_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_6593_cast_fp16 = sub(x = value_token_241_cast_fp16, y = memory_241_cast_fp16)[name = string("op_6593_cast_fp16")];
            tensor<int32, [3]> var_6596_begin_0 = const()[name = string("op_6596_begin_0"), val = tensor<int32, [3]>([0, 0, 8])];
            tensor<int32, [3]> var_6596_end_0 = const()[name = string("op_6596_end_0"), val = tensor<int32, [3]>([1, 16, 9])];
            tensor<bool, [3]> var_6596_end_mask_0 = const()[name = string("op_6596_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_6596_squeeze_mask_0 = const()[name = string("op_6596_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_6596_cast_fp16 = slice_by_index(begin = var_6596_begin_0, end = var_6596_end_0, end_mask = var_6596_end_mask_0, squeeze_mask = var_6596_squeeze_mask_0, x = beta_15_cast_fp16)[name = string("op_6596_cast_fp16")];
            tensor<int32, [1]> var_6597_axes_0 = const()[name = string("op_6597_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_6597_cast_fp16 = expand_dims(axes = var_6597_axes_0, x = var_6596_cast_fp16)[name = string("op_6597_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_241_cast_fp16 = mul(x = var_6593_cast_fp16, y = var_6597_cast_fp16)[name = string("delta_241_cast_fp16")];
            tensor<int32, [1]> var_6600_axes_0 = const()[name = string("op_6600_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_6600_cast_fp16 = expand_dims(axes = var_6600_axes_0, x = delta_241_cast_fp16)[name = string("op_6600_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6601_cast_fp16 = mul(x = var_6589_cast_fp16, y = var_6600_cast_fp16)[name = string("op_6601_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_483_cast_fp16 = add(x = state_481_cast_fp16, y = var_6601_cast_fp16)[name = string("state_483_cast_fp16")];
            tensor<int32, [4]> var_6605_begin_0 = const()[name = string("op_6605_begin_0"), val = tensor<int32, [4]>([0, 0, 8, 0])];
            tensor<int32, [4]> var_6605_end_0 = const()[name = string("op_6605_end_0"), val = tensor<int32, [4]>([1, 16, 9, 128])];
            tensor<bool, [4]> var_6605_end_mask_0 = const()[name = string("op_6605_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_6605_squeeze_mask_0 = const()[name = string("op_6605_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_6605_cast_fp16 = slice_by_index(begin = var_6605_begin_0, end = var_6605_end_0, end_mask = var_6605_end_mask_0, squeeze_mask = var_6605_squeeze_mask_0, x = _inversed_query_39_cast_fp16)[name = string("op_6605_cast_fp16")];
            tensor<int32, [1]> var_6606_axes_0 = const()[name = string("op_6606_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_6606_cast_fp16 = expand_dims(axes = var_6606_axes_0, x = var_6605_cast_fp16)[name = string("op_6606_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6607_cast_fp16 = mul(x = state_483_cast_fp16, y = var_6606_cast_fp16)[name = string("op_6607_cast_fp16")];
            tensor<int32, [1]> var_6609_axes_0 = const()[name = string("op_6609_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_6609_keep_dims_0 = const()[name = string("op_6609_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_6609_cast_fp16 = reduce_sum(axes = var_6609_axes_0, keep_dims = var_6609_keep_dims_0, x = var_6607_cast_fp16)[name = string("op_6609_cast_fp16")];
            tensor<int32, [3]> var_6612_begin_0 = const()[name = string("op_6612_begin_0"), val = tensor<int32, [3]>([0, 0, 9])];
            tensor<int32, [3]> var_6612_end_0 = const()[name = string("op_6612_end_0"), val = tensor<int32, [3]>([1, 16, 10])];
            tensor<bool, [3]> var_6612_end_mask_0 = const()[name = string("op_6612_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_6612_squeeze_mask_0 = const()[name = string("op_6612_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_6612_cast_fp16 = slice_by_index(begin = var_6612_begin_0, end = var_6612_end_0, end_mask = var_6612_end_mask_0, squeeze_mask = var_6612_squeeze_mask_0, x = decay_15_cast_fp16)[name = string("op_6612_cast_fp16")];
            tensor<fp16, [1, 16]> var_6613_cast_fp16 = exp(x = var_6612_cast_fp16)[name = string("op_6613_cast_fp16")];
            tensor<int32, [1]> var_6614_axes_0 = const()[name = string("op_6614_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_6614_cast_fp16 = expand_dims(axes = var_6614_axes_0, x = var_6613_cast_fp16)[name = string("op_6614_cast_fp16")];
            tensor<int32, [1]> var_6615_axes_0 = const()[name = string("op_6615_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_6615_cast_fp16 = expand_dims(axes = var_6615_axes_0, x = var_6614_cast_fp16)[name = string("op_6615_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_485_cast_fp16 = mul(x = state_483_cast_fp16, y = var_6615_cast_fp16)[name = string("state_485_cast_fp16")];
            tensor<int32, [4]> key_token_243_begin_0 = const()[name = string("key_token_243_begin_0"), val = tensor<int32, [4]>([0, 0, 9, 0])];
            tensor<int32, [4]> key_token_243_end_0 = const()[name = string("key_token_243_end_0"), val = tensor<int32, [4]>([1, 16, 10, 128])];
            tensor<bool, [4]> key_token_243_end_mask_0 = const()[name = string("key_token_243_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_243_squeeze_mask_0 = const()[name = string("key_token_243_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_243_cast_fp16 = slice_by_index(begin = key_token_243_begin_0, end = key_token_243_end_0, end_mask = key_token_243_end_mask_0, squeeze_mask = key_token_243_squeeze_mask_0, x = key_39_cast_fp16)[name = string("key_token_243_cast_fp16")];
            tensor<int32, [4]> value_token_243_begin_0 = const()[name = string("value_token_243_begin_0"), val = tensor<int32, [4]>([0, 0, 9, 0])];
            tensor<int32, [4]> value_token_243_end_0 = const()[name = string("value_token_243_end_0"), val = tensor<int32, [4]>([1, 16, 10, 128])];
            tensor<bool, [4]> value_token_243_end_mask_0 = const()[name = string("value_token_243_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_243_squeeze_mask_0 = const()[name = string("value_token_243_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_243_cast_fp16 = slice_by_index(begin = value_token_243_begin_0, end = value_token_243_end_0, end_mask = value_token_243_end_mask_0, squeeze_mask = value_token_243_squeeze_mask_0, x = value_141_cast_fp16)[name = string("value_token_243_cast_fp16")];
            tensor<int32, [1]> var_6623_axes_0 = const()[name = string("op_6623_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_6623_cast_fp16 = expand_dims(axes = var_6623_axes_0, x = key_token_243_cast_fp16)[name = string("op_6623_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6624_cast_fp16 = mul(x = state_485_cast_fp16, y = var_6623_cast_fp16)[name = string("op_6624_cast_fp16")];
            tensor<int32, [1]> memory_243_axes_0 = const()[name = string("memory_243_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_243_keep_dims_0 = const()[name = string("memory_243_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_243_cast_fp16 = reduce_sum(axes = memory_243_axes_0, keep_dims = memory_243_keep_dims_0, x = var_6624_cast_fp16)[name = string("memory_243_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_6627_cast_fp16 = sub(x = value_token_243_cast_fp16, y = memory_243_cast_fp16)[name = string("op_6627_cast_fp16")];
            tensor<int32, [3]> var_6630_begin_0 = const()[name = string("op_6630_begin_0"), val = tensor<int32, [3]>([0, 0, 9])];
            tensor<int32, [3]> var_6630_end_0 = const()[name = string("op_6630_end_0"), val = tensor<int32, [3]>([1, 16, 10])];
            tensor<bool, [3]> var_6630_end_mask_0 = const()[name = string("op_6630_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_6630_squeeze_mask_0 = const()[name = string("op_6630_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_6630_cast_fp16 = slice_by_index(begin = var_6630_begin_0, end = var_6630_end_0, end_mask = var_6630_end_mask_0, squeeze_mask = var_6630_squeeze_mask_0, x = beta_15_cast_fp16)[name = string("op_6630_cast_fp16")];
            tensor<int32, [1]> var_6631_axes_0 = const()[name = string("op_6631_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_6631_cast_fp16 = expand_dims(axes = var_6631_axes_0, x = var_6630_cast_fp16)[name = string("op_6631_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_243_cast_fp16 = mul(x = var_6627_cast_fp16, y = var_6631_cast_fp16)[name = string("delta_243_cast_fp16")];
            tensor<int32, [1]> var_6634_axes_0 = const()[name = string("op_6634_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_6634_cast_fp16 = expand_dims(axes = var_6634_axes_0, x = delta_243_cast_fp16)[name = string("op_6634_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6635_cast_fp16 = mul(x = var_6623_cast_fp16, y = var_6634_cast_fp16)[name = string("op_6635_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_487_cast_fp16 = add(x = state_485_cast_fp16, y = var_6635_cast_fp16)[name = string("state_487_cast_fp16")];
            tensor<int32, [4]> var_6639_begin_0 = const()[name = string("op_6639_begin_0"), val = tensor<int32, [4]>([0, 0, 9, 0])];
            tensor<int32, [4]> var_6639_end_0 = const()[name = string("op_6639_end_0"), val = tensor<int32, [4]>([1, 16, 10, 128])];
            tensor<bool, [4]> var_6639_end_mask_0 = const()[name = string("op_6639_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_6639_squeeze_mask_0 = const()[name = string("op_6639_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_6639_cast_fp16 = slice_by_index(begin = var_6639_begin_0, end = var_6639_end_0, end_mask = var_6639_end_mask_0, squeeze_mask = var_6639_squeeze_mask_0, x = _inversed_query_39_cast_fp16)[name = string("op_6639_cast_fp16")];
            tensor<int32, [1]> var_6640_axes_0 = const()[name = string("op_6640_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_6640_cast_fp16 = expand_dims(axes = var_6640_axes_0, x = var_6639_cast_fp16)[name = string("op_6640_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6641_cast_fp16 = mul(x = state_487_cast_fp16, y = var_6640_cast_fp16)[name = string("op_6641_cast_fp16")];
            tensor<int32, [1]> var_6643_axes_0 = const()[name = string("op_6643_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_6643_keep_dims_0 = const()[name = string("op_6643_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_6643_cast_fp16 = reduce_sum(axes = var_6643_axes_0, keep_dims = var_6643_keep_dims_0, x = var_6641_cast_fp16)[name = string("op_6643_cast_fp16")];
            tensor<int32, [3]> var_6646_begin_0 = const()[name = string("op_6646_begin_0"), val = tensor<int32, [3]>([0, 0, 10])];
            tensor<int32, [3]> var_6646_end_0 = const()[name = string("op_6646_end_0"), val = tensor<int32, [3]>([1, 16, 11])];
            tensor<bool, [3]> var_6646_end_mask_0 = const()[name = string("op_6646_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_6646_squeeze_mask_0 = const()[name = string("op_6646_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_6646_cast_fp16 = slice_by_index(begin = var_6646_begin_0, end = var_6646_end_0, end_mask = var_6646_end_mask_0, squeeze_mask = var_6646_squeeze_mask_0, x = decay_15_cast_fp16)[name = string("op_6646_cast_fp16")];
            tensor<fp16, [1, 16]> var_6647_cast_fp16 = exp(x = var_6646_cast_fp16)[name = string("op_6647_cast_fp16")];
            tensor<int32, [1]> var_6648_axes_0 = const()[name = string("op_6648_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_6648_cast_fp16 = expand_dims(axes = var_6648_axes_0, x = var_6647_cast_fp16)[name = string("op_6648_cast_fp16")];
            tensor<int32, [1]> var_6649_axes_0 = const()[name = string("op_6649_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_6649_cast_fp16 = expand_dims(axes = var_6649_axes_0, x = var_6648_cast_fp16)[name = string("op_6649_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_489_cast_fp16 = mul(x = state_487_cast_fp16, y = var_6649_cast_fp16)[name = string("state_489_cast_fp16")];
            tensor<int32, [4]> key_token_245_begin_0 = const()[name = string("key_token_245_begin_0"), val = tensor<int32, [4]>([0, 0, 10, 0])];
            tensor<int32, [4]> key_token_245_end_0 = const()[name = string("key_token_245_end_0"), val = tensor<int32, [4]>([1, 16, 11, 128])];
            tensor<bool, [4]> key_token_245_end_mask_0 = const()[name = string("key_token_245_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_245_squeeze_mask_0 = const()[name = string("key_token_245_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_245_cast_fp16 = slice_by_index(begin = key_token_245_begin_0, end = key_token_245_end_0, end_mask = key_token_245_end_mask_0, squeeze_mask = key_token_245_squeeze_mask_0, x = key_39_cast_fp16)[name = string("key_token_245_cast_fp16")];
            tensor<int32, [4]> value_token_245_begin_0 = const()[name = string("value_token_245_begin_0"), val = tensor<int32, [4]>([0, 0, 10, 0])];
            tensor<int32, [4]> value_token_245_end_0 = const()[name = string("value_token_245_end_0"), val = tensor<int32, [4]>([1, 16, 11, 128])];
            tensor<bool, [4]> value_token_245_end_mask_0 = const()[name = string("value_token_245_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_245_squeeze_mask_0 = const()[name = string("value_token_245_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_245_cast_fp16 = slice_by_index(begin = value_token_245_begin_0, end = value_token_245_end_0, end_mask = value_token_245_end_mask_0, squeeze_mask = value_token_245_squeeze_mask_0, x = value_141_cast_fp16)[name = string("value_token_245_cast_fp16")];
            tensor<int32, [1]> var_6657_axes_0 = const()[name = string("op_6657_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_6657_cast_fp16 = expand_dims(axes = var_6657_axes_0, x = key_token_245_cast_fp16)[name = string("op_6657_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6658_cast_fp16 = mul(x = state_489_cast_fp16, y = var_6657_cast_fp16)[name = string("op_6658_cast_fp16")];
            tensor<int32, [1]> memory_245_axes_0 = const()[name = string("memory_245_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_245_keep_dims_0 = const()[name = string("memory_245_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_245_cast_fp16 = reduce_sum(axes = memory_245_axes_0, keep_dims = memory_245_keep_dims_0, x = var_6658_cast_fp16)[name = string("memory_245_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_6661_cast_fp16 = sub(x = value_token_245_cast_fp16, y = memory_245_cast_fp16)[name = string("op_6661_cast_fp16")];
            tensor<int32, [3]> var_6664_begin_0 = const()[name = string("op_6664_begin_0"), val = tensor<int32, [3]>([0, 0, 10])];
            tensor<int32, [3]> var_6664_end_0 = const()[name = string("op_6664_end_0"), val = tensor<int32, [3]>([1, 16, 11])];
            tensor<bool, [3]> var_6664_end_mask_0 = const()[name = string("op_6664_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_6664_squeeze_mask_0 = const()[name = string("op_6664_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_6664_cast_fp16 = slice_by_index(begin = var_6664_begin_0, end = var_6664_end_0, end_mask = var_6664_end_mask_0, squeeze_mask = var_6664_squeeze_mask_0, x = beta_15_cast_fp16)[name = string("op_6664_cast_fp16")];
            tensor<int32, [1]> var_6665_axes_0 = const()[name = string("op_6665_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_6665_cast_fp16 = expand_dims(axes = var_6665_axes_0, x = var_6664_cast_fp16)[name = string("op_6665_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_245_cast_fp16 = mul(x = var_6661_cast_fp16, y = var_6665_cast_fp16)[name = string("delta_245_cast_fp16")];
            tensor<int32, [1]> var_6668_axes_0 = const()[name = string("op_6668_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_6668_cast_fp16 = expand_dims(axes = var_6668_axes_0, x = delta_245_cast_fp16)[name = string("op_6668_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6669_cast_fp16 = mul(x = var_6657_cast_fp16, y = var_6668_cast_fp16)[name = string("op_6669_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_491_cast_fp16 = add(x = state_489_cast_fp16, y = var_6669_cast_fp16)[name = string("state_491_cast_fp16")];
            tensor<int32, [4]> var_6673_begin_0 = const()[name = string("op_6673_begin_0"), val = tensor<int32, [4]>([0, 0, 10, 0])];
            tensor<int32, [4]> var_6673_end_0 = const()[name = string("op_6673_end_0"), val = tensor<int32, [4]>([1, 16, 11, 128])];
            tensor<bool, [4]> var_6673_end_mask_0 = const()[name = string("op_6673_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_6673_squeeze_mask_0 = const()[name = string("op_6673_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_6673_cast_fp16 = slice_by_index(begin = var_6673_begin_0, end = var_6673_end_0, end_mask = var_6673_end_mask_0, squeeze_mask = var_6673_squeeze_mask_0, x = _inversed_query_39_cast_fp16)[name = string("op_6673_cast_fp16")];
            tensor<int32, [1]> var_6674_axes_0 = const()[name = string("op_6674_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_6674_cast_fp16 = expand_dims(axes = var_6674_axes_0, x = var_6673_cast_fp16)[name = string("op_6674_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6675_cast_fp16 = mul(x = state_491_cast_fp16, y = var_6674_cast_fp16)[name = string("op_6675_cast_fp16")];
            tensor<int32, [1]> var_6677_axes_0 = const()[name = string("op_6677_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_6677_keep_dims_0 = const()[name = string("op_6677_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_6677_cast_fp16 = reduce_sum(axes = var_6677_axes_0, keep_dims = var_6677_keep_dims_0, x = var_6675_cast_fp16)[name = string("op_6677_cast_fp16")];
            tensor<int32, [3]> var_6680_begin_0 = const()[name = string("op_6680_begin_0"), val = tensor<int32, [3]>([0, 0, 11])];
            tensor<int32, [3]> var_6680_end_0 = const()[name = string("op_6680_end_0"), val = tensor<int32, [3]>([1, 16, 12])];
            tensor<bool, [3]> var_6680_end_mask_0 = const()[name = string("op_6680_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_6680_squeeze_mask_0 = const()[name = string("op_6680_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_6680_cast_fp16 = slice_by_index(begin = var_6680_begin_0, end = var_6680_end_0, end_mask = var_6680_end_mask_0, squeeze_mask = var_6680_squeeze_mask_0, x = decay_15_cast_fp16)[name = string("op_6680_cast_fp16")];
            tensor<fp16, [1, 16]> var_6681_cast_fp16 = exp(x = var_6680_cast_fp16)[name = string("op_6681_cast_fp16")];
            tensor<int32, [1]> var_6682_axes_0 = const()[name = string("op_6682_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_6682_cast_fp16 = expand_dims(axes = var_6682_axes_0, x = var_6681_cast_fp16)[name = string("op_6682_cast_fp16")];
            tensor<int32, [1]> var_6683_axes_0 = const()[name = string("op_6683_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_6683_cast_fp16 = expand_dims(axes = var_6683_axes_0, x = var_6682_cast_fp16)[name = string("op_6683_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_493_cast_fp16 = mul(x = state_491_cast_fp16, y = var_6683_cast_fp16)[name = string("state_493_cast_fp16")];
            tensor<int32, [4]> key_token_247_begin_0 = const()[name = string("key_token_247_begin_0"), val = tensor<int32, [4]>([0, 0, 11, 0])];
            tensor<int32, [4]> key_token_247_end_0 = const()[name = string("key_token_247_end_0"), val = tensor<int32, [4]>([1, 16, 12, 128])];
            tensor<bool, [4]> key_token_247_end_mask_0 = const()[name = string("key_token_247_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_247_squeeze_mask_0 = const()[name = string("key_token_247_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_247_cast_fp16 = slice_by_index(begin = key_token_247_begin_0, end = key_token_247_end_0, end_mask = key_token_247_end_mask_0, squeeze_mask = key_token_247_squeeze_mask_0, x = key_39_cast_fp16)[name = string("key_token_247_cast_fp16")];
            tensor<int32, [4]> value_token_247_begin_0 = const()[name = string("value_token_247_begin_0"), val = tensor<int32, [4]>([0, 0, 11, 0])];
            tensor<int32, [4]> value_token_247_end_0 = const()[name = string("value_token_247_end_0"), val = tensor<int32, [4]>([1, 16, 12, 128])];
            tensor<bool, [4]> value_token_247_end_mask_0 = const()[name = string("value_token_247_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_247_squeeze_mask_0 = const()[name = string("value_token_247_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_247_cast_fp16 = slice_by_index(begin = value_token_247_begin_0, end = value_token_247_end_0, end_mask = value_token_247_end_mask_0, squeeze_mask = value_token_247_squeeze_mask_0, x = value_141_cast_fp16)[name = string("value_token_247_cast_fp16")];
            tensor<int32, [1]> var_6691_axes_0 = const()[name = string("op_6691_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_6691_cast_fp16 = expand_dims(axes = var_6691_axes_0, x = key_token_247_cast_fp16)[name = string("op_6691_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6692_cast_fp16 = mul(x = state_493_cast_fp16, y = var_6691_cast_fp16)[name = string("op_6692_cast_fp16")];
            tensor<int32, [1]> memory_247_axes_0 = const()[name = string("memory_247_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_247_keep_dims_0 = const()[name = string("memory_247_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_247_cast_fp16 = reduce_sum(axes = memory_247_axes_0, keep_dims = memory_247_keep_dims_0, x = var_6692_cast_fp16)[name = string("memory_247_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_6695_cast_fp16 = sub(x = value_token_247_cast_fp16, y = memory_247_cast_fp16)[name = string("op_6695_cast_fp16")];
            tensor<int32, [3]> var_6698_begin_0 = const()[name = string("op_6698_begin_0"), val = tensor<int32, [3]>([0, 0, 11])];
            tensor<int32, [3]> var_6698_end_0 = const()[name = string("op_6698_end_0"), val = tensor<int32, [3]>([1, 16, 12])];
            tensor<bool, [3]> var_6698_end_mask_0 = const()[name = string("op_6698_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_6698_squeeze_mask_0 = const()[name = string("op_6698_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_6698_cast_fp16 = slice_by_index(begin = var_6698_begin_0, end = var_6698_end_0, end_mask = var_6698_end_mask_0, squeeze_mask = var_6698_squeeze_mask_0, x = beta_15_cast_fp16)[name = string("op_6698_cast_fp16")];
            tensor<int32, [1]> var_6699_axes_0 = const()[name = string("op_6699_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_6699_cast_fp16 = expand_dims(axes = var_6699_axes_0, x = var_6698_cast_fp16)[name = string("op_6699_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_247_cast_fp16 = mul(x = var_6695_cast_fp16, y = var_6699_cast_fp16)[name = string("delta_247_cast_fp16")];
            tensor<int32, [1]> var_6702_axes_0 = const()[name = string("op_6702_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_6702_cast_fp16 = expand_dims(axes = var_6702_axes_0, x = delta_247_cast_fp16)[name = string("op_6702_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6703_cast_fp16 = mul(x = var_6691_cast_fp16, y = var_6702_cast_fp16)[name = string("op_6703_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_495_cast_fp16 = add(x = state_493_cast_fp16, y = var_6703_cast_fp16)[name = string("state_495_cast_fp16")];
            tensor<int32, [4]> var_6707_begin_0 = const()[name = string("op_6707_begin_0"), val = tensor<int32, [4]>([0, 0, 11, 0])];
            tensor<int32, [4]> var_6707_end_0 = const()[name = string("op_6707_end_0"), val = tensor<int32, [4]>([1, 16, 12, 128])];
            tensor<bool, [4]> var_6707_end_mask_0 = const()[name = string("op_6707_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_6707_squeeze_mask_0 = const()[name = string("op_6707_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_6707_cast_fp16 = slice_by_index(begin = var_6707_begin_0, end = var_6707_end_0, end_mask = var_6707_end_mask_0, squeeze_mask = var_6707_squeeze_mask_0, x = _inversed_query_39_cast_fp16)[name = string("op_6707_cast_fp16")];
            tensor<int32, [1]> var_6708_axes_0 = const()[name = string("op_6708_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_6708_cast_fp16 = expand_dims(axes = var_6708_axes_0, x = var_6707_cast_fp16)[name = string("op_6708_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6709_cast_fp16 = mul(x = state_495_cast_fp16, y = var_6708_cast_fp16)[name = string("op_6709_cast_fp16")];
            tensor<int32, [1]> var_6711_axes_0 = const()[name = string("op_6711_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_6711_keep_dims_0 = const()[name = string("op_6711_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_6711_cast_fp16 = reduce_sum(axes = var_6711_axes_0, keep_dims = var_6711_keep_dims_0, x = var_6709_cast_fp16)[name = string("op_6711_cast_fp16")];
            tensor<int32, [3]> var_6714_begin_0 = const()[name = string("op_6714_begin_0"), val = tensor<int32, [3]>([0, 0, 12])];
            tensor<int32, [3]> var_6714_end_0 = const()[name = string("op_6714_end_0"), val = tensor<int32, [3]>([1, 16, 13])];
            tensor<bool, [3]> var_6714_end_mask_0 = const()[name = string("op_6714_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_6714_squeeze_mask_0 = const()[name = string("op_6714_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_6714_cast_fp16 = slice_by_index(begin = var_6714_begin_0, end = var_6714_end_0, end_mask = var_6714_end_mask_0, squeeze_mask = var_6714_squeeze_mask_0, x = decay_15_cast_fp16)[name = string("op_6714_cast_fp16")];
            tensor<fp16, [1, 16]> var_6715_cast_fp16 = exp(x = var_6714_cast_fp16)[name = string("op_6715_cast_fp16")];
            tensor<int32, [1]> var_6716_axes_0 = const()[name = string("op_6716_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_6716_cast_fp16 = expand_dims(axes = var_6716_axes_0, x = var_6715_cast_fp16)[name = string("op_6716_cast_fp16")];
            tensor<int32, [1]> var_6717_axes_0 = const()[name = string("op_6717_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_6717_cast_fp16 = expand_dims(axes = var_6717_axes_0, x = var_6716_cast_fp16)[name = string("op_6717_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_497_cast_fp16 = mul(x = state_495_cast_fp16, y = var_6717_cast_fp16)[name = string("state_497_cast_fp16")];
            tensor<int32, [4]> key_token_249_begin_0 = const()[name = string("key_token_249_begin_0"), val = tensor<int32, [4]>([0, 0, 12, 0])];
            tensor<int32, [4]> key_token_249_end_0 = const()[name = string("key_token_249_end_0"), val = tensor<int32, [4]>([1, 16, 13, 128])];
            tensor<bool, [4]> key_token_249_end_mask_0 = const()[name = string("key_token_249_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_249_squeeze_mask_0 = const()[name = string("key_token_249_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_249_cast_fp16 = slice_by_index(begin = key_token_249_begin_0, end = key_token_249_end_0, end_mask = key_token_249_end_mask_0, squeeze_mask = key_token_249_squeeze_mask_0, x = key_39_cast_fp16)[name = string("key_token_249_cast_fp16")];
            tensor<int32, [4]> value_token_249_begin_0 = const()[name = string("value_token_249_begin_0"), val = tensor<int32, [4]>([0, 0, 12, 0])];
            tensor<int32, [4]> value_token_249_end_0 = const()[name = string("value_token_249_end_0"), val = tensor<int32, [4]>([1, 16, 13, 128])];
            tensor<bool, [4]> value_token_249_end_mask_0 = const()[name = string("value_token_249_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_249_squeeze_mask_0 = const()[name = string("value_token_249_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_249_cast_fp16 = slice_by_index(begin = value_token_249_begin_0, end = value_token_249_end_0, end_mask = value_token_249_end_mask_0, squeeze_mask = value_token_249_squeeze_mask_0, x = value_141_cast_fp16)[name = string("value_token_249_cast_fp16")];
            tensor<int32, [1]> var_6725_axes_0 = const()[name = string("op_6725_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_6725_cast_fp16 = expand_dims(axes = var_6725_axes_0, x = key_token_249_cast_fp16)[name = string("op_6725_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6726_cast_fp16 = mul(x = state_497_cast_fp16, y = var_6725_cast_fp16)[name = string("op_6726_cast_fp16")];
            tensor<int32, [1]> memory_249_axes_0 = const()[name = string("memory_249_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_249_keep_dims_0 = const()[name = string("memory_249_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_249_cast_fp16 = reduce_sum(axes = memory_249_axes_0, keep_dims = memory_249_keep_dims_0, x = var_6726_cast_fp16)[name = string("memory_249_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_6729_cast_fp16 = sub(x = value_token_249_cast_fp16, y = memory_249_cast_fp16)[name = string("op_6729_cast_fp16")];
            tensor<int32, [3]> var_6732_begin_0 = const()[name = string("op_6732_begin_0"), val = tensor<int32, [3]>([0, 0, 12])];
            tensor<int32, [3]> var_6732_end_0 = const()[name = string("op_6732_end_0"), val = tensor<int32, [3]>([1, 16, 13])];
            tensor<bool, [3]> var_6732_end_mask_0 = const()[name = string("op_6732_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_6732_squeeze_mask_0 = const()[name = string("op_6732_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_6732_cast_fp16 = slice_by_index(begin = var_6732_begin_0, end = var_6732_end_0, end_mask = var_6732_end_mask_0, squeeze_mask = var_6732_squeeze_mask_0, x = beta_15_cast_fp16)[name = string("op_6732_cast_fp16")];
            tensor<int32, [1]> var_6733_axes_0 = const()[name = string("op_6733_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_6733_cast_fp16 = expand_dims(axes = var_6733_axes_0, x = var_6732_cast_fp16)[name = string("op_6733_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_249_cast_fp16 = mul(x = var_6729_cast_fp16, y = var_6733_cast_fp16)[name = string("delta_249_cast_fp16")];
            tensor<int32, [1]> var_6736_axes_0 = const()[name = string("op_6736_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_6736_cast_fp16 = expand_dims(axes = var_6736_axes_0, x = delta_249_cast_fp16)[name = string("op_6736_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6737_cast_fp16 = mul(x = var_6725_cast_fp16, y = var_6736_cast_fp16)[name = string("op_6737_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_499_cast_fp16 = add(x = state_497_cast_fp16, y = var_6737_cast_fp16)[name = string("state_499_cast_fp16")];
            tensor<int32, [4]> var_6741_begin_0 = const()[name = string("op_6741_begin_0"), val = tensor<int32, [4]>([0, 0, 12, 0])];
            tensor<int32, [4]> var_6741_end_0 = const()[name = string("op_6741_end_0"), val = tensor<int32, [4]>([1, 16, 13, 128])];
            tensor<bool, [4]> var_6741_end_mask_0 = const()[name = string("op_6741_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_6741_squeeze_mask_0 = const()[name = string("op_6741_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_6741_cast_fp16 = slice_by_index(begin = var_6741_begin_0, end = var_6741_end_0, end_mask = var_6741_end_mask_0, squeeze_mask = var_6741_squeeze_mask_0, x = _inversed_query_39_cast_fp16)[name = string("op_6741_cast_fp16")];
            tensor<int32, [1]> var_6742_axes_0 = const()[name = string("op_6742_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_6742_cast_fp16 = expand_dims(axes = var_6742_axes_0, x = var_6741_cast_fp16)[name = string("op_6742_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6743_cast_fp16 = mul(x = state_499_cast_fp16, y = var_6742_cast_fp16)[name = string("op_6743_cast_fp16")];
            tensor<int32, [1]> var_6745_axes_0 = const()[name = string("op_6745_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_6745_keep_dims_0 = const()[name = string("op_6745_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_6745_cast_fp16 = reduce_sum(axes = var_6745_axes_0, keep_dims = var_6745_keep_dims_0, x = var_6743_cast_fp16)[name = string("op_6745_cast_fp16")];
            tensor<int32, [3]> var_6748_begin_0 = const()[name = string("op_6748_begin_0"), val = tensor<int32, [3]>([0, 0, 13])];
            tensor<int32, [3]> var_6748_end_0 = const()[name = string("op_6748_end_0"), val = tensor<int32, [3]>([1, 16, 14])];
            tensor<bool, [3]> var_6748_end_mask_0 = const()[name = string("op_6748_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_6748_squeeze_mask_0 = const()[name = string("op_6748_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_6748_cast_fp16 = slice_by_index(begin = var_6748_begin_0, end = var_6748_end_0, end_mask = var_6748_end_mask_0, squeeze_mask = var_6748_squeeze_mask_0, x = decay_15_cast_fp16)[name = string("op_6748_cast_fp16")];
            tensor<fp16, [1, 16]> var_6749_cast_fp16 = exp(x = var_6748_cast_fp16)[name = string("op_6749_cast_fp16")];
            tensor<int32, [1]> var_6750_axes_0 = const()[name = string("op_6750_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_6750_cast_fp16 = expand_dims(axes = var_6750_axes_0, x = var_6749_cast_fp16)[name = string("op_6750_cast_fp16")];
            tensor<int32, [1]> var_6751_axes_0 = const()[name = string("op_6751_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_6751_cast_fp16 = expand_dims(axes = var_6751_axes_0, x = var_6750_cast_fp16)[name = string("op_6751_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_501_cast_fp16 = mul(x = state_499_cast_fp16, y = var_6751_cast_fp16)[name = string("state_501_cast_fp16")];
            tensor<int32, [4]> key_token_251_begin_0 = const()[name = string("key_token_251_begin_0"), val = tensor<int32, [4]>([0, 0, 13, 0])];
            tensor<int32, [4]> key_token_251_end_0 = const()[name = string("key_token_251_end_0"), val = tensor<int32, [4]>([1, 16, 14, 128])];
            tensor<bool, [4]> key_token_251_end_mask_0 = const()[name = string("key_token_251_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_251_squeeze_mask_0 = const()[name = string("key_token_251_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_251_cast_fp16 = slice_by_index(begin = key_token_251_begin_0, end = key_token_251_end_0, end_mask = key_token_251_end_mask_0, squeeze_mask = key_token_251_squeeze_mask_0, x = key_39_cast_fp16)[name = string("key_token_251_cast_fp16")];
            tensor<int32, [4]> value_token_251_begin_0 = const()[name = string("value_token_251_begin_0"), val = tensor<int32, [4]>([0, 0, 13, 0])];
            tensor<int32, [4]> value_token_251_end_0 = const()[name = string("value_token_251_end_0"), val = tensor<int32, [4]>([1, 16, 14, 128])];
            tensor<bool, [4]> value_token_251_end_mask_0 = const()[name = string("value_token_251_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_251_squeeze_mask_0 = const()[name = string("value_token_251_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_251_cast_fp16 = slice_by_index(begin = value_token_251_begin_0, end = value_token_251_end_0, end_mask = value_token_251_end_mask_0, squeeze_mask = value_token_251_squeeze_mask_0, x = value_141_cast_fp16)[name = string("value_token_251_cast_fp16")];
            tensor<int32, [1]> var_6759_axes_0 = const()[name = string("op_6759_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_6759_cast_fp16 = expand_dims(axes = var_6759_axes_0, x = key_token_251_cast_fp16)[name = string("op_6759_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6760_cast_fp16 = mul(x = state_501_cast_fp16, y = var_6759_cast_fp16)[name = string("op_6760_cast_fp16")];
            tensor<int32, [1]> memory_251_axes_0 = const()[name = string("memory_251_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_251_keep_dims_0 = const()[name = string("memory_251_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_251_cast_fp16 = reduce_sum(axes = memory_251_axes_0, keep_dims = memory_251_keep_dims_0, x = var_6760_cast_fp16)[name = string("memory_251_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_6763_cast_fp16 = sub(x = value_token_251_cast_fp16, y = memory_251_cast_fp16)[name = string("op_6763_cast_fp16")];
            tensor<int32, [3]> var_6766_begin_0 = const()[name = string("op_6766_begin_0"), val = tensor<int32, [3]>([0, 0, 13])];
            tensor<int32, [3]> var_6766_end_0 = const()[name = string("op_6766_end_0"), val = tensor<int32, [3]>([1, 16, 14])];
            tensor<bool, [3]> var_6766_end_mask_0 = const()[name = string("op_6766_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_6766_squeeze_mask_0 = const()[name = string("op_6766_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_6766_cast_fp16 = slice_by_index(begin = var_6766_begin_0, end = var_6766_end_0, end_mask = var_6766_end_mask_0, squeeze_mask = var_6766_squeeze_mask_0, x = beta_15_cast_fp16)[name = string("op_6766_cast_fp16")];
            tensor<int32, [1]> var_6767_axes_0 = const()[name = string("op_6767_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_6767_cast_fp16 = expand_dims(axes = var_6767_axes_0, x = var_6766_cast_fp16)[name = string("op_6767_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_251_cast_fp16 = mul(x = var_6763_cast_fp16, y = var_6767_cast_fp16)[name = string("delta_251_cast_fp16")];
            tensor<int32, [1]> var_6770_axes_0 = const()[name = string("op_6770_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_6770_cast_fp16 = expand_dims(axes = var_6770_axes_0, x = delta_251_cast_fp16)[name = string("op_6770_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6771_cast_fp16 = mul(x = var_6759_cast_fp16, y = var_6770_cast_fp16)[name = string("op_6771_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_503_cast_fp16 = add(x = state_501_cast_fp16, y = var_6771_cast_fp16)[name = string("state_503_cast_fp16")];
            tensor<int32, [4]> var_6775_begin_0 = const()[name = string("op_6775_begin_0"), val = tensor<int32, [4]>([0, 0, 13, 0])];
            tensor<int32, [4]> var_6775_end_0 = const()[name = string("op_6775_end_0"), val = tensor<int32, [4]>([1, 16, 14, 128])];
            tensor<bool, [4]> var_6775_end_mask_0 = const()[name = string("op_6775_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_6775_squeeze_mask_0 = const()[name = string("op_6775_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_6775_cast_fp16 = slice_by_index(begin = var_6775_begin_0, end = var_6775_end_0, end_mask = var_6775_end_mask_0, squeeze_mask = var_6775_squeeze_mask_0, x = _inversed_query_39_cast_fp16)[name = string("op_6775_cast_fp16")];
            tensor<int32, [1]> var_6776_axes_0 = const()[name = string("op_6776_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_6776_cast_fp16 = expand_dims(axes = var_6776_axes_0, x = var_6775_cast_fp16)[name = string("op_6776_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6777_cast_fp16 = mul(x = state_503_cast_fp16, y = var_6776_cast_fp16)[name = string("op_6777_cast_fp16")];
            tensor<int32, [1]> var_6779_axes_0 = const()[name = string("op_6779_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_6779_keep_dims_0 = const()[name = string("op_6779_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_6779_cast_fp16 = reduce_sum(axes = var_6779_axes_0, keep_dims = var_6779_keep_dims_0, x = var_6777_cast_fp16)[name = string("op_6779_cast_fp16")];
            tensor<int32, [3]> var_6782_begin_0 = const()[name = string("op_6782_begin_0"), val = tensor<int32, [3]>([0, 0, 14])];
            tensor<int32, [3]> var_6782_end_0 = const()[name = string("op_6782_end_0"), val = tensor<int32, [3]>([1, 16, 15])];
            tensor<bool, [3]> var_6782_end_mask_0 = const()[name = string("op_6782_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_6782_squeeze_mask_0 = const()[name = string("op_6782_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_6782_cast_fp16 = slice_by_index(begin = var_6782_begin_0, end = var_6782_end_0, end_mask = var_6782_end_mask_0, squeeze_mask = var_6782_squeeze_mask_0, x = decay_15_cast_fp16)[name = string("op_6782_cast_fp16")];
            tensor<fp16, [1, 16]> var_6783_cast_fp16 = exp(x = var_6782_cast_fp16)[name = string("op_6783_cast_fp16")];
            tensor<int32, [1]> var_6784_axes_0 = const()[name = string("op_6784_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_6784_cast_fp16 = expand_dims(axes = var_6784_axes_0, x = var_6783_cast_fp16)[name = string("op_6784_cast_fp16")];
            tensor<int32, [1]> var_6785_axes_0 = const()[name = string("op_6785_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_6785_cast_fp16 = expand_dims(axes = var_6785_axes_0, x = var_6784_cast_fp16)[name = string("op_6785_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_505_cast_fp16 = mul(x = state_503_cast_fp16, y = var_6785_cast_fp16)[name = string("state_505_cast_fp16")];
            tensor<int32, [4]> key_token_253_begin_0 = const()[name = string("key_token_253_begin_0"), val = tensor<int32, [4]>([0, 0, 14, 0])];
            tensor<int32, [4]> key_token_253_end_0 = const()[name = string("key_token_253_end_0"), val = tensor<int32, [4]>([1, 16, 15, 128])];
            tensor<bool, [4]> key_token_253_end_mask_0 = const()[name = string("key_token_253_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_253_squeeze_mask_0 = const()[name = string("key_token_253_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_253_cast_fp16 = slice_by_index(begin = key_token_253_begin_0, end = key_token_253_end_0, end_mask = key_token_253_end_mask_0, squeeze_mask = key_token_253_squeeze_mask_0, x = key_39_cast_fp16)[name = string("key_token_253_cast_fp16")];
            tensor<int32, [4]> value_token_253_begin_0 = const()[name = string("value_token_253_begin_0"), val = tensor<int32, [4]>([0, 0, 14, 0])];
            tensor<int32, [4]> value_token_253_end_0 = const()[name = string("value_token_253_end_0"), val = tensor<int32, [4]>([1, 16, 15, 128])];
            tensor<bool, [4]> value_token_253_end_mask_0 = const()[name = string("value_token_253_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_253_squeeze_mask_0 = const()[name = string("value_token_253_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_253_cast_fp16 = slice_by_index(begin = value_token_253_begin_0, end = value_token_253_end_0, end_mask = value_token_253_end_mask_0, squeeze_mask = value_token_253_squeeze_mask_0, x = value_141_cast_fp16)[name = string("value_token_253_cast_fp16")];
            tensor<int32, [1]> var_6793_axes_0 = const()[name = string("op_6793_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_6793_cast_fp16 = expand_dims(axes = var_6793_axes_0, x = key_token_253_cast_fp16)[name = string("op_6793_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6794_cast_fp16 = mul(x = state_505_cast_fp16, y = var_6793_cast_fp16)[name = string("op_6794_cast_fp16")];
            tensor<int32, [1]> memory_253_axes_0 = const()[name = string("memory_253_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_253_keep_dims_0 = const()[name = string("memory_253_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_253_cast_fp16 = reduce_sum(axes = memory_253_axes_0, keep_dims = memory_253_keep_dims_0, x = var_6794_cast_fp16)[name = string("memory_253_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_6797_cast_fp16 = sub(x = value_token_253_cast_fp16, y = memory_253_cast_fp16)[name = string("op_6797_cast_fp16")];
            tensor<int32, [3]> var_6800_begin_0 = const()[name = string("op_6800_begin_0"), val = tensor<int32, [3]>([0, 0, 14])];
            tensor<int32, [3]> var_6800_end_0 = const()[name = string("op_6800_end_0"), val = tensor<int32, [3]>([1, 16, 15])];
            tensor<bool, [3]> var_6800_end_mask_0 = const()[name = string("op_6800_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_6800_squeeze_mask_0 = const()[name = string("op_6800_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_6800_cast_fp16 = slice_by_index(begin = var_6800_begin_0, end = var_6800_end_0, end_mask = var_6800_end_mask_0, squeeze_mask = var_6800_squeeze_mask_0, x = beta_15_cast_fp16)[name = string("op_6800_cast_fp16")];
            tensor<int32, [1]> var_6801_axes_0 = const()[name = string("op_6801_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_6801_cast_fp16 = expand_dims(axes = var_6801_axes_0, x = var_6800_cast_fp16)[name = string("op_6801_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_253_cast_fp16 = mul(x = var_6797_cast_fp16, y = var_6801_cast_fp16)[name = string("delta_253_cast_fp16")];
            tensor<int32, [1]> var_6804_axes_0 = const()[name = string("op_6804_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_6804_cast_fp16 = expand_dims(axes = var_6804_axes_0, x = delta_253_cast_fp16)[name = string("op_6804_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6805_cast_fp16 = mul(x = var_6793_cast_fp16, y = var_6804_cast_fp16)[name = string("op_6805_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_507_cast_fp16 = add(x = state_505_cast_fp16, y = var_6805_cast_fp16)[name = string("state_507_cast_fp16")];
            tensor<int32, [4]> var_6809_begin_0 = const()[name = string("op_6809_begin_0"), val = tensor<int32, [4]>([0, 0, 14, 0])];
            tensor<int32, [4]> var_6809_end_0 = const()[name = string("op_6809_end_0"), val = tensor<int32, [4]>([1, 16, 15, 128])];
            tensor<bool, [4]> var_6809_end_mask_0 = const()[name = string("op_6809_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_6809_squeeze_mask_0 = const()[name = string("op_6809_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_6809_cast_fp16 = slice_by_index(begin = var_6809_begin_0, end = var_6809_end_0, end_mask = var_6809_end_mask_0, squeeze_mask = var_6809_squeeze_mask_0, x = _inversed_query_39_cast_fp16)[name = string("op_6809_cast_fp16")];
            tensor<int32, [1]> var_6810_axes_0 = const()[name = string("op_6810_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_6810_cast_fp16 = expand_dims(axes = var_6810_axes_0, x = var_6809_cast_fp16)[name = string("op_6810_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6811_cast_fp16 = mul(x = state_507_cast_fp16, y = var_6810_cast_fp16)[name = string("op_6811_cast_fp16")];
            tensor<int32, [1]> var_6813_axes_0 = const()[name = string("op_6813_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_6813_keep_dims_0 = const()[name = string("op_6813_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_6813_cast_fp16 = reduce_sum(axes = var_6813_axes_0, keep_dims = var_6813_keep_dims_0, x = var_6811_cast_fp16)[name = string("op_6813_cast_fp16")];
            tensor<int32, [3]> var_6816_begin_0 = const()[name = string("op_6816_begin_0"), val = tensor<int32, [3]>([0, 0, 15])];
            tensor<int32, [3]> var_6816_end_0 = const()[name = string("op_6816_end_0"), val = tensor<int32, [3]>([1, 16, 16])];
            tensor<bool, [3]> var_6816_end_mask_0 = const()[name = string("op_6816_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_6816_squeeze_mask_0 = const()[name = string("op_6816_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_6816_cast_fp16 = slice_by_index(begin = var_6816_begin_0, end = var_6816_end_0, end_mask = var_6816_end_mask_0, squeeze_mask = var_6816_squeeze_mask_0, x = decay_15_cast_fp16)[name = string("op_6816_cast_fp16")];
            tensor<fp16, [1, 16]> var_6817_cast_fp16 = exp(x = var_6816_cast_fp16)[name = string("op_6817_cast_fp16")];
            tensor<int32, [1]> var_6818_axes_0 = const()[name = string("op_6818_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_6818_cast_fp16 = expand_dims(axes = var_6818_axes_0, x = var_6817_cast_fp16)[name = string("op_6818_cast_fp16")];
            tensor<int32, [1]> var_6819_axes_0 = const()[name = string("op_6819_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_6819_cast_fp16 = expand_dims(axes = var_6819_axes_0, x = var_6818_cast_fp16)[name = string("op_6819_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_509_cast_fp16 = mul(x = state_507_cast_fp16, y = var_6819_cast_fp16)[name = string("state_509_cast_fp16")];
            tensor<int32, [4]> key_token_255_begin_0 = const()[name = string("key_token_255_begin_0"), val = tensor<int32, [4]>([0, 0, 15, 0])];
            tensor<int32, [4]> key_token_255_end_0 = const()[name = string("key_token_255_end_0"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<bool, [4]> key_token_255_end_mask_0 = const()[name = string("key_token_255_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_255_squeeze_mask_0 = const()[name = string("key_token_255_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_255_cast_fp16 = slice_by_index(begin = key_token_255_begin_0, end = key_token_255_end_0, end_mask = key_token_255_end_mask_0, squeeze_mask = key_token_255_squeeze_mask_0, x = key_39_cast_fp16)[name = string("key_token_255_cast_fp16")];
            tensor<int32, [4]> value_token_255_begin_0 = const()[name = string("value_token_255_begin_0"), val = tensor<int32, [4]>([0, 0, 15, 0])];
            tensor<int32, [4]> value_token_255_end_0 = const()[name = string("value_token_255_end_0"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<bool, [4]> value_token_255_end_mask_0 = const()[name = string("value_token_255_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_255_squeeze_mask_0 = const()[name = string("value_token_255_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_255_cast_fp16 = slice_by_index(begin = value_token_255_begin_0, end = value_token_255_end_0, end_mask = value_token_255_end_mask_0, squeeze_mask = value_token_255_squeeze_mask_0, x = value_141_cast_fp16)[name = string("value_token_255_cast_fp16")];
            tensor<int32, [1]> var_6827_axes_0 = const()[name = string("op_6827_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_6827_cast_fp16 = expand_dims(axes = var_6827_axes_0, x = key_token_255_cast_fp16)[name = string("op_6827_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6828_cast_fp16 = mul(x = state_509_cast_fp16, y = var_6827_cast_fp16)[name = string("op_6828_cast_fp16")];
            tensor<int32, [1]> memory_255_axes_0 = const()[name = string("memory_255_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_255_keep_dims_0 = const()[name = string("memory_255_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_255_cast_fp16 = reduce_sum(axes = memory_255_axes_0, keep_dims = memory_255_keep_dims_0, x = var_6828_cast_fp16)[name = string("memory_255_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_6831_cast_fp16 = sub(x = value_token_255_cast_fp16, y = memory_255_cast_fp16)[name = string("op_6831_cast_fp16")];
            tensor<int32, [3]> var_6834_begin_0 = const()[name = string("op_6834_begin_0"), val = tensor<int32, [3]>([0, 0, 15])];
            tensor<int32, [3]> var_6834_end_0 = const()[name = string("op_6834_end_0"), val = tensor<int32, [3]>([1, 16, 16])];
            tensor<bool, [3]> var_6834_end_mask_0 = const()[name = string("op_6834_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_6834_squeeze_mask_0 = const()[name = string("op_6834_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_6834_cast_fp16 = slice_by_index(begin = var_6834_begin_0, end = var_6834_end_0, end_mask = var_6834_end_mask_0, squeeze_mask = var_6834_squeeze_mask_0, x = beta_15_cast_fp16)[name = string("op_6834_cast_fp16")];
            tensor<int32, [1]> var_6835_axes_0 = const()[name = string("op_6835_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_6835_cast_fp16 = expand_dims(axes = var_6835_axes_0, x = var_6834_cast_fp16)[name = string("op_6835_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_255_cast_fp16 = mul(x = var_6831_cast_fp16, y = var_6835_cast_fp16)[name = string("delta_255_cast_fp16")];
            tensor<int32, [1]> var_6838_axes_0 = const()[name = string("op_6838_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_6838_cast_fp16 = expand_dims(axes = var_6838_axes_0, x = delta_255_cast_fp16)[name = string("op_6838_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6839_cast_fp16 = mul(x = var_6827_cast_fp16, y = var_6838_cast_fp16)[name = string("op_6839_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> rec9_out = add(x = state_509_cast_fp16, y = var_6839_cast_fp16)[name = string("state_511_cast_fp16")];
            tensor<int32, [4]> var_6843_begin_0 = const()[name = string("op_6843_begin_0"), val = tensor<int32, [4]>([0, 0, 15, 0])];
            tensor<int32, [4]> var_6843_end_0 = const()[name = string("op_6843_end_0"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<bool, [4]> var_6843_end_mask_0 = const()[name = string("op_6843_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_6843_squeeze_mask_0 = const()[name = string("op_6843_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_6843_cast_fp16 = slice_by_index(begin = var_6843_begin_0, end = var_6843_end_0, end_mask = var_6843_end_mask_0, squeeze_mask = var_6843_squeeze_mask_0, x = _inversed_query_39_cast_fp16)[name = string("op_6843_cast_fp16")];
            tensor<int32, [1]> var_6844_axes_0 = const()[name = string("op_6844_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_6844_cast_fp16 = expand_dims(axes = var_6844_axes_0, x = var_6843_cast_fp16)[name = string("op_6844_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6845_cast_fp16 = mul(x = rec9_out, y = var_6844_cast_fp16)[name = string("op_6845_cast_fp16")];
            tensor<int32, [1]> var_6847_axes_0 = const()[name = string("op_6847_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_6847_keep_dims_0 = const()[name = string("op_6847_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_6847_cast_fp16 = reduce_sum(axes = var_6847_axes_0, keep_dims = var_6847_keep_dims_0, x = var_6845_cast_fp16)[name = string("op_6847_cast_fp16")];
            int32 attention_71_axis_0 = const()[name = string("attention_71_axis_0"), val = int32(1)];
            tensor<fp16, [1, 16, 16, 128]> attention_71_cast_fp16 = stack(axis = attention_71_axis_0, values = (var_6337_cast_fp16, var_6371_cast_fp16, var_6405_cast_fp16, var_6439_cast_fp16, var_6473_cast_fp16, var_6507_cast_fp16, var_6541_cast_fp16, var_6575_cast_fp16, var_6609_cast_fp16, var_6643_cast_fp16, var_6677_cast_fp16, var_6711_cast_fp16, var_6745_cast_fp16, var_6779_cast_fp16, var_6813_cast_fp16, var_6847_cast_fp16))[name = string("attention_71_cast_fp16")];
            tensor<int32, [2]> var_6850 = const()[name = string("op_6850"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [256, 128]> attention_73_cast_fp16 = reshape(shape = var_6850, x = attention_71_cast_fp16)[name = string("attention_73_cast_fp16")];
            tensor<int32, [2]> var_6852 = const()[name = string("op_6852"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [256, 128]> input_123_cast_fp16 = reshape(shape = var_6852, x = linear_73_cast_fp16)[name = string("input_123_cast_fp16")];
            tensor<fp16, [256, 128]> var_6854_cast_fp16 = mul(x = attention_73_cast_fp16, y = attention_73_cast_fp16)[name = string("op_6854_cast_fp16")];
            tensor<int32, [1]> variance_61_axes_0 = const()[name = string("variance_61_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_61_keep_dims_0 = const()[name = string("variance_61_keep_dims_0"), val = bool(true)];
            tensor<fp16, [256, 1]> variance_61_cast_fp16 = reduce_mean(axes = variance_61_axes_0, keep_dims = variance_61_keep_dims_0, x = var_6854_cast_fp16)[name = string("variance_61_cast_fp16")];
            fp16 var_6857_to_fp16 = const()[name = string("op_6857_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [256, 1]> var_6858_cast_fp16 = add(x = variance_61_cast_fp16, y = var_6857_to_fp16)[name = string("op_6858_cast_fp16")];
            fp32 var_6859_epsilon_0 = const()[name = string("op_6859_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [256, 1]> var_6859_cast_fp16 = rsqrt(epsilon = var_6859_epsilon_0, x = var_6858_cast_fp16)[name = string("op_6859_cast_fp16")];
            tensor<fp16, [256, 128]> attention_75_cast_fp16 = mul(x = attention_73_cast_fp16, y = var_6859_cast_fp16)[name = string("attention_75_cast_fp16")];
            tensor<fp16, [128]> groups_2_1_gated_norm_promoted_to_fp16 = const()[name = string("groups_2_1_gated_norm_promoted_to_fp16"), val = tensor<fp16, [128]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(148771264)))];
            tensor<fp16, [256, 128]> attention_77_cast_fp16 = mul(x = attention_75_cast_fp16, y = groups_2_1_gated_norm_promoted_to_fp16)[name = string("attention_77_cast_fp16")];
            tensor<fp16, [256, 128]> var_6862_cast_fp16 = silu(x = input_123_cast_fp16)[name = string("op_6862_cast_fp16")];
            tensor<fp16, [256, 128]> attention_79_cast_fp16 = mul(x = attention_77_cast_fp16, y = var_6862_cast_fp16)[name = string("attention_79_cast_fp16")];
            tensor<int32, [2]> var_6864 = const()[name = string("op_6864"), val = tensor<int32, [2]>([16, 2048])];
            tensor<fp16, [16, 2048]> input_125_cast_fp16 = reshape(shape = var_6864, x = attention_79_cast_fp16)[name = string("input_125_cast_fp16")];
            tensor<fp16, [1024, 2048]> groups_2_1_out_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 2048]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(148771584))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(150344512))))[name = string("groups_2_1_out_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 1024]> linear_74_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = groups_2_1_out_proj_weight_promoted_to_fp16_palettized, x = input_125_cast_fp16)[name = string("linear_74_cast_fp16")];
            tensor<fp16, [16, 1024]> value_143_cast_fp16 = add(x = value_131_cast_fp16, y = linear_74_cast_fp16)[name = string("value_143_cast_fp16")];
            tensor<fp16, [16, 1024]> var_6869_cast_fp16 = mul(x = value_143_cast_fp16, y = value_143_cast_fp16)[name = string("op_6869_cast_fp16")];
            tensor<int32, [1]> variance_63_axes_0 = const()[name = string("variance_63_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_63_keep_dims_0 = const()[name = string("variance_63_keep_dims_0"), val = bool(true)];
            tensor<fp16, [16, 1]> variance_63_cast_fp16 = reduce_mean(axes = variance_63_axes_0, keep_dims = variance_63_keep_dims_0, x = var_6869_cast_fp16)[name = string("variance_63_cast_fp16")];
            fp16 var_6872_to_fp16 = const()[name = string("op_6872_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [16, 1]> var_6873_cast_fp16 = add(x = variance_63_cast_fp16, y = var_6872_to_fp16)[name = string("op_6873_cast_fp16")];
            fp32 var_6874_epsilon_0 = const()[name = string("op_6874_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [16, 1]> var_6874_cast_fp16 = rsqrt(epsilon = var_6874_epsilon_0, x = var_6873_cast_fp16)[name = string("op_6874_cast_fp16")];
            tensor<fp16, [16, 1024]> var_6875_cast_fp16 = mul(x = value_143_cast_fp16, y = var_6874_cast_fp16)[name = string("op_6875_cast_fp16")];
            tensor<fp16, [1024]> var_6877_to_fp16 = const()[name = string("op_6877_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(150352768)))];
            tensor<fp16, [16, 1024]> input_127_cast_fp16 = mul(x = var_6875_cast_fp16, y = var_6877_to_fp16)[name = string("input_127_cast_fp16")];
            tensor<fp16, [3584, 1024]> groups_2_1_gate_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(150354880))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(153107456))))[name = string("groups_2_1_gate_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 3584]> linear_75_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = groups_2_1_gate_proj_weight_promoted_to_fp16_palettized, x = input_127_cast_fp16)[name = string("linear_75_cast_fp16")];
            tensor<fp16, [16, 3584]> var_6881_cast_fp16 = silu(x = linear_75_cast_fp16)[name = string("op_6881_cast_fp16")];
            tensor<fp16, [3584, 1024]> groups_2_1_up_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(153136192))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(155888768))))[name = string("groups_2_1_up_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 3584]> linear_76_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = groups_2_1_up_proj_weight_promoted_to_fp16_palettized, x = input_127_cast_fp16)[name = string("linear_76_cast_fp16")];
            tensor<fp16, [16, 3584]> input_131_cast_fp16 = mul(x = var_6881_cast_fp16, y = linear_76_cast_fp16)[name = string("input_131_cast_fp16")];
            tensor<fp16, [1024, 3584]> groups_2_1_down_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 3584]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(155917504))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(158670080))))[name = string("groups_2_1_down_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 1024]> linear_77_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = groups_2_1_down_proj_weight_promoted_to_fp16_palettized, x = input_131_cast_fp16)[name = string("linear_77_cast_fp16")];
            tensor<fp16, [16, 1024]> value_145_cast_fp16 = add(x = value_143_cast_fp16, y = linear_77_cast_fp16)[name = string("value_145_cast_fp16")];
            int32 var_6919 = const()[name = string("op_6919"), val = int32(-1)];
            tensor<fp16, [16, 1024]> var_6934_cast_fp16 = mul(x = value_145_cast_fp16, y = value_145_cast_fp16)[name = string("op_6934_cast_fp16")];
            tensor<int32, [1]> variance_65_axes_0 = const()[name = string("variance_65_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_65_keep_dims_0 = const()[name = string("variance_65_keep_dims_0"), val = bool(true)];
            tensor<fp16, [16, 1]> variance_65_cast_fp16 = reduce_mean(axes = variance_65_axes_0, keep_dims = variance_65_keep_dims_0, x = var_6934_cast_fp16)[name = string("variance_65_cast_fp16")];
            fp16 var_6937_to_fp16 = const()[name = string("op_6937_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [16, 1]> var_6938_cast_fp16 = add(x = variance_65_cast_fp16, y = var_6937_to_fp16)[name = string("op_6938_cast_fp16")];
            fp32 var_6939_epsilon_0 = const()[name = string("op_6939_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [16, 1]> var_6939_cast_fp16 = rsqrt(epsilon = var_6939_epsilon_0, x = var_6938_cast_fp16)[name = string("op_6939_cast_fp16")];
            tensor<fp16, [16, 1024]> var_6940_cast_fp16 = mul(x = value_145_cast_fp16, y = var_6939_cast_fp16)[name = string("op_6940_cast_fp16")];
            tensor<fp16, [1024]> var_6942_to_fp16 = const()[name = string("op_6942_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(158678336)))];
            tensor<fp16, [16, 1024]> input_133_cast_fp16 = mul(x = var_6940_cast_fp16, y = var_6942_to_fp16)[name = string("input_133_cast_fp16")];
            tensor<fp16, [6144, 1024]> groups_2_2_in_proj_qkv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(158680448))), lut = tensor<fp16, [384, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(163399104))))[name = string("groups_2_2_in_proj_qkv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 6144]> linear_78_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = groups_2_2_in_proj_qkv_weight_promoted_to_fp16_palettized, x = input_133_cast_fp16)[name = string("linear_78_cast_fp16")];
            tensor<int32, [2]> var_6946_perm_0 = const()[name = string("op_6946_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> mixed_17_axes_0 = const()[name = string("mixed_17_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [6144, 16]> var_6946_cast_fp16 = transpose(perm = var_6946_perm_0, x = linear_78_cast_fp16)[name = string("transpose_68")];
            tensor<fp16, [1, 6144, 16]> mixed_17_cast_fp16 = expand_dims(axes = mixed_17_axes_0, x = var_6946_cast_fp16)[name = string("mixed_17_cast_fp16")];
            bool convolution_input_17_interleave_0 = const()[name = string("convolution_input_17_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 6144, 19]> convolution_input_17_cast_fp16 = concat(axis = var_6919, interleave = convolution_input_17_interleave_0, values = (conv10, mixed_17_cast_fp16))[name = string("convolution_input_17_cast_fp16")];
            string input_135_pad_type_0 = const()[name = string("input_135_pad_type_0"), val = string("valid")];
            int32 input_135_groups_0 = const()[name = string("input_135_groups_0"), val = int32(6144)];
            tensor<int32, [1]> input_135_strides_0 = const()[name = string("input_135_strides_0"), val = tensor<int32, [1]>([1])];
            tensor<int32, [2]> input_135_pad_0 = const()[name = string("input_135_pad_0"), val = tensor<int32, [2]>([0, 0])];
            tensor<int32, [1]> input_135_dilations_0 = const()[name = string("input_135_dilations_0"), val = tensor<int32, [1]>([1])];
            tensor<fp16, [6144, 1, 4]> groups_2_2_conv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1, 4]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(163448320))), lut = tensor<fp16, [384, 1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(163466816))))[name = string("groups_2_2_conv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 6144, 16]> input_135_cast_fp16 = conv(dilations = input_135_dilations_0, groups = input_135_groups_0, pad = input_135_pad_0, pad_type = input_135_pad_type_0, strides = input_135_strides_0, weight = groups_2_2_conv_weight_promoted_to_fp16_palettized, x = convolution_input_17_cast_fp16)[name = string("input_135_cast_fp16")];
            tensor<fp16, [1, 6144, 16]> var_6955_cast_fp16 = silu(x = input_135_cast_fp16)[name = string("op_6955_cast_fp16")];
            tensor<int32, [3]> var_6956_perm_0 = const()[name = string("op_6956_perm_0"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [3]> var_6959_begin_0 = const()[name = string("op_6959_begin_0"), val = tensor<int32, [3]>([0, 0, 16])];
            tensor<int32, [3]> var_6959_end_0 = const()[name = string("op_6959_end_0"), val = tensor<int32, [3]>([1, 6144, 19])];
            tensor<bool, [3]> var_6959_end_mask_0 = const()[name = string("op_6959_end_mask_0"), val = tensor<bool, [3]>([true, true, true])];
            tensor<fp16, [1, 6144, 3]> conv10_out = slice_by_index(begin = var_6959_begin_0, end = var_6959_end_0, end_mask = var_6959_end_mask_0, x = convolution_input_17_cast_fp16)[name = string("op_6959_cast_fp16")];
            tensor<int32, [3]> var_6960 = const()[name = string("op_6960"), val = tensor<int32, [3]>([2048, 2048, 2048])];
            int32 var_6961_axis_0 = const()[name = string("op_6961_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 16, 6144]> var_6956_cast_fp16 = transpose(perm = var_6956_perm_0, x = var_6955_cast_fp16)[name = string("transpose_67")];
            tensor<fp16, [1, 16, 2048]> var_6961_cast_fp16_0, tensor<fp16, [1, 16, 2048]> var_6961_cast_fp16_1, tensor<fp16, [1, 16, 2048]> var_6961_cast_fp16_2 = split(axis = var_6961_axis_0, split_sizes = var_6960, x = var_6956_cast_fp16)[name = string("op_6961_cast_fp16")];
            tensor<int32, [4]> var_6965 = const()[name = string("op_6965"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<fp16, [1, 16, 16, 128]> value_149_cast_fp16 = reshape(shape = var_6965, x = var_6961_cast_fp16_0)[name = string("value_149_cast_fp16")];
            tensor<int32, [4]> var_6967 = const()[name = string("op_6967"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<fp16, [1, 16, 16, 128]> value_151_cast_fp16 = reshape(shape = var_6967, x = var_6961_cast_fp16_1)[name = string("value_151_cast_fp16")];
            tensor<int32, [4]> var_6969 = const()[name = string("op_6969"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<fp16, [1, 16, 16, 128]> value_153_cast_fp16 = reshape(shape = var_6969, x = var_6961_cast_fp16_2)[name = string("value_153_cast_fp16")];
            tensor<fp16, [1, 16, 16, 128]> var_6971_cast_fp16 = mul(x = value_149_cast_fp16, y = value_149_cast_fp16)[name = string("op_6971_cast_fp16")];
            tensor<int32, [1]> var_6973_axes_0 = const()[name = string("op_6973_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_6973_keep_dims_0 = const()[name = string("op_6973_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 16, 16, 1]> var_6973_cast_fp16 = reduce_sum(axes = var_6973_axes_0, keep_dims = var_6973_keep_dims_0, x = var_6971_cast_fp16)[name = string("op_6973_cast_fp16")];
            fp16 var_6974_to_fp16 = const()[name = string("op_6974_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 16, 16, 1]> var_6975_cast_fp16 = add(x = var_6973_cast_fp16, y = var_6974_to_fp16)[name = string("op_6975_cast_fp16")];
            fp32 var_6976_epsilon_0 = const()[name = string("op_6976_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 16, 16, 1]> var_6976_cast_fp16 = rsqrt(epsilon = var_6976_epsilon_0, x = var_6975_cast_fp16)[name = string("op_6976_cast_fp16")];
            tensor<fp16, [1, 16, 16, 128]> var_6977_cast_fp16 = mul(x = value_149_cast_fp16, y = var_6976_cast_fp16)[name = string("op_6977_cast_fp16")];
            tensor<int32, [4]> var_6978_perm_0 = const()[name = string("op_6978_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            fp16 _inversed_query_43_y_0_to_fp16 = const()[name = string("_inversed_query_43_y_0_to_fp16"), val = fp16(0x1.6ap-4)];
            tensor<fp16, [1, 16, 16, 128]> var_6978_cast_fp16 = transpose(perm = var_6978_perm_0, x = var_6977_cast_fp16)[name = string("transpose_66")];
            tensor<fp16, [1, 16, 16, 128]> _inversed_query_43_cast_fp16 = mul(x = var_6978_cast_fp16, y = _inversed_query_43_y_0_to_fp16)[name = string("_inversed_query_43_cast_fp16")];
            tensor<fp16, [1, 16, 16, 128]> var_6981_cast_fp16 = mul(x = value_151_cast_fp16, y = value_151_cast_fp16)[name = string("op_6981_cast_fp16")];
            tensor<int32, [1]> var_6983_axes_0 = const()[name = string("op_6983_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_6983_keep_dims_0 = const()[name = string("op_6983_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 16, 16, 1]> var_6983_cast_fp16 = reduce_sum(axes = var_6983_axes_0, keep_dims = var_6983_keep_dims_0, x = var_6981_cast_fp16)[name = string("op_6983_cast_fp16")];
            fp16 var_6984_to_fp16 = const()[name = string("op_6984_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 16, 16, 1]> var_6985_cast_fp16 = add(x = var_6983_cast_fp16, y = var_6984_to_fp16)[name = string("op_6985_cast_fp16")];
            fp32 var_6986_epsilon_0 = const()[name = string("op_6986_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 16, 16, 1]> var_6986_cast_fp16 = rsqrt(epsilon = var_6986_epsilon_0, x = var_6985_cast_fp16)[name = string("op_6986_cast_fp16")];
            tensor<fp16, [1, 16, 16, 128]> var_6987_cast_fp16 = mul(x = value_151_cast_fp16, y = var_6986_cast_fp16)[name = string("op_6987_cast_fp16")];
            tensor<int32, [4]> key_43_perm_0 = const()[name = string("key_43_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<int32, [4]> value_155_perm_0 = const()[name = string("value_155_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<fp16, [16, 1024]> groups_2_2_in_proj_b_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(163516032))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(163528384))))[name = string("groups_2_2_in_proj_b_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16]> linear_79_bias_0_to_fp16 = const()[name = string("linear_79_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(4862656)))];
            tensor<fp16, [16, 16]> linear_79_cast_fp16 = linear(bias = linear_79_bias_0_to_fp16, weight = groups_2_2_in_proj_b_weight_promoted_to_fp16_palettized, x = input_133_cast_fp16)[name = string("linear_79_cast_fp16")];
            tensor<fp16, [16, 16]> var_6992_cast_fp16 = sigmoid(x = linear_79_cast_fp16)[name = string("op_6992_cast_fp16")];
            tensor<int32, [2]> var_6993_perm_0 = const()[name = string("op_6993_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> beta_17_axes_0 = const()[name = string("beta_17_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 16]> var_6993_cast_fp16 = transpose(perm = var_6993_perm_0, x = var_6992_cast_fp16)[name = string("transpose_65")];
            tensor<fp16, [1, 16, 16]> beta_17_cast_fp16 = expand_dims(axes = beta_17_axes_0, x = var_6993_cast_fp16)[name = string("beta_17_cast_fp16")];
            tensor<fp16, [16, 1024]> op_6999_weight_0_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(163528576))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(163540928))))[name = string("op_6999_weight_0_to_fp16_palettized")];
            tensor<fp16, [16]> var_6999_bias_0_to_fp16 = const()[name = string("op_6999_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(163541120)))];
            tensor<fp16, [16, 16]> var_6999_cast_fp16 = linear(bias = var_6999_bias_0_to_fp16, weight = op_6999_weight_0_to_fp16_palettized, x = input_133_cast_fp16)[name = string("op_6999_cast_fp16")];
            tensor<fp16, [16, 16]> var_7000_cast_fp16 = softplus(x = var_6999_cast_fp16)[name = string("op_7000_cast_fp16")];
            tensor<fp16, [16]> var_6996_promoted_to_fp16 = const()[name = string("op_6996_promoted_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(163541248)))];
            tensor<fp16, [16, 16]> var_7001_cast_fp16 = mul(x = var_6996_promoted_to_fp16, y = var_7000_cast_fp16)[name = string("op_7001_cast_fp16")];
            tensor<int32, [2]> var_7002_perm_0 = const()[name = string("op_7002_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> decay_17_axes_0 = const()[name = string("decay_17_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 16]> var_7002_cast_fp16 = transpose(perm = var_7002_perm_0, x = var_7001_cast_fp16)[name = string("transpose_64")];
            tensor<fp16, [1, 16, 16]> decay_17_cast_fp16 = expand_dims(axes = decay_17_axes_0, x = var_7002_cast_fp16)[name = string("decay_17_cast_fp16")];
            tensor<fp16, [2048, 1024]> groups_2_2_in_proj_z_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [2048, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(163541376))), lut = tensor<fp16, [128, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(165114304))))[name = string("groups_2_2_in_proj_z_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 2048]> linear_81_cast_fp16 = linear(bias = linear_3_bias_0_to_fp16, weight = groups_2_2_in_proj_z_weight_promoted_to_fp16_palettized, x = input_133_cast_fp16)[name = string("linear_81_cast_fp16")];
            tensor<int32, [3]> var_7010_begin_0 = const()[name = string("op_7010_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_7010_end_0 = const()[name = string("op_7010_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_7010_end_mask_0 = const()[name = string("op_7010_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_7010_squeeze_mask_0 = const()[name = string("op_7010_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_7010_cast_fp16 = slice_by_index(begin = var_7010_begin_0, end = var_7010_end_0, end_mask = var_7010_end_mask_0, squeeze_mask = var_7010_squeeze_mask_0, x = decay_17_cast_fp16)[name = string("op_7010_cast_fp16")];
            tensor<fp16, [1, 16]> var_7011_cast_fp16 = exp(x = var_7010_cast_fp16)[name = string("op_7011_cast_fp16")];
            tensor<int32, [1]> var_7012_axes_0 = const()[name = string("op_7012_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_7012_cast_fp16 = expand_dims(axes = var_7012_axes_0, x = var_7011_cast_fp16)[name = string("op_7012_cast_fp16")];
            tensor<int32, [1]> var_7013_axes_0 = const()[name = string("op_7013_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_7013_cast_fp16 = expand_dims(axes = var_7013_axes_0, x = var_7012_cast_fp16)[name = string("op_7013_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_513_cast_fp16 = mul(x = rec10, y = var_7013_cast_fp16)[name = string("state_513_cast_fp16")];
            tensor<int32, [4]> key_token_257_begin_0 = const()[name = string("key_token_257_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> key_token_257_end_0 = const()[name = string("key_token_257_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> key_token_257_end_mask_0 = const()[name = string("key_token_257_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_257_squeeze_mask_0 = const()[name = string("key_token_257_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 16, 128]> key_43_cast_fp16 = transpose(perm = key_43_perm_0, x = var_6987_cast_fp16)[name = string("transpose_63")];
            tensor<fp16, [1, 16, 128]> key_token_257_cast_fp16 = slice_by_index(begin = key_token_257_begin_0, end = key_token_257_end_0, end_mask = key_token_257_end_mask_0, squeeze_mask = key_token_257_squeeze_mask_0, x = key_43_cast_fp16)[name = string("key_token_257_cast_fp16")];
            tensor<int32, [4]> value_token_257_begin_0 = const()[name = string("value_token_257_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> value_token_257_end_0 = const()[name = string("value_token_257_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> value_token_257_end_mask_0 = const()[name = string("value_token_257_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_257_squeeze_mask_0 = const()[name = string("value_token_257_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 16, 128]> value_155_cast_fp16 = transpose(perm = value_155_perm_0, x = value_153_cast_fp16)[name = string("transpose_62")];
            tensor<fp16, [1, 16, 128]> value_token_257_cast_fp16 = slice_by_index(begin = value_token_257_begin_0, end = value_token_257_end_0, end_mask = value_token_257_end_mask_0, squeeze_mask = value_token_257_squeeze_mask_0, x = value_155_cast_fp16)[name = string("value_token_257_cast_fp16")];
            tensor<int32, [1]> var_7021_axes_0 = const()[name = string("op_7021_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_7021_cast_fp16 = expand_dims(axes = var_7021_axes_0, x = key_token_257_cast_fp16)[name = string("op_7021_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_7022_cast_fp16 = mul(x = state_513_cast_fp16, y = var_7021_cast_fp16)[name = string("op_7022_cast_fp16")];
            tensor<int32, [1]> memory_257_axes_0 = const()[name = string("memory_257_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_257_keep_dims_0 = const()[name = string("memory_257_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_257_cast_fp16 = reduce_sum(axes = memory_257_axes_0, keep_dims = memory_257_keep_dims_0, x = var_7022_cast_fp16)[name = string("memory_257_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_7025_cast_fp16 = sub(x = value_token_257_cast_fp16, y = memory_257_cast_fp16)[name = string("op_7025_cast_fp16")];
            tensor<int32, [3]> var_7028_begin_0 = const()[name = string("op_7028_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_7028_end_0 = const()[name = string("op_7028_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_7028_end_mask_0 = const()[name = string("op_7028_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_7028_squeeze_mask_0 = const()[name = string("op_7028_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_7028_cast_fp16 = slice_by_index(begin = var_7028_begin_0, end = var_7028_end_0, end_mask = var_7028_end_mask_0, squeeze_mask = var_7028_squeeze_mask_0, x = beta_17_cast_fp16)[name = string("op_7028_cast_fp16")];
            tensor<int32, [1]> var_7029_axes_0 = const()[name = string("op_7029_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_7029_cast_fp16 = expand_dims(axes = var_7029_axes_0, x = var_7028_cast_fp16)[name = string("op_7029_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_257_cast_fp16 = mul(x = var_7025_cast_fp16, y = var_7029_cast_fp16)[name = string("delta_257_cast_fp16")];
            tensor<int32, [1]> var_7032_axes_0 = const()[name = string("op_7032_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_7032_cast_fp16 = expand_dims(axes = var_7032_axes_0, x = delta_257_cast_fp16)[name = string("op_7032_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_7033_cast_fp16 = mul(x = var_7021_cast_fp16, y = var_7032_cast_fp16)[name = string("op_7033_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_515_cast_fp16 = add(x = state_513_cast_fp16, y = var_7033_cast_fp16)[name = string("state_515_cast_fp16")];
            tensor<int32, [4]> var_7037_begin_0 = const()[name = string("op_7037_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> var_7037_end_0 = const()[name = string("op_7037_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> var_7037_end_mask_0 = const()[name = string("op_7037_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_7037_squeeze_mask_0 = const()[name = string("op_7037_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_7037_cast_fp16 = slice_by_index(begin = var_7037_begin_0, end = var_7037_end_0, end_mask = var_7037_end_mask_0, squeeze_mask = var_7037_squeeze_mask_0, x = _inversed_query_43_cast_fp16)[name = string("op_7037_cast_fp16")];
            tensor<int32, [1]> var_7038_axes_0 = const()[name = string("op_7038_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_7038_cast_fp16 = expand_dims(axes = var_7038_axes_0, x = var_7037_cast_fp16)[name = string("op_7038_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_7039_cast_fp16 = mul(x = state_515_cast_fp16, y = var_7038_cast_fp16)[name = string("op_7039_cast_fp16")];
            tensor<int32, [1]> var_7041_axes_0 = const()[name = string("op_7041_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_7041_keep_dims_0 = const()[name = string("op_7041_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_7041_cast_fp16 = reduce_sum(axes = var_7041_axes_0, keep_dims = var_7041_keep_dims_0, x = var_7039_cast_fp16)[name = string("op_7041_cast_fp16")];
            tensor<int32, [3]> var_7044_begin_0 = const()[name = string("op_7044_begin_0"), val = tensor<int32, [3]>([0, 0, 1])];
            tensor<int32, [3]> var_7044_end_0 = const()[name = string("op_7044_end_0"), val = tensor<int32, [3]>([1, 16, 2])];
            tensor<bool, [3]> var_7044_end_mask_0 = const()[name = string("op_7044_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_7044_squeeze_mask_0 = const()[name = string("op_7044_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_7044_cast_fp16 = slice_by_index(begin = var_7044_begin_0, end = var_7044_end_0, end_mask = var_7044_end_mask_0, squeeze_mask = var_7044_squeeze_mask_0, x = decay_17_cast_fp16)[name = string("op_7044_cast_fp16")];
            tensor<fp16, [1, 16]> var_7045_cast_fp16 = exp(x = var_7044_cast_fp16)[name = string("op_7045_cast_fp16")];
            tensor<int32, [1]> var_7046_axes_0 = const()[name = string("op_7046_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_7046_cast_fp16 = expand_dims(axes = var_7046_axes_0, x = var_7045_cast_fp16)[name = string("op_7046_cast_fp16")];
            tensor<int32, [1]> var_7047_axes_0 = const()[name = string("op_7047_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_7047_cast_fp16 = expand_dims(axes = var_7047_axes_0, x = var_7046_cast_fp16)[name = string("op_7047_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_517_cast_fp16 = mul(x = state_515_cast_fp16, y = var_7047_cast_fp16)[name = string("state_517_cast_fp16")];
            tensor<int32, [4]> key_token_259_begin_0 = const()[name = string("key_token_259_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> key_token_259_end_0 = const()[name = string("key_token_259_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> key_token_259_end_mask_0 = const()[name = string("key_token_259_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_259_squeeze_mask_0 = const()[name = string("key_token_259_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_259_cast_fp16 = slice_by_index(begin = key_token_259_begin_0, end = key_token_259_end_0, end_mask = key_token_259_end_mask_0, squeeze_mask = key_token_259_squeeze_mask_0, x = key_43_cast_fp16)[name = string("key_token_259_cast_fp16")];
            tensor<int32, [4]> value_token_259_begin_0 = const()[name = string("value_token_259_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> value_token_259_end_0 = const()[name = string("value_token_259_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> value_token_259_end_mask_0 = const()[name = string("value_token_259_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_259_squeeze_mask_0 = const()[name = string("value_token_259_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_259_cast_fp16 = slice_by_index(begin = value_token_259_begin_0, end = value_token_259_end_0, end_mask = value_token_259_end_mask_0, squeeze_mask = value_token_259_squeeze_mask_0, x = value_155_cast_fp16)[name = string("value_token_259_cast_fp16")];
            tensor<int32, [1]> var_7055_axes_0 = const()[name = string("op_7055_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_7055_cast_fp16 = expand_dims(axes = var_7055_axes_0, x = key_token_259_cast_fp16)[name = string("op_7055_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_7056_cast_fp16 = mul(x = state_517_cast_fp16, y = var_7055_cast_fp16)[name = string("op_7056_cast_fp16")];
            tensor<int32, [1]> memory_259_axes_0 = const()[name = string("memory_259_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_259_keep_dims_0 = const()[name = string("memory_259_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_259_cast_fp16 = reduce_sum(axes = memory_259_axes_0, keep_dims = memory_259_keep_dims_0, x = var_7056_cast_fp16)[name = string("memory_259_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_7059_cast_fp16 = sub(x = value_token_259_cast_fp16, y = memory_259_cast_fp16)[name = string("op_7059_cast_fp16")];
            tensor<int32, [3]> var_7062_begin_0 = const()[name = string("op_7062_begin_0"), val = tensor<int32, [3]>([0, 0, 1])];
            tensor<int32, [3]> var_7062_end_0 = const()[name = string("op_7062_end_0"), val = tensor<int32, [3]>([1, 16, 2])];
            tensor<bool, [3]> var_7062_end_mask_0 = const()[name = string("op_7062_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_7062_squeeze_mask_0 = const()[name = string("op_7062_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_7062_cast_fp16 = slice_by_index(begin = var_7062_begin_0, end = var_7062_end_0, end_mask = var_7062_end_mask_0, squeeze_mask = var_7062_squeeze_mask_0, x = beta_17_cast_fp16)[name = string("op_7062_cast_fp16")];
            tensor<int32, [1]> var_7063_axes_0 = const()[name = string("op_7063_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_7063_cast_fp16 = expand_dims(axes = var_7063_axes_0, x = var_7062_cast_fp16)[name = string("op_7063_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_259_cast_fp16 = mul(x = var_7059_cast_fp16, y = var_7063_cast_fp16)[name = string("delta_259_cast_fp16")];
            tensor<int32, [1]> var_7066_axes_0 = const()[name = string("op_7066_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_7066_cast_fp16 = expand_dims(axes = var_7066_axes_0, x = delta_259_cast_fp16)[name = string("op_7066_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_7067_cast_fp16 = mul(x = var_7055_cast_fp16, y = var_7066_cast_fp16)[name = string("op_7067_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_519_cast_fp16 = add(x = state_517_cast_fp16, y = var_7067_cast_fp16)[name = string("state_519_cast_fp16")];
            tensor<int32, [4]> var_7071_begin_0 = const()[name = string("op_7071_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> var_7071_end_0 = const()[name = string("op_7071_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> var_7071_end_mask_0 = const()[name = string("op_7071_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_7071_squeeze_mask_0 = const()[name = string("op_7071_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_7071_cast_fp16 = slice_by_index(begin = var_7071_begin_0, end = var_7071_end_0, end_mask = var_7071_end_mask_0, squeeze_mask = var_7071_squeeze_mask_0, x = _inversed_query_43_cast_fp16)[name = string("op_7071_cast_fp16")];
            tensor<int32, [1]> var_7072_axes_0 = const()[name = string("op_7072_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_7072_cast_fp16 = expand_dims(axes = var_7072_axes_0, x = var_7071_cast_fp16)[name = string("op_7072_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_7073_cast_fp16 = mul(x = state_519_cast_fp16, y = var_7072_cast_fp16)[name = string("op_7073_cast_fp16")];
            tensor<int32, [1]> var_7075_axes_0 = const()[name = string("op_7075_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_7075_keep_dims_0 = const()[name = string("op_7075_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_7075_cast_fp16 = reduce_sum(axes = var_7075_axes_0, keep_dims = var_7075_keep_dims_0, x = var_7073_cast_fp16)[name = string("op_7075_cast_fp16")];
            tensor<int32, [3]> var_7078_begin_0 = const()[name = string("op_7078_begin_0"), val = tensor<int32, [3]>([0, 0, 2])];
            tensor<int32, [3]> var_7078_end_0 = const()[name = string("op_7078_end_0"), val = tensor<int32, [3]>([1, 16, 3])];
            tensor<bool, [3]> var_7078_end_mask_0 = const()[name = string("op_7078_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_7078_squeeze_mask_0 = const()[name = string("op_7078_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_7078_cast_fp16 = slice_by_index(begin = var_7078_begin_0, end = var_7078_end_0, end_mask = var_7078_end_mask_0, squeeze_mask = var_7078_squeeze_mask_0, x = decay_17_cast_fp16)[name = string("op_7078_cast_fp16")];
            tensor<fp16, [1, 16]> var_7079_cast_fp16 = exp(x = var_7078_cast_fp16)[name = string("op_7079_cast_fp16")];
            tensor<int32, [1]> var_7080_axes_0 = const()[name = string("op_7080_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_7080_cast_fp16 = expand_dims(axes = var_7080_axes_0, x = var_7079_cast_fp16)[name = string("op_7080_cast_fp16")];
            tensor<int32, [1]> var_7081_axes_0 = const()[name = string("op_7081_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_7081_cast_fp16 = expand_dims(axes = var_7081_axes_0, x = var_7080_cast_fp16)[name = string("op_7081_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_521_cast_fp16 = mul(x = state_519_cast_fp16, y = var_7081_cast_fp16)[name = string("state_521_cast_fp16")];
            tensor<int32, [4]> key_token_261_begin_0 = const()[name = string("key_token_261_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> key_token_261_end_0 = const()[name = string("key_token_261_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> key_token_261_end_mask_0 = const()[name = string("key_token_261_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_261_squeeze_mask_0 = const()[name = string("key_token_261_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_261_cast_fp16 = slice_by_index(begin = key_token_261_begin_0, end = key_token_261_end_0, end_mask = key_token_261_end_mask_0, squeeze_mask = key_token_261_squeeze_mask_0, x = key_43_cast_fp16)[name = string("key_token_261_cast_fp16")];
            tensor<int32, [4]> value_token_261_begin_0 = const()[name = string("value_token_261_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> value_token_261_end_0 = const()[name = string("value_token_261_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> value_token_261_end_mask_0 = const()[name = string("value_token_261_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_261_squeeze_mask_0 = const()[name = string("value_token_261_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_261_cast_fp16 = slice_by_index(begin = value_token_261_begin_0, end = value_token_261_end_0, end_mask = value_token_261_end_mask_0, squeeze_mask = value_token_261_squeeze_mask_0, x = value_155_cast_fp16)[name = string("value_token_261_cast_fp16")];
            tensor<int32, [1]> var_7089_axes_0 = const()[name = string("op_7089_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_7089_cast_fp16 = expand_dims(axes = var_7089_axes_0, x = key_token_261_cast_fp16)[name = string("op_7089_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_7090_cast_fp16 = mul(x = state_521_cast_fp16, y = var_7089_cast_fp16)[name = string("op_7090_cast_fp16")];
            tensor<int32, [1]> memory_261_axes_0 = const()[name = string("memory_261_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_261_keep_dims_0 = const()[name = string("memory_261_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_261_cast_fp16 = reduce_sum(axes = memory_261_axes_0, keep_dims = memory_261_keep_dims_0, x = var_7090_cast_fp16)[name = string("memory_261_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_7093_cast_fp16 = sub(x = value_token_261_cast_fp16, y = memory_261_cast_fp16)[name = string("op_7093_cast_fp16")];
            tensor<int32, [3]> var_7096_begin_0 = const()[name = string("op_7096_begin_0"), val = tensor<int32, [3]>([0, 0, 2])];
            tensor<int32, [3]> var_7096_end_0 = const()[name = string("op_7096_end_0"), val = tensor<int32, [3]>([1, 16, 3])];
            tensor<bool, [3]> var_7096_end_mask_0 = const()[name = string("op_7096_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_7096_squeeze_mask_0 = const()[name = string("op_7096_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_7096_cast_fp16 = slice_by_index(begin = var_7096_begin_0, end = var_7096_end_0, end_mask = var_7096_end_mask_0, squeeze_mask = var_7096_squeeze_mask_0, x = beta_17_cast_fp16)[name = string("op_7096_cast_fp16")];
            tensor<int32, [1]> var_7097_axes_0 = const()[name = string("op_7097_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_7097_cast_fp16 = expand_dims(axes = var_7097_axes_0, x = var_7096_cast_fp16)[name = string("op_7097_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_261_cast_fp16 = mul(x = var_7093_cast_fp16, y = var_7097_cast_fp16)[name = string("delta_261_cast_fp16")];
            tensor<int32, [1]> var_7100_axes_0 = const()[name = string("op_7100_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_7100_cast_fp16 = expand_dims(axes = var_7100_axes_0, x = delta_261_cast_fp16)[name = string("op_7100_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_7101_cast_fp16 = mul(x = var_7089_cast_fp16, y = var_7100_cast_fp16)[name = string("op_7101_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_523_cast_fp16 = add(x = state_521_cast_fp16, y = var_7101_cast_fp16)[name = string("state_523_cast_fp16")];
            tensor<int32, [4]> var_7105_begin_0 = const()[name = string("op_7105_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> var_7105_end_0 = const()[name = string("op_7105_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> var_7105_end_mask_0 = const()[name = string("op_7105_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_7105_squeeze_mask_0 = const()[name = string("op_7105_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_7105_cast_fp16 = slice_by_index(begin = var_7105_begin_0, end = var_7105_end_0, end_mask = var_7105_end_mask_0, squeeze_mask = var_7105_squeeze_mask_0, x = _inversed_query_43_cast_fp16)[name = string("op_7105_cast_fp16")];
            tensor<int32, [1]> var_7106_axes_0 = const()[name = string("op_7106_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_7106_cast_fp16 = expand_dims(axes = var_7106_axes_0, x = var_7105_cast_fp16)[name = string("op_7106_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_7107_cast_fp16 = mul(x = state_523_cast_fp16, y = var_7106_cast_fp16)[name = string("op_7107_cast_fp16")];
            tensor<int32, [1]> var_7109_axes_0 = const()[name = string("op_7109_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_7109_keep_dims_0 = const()[name = string("op_7109_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_7109_cast_fp16 = reduce_sum(axes = var_7109_axes_0, keep_dims = var_7109_keep_dims_0, x = var_7107_cast_fp16)[name = string("op_7109_cast_fp16")];
            tensor<int32, [3]> var_7112_begin_0 = const()[name = string("op_7112_begin_0"), val = tensor<int32, [3]>([0, 0, 3])];
            tensor<int32, [3]> var_7112_end_0 = const()[name = string("op_7112_end_0"), val = tensor<int32, [3]>([1, 16, 4])];
            tensor<bool, [3]> var_7112_end_mask_0 = const()[name = string("op_7112_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_7112_squeeze_mask_0 = const()[name = string("op_7112_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_7112_cast_fp16 = slice_by_index(begin = var_7112_begin_0, end = var_7112_end_0, end_mask = var_7112_end_mask_0, squeeze_mask = var_7112_squeeze_mask_0, x = decay_17_cast_fp16)[name = string("op_7112_cast_fp16")];
            tensor<fp16, [1, 16]> var_7113_cast_fp16 = exp(x = var_7112_cast_fp16)[name = string("op_7113_cast_fp16")];
            tensor<int32, [1]> var_7114_axes_0 = const()[name = string("op_7114_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_7114_cast_fp16 = expand_dims(axes = var_7114_axes_0, x = var_7113_cast_fp16)[name = string("op_7114_cast_fp16")];
            tensor<int32, [1]> var_7115_axes_0 = const()[name = string("op_7115_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_7115_cast_fp16 = expand_dims(axes = var_7115_axes_0, x = var_7114_cast_fp16)[name = string("op_7115_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_525_cast_fp16 = mul(x = state_523_cast_fp16, y = var_7115_cast_fp16)[name = string("state_525_cast_fp16")];
            tensor<int32, [4]> key_token_263_begin_0 = const()[name = string("key_token_263_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> key_token_263_end_0 = const()[name = string("key_token_263_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> key_token_263_end_mask_0 = const()[name = string("key_token_263_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_263_squeeze_mask_0 = const()[name = string("key_token_263_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_263_cast_fp16 = slice_by_index(begin = key_token_263_begin_0, end = key_token_263_end_0, end_mask = key_token_263_end_mask_0, squeeze_mask = key_token_263_squeeze_mask_0, x = key_43_cast_fp16)[name = string("key_token_263_cast_fp16")];
            tensor<int32, [4]> value_token_263_begin_0 = const()[name = string("value_token_263_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> value_token_263_end_0 = const()[name = string("value_token_263_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> value_token_263_end_mask_0 = const()[name = string("value_token_263_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_263_squeeze_mask_0 = const()[name = string("value_token_263_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_263_cast_fp16 = slice_by_index(begin = value_token_263_begin_0, end = value_token_263_end_0, end_mask = value_token_263_end_mask_0, squeeze_mask = value_token_263_squeeze_mask_0, x = value_155_cast_fp16)[name = string("value_token_263_cast_fp16")];
            tensor<int32, [1]> var_7123_axes_0 = const()[name = string("op_7123_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_7123_cast_fp16 = expand_dims(axes = var_7123_axes_0, x = key_token_263_cast_fp16)[name = string("op_7123_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_7124_cast_fp16 = mul(x = state_525_cast_fp16, y = var_7123_cast_fp16)[name = string("op_7124_cast_fp16")];
            tensor<int32, [1]> memory_263_axes_0 = const()[name = string("memory_263_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_263_keep_dims_0 = const()[name = string("memory_263_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_263_cast_fp16 = reduce_sum(axes = memory_263_axes_0, keep_dims = memory_263_keep_dims_0, x = var_7124_cast_fp16)[name = string("memory_263_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_7127_cast_fp16 = sub(x = value_token_263_cast_fp16, y = memory_263_cast_fp16)[name = string("op_7127_cast_fp16")];
            tensor<int32, [3]> var_7130_begin_0 = const()[name = string("op_7130_begin_0"), val = tensor<int32, [3]>([0, 0, 3])];
            tensor<int32, [3]> var_7130_end_0 = const()[name = string("op_7130_end_0"), val = tensor<int32, [3]>([1, 16, 4])];
            tensor<bool, [3]> var_7130_end_mask_0 = const()[name = string("op_7130_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_7130_squeeze_mask_0 = const()[name = string("op_7130_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_7130_cast_fp16 = slice_by_index(begin = var_7130_begin_0, end = var_7130_end_0, end_mask = var_7130_end_mask_0, squeeze_mask = var_7130_squeeze_mask_0, x = beta_17_cast_fp16)[name = string("op_7130_cast_fp16")];
            tensor<int32, [1]> var_7131_axes_0 = const()[name = string("op_7131_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_7131_cast_fp16 = expand_dims(axes = var_7131_axes_0, x = var_7130_cast_fp16)[name = string("op_7131_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_263_cast_fp16 = mul(x = var_7127_cast_fp16, y = var_7131_cast_fp16)[name = string("delta_263_cast_fp16")];
            tensor<int32, [1]> var_7134_axes_0 = const()[name = string("op_7134_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_7134_cast_fp16 = expand_dims(axes = var_7134_axes_0, x = delta_263_cast_fp16)[name = string("op_7134_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_7135_cast_fp16 = mul(x = var_7123_cast_fp16, y = var_7134_cast_fp16)[name = string("op_7135_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_527_cast_fp16 = add(x = state_525_cast_fp16, y = var_7135_cast_fp16)[name = string("state_527_cast_fp16")];
            tensor<int32, [4]> var_7139_begin_0 = const()[name = string("op_7139_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> var_7139_end_0 = const()[name = string("op_7139_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> var_7139_end_mask_0 = const()[name = string("op_7139_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_7139_squeeze_mask_0 = const()[name = string("op_7139_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_7139_cast_fp16 = slice_by_index(begin = var_7139_begin_0, end = var_7139_end_0, end_mask = var_7139_end_mask_0, squeeze_mask = var_7139_squeeze_mask_0, x = _inversed_query_43_cast_fp16)[name = string("op_7139_cast_fp16")];
            tensor<int32, [1]> var_7140_axes_0 = const()[name = string("op_7140_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_7140_cast_fp16 = expand_dims(axes = var_7140_axes_0, x = var_7139_cast_fp16)[name = string("op_7140_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_7141_cast_fp16 = mul(x = state_527_cast_fp16, y = var_7140_cast_fp16)[name = string("op_7141_cast_fp16")];
            tensor<int32, [1]> var_7143_axes_0 = const()[name = string("op_7143_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_7143_keep_dims_0 = const()[name = string("op_7143_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_7143_cast_fp16 = reduce_sum(axes = var_7143_axes_0, keep_dims = var_7143_keep_dims_0, x = var_7141_cast_fp16)[name = string("op_7143_cast_fp16")];
            tensor<int32, [3]> var_7146_begin_0 = const()[name = string("op_7146_begin_0"), val = tensor<int32, [3]>([0, 0, 4])];
            tensor<int32, [3]> var_7146_end_0 = const()[name = string("op_7146_end_0"), val = tensor<int32, [3]>([1, 16, 5])];
            tensor<bool, [3]> var_7146_end_mask_0 = const()[name = string("op_7146_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_7146_squeeze_mask_0 = const()[name = string("op_7146_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_7146_cast_fp16 = slice_by_index(begin = var_7146_begin_0, end = var_7146_end_0, end_mask = var_7146_end_mask_0, squeeze_mask = var_7146_squeeze_mask_0, x = decay_17_cast_fp16)[name = string("op_7146_cast_fp16")];
            tensor<fp16, [1, 16]> var_7147_cast_fp16 = exp(x = var_7146_cast_fp16)[name = string("op_7147_cast_fp16")];
            tensor<int32, [1]> var_7148_axes_0 = const()[name = string("op_7148_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_7148_cast_fp16 = expand_dims(axes = var_7148_axes_0, x = var_7147_cast_fp16)[name = string("op_7148_cast_fp16")];
            tensor<int32, [1]> var_7149_axes_0 = const()[name = string("op_7149_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_7149_cast_fp16 = expand_dims(axes = var_7149_axes_0, x = var_7148_cast_fp16)[name = string("op_7149_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_529_cast_fp16 = mul(x = state_527_cast_fp16, y = var_7149_cast_fp16)[name = string("state_529_cast_fp16")];
            tensor<int32, [4]> key_token_265_begin_0 = const()[name = string("key_token_265_begin_0"), val = tensor<int32, [4]>([0, 0, 4, 0])];
            tensor<int32, [4]> key_token_265_end_0 = const()[name = string("key_token_265_end_0"), val = tensor<int32, [4]>([1, 16, 5, 128])];
            tensor<bool, [4]> key_token_265_end_mask_0 = const()[name = string("key_token_265_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_265_squeeze_mask_0 = const()[name = string("key_token_265_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_265_cast_fp16 = slice_by_index(begin = key_token_265_begin_0, end = key_token_265_end_0, end_mask = key_token_265_end_mask_0, squeeze_mask = key_token_265_squeeze_mask_0, x = key_43_cast_fp16)[name = string("key_token_265_cast_fp16")];
            tensor<int32, [4]> value_token_265_begin_0 = const()[name = string("value_token_265_begin_0"), val = tensor<int32, [4]>([0, 0, 4, 0])];
            tensor<int32, [4]> value_token_265_end_0 = const()[name = string("value_token_265_end_0"), val = tensor<int32, [4]>([1, 16, 5, 128])];
            tensor<bool, [4]> value_token_265_end_mask_0 = const()[name = string("value_token_265_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_265_squeeze_mask_0 = const()[name = string("value_token_265_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_265_cast_fp16 = slice_by_index(begin = value_token_265_begin_0, end = value_token_265_end_0, end_mask = value_token_265_end_mask_0, squeeze_mask = value_token_265_squeeze_mask_0, x = value_155_cast_fp16)[name = string("value_token_265_cast_fp16")];
            tensor<int32, [1]> var_7157_axes_0 = const()[name = string("op_7157_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_7157_cast_fp16 = expand_dims(axes = var_7157_axes_0, x = key_token_265_cast_fp16)[name = string("op_7157_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_7158_cast_fp16 = mul(x = state_529_cast_fp16, y = var_7157_cast_fp16)[name = string("op_7158_cast_fp16")];
            tensor<int32, [1]> memory_265_axes_0 = const()[name = string("memory_265_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_265_keep_dims_0 = const()[name = string("memory_265_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_265_cast_fp16 = reduce_sum(axes = memory_265_axes_0, keep_dims = memory_265_keep_dims_0, x = var_7158_cast_fp16)[name = string("memory_265_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_7161_cast_fp16 = sub(x = value_token_265_cast_fp16, y = memory_265_cast_fp16)[name = string("op_7161_cast_fp16")];
            tensor<int32, [3]> var_7164_begin_0 = const()[name = string("op_7164_begin_0"), val = tensor<int32, [3]>([0, 0, 4])];
            tensor<int32, [3]> var_7164_end_0 = const()[name = string("op_7164_end_0"), val = tensor<int32, [3]>([1, 16, 5])];
            tensor<bool, [3]> var_7164_end_mask_0 = const()[name = string("op_7164_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_7164_squeeze_mask_0 = const()[name = string("op_7164_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_7164_cast_fp16 = slice_by_index(begin = var_7164_begin_0, end = var_7164_end_0, end_mask = var_7164_end_mask_0, squeeze_mask = var_7164_squeeze_mask_0, x = beta_17_cast_fp16)[name = string("op_7164_cast_fp16")];
            tensor<int32, [1]> var_7165_axes_0 = const()[name = string("op_7165_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_7165_cast_fp16 = expand_dims(axes = var_7165_axes_0, x = var_7164_cast_fp16)[name = string("op_7165_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_265_cast_fp16 = mul(x = var_7161_cast_fp16, y = var_7165_cast_fp16)[name = string("delta_265_cast_fp16")];
            tensor<int32, [1]> var_7168_axes_0 = const()[name = string("op_7168_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_7168_cast_fp16 = expand_dims(axes = var_7168_axes_0, x = delta_265_cast_fp16)[name = string("op_7168_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_7169_cast_fp16 = mul(x = var_7157_cast_fp16, y = var_7168_cast_fp16)[name = string("op_7169_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_531_cast_fp16 = add(x = state_529_cast_fp16, y = var_7169_cast_fp16)[name = string("state_531_cast_fp16")];
            tensor<int32, [4]> var_7173_begin_0 = const()[name = string("op_7173_begin_0"), val = tensor<int32, [4]>([0, 0, 4, 0])];
            tensor<int32, [4]> var_7173_end_0 = const()[name = string("op_7173_end_0"), val = tensor<int32, [4]>([1, 16, 5, 128])];
            tensor<bool, [4]> var_7173_end_mask_0 = const()[name = string("op_7173_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_7173_squeeze_mask_0 = const()[name = string("op_7173_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_7173_cast_fp16 = slice_by_index(begin = var_7173_begin_0, end = var_7173_end_0, end_mask = var_7173_end_mask_0, squeeze_mask = var_7173_squeeze_mask_0, x = _inversed_query_43_cast_fp16)[name = string("op_7173_cast_fp16")];
            tensor<int32, [1]> var_7174_axes_0 = const()[name = string("op_7174_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_7174_cast_fp16 = expand_dims(axes = var_7174_axes_0, x = var_7173_cast_fp16)[name = string("op_7174_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_7175_cast_fp16 = mul(x = state_531_cast_fp16, y = var_7174_cast_fp16)[name = string("op_7175_cast_fp16")];
            tensor<int32, [1]> var_7177_axes_0 = const()[name = string("op_7177_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_7177_keep_dims_0 = const()[name = string("op_7177_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_7177_cast_fp16 = reduce_sum(axes = var_7177_axes_0, keep_dims = var_7177_keep_dims_0, x = var_7175_cast_fp16)[name = string("op_7177_cast_fp16")];
            tensor<int32, [3]> var_7180_begin_0 = const()[name = string("op_7180_begin_0"), val = tensor<int32, [3]>([0, 0, 5])];
            tensor<int32, [3]> var_7180_end_0 = const()[name = string("op_7180_end_0"), val = tensor<int32, [3]>([1, 16, 6])];
            tensor<bool, [3]> var_7180_end_mask_0 = const()[name = string("op_7180_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_7180_squeeze_mask_0 = const()[name = string("op_7180_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_7180_cast_fp16 = slice_by_index(begin = var_7180_begin_0, end = var_7180_end_0, end_mask = var_7180_end_mask_0, squeeze_mask = var_7180_squeeze_mask_0, x = decay_17_cast_fp16)[name = string("op_7180_cast_fp16")];
            tensor<fp16, [1, 16]> var_7181_cast_fp16 = exp(x = var_7180_cast_fp16)[name = string("op_7181_cast_fp16")];
            tensor<int32, [1]> var_7182_axes_0 = const()[name = string("op_7182_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_7182_cast_fp16 = expand_dims(axes = var_7182_axes_0, x = var_7181_cast_fp16)[name = string("op_7182_cast_fp16")];
            tensor<int32, [1]> var_7183_axes_0 = const()[name = string("op_7183_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_7183_cast_fp16 = expand_dims(axes = var_7183_axes_0, x = var_7182_cast_fp16)[name = string("op_7183_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_533_cast_fp16 = mul(x = state_531_cast_fp16, y = var_7183_cast_fp16)[name = string("state_533_cast_fp16")];
            tensor<int32, [4]> key_token_267_begin_0 = const()[name = string("key_token_267_begin_0"), val = tensor<int32, [4]>([0, 0, 5, 0])];
            tensor<int32, [4]> key_token_267_end_0 = const()[name = string("key_token_267_end_0"), val = tensor<int32, [4]>([1, 16, 6, 128])];
            tensor<bool, [4]> key_token_267_end_mask_0 = const()[name = string("key_token_267_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_267_squeeze_mask_0 = const()[name = string("key_token_267_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_267_cast_fp16 = slice_by_index(begin = key_token_267_begin_0, end = key_token_267_end_0, end_mask = key_token_267_end_mask_0, squeeze_mask = key_token_267_squeeze_mask_0, x = key_43_cast_fp16)[name = string("key_token_267_cast_fp16")];
            tensor<int32, [4]> value_token_267_begin_0 = const()[name = string("value_token_267_begin_0"), val = tensor<int32, [4]>([0, 0, 5, 0])];
            tensor<int32, [4]> value_token_267_end_0 = const()[name = string("value_token_267_end_0"), val = tensor<int32, [4]>([1, 16, 6, 128])];
            tensor<bool, [4]> value_token_267_end_mask_0 = const()[name = string("value_token_267_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_267_squeeze_mask_0 = const()[name = string("value_token_267_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_267_cast_fp16 = slice_by_index(begin = value_token_267_begin_0, end = value_token_267_end_0, end_mask = value_token_267_end_mask_0, squeeze_mask = value_token_267_squeeze_mask_0, x = value_155_cast_fp16)[name = string("value_token_267_cast_fp16")];
            tensor<int32, [1]> var_7191_axes_0 = const()[name = string("op_7191_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_7191_cast_fp16 = expand_dims(axes = var_7191_axes_0, x = key_token_267_cast_fp16)[name = string("op_7191_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_7192_cast_fp16 = mul(x = state_533_cast_fp16, y = var_7191_cast_fp16)[name = string("op_7192_cast_fp16")];
            tensor<int32, [1]> memory_267_axes_0 = const()[name = string("memory_267_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_267_keep_dims_0 = const()[name = string("memory_267_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_267_cast_fp16 = reduce_sum(axes = memory_267_axes_0, keep_dims = memory_267_keep_dims_0, x = var_7192_cast_fp16)[name = string("memory_267_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_7195_cast_fp16 = sub(x = value_token_267_cast_fp16, y = memory_267_cast_fp16)[name = string("op_7195_cast_fp16")];
            tensor<int32, [3]> var_7198_begin_0 = const()[name = string("op_7198_begin_0"), val = tensor<int32, [3]>([0, 0, 5])];
            tensor<int32, [3]> var_7198_end_0 = const()[name = string("op_7198_end_0"), val = tensor<int32, [3]>([1, 16, 6])];
            tensor<bool, [3]> var_7198_end_mask_0 = const()[name = string("op_7198_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_7198_squeeze_mask_0 = const()[name = string("op_7198_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_7198_cast_fp16 = slice_by_index(begin = var_7198_begin_0, end = var_7198_end_0, end_mask = var_7198_end_mask_0, squeeze_mask = var_7198_squeeze_mask_0, x = beta_17_cast_fp16)[name = string("op_7198_cast_fp16")];
            tensor<int32, [1]> var_7199_axes_0 = const()[name = string("op_7199_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_7199_cast_fp16 = expand_dims(axes = var_7199_axes_0, x = var_7198_cast_fp16)[name = string("op_7199_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_267_cast_fp16 = mul(x = var_7195_cast_fp16, y = var_7199_cast_fp16)[name = string("delta_267_cast_fp16")];
            tensor<int32, [1]> var_7202_axes_0 = const()[name = string("op_7202_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_7202_cast_fp16 = expand_dims(axes = var_7202_axes_0, x = delta_267_cast_fp16)[name = string("op_7202_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_7203_cast_fp16 = mul(x = var_7191_cast_fp16, y = var_7202_cast_fp16)[name = string("op_7203_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_535_cast_fp16 = add(x = state_533_cast_fp16, y = var_7203_cast_fp16)[name = string("state_535_cast_fp16")];
            tensor<int32, [4]> var_7207_begin_0 = const()[name = string("op_7207_begin_0"), val = tensor<int32, [4]>([0, 0, 5, 0])];
            tensor<int32, [4]> var_7207_end_0 = const()[name = string("op_7207_end_0"), val = tensor<int32, [4]>([1, 16, 6, 128])];
            tensor<bool, [4]> var_7207_end_mask_0 = const()[name = string("op_7207_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_7207_squeeze_mask_0 = const()[name = string("op_7207_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_7207_cast_fp16 = slice_by_index(begin = var_7207_begin_0, end = var_7207_end_0, end_mask = var_7207_end_mask_0, squeeze_mask = var_7207_squeeze_mask_0, x = _inversed_query_43_cast_fp16)[name = string("op_7207_cast_fp16")];
            tensor<int32, [1]> var_7208_axes_0 = const()[name = string("op_7208_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_7208_cast_fp16 = expand_dims(axes = var_7208_axes_0, x = var_7207_cast_fp16)[name = string("op_7208_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_7209_cast_fp16 = mul(x = state_535_cast_fp16, y = var_7208_cast_fp16)[name = string("op_7209_cast_fp16")];
            tensor<int32, [1]> var_7211_axes_0 = const()[name = string("op_7211_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_7211_keep_dims_0 = const()[name = string("op_7211_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_7211_cast_fp16 = reduce_sum(axes = var_7211_axes_0, keep_dims = var_7211_keep_dims_0, x = var_7209_cast_fp16)[name = string("op_7211_cast_fp16")];
            tensor<int32, [3]> var_7214_begin_0 = const()[name = string("op_7214_begin_0"), val = tensor<int32, [3]>([0, 0, 6])];
            tensor<int32, [3]> var_7214_end_0 = const()[name = string("op_7214_end_0"), val = tensor<int32, [3]>([1, 16, 7])];
            tensor<bool, [3]> var_7214_end_mask_0 = const()[name = string("op_7214_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_7214_squeeze_mask_0 = const()[name = string("op_7214_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_7214_cast_fp16 = slice_by_index(begin = var_7214_begin_0, end = var_7214_end_0, end_mask = var_7214_end_mask_0, squeeze_mask = var_7214_squeeze_mask_0, x = decay_17_cast_fp16)[name = string("op_7214_cast_fp16")];
            tensor<fp16, [1, 16]> var_7215_cast_fp16 = exp(x = var_7214_cast_fp16)[name = string("op_7215_cast_fp16")];
            tensor<int32, [1]> var_7216_axes_0 = const()[name = string("op_7216_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_7216_cast_fp16 = expand_dims(axes = var_7216_axes_0, x = var_7215_cast_fp16)[name = string("op_7216_cast_fp16")];
            tensor<int32, [1]> var_7217_axes_0 = const()[name = string("op_7217_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_7217_cast_fp16 = expand_dims(axes = var_7217_axes_0, x = var_7216_cast_fp16)[name = string("op_7217_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_537_cast_fp16 = mul(x = state_535_cast_fp16, y = var_7217_cast_fp16)[name = string("state_537_cast_fp16")];
            tensor<int32, [4]> key_token_269_begin_0 = const()[name = string("key_token_269_begin_0"), val = tensor<int32, [4]>([0, 0, 6, 0])];
            tensor<int32, [4]> key_token_269_end_0 = const()[name = string("key_token_269_end_0"), val = tensor<int32, [4]>([1, 16, 7, 128])];
            tensor<bool, [4]> key_token_269_end_mask_0 = const()[name = string("key_token_269_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_269_squeeze_mask_0 = const()[name = string("key_token_269_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_269_cast_fp16 = slice_by_index(begin = key_token_269_begin_0, end = key_token_269_end_0, end_mask = key_token_269_end_mask_0, squeeze_mask = key_token_269_squeeze_mask_0, x = key_43_cast_fp16)[name = string("key_token_269_cast_fp16")];
            tensor<int32, [4]> value_token_269_begin_0 = const()[name = string("value_token_269_begin_0"), val = tensor<int32, [4]>([0, 0, 6, 0])];
            tensor<int32, [4]> value_token_269_end_0 = const()[name = string("value_token_269_end_0"), val = tensor<int32, [4]>([1, 16, 7, 128])];
            tensor<bool, [4]> value_token_269_end_mask_0 = const()[name = string("value_token_269_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_269_squeeze_mask_0 = const()[name = string("value_token_269_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_269_cast_fp16 = slice_by_index(begin = value_token_269_begin_0, end = value_token_269_end_0, end_mask = value_token_269_end_mask_0, squeeze_mask = value_token_269_squeeze_mask_0, x = value_155_cast_fp16)[name = string("value_token_269_cast_fp16")];
            tensor<int32, [1]> var_7225_axes_0 = const()[name = string("op_7225_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_7225_cast_fp16 = expand_dims(axes = var_7225_axes_0, x = key_token_269_cast_fp16)[name = string("op_7225_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_7226_cast_fp16 = mul(x = state_537_cast_fp16, y = var_7225_cast_fp16)[name = string("op_7226_cast_fp16")];
            tensor<int32, [1]> memory_269_axes_0 = const()[name = string("memory_269_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_269_keep_dims_0 = const()[name = string("memory_269_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_269_cast_fp16 = reduce_sum(axes = memory_269_axes_0, keep_dims = memory_269_keep_dims_0, x = var_7226_cast_fp16)[name = string("memory_269_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_7229_cast_fp16 = sub(x = value_token_269_cast_fp16, y = memory_269_cast_fp16)[name = string("op_7229_cast_fp16")];
            tensor<int32, [3]> var_7232_begin_0 = const()[name = string("op_7232_begin_0"), val = tensor<int32, [3]>([0, 0, 6])];
            tensor<int32, [3]> var_7232_end_0 = const()[name = string("op_7232_end_0"), val = tensor<int32, [3]>([1, 16, 7])];
            tensor<bool, [3]> var_7232_end_mask_0 = const()[name = string("op_7232_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_7232_squeeze_mask_0 = const()[name = string("op_7232_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_7232_cast_fp16 = slice_by_index(begin = var_7232_begin_0, end = var_7232_end_0, end_mask = var_7232_end_mask_0, squeeze_mask = var_7232_squeeze_mask_0, x = beta_17_cast_fp16)[name = string("op_7232_cast_fp16")];
            tensor<int32, [1]> var_7233_axes_0 = const()[name = string("op_7233_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_7233_cast_fp16 = expand_dims(axes = var_7233_axes_0, x = var_7232_cast_fp16)[name = string("op_7233_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_269_cast_fp16 = mul(x = var_7229_cast_fp16, y = var_7233_cast_fp16)[name = string("delta_269_cast_fp16")];
            tensor<int32, [1]> var_7236_axes_0 = const()[name = string("op_7236_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_7236_cast_fp16 = expand_dims(axes = var_7236_axes_0, x = delta_269_cast_fp16)[name = string("op_7236_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_7237_cast_fp16 = mul(x = var_7225_cast_fp16, y = var_7236_cast_fp16)[name = string("op_7237_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_539_cast_fp16 = add(x = state_537_cast_fp16, y = var_7237_cast_fp16)[name = string("state_539_cast_fp16")];
            tensor<int32, [4]> var_7241_begin_0 = const()[name = string("op_7241_begin_0"), val = tensor<int32, [4]>([0, 0, 6, 0])];
            tensor<int32, [4]> var_7241_end_0 = const()[name = string("op_7241_end_0"), val = tensor<int32, [4]>([1, 16, 7, 128])];
            tensor<bool, [4]> var_7241_end_mask_0 = const()[name = string("op_7241_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_7241_squeeze_mask_0 = const()[name = string("op_7241_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_7241_cast_fp16 = slice_by_index(begin = var_7241_begin_0, end = var_7241_end_0, end_mask = var_7241_end_mask_0, squeeze_mask = var_7241_squeeze_mask_0, x = _inversed_query_43_cast_fp16)[name = string("op_7241_cast_fp16")];
            tensor<int32, [1]> var_7242_axes_0 = const()[name = string("op_7242_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_7242_cast_fp16 = expand_dims(axes = var_7242_axes_0, x = var_7241_cast_fp16)[name = string("op_7242_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_7243_cast_fp16 = mul(x = state_539_cast_fp16, y = var_7242_cast_fp16)[name = string("op_7243_cast_fp16")];
            tensor<int32, [1]> var_7245_axes_0 = const()[name = string("op_7245_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_7245_keep_dims_0 = const()[name = string("op_7245_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_7245_cast_fp16 = reduce_sum(axes = var_7245_axes_0, keep_dims = var_7245_keep_dims_0, x = var_7243_cast_fp16)[name = string("op_7245_cast_fp16")];
            tensor<int32, [3]> var_7248_begin_0 = const()[name = string("op_7248_begin_0"), val = tensor<int32, [3]>([0, 0, 7])];
            tensor<int32, [3]> var_7248_end_0 = const()[name = string("op_7248_end_0"), val = tensor<int32, [3]>([1, 16, 8])];
            tensor<bool, [3]> var_7248_end_mask_0 = const()[name = string("op_7248_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_7248_squeeze_mask_0 = const()[name = string("op_7248_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_7248_cast_fp16 = slice_by_index(begin = var_7248_begin_0, end = var_7248_end_0, end_mask = var_7248_end_mask_0, squeeze_mask = var_7248_squeeze_mask_0, x = decay_17_cast_fp16)[name = string("op_7248_cast_fp16")];
            tensor<fp16, [1, 16]> var_7249_cast_fp16 = exp(x = var_7248_cast_fp16)[name = string("op_7249_cast_fp16")];
            tensor<int32, [1]> var_7250_axes_0 = const()[name = string("op_7250_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_7250_cast_fp16 = expand_dims(axes = var_7250_axes_0, x = var_7249_cast_fp16)[name = string("op_7250_cast_fp16")];
            tensor<int32, [1]> var_7251_axes_0 = const()[name = string("op_7251_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_7251_cast_fp16 = expand_dims(axes = var_7251_axes_0, x = var_7250_cast_fp16)[name = string("op_7251_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_541_cast_fp16 = mul(x = state_539_cast_fp16, y = var_7251_cast_fp16)[name = string("state_541_cast_fp16")];
            tensor<int32, [4]> key_token_271_begin_0 = const()[name = string("key_token_271_begin_0"), val = tensor<int32, [4]>([0, 0, 7, 0])];
            tensor<int32, [4]> key_token_271_end_0 = const()[name = string("key_token_271_end_0"), val = tensor<int32, [4]>([1, 16, 8, 128])];
            tensor<bool, [4]> key_token_271_end_mask_0 = const()[name = string("key_token_271_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_271_squeeze_mask_0 = const()[name = string("key_token_271_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_271_cast_fp16 = slice_by_index(begin = key_token_271_begin_0, end = key_token_271_end_0, end_mask = key_token_271_end_mask_0, squeeze_mask = key_token_271_squeeze_mask_0, x = key_43_cast_fp16)[name = string("key_token_271_cast_fp16")];
            tensor<int32, [4]> value_token_271_begin_0 = const()[name = string("value_token_271_begin_0"), val = tensor<int32, [4]>([0, 0, 7, 0])];
            tensor<int32, [4]> value_token_271_end_0 = const()[name = string("value_token_271_end_0"), val = tensor<int32, [4]>([1, 16, 8, 128])];
            tensor<bool, [4]> value_token_271_end_mask_0 = const()[name = string("value_token_271_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_271_squeeze_mask_0 = const()[name = string("value_token_271_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_271_cast_fp16 = slice_by_index(begin = value_token_271_begin_0, end = value_token_271_end_0, end_mask = value_token_271_end_mask_0, squeeze_mask = value_token_271_squeeze_mask_0, x = value_155_cast_fp16)[name = string("value_token_271_cast_fp16")];
            tensor<int32, [1]> var_7259_axes_0 = const()[name = string("op_7259_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_7259_cast_fp16 = expand_dims(axes = var_7259_axes_0, x = key_token_271_cast_fp16)[name = string("op_7259_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_7260_cast_fp16 = mul(x = state_541_cast_fp16, y = var_7259_cast_fp16)[name = string("op_7260_cast_fp16")];
            tensor<int32, [1]> memory_271_axes_0 = const()[name = string("memory_271_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_271_keep_dims_0 = const()[name = string("memory_271_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_271_cast_fp16 = reduce_sum(axes = memory_271_axes_0, keep_dims = memory_271_keep_dims_0, x = var_7260_cast_fp16)[name = string("memory_271_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_7263_cast_fp16 = sub(x = value_token_271_cast_fp16, y = memory_271_cast_fp16)[name = string("op_7263_cast_fp16")];
            tensor<int32, [3]> var_7266_begin_0 = const()[name = string("op_7266_begin_0"), val = tensor<int32, [3]>([0, 0, 7])];
            tensor<int32, [3]> var_7266_end_0 = const()[name = string("op_7266_end_0"), val = tensor<int32, [3]>([1, 16, 8])];
            tensor<bool, [3]> var_7266_end_mask_0 = const()[name = string("op_7266_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_7266_squeeze_mask_0 = const()[name = string("op_7266_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_7266_cast_fp16 = slice_by_index(begin = var_7266_begin_0, end = var_7266_end_0, end_mask = var_7266_end_mask_0, squeeze_mask = var_7266_squeeze_mask_0, x = beta_17_cast_fp16)[name = string("op_7266_cast_fp16")];
            tensor<int32, [1]> var_7267_axes_0 = const()[name = string("op_7267_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_7267_cast_fp16 = expand_dims(axes = var_7267_axes_0, x = var_7266_cast_fp16)[name = string("op_7267_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_271_cast_fp16 = mul(x = var_7263_cast_fp16, y = var_7267_cast_fp16)[name = string("delta_271_cast_fp16")];
            tensor<int32, [1]> var_7270_axes_0 = const()[name = string("op_7270_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_7270_cast_fp16 = expand_dims(axes = var_7270_axes_0, x = delta_271_cast_fp16)[name = string("op_7270_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_7271_cast_fp16 = mul(x = var_7259_cast_fp16, y = var_7270_cast_fp16)[name = string("op_7271_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_543_cast_fp16 = add(x = state_541_cast_fp16, y = var_7271_cast_fp16)[name = string("state_543_cast_fp16")];
            tensor<int32, [4]> var_7275_begin_0 = const()[name = string("op_7275_begin_0"), val = tensor<int32, [4]>([0, 0, 7, 0])];
            tensor<int32, [4]> var_7275_end_0 = const()[name = string("op_7275_end_0"), val = tensor<int32, [4]>([1, 16, 8, 128])];
            tensor<bool, [4]> var_7275_end_mask_0 = const()[name = string("op_7275_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_7275_squeeze_mask_0 = const()[name = string("op_7275_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_7275_cast_fp16 = slice_by_index(begin = var_7275_begin_0, end = var_7275_end_0, end_mask = var_7275_end_mask_0, squeeze_mask = var_7275_squeeze_mask_0, x = _inversed_query_43_cast_fp16)[name = string("op_7275_cast_fp16")];
            tensor<int32, [1]> var_7276_axes_0 = const()[name = string("op_7276_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_7276_cast_fp16 = expand_dims(axes = var_7276_axes_0, x = var_7275_cast_fp16)[name = string("op_7276_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_7277_cast_fp16 = mul(x = state_543_cast_fp16, y = var_7276_cast_fp16)[name = string("op_7277_cast_fp16")];
            tensor<int32, [1]> var_7279_axes_0 = const()[name = string("op_7279_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_7279_keep_dims_0 = const()[name = string("op_7279_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_7279_cast_fp16 = reduce_sum(axes = var_7279_axes_0, keep_dims = var_7279_keep_dims_0, x = var_7277_cast_fp16)[name = string("op_7279_cast_fp16")];
            tensor<int32, [3]> var_7282_begin_0 = const()[name = string("op_7282_begin_0"), val = tensor<int32, [3]>([0, 0, 8])];
            tensor<int32, [3]> var_7282_end_0 = const()[name = string("op_7282_end_0"), val = tensor<int32, [3]>([1, 16, 9])];
            tensor<bool, [3]> var_7282_end_mask_0 = const()[name = string("op_7282_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_7282_squeeze_mask_0 = const()[name = string("op_7282_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_7282_cast_fp16 = slice_by_index(begin = var_7282_begin_0, end = var_7282_end_0, end_mask = var_7282_end_mask_0, squeeze_mask = var_7282_squeeze_mask_0, x = decay_17_cast_fp16)[name = string("op_7282_cast_fp16")];
            tensor<fp16, [1, 16]> var_7283_cast_fp16 = exp(x = var_7282_cast_fp16)[name = string("op_7283_cast_fp16")];
            tensor<int32, [1]> var_7284_axes_0 = const()[name = string("op_7284_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_7284_cast_fp16 = expand_dims(axes = var_7284_axes_0, x = var_7283_cast_fp16)[name = string("op_7284_cast_fp16")];
            tensor<int32, [1]> var_7285_axes_0 = const()[name = string("op_7285_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_7285_cast_fp16 = expand_dims(axes = var_7285_axes_0, x = var_7284_cast_fp16)[name = string("op_7285_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_545_cast_fp16 = mul(x = state_543_cast_fp16, y = var_7285_cast_fp16)[name = string("state_545_cast_fp16")];
            tensor<int32, [4]> key_token_273_begin_0 = const()[name = string("key_token_273_begin_0"), val = tensor<int32, [4]>([0, 0, 8, 0])];
            tensor<int32, [4]> key_token_273_end_0 = const()[name = string("key_token_273_end_0"), val = tensor<int32, [4]>([1, 16, 9, 128])];
            tensor<bool, [4]> key_token_273_end_mask_0 = const()[name = string("key_token_273_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_273_squeeze_mask_0 = const()[name = string("key_token_273_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_273_cast_fp16 = slice_by_index(begin = key_token_273_begin_0, end = key_token_273_end_0, end_mask = key_token_273_end_mask_0, squeeze_mask = key_token_273_squeeze_mask_0, x = key_43_cast_fp16)[name = string("key_token_273_cast_fp16")];
            tensor<int32, [4]> value_token_273_begin_0 = const()[name = string("value_token_273_begin_0"), val = tensor<int32, [4]>([0, 0, 8, 0])];
            tensor<int32, [4]> value_token_273_end_0 = const()[name = string("value_token_273_end_0"), val = tensor<int32, [4]>([1, 16, 9, 128])];
            tensor<bool, [4]> value_token_273_end_mask_0 = const()[name = string("value_token_273_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_273_squeeze_mask_0 = const()[name = string("value_token_273_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_273_cast_fp16 = slice_by_index(begin = value_token_273_begin_0, end = value_token_273_end_0, end_mask = value_token_273_end_mask_0, squeeze_mask = value_token_273_squeeze_mask_0, x = value_155_cast_fp16)[name = string("value_token_273_cast_fp16")];
            tensor<int32, [1]> var_7293_axes_0 = const()[name = string("op_7293_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_7293_cast_fp16 = expand_dims(axes = var_7293_axes_0, x = key_token_273_cast_fp16)[name = string("op_7293_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_7294_cast_fp16 = mul(x = state_545_cast_fp16, y = var_7293_cast_fp16)[name = string("op_7294_cast_fp16")];
            tensor<int32, [1]> memory_273_axes_0 = const()[name = string("memory_273_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_273_keep_dims_0 = const()[name = string("memory_273_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_273_cast_fp16 = reduce_sum(axes = memory_273_axes_0, keep_dims = memory_273_keep_dims_0, x = var_7294_cast_fp16)[name = string("memory_273_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_7297_cast_fp16 = sub(x = value_token_273_cast_fp16, y = memory_273_cast_fp16)[name = string("op_7297_cast_fp16")];
            tensor<int32, [3]> var_7300_begin_0 = const()[name = string("op_7300_begin_0"), val = tensor<int32, [3]>([0, 0, 8])];
            tensor<int32, [3]> var_7300_end_0 = const()[name = string("op_7300_end_0"), val = tensor<int32, [3]>([1, 16, 9])];
            tensor<bool, [3]> var_7300_end_mask_0 = const()[name = string("op_7300_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_7300_squeeze_mask_0 = const()[name = string("op_7300_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_7300_cast_fp16 = slice_by_index(begin = var_7300_begin_0, end = var_7300_end_0, end_mask = var_7300_end_mask_0, squeeze_mask = var_7300_squeeze_mask_0, x = beta_17_cast_fp16)[name = string("op_7300_cast_fp16")];
            tensor<int32, [1]> var_7301_axes_0 = const()[name = string("op_7301_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_7301_cast_fp16 = expand_dims(axes = var_7301_axes_0, x = var_7300_cast_fp16)[name = string("op_7301_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_273_cast_fp16 = mul(x = var_7297_cast_fp16, y = var_7301_cast_fp16)[name = string("delta_273_cast_fp16")];
            tensor<int32, [1]> var_7304_axes_0 = const()[name = string("op_7304_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_7304_cast_fp16 = expand_dims(axes = var_7304_axes_0, x = delta_273_cast_fp16)[name = string("op_7304_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_7305_cast_fp16 = mul(x = var_7293_cast_fp16, y = var_7304_cast_fp16)[name = string("op_7305_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_547_cast_fp16 = add(x = state_545_cast_fp16, y = var_7305_cast_fp16)[name = string("state_547_cast_fp16")];
            tensor<int32, [4]> var_7309_begin_0 = const()[name = string("op_7309_begin_0"), val = tensor<int32, [4]>([0, 0, 8, 0])];
            tensor<int32, [4]> var_7309_end_0 = const()[name = string("op_7309_end_0"), val = tensor<int32, [4]>([1, 16, 9, 128])];
            tensor<bool, [4]> var_7309_end_mask_0 = const()[name = string("op_7309_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_7309_squeeze_mask_0 = const()[name = string("op_7309_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_7309_cast_fp16 = slice_by_index(begin = var_7309_begin_0, end = var_7309_end_0, end_mask = var_7309_end_mask_0, squeeze_mask = var_7309_squeeze_mask_0, x = _inversed_query_43_cast_fp16)[name = string("op_7309_cast_fp16")];
            tensor<int32, [1]> var_7310_axes_0 = const()[name = string("op_7310_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_7310_cast_fp16 = expand_dims(axes = var_7310_axes_0, x = var_7309_cast_fp16)[name = string("op_7310_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_7311_cast_fp16 = mul(x = state_547_cast_fp16, y = var_7310_cast_fp16)[name = string("op_7311_cast_fp16")];
            tensor<int32, [1]> var_7313_axes_0 = const()[name = string("op_7313_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_7313_keep_dims_0 = const()[name = string("op_7313_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_7313_cast_fp16 = reduce_sum(axes = var_7313_axes_0, keep_dims = var_7313_keep_dims_0, x = var_7311_cast_fp16)[name = string("op_7313_cast_fp16")];
            tensor<int32, [3]> var_7316_begin_0 = const()[name = string("op_7316_begin_0"), val = tensor<int32, [3]>([0, 0, 9])];
            tensor<int32, [3]> var_7316_end_0 = const()[name = string("op_7316_end_0"), val = tensor<int32, [3]>([1, 16, 10])];
            tensor<bool, [3]> var_7316_end_mask_0 = const()[name = string("op_7316_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_7316_squeeze_mask_0 = const()[name = string("op_7316_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_7316_cast_fp16 = slice_by_index(begin = var_7316_begin_0, end = var_7316_end_0, end_mask = var_7316_end_mask_0, squeeze_mask = var_7316_squeeze_mask_0, x = decay_17_cast_fp16)[name = string("op_7316_cast_fp16")];
            tensor<fp16, [1, 16]> var_7317_cast_fp16 = exp(x = var_7316_cast_fp16)[name = string("op_7317_cast_fp16")];
            tensor<int32, [1]> var_7318_axes_0 = const()[name = string("op_7318_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_7318_cast_fp16 = expand_dims(axes = var_7318_axes_0, x = var_7317_cast_fp16)[name = string("op_7318_cast_fp16")];
            tensor<int32, [1]> var_7319_axes_0 = const()[name = string("op_7319_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_7319_cast_fp16 = expand_dims(axes = var_7319_axes_0, x = var_7318_cast_fp16)[name = string("op_7319_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_549_cast_fp16 = mul(x = state_547_cast_fp16, y = var_7319_cast_fp16)[name = string("state_549_cast_fp16")];
            tensor<int32, [4]> key_token_275_begin_0 = const()[name = string("key_token_275_begin_0"), val = tensor<int32, [4]>([0, 0, 9, 0])];
            tensor<int32, [4]> key_token_275_end_0 = const()[name = string("key_token_275_end_0"), val = tensor<int32, [4]>([1, 16, 10, 128])];
            tensor<bool, [4]> key_token_275_end_mask_0 = const()[name = string("key_token_275_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_275_squeeze_mask_0 = const()[name = string("key_token_275_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_275_cast_fp16 = slice_by_index(begin = key_token_275_begin_0, end = key_token_275_end_0, end_mask = key_token_275_end_mask_0, squeeze_mask = key_token_275_squeeze_mask_0, x = key_43_cast_fp16)[name = string("key_token_275_cast_fp16")];
            tensor<int32, [4]> value_token_275_begin_0 = const()[name = string("value_token_275_begin_0"), val = tensor<int32, [4]>([0, 0, 9, 0])];
            tensor<int32, [4]> value_token_275_end_0 = const()[name = string("value_token_275_end_0"), val = tensor<int32, [4]>([1, 16, 10, 128])];
            tensor<bool, [4]> value_token_275_end_mask_0 = const()[name = string("value_token_275_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_275_squeeze_mask_0 = const()[name = string("value_token_275_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_275_cast_fp16 = slice_by_index(begin = value_token_275_begin_0, end = value_token_275_end_0, end_mask = value_token_275_end_mask_0, squeeze_mask = value_token_275_squeeze_mask_0, x = value_155_cast_fp16)[name = string("value_token_275_cast_fp16")];
            tensor<int32, [1]> var_7327_axes_0 = const()[name = string("op_7327_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_7327_cast_fp16 = expand_dims(axes = var_7327_axes_0, x = key_token_275_cast_fp16)[name = string("op_7327_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_7328_cast_fp16 = mul(x = state_549_cast_fp16, y = var_7327_cast_fp16)[name = string("op_7328_cast_fp16")];
            tensor<int32, [1]> memory_275_axes_0 = const()[name = string("memory_275_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_275_keep_dims_0 = const()[name = string("memory_275_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_275_cast_fp16 = reduce_sum(axes = memory_275_axes_0, keep_dims = memory_275_keep_dims_0, x = var_7328_cast_fp16)[name = string("memory_275_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_7331_cast_fp16 = sub(x = value_token_275_cast_fp16, y = memory_275_cast_fp16)[name = string("op_7331_cast_fp16")];
            tensor<int32, [3]> var_7334_begin_0 = const()[name = string("op_7334_begin_0"), val = tensor<int32, [3]>([0, 0, 9])];
            tensor<int32, [3]> var_7334_end_0 = const()[name = string("op_7334_end_0"), val = tensor<int32, [3]>([1, 16, 10])];
            tensor<bool, [3]> var_7334_end_mask_0 = const()[name = string("op_7334_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_7334_squeeze_mask_0 = const()[name = string("op_7334_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_7334_cast_fp16 = slice_by_index(begin = var_7334_begin_0, end = var_7334_end_0, end_mask = var_7334_end_mask_0, squeeze_mask = var_7334_squeeze_mask_0, x = beta_17_cast_fp16)[name = string("op_7334_cast_fp16")];
            tensor<int32, [1]> var_7335_axes_0 = const()[name = string("op_7335_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_7335_cast_fp16 = expand_dims(axes = var_7335_axes_0, x = var_7334_cast_fp16)[name = string("op_7335_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_275_cast_fp16 = mul(x = var_7331_cast_fp16, y = var_7335_cast_fp16)[name = string("delta_275_cast_fp16")];
            tensor<int32, [1]> var_7338_axes_0 = const()[name = string("op_7338_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_7338_cast_fp16 = expand_dims(axes = var_7338_axes_0, x = delta_275_cast_fp16)[name = string("op_7338_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_7339_cast_fp16 = mul(x = var_7327_cast_fp16, y = var_7338_cast_fp16)[name = string("op_7339_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_551_cast_fp16 = add(x = state_549_cast_fp16, y = var_7339_cast_fp16)[name = string("state_551_cast_fp16")];
            tensor<int32, [4]> var_7343_begin_0 = const()[name = string("op_7343_begin_0"), val = tensor<int32, [4]>([0, 0, 9, 0])];
            tensor<int32, [4]> var_7343_end_0 = const()[name = string("op_7343_end_0"), val = tensor<int32, [4]>([1, 16, 10, 128])];
            tensor<bool, [4]> var_7343_end_mask_0 = const()[name = string("op_7343_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_7343_squeeze_mask_0 = const()[name = string("op_7343_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_7343_cast_fp16 = slice_by_index(begin = var_7343_begin_0, end = var_7343_end_0, end_mask = var_7343_end_mask_0, squeeze_mask = var_7343_squeeze_mask_0, x = _inversed_query_43_cast_fp16)[name = string("op_7343_cast_fp16")];
            tensor<int32, [1]> var_7344_axes_0 = const()[name = string("op_7344_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_7344_cast_fp16 = expand_dims(axes = var_7344_axes_0, x = var_7343_cast_fp16)[name = string("op_7344_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_7345_cast_fp16 = mul(x = state_551_cast_fp16, y = var_7344_cast_fp16)[name = string("op_7345_cast_fp16")];
            tensor<int32, [1]> var_7347_axes_0 = const()[name = string("op_7347_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_7347_keep_dims_0 = const()[name = string("op_7347_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_7347_cast_fp16 = reduce_sum(axes = var_7347_axes_0, keep_dims = var_7347_keep_dims_0, x = var_7345_cast_fp16)[name = string("op_7347_cast_fp16")];
            tensor<int32, [3]> var_7350_begin_0 = const()[name = string("op_7350_begin_0"), val = tensor<int32, [3]>([0, 0, 10])];
            tensor<int32, [3]> var_7350_end_0 = const()[name = string("op_7350_end_0"), val = tensor<int32, [3]>([1, 16, 11])];
            tensor<bool, [3]> var_7350_end_mask_0 = const()[name = string("op_7350_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_7350_squeeze_mask_0 = const()[name = string("op_7350_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_7350_cast_fp16 = slice_by_index(begin = var_7350_begin_0, end = var_7350_end_0, end_mask = var_7350_end_mask_0, squeeze_mask = var_7350_squeeze_mask_0, x = decay_17_cast_fp16)[name = string("op_7350_cast_fp16")];
            tensor<fp16, [1, 16]> var_7351_cast_fp16 = exp(x = var_7350_cast_fp16)[name = string("op_7351_cast_fp16")];
            tensor<int32, [1]> var_7352_axes_0 = const()[name = string("op_7352_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_7352_cast_fp16 = expand_dims(axes = var_7352_axes_0, x = var_7351_cast_fp16)[name = string("op_7352_cast_fp16")];
            tensor<int32, [1]> var_7353_axes_0 = const()[name = string("op_7353_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_7353_cast_fp16 = expand_dims(axes = var_7353_axes_0, x = var_7352_cast_fp16)[name = string("op_7353_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_553_cast_fp16 = mul(x = state_551_cast_fp16, y = var_7353_cast_fp16)[name = string("state_553_cast_fp16")];
            tensor<int32, [4]> key_token_277_begin_0 = const()[name = string("key_token_277_begin_0"), val = tensor<int32, [4]>([0, 0, 10, 0])];
            tensor<int32, [4]> key_token_277_end_0 = const()[name = string("key_token_277_end_0"), val = tensor<int32, [4]>([1, 16, 11, 128])];
            tensor<bool, [4]> key_token_277_end_mask_0 = const()[name = string("key_token_277_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_277_squeeze_mask_0 = const()[name = string("key_token_277_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_277_cast_fp16 = slice_by_index(begin = key_token_277_begin_0, end = key_token_277_end_0, end_mask = key_token_277_end_mask_0, squeeze_mask = key_token_277_squeeze_mask_0, x = key_43_cast_fp16)[name = string("key_token_277_cast_fp16")];
            tensor<int32, [4]> value_token_277_begin_0 = const()[name = string("value_token_277_begin_0"), val = tensor<int32, [4]>([0, 0, 10, 0])];
            tensor<int32, [4]> value_token_277_end_0 = const()[name = string("value_token_277_end_0"), val = tensor<int32, [4]>([1, 16, 11, 128])];
            tensor<bool, [4]> value_token_277_end_mask_0 = const()[name = string("value_token_277_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_277_squeeze_mask_0 = const()[name = string("value_token_277_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_277_cast_fp16 = slice_by_index(begin = value_token_277_begin_0, end = value_token_277_end_0, end_mask = value_token_277_end_mask_0, squeeze_mask = value_token_277_squeeze_mask_0, x = value_155_cast_fp16)[name = string("value_token_277_cast_fp16")];
            tensor<int32, [1]> var_7361_axes_0 = const()[name = string("op_7361_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_7361_cast_fp16 = expand_dims(axes = var_7361_axes_0, x = key_token_277_cast_fp16)[name = string("op_7361_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_7362_cast_fp16 = mul(x = state_553_cast_fp16, y = var_7361_cast_fp16)[name = string("op_7362_cast_fp16")];
            tensor<int32, [1]> memory_277_axes_0 = const()[name = string("memory_277_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_277_keep_dims_0 = const()[name = string("memory_277_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_277_cast_fp16 = reduce_sum(axes = memory_277_axes_0, keep_dims = memory_277_keep_dims_0, x = var_7362_cast_fp16)[name = string("memory_277_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_7365_cast_fp16 = sub(x = value_token_277_cast_fp16, y = memory_277_cast_fp16)[name = string("op_7365_cast_fp16")];
            tensor<int32, [3]> var_7368_begin_0 = const()[name = string("op_7368_begin_0"), val = tensor<int32, [3]>([0, 0, 10])];
            tensor<int32, [3]> var_7368_end_0 = const()[name = string("op_7368_end_0"), val = tensor<int32, [3]>([1, 16, 11])];
            tensor<bool, [3]> var_7368_end_mask_0 = const()[name = string("op_7368_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_7368_squeeze_mask_0 = const()[name = string("op_7368_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_7368_cast_fp16 = slice_by_index(begin = var_7368_begin_0, end = var_7368_end_0, end_mask = var_7368_end_mask_0, squeeze_mask = var_7368_squeeze_mask_0, x = beta_17_cast_fp16)[name = string("op_7368_cast_fp16")];
            tensor<int32, [1]> var_7369_axes_0 = const()[name = string("op_7369_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_7369_cast_fp16 = expand_dims(axes = var_7369_axes_0, x = var_7368_cast_fp16)[name = string("op_7369_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_277_cast_fp16 = mul(x = var_7365_cast_fp16, y = var_7369_cast_fp16)[name = string("delta_277_cast_fp16")];
            tensor<int32, [1]> var_7372_axes_0 = const()[name = string("op_7372_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_7372_cast_fp16 = expand_dims(axes = var_7372_axes_0, x = delta_277_cast_fp16)[name = string("op_7372_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_7373_cast_fp16 = mul(x = var_7361_cast_fp16, y = var_7372_cast_fp16)[name = string("op_7373_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_555_cast_fp16 = add(x = state_553_cast_fp16, y = var_7373_cast_fp16)[name = string("state_555_cast_fp16")];
            tensor<int32, [4]> var_7377_begin_0 = const()[name = string("op_7377_begin_0"), val = tensor<int32, [4]>([0, 0, 10, 0])];
            tensor<int32, [4]> var_7377_end_0 = const()[name = string("op_7377_end_0"), val = tensor<int32, [4]>([1, 16, 11, 128])];
            tensor<bool, [4]> var_7377_end_mask_0 = const()[name = string("op_7377_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_7377_squeeze_mask_0 = const()[name = string("op_7377_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_7377_cast_fp16 = slice_by_index(begin = var_7377_begin_0, end = var_7377_end_0, end_mask = var_7377_end_mask_0, squeeze_mask = var_7377_squeeze_mask_0, x = _inversed_query_43_cast_fp16)[name = string("op_7377_cast_fp16")];
            tensor<int32, [1]> var_7378_axes_0 = const()[name = string("op_7378_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_7378_cast_fp16 = expand_dims(axes = var_7378_axes_0, x = var_7377_cast_fp16)[name = string("op_7378_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_7379_cast_fp16 = mul(x = state_555_cast_fp16, y = var_7378_cast_fp16)[name = string("op_7379_cast_fp16")];
            tensor<int32, [1]> var_7381_axes_0 = const()[name = string("op_7381_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_7381_keep_dims_0 = const()[name = string("op_7381_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_7381_cast_fp16 = reduce_sum(axes = var_7381_axes_0, keep_dims = var_7381_keep_dims_0, x = var_7379_cast_fp16)[name = string("op_7381_cast_fp16")];
            tensor<int32, [3]> var_7384_begin_0 = const()[name = string("op_7384_begin_0"), val = tensor<int32, [3]>([0, 0, 11])];
            tensor<int32, [3]> var_7384_end_0 = const()[name = string("op_7384_end_0"), val = tensor<int32, [3]>([1, 16, 12])];
            tensor<bool, [3]> var_7384_end_mask_0 = const()[name = string("op_7384_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_7384_squeeze_mask_0 = const()[name = string("op_7384_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_7384_cast_fp16 = slice_by_index(begin = var_7384_begin_0, end = var_7384_end_0, end_mask = var_7384_end_mask_0, squeeze_mask = var_7384_squeeze_mask_0, x = decay_17_cast_fp16)[name = string("op_7384_cast_fp16")];
            tensor<fp16, [1, 16]> var_7385_cast_fp16 = exp(x = var_7384_cast_fp16)[name = string("op_7385_cast_fp16")];
            tensor<int32, [1]> var_7386_axes_0 = const()[name = string("op_7386_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_7386_cast_fp16 = expand_dims(axes = var_7386_axes_0, x = var_7385_cast_fp16)[name = string("op_7386_cast_fp16")];
            tensor<int32, [1]> var_7387_axes_0 = const()[name = string("op_7387_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_7387_cast_fp16 = expand_dims(axes = var_7387_axes_0, x = var_7386_cast_fp16)[name = string("op_7387_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_557_cast_fp16 = mul(x = state_555_cast_fp16, y = var_7387_cast_fp16)[name = string("state_557_cast_fp16")];
            tensor<int32, [4]> key_token_279_begin_0 = const()[name = string("key_token_279_begin_0"), val = tensor<int32, [4]>([0, 0, 11, 0])];
            tensor<int32, [4]> key_token_279_end_0 = const()[name = string("key_token_279_end_0"), val = tensor<int32, [4]>([1, 16, 12, 128])];
            tensor<bool, [4]> key_token_279_end_mask_0 = const()[name = string("key_token_279_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_279_squeeze_mask_0 = const()[name = string("key_token_279_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_279_cast_fp16 = slice_by_index(begin = key_token_279_begin_0, end = key_token_279_end_0, end_mask = key_token_279_end_mask_0, squeeze_mask = key_token_279_squeeze_mask_0, x = key_43_cast_fp16)[name = string("key_token_279_cast_fp16")];
            tensor<int32, [4]> value_token_279_begin_0 = const()[name = string("value_token_279_begin_0"), val = tensor<int32, [4]>([0, 0, 11, 0])];
            tensor<int32, [4]> value_token_279_end_0 = const()[name = string("value_token_279_end_0"), val = tensor<int32, [4]>([1, 16, 12, 128])];
            tensor<bool, [4]> value_token_279_end_mask_0 = const()[name = string("value_token_279_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_279_squeeze_mask_0 = const()[name = string("value_token_279_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_279_cast_fp16 = slice_by_index(begin = value_token_279_begin_0, end = value_token_279_end_0, end_mask = value_token_279_end_mask_0, squeeze_mask = value_token_279_squeeze_mask_0, x = value_155_cast_fp16)[name = string("value_token_279_cast_fp16")];
            tensor<int32, [1]> var_7395_axes_0 = const()[name = string("op_7395_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_7395_cast_fp16 = expand_dims(axes = var_7395_axes_0, x = key_token_279_cast_fp16)[name = string("op_7395_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_7396_cast_fp16 = mul(x = state_557_cast_fp16, y = var_7395_cast_fp16)[name = string("op_7396_cast_fp16")];
            tensor<int32, [1]> memory_279_axes_0 = const()[name = string("memory_279_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_279_keep_dims_0 = const()[name = string("memory_279_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_279_cast_fp16 = reduce_sum(axes = memory_279_axes_0, keep_dims = memory_279_keep_dims_0, x = var_7396_cast_fp16)[name = string("memory_279_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_7399_cast_fp16 = sub(x = value_token_279_cast_fp16, y = memory_279_cast_fp16)[name = string("op_7399_cast_fp16")];
            tensor<int32, [3]> var_7402_begin_0 = const()[name = string("op_7402_begin_0"), val = tensor<int32, [3]>([0, 0, 11])];
            tensor<int32, [3]> var_7402_end_0 = const()[name = string("op_7402_end_0"), val = tensor<int32, [3]>([1, 16, 12])];
            tensor<bool, [3]> var_7402_end_mask_0 = const()[name = string("op_7402_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_7402_squeeze_mask_0 = const()[name = string("op_7402_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_7402_cast_fp16 = slice_by_index(begin = var_7402_begin_0, end = var_7402_end_0, end_mask = var_7402_end_mask_0, squeeze_mask = var_7402_squeeze_mask_0, x = beta_17_cast_fp16)[name = string("op_7402_cast_fp16")];
            tensor<int32, [1]> var_7403_axes_0 = const()[name = string("op_7403_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_7403_cast_fp16 = expand_dims(axes = var_7403_axes_0, x = var_7402_cast_fp16)[name = string("op_7403_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_279_cast_fp16 = mul(x = var_7399_cast_fp16, y = var_7403_cast_fp16)[name = string("delta_279_cast_fp16")];
            tensor<int32, [1]> var_7406_axes_0 = const()[name = string("op_7406_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_7406_cast_fp16 = expand_dims(axes = var_7406_axes_0, x = delta_279_cast_fp16)[name = string("op_7406_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_7407_cast_fp16 = mul(x = var_7395_cast_fp16, y = var_7406_cast_fp16)[name = string("op_7407_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_559_cast_fp16 = add(x = state_557_cast_fp16, y = var_7407_cast_fp16)[name = string("state_559_cast_fp16")];
            tensor<int32, [4]> var_7411_begin_0 = const()[name = string("op_7411_begin_0"), val = tensor<int32, [4]>([0, 0, 11, 0])];
            tensor<int32, [4]> var_7411_end_0 = const()[name = string("op_7411_end_0"), val = tensor<int32, [4]>([1, 16, 12, 128])];
            tensor<bool, [4]> var_7411_end_mask_0 = const()[name = string("op_7411_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_7411_squeeze_mask_0 = const()[name = string("op_7411_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_7411_cast_fp16 = slice_by_index(begin = var_7411_begin_0, end = var_7411_end_0, end_mask = var_7411_end_mask_0, squeeze_mask = var_7411_squeeze_mask_0, x = _inversed_query_43_cast_fp16)[name = string("op_7411_cast_fp16")];
            tensor<int32, [1]> var_7412_axes_0 = const()[name = string("op_7412_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_7412_cast_fp16 = expand_dims(axes = var_7412_axes_0, x = var_7411_cast_fp16)[name = string("op_7412_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_7413_cast_fp16 = mul(x = state_559_cast_fp16, y = var_7412_cast_fp16)[name = string("op_7413_cast_fp16")];
            tensor<int32, [1]> var_7415_axes_0 = const()[name = string("op_7415_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_7415_keep_dims_0 = const()[name = string("op_7415_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_7415_cast_fp16 = reduce_sum(axes = var_7415_axes_0, keep_dims = var_7415_keep_dims_0, x = var_7413_cast_fp16)[name = string("op_7415_cast_fp16")];
            tensor<int32, [3]> var_7418_begin_0 = const()[name = string("op_7418_begin_0"), val = tensor<int32, [3]>([0, 0, 12])];
            tensor<int32, [3]> var_7418_end_0 = const()[name = string("op_7418_end_0"), val = tensor<int32, [3]>([1, 16, 13])];
            tensor<bool, [3]> var_7418_end_mask_0 = const()[name = string("op_7418_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_7418_squeeze_mask_0 = const()[name = string("op_7418_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_7418_cast_fp16 = slice_by_index(begin = var_7418_begin_0, end = var_7418_end_0, end_mask = var_7418_end_mask_0, squeeze_mask = var_7418_squeeze_mask_0, x = decay_17_cast_fp16)[name = string("op_7418_cast_fp16")];
            tensor<fp16, [1, 16]> var_7419_cast_fp16 = exp(x = var_7418_cast_fp16)[name = string("op_7419_cast_fp16")];
            tensor<int32, [1]> var_7420_axes_0 = const()[name = string("op_7420_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_7420_cast_fp16 = expand_dims(axes = var_7420_axes_0, x = var_7419_cast_fp16)[name = string("op_7420_cast_fp16")];
            tensor<int32, [1]> var_7421_axes_0 = const()[name = string("op_7421_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_7421_cast_fp16 = expand_dims(axes = var_7421_axes_0, x = var_7420_cast_fp16)[name = string("op_7421_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_561_cast_fp16 = mul(x = state_559_cast_fp16, y = var_7421_cast_fp16)[name = string("state_561_cast_fp16")];
            tensor<int32, [4]> key_token_281_begin_0 = const()[name = string("key_token_281_begin_0"), val = tensor<int32, [4]>([0, 0, 12, 0])];
            tensor<int32, [4]> key_token_281_end_0 = const()[name = string("key_token_281_end_0"), val = tensor<int32, [4]>([1, 16, 13, 128])];
            tensor<bool, [4]> key_token_281_end_mask_0 = const()[name = string("key_token_281_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_281_squeeze_mask_0 = const()[name = string("key_token_281_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_281_cast_fp16 = slice_by_index(begin = key_token_281_begin_0, end = key_token_281_end_0, end_mask = key_token_281_end_mask_0, squeeze_mask = key_token_281_squeeze_mask_0, x = key_43_cast_fp16)[name = string("key_token_281_cast_fp16")];
            tensor<int32, [4]> value_token_281_begin_0 = const()[name = string("value_token_281_begin_0"), val = tensor<int32, [4]>([0, 0, 12, 0])];
            tensor<int32, [4]> value_token_281_end_0 = const()[name = string("value_token_281_end_0"), val = tensor<int32, [4]>([1, 16, 13, 128])];
            tensor<bool, [4]> value_token_281_end_mask_0 = const()[name = string("value_token_281_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_281_squeeze_mask_0 = const()[name = string("value_token_281_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_281_cast_fp16 = slice_by_index(begin = value_token_281_begin_0, end = value_token_281_end_0, end_mask = value_token_281_end_mask_0, squeeze_mask = value_token_281_squeeze_mask_0, x = value_155_cast_fp16)[name = string("value_token_281_cast_fp16")];
            tensor<int32, [1]> var_7429_axes_0 = const()[name = string("op_7429_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_7429_cast_fp16 = expand_dims(axes = var_7429_axes_0, x = key_token_281_cast_fp16)[name = string("op_7429_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_7430_cast_fp16 = mul(x = state_561_cast_fp16, y = var_7429_cast_fp16)[name = string("op_7430_cast_fp16")];
            tensor<int32, [1]> memory_281_axes_0 = const()[name = string("memory_281_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_281_keep_dims_0 = const()[name = string("memory_281_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_281_cast_fp16 = reduce_sum(axes = memory_281_axes_0, keep_dims = memory_281_keep_dims_0, x = var_7430_cast_fp16)[name = string("memory_281_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_7433_cast_fp16 = sub(x = value_token_281_cast_fp16, y = memory_281_cast_fp16)[name = string("op_7433_cast_fp16")];
            tensor<int32, [3]> var_7436_begin_0 = const()[name = string("op_7436_begin_0"), val = tensor<int32, [3]>([0, 0, 12])];
            tensor<int32, [3]> var_7436_end_0 = const()[name = string("op_7436_end_0"), val = tensor<int32, [3]>([1, 16, 13])];
            tensor<bool, [3]> var_7436_end_mask_0 = const()[name = string("op_7436_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_7436_squeeze_mask_0 = const()[name = string("op_7436_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_7436_cast_fp16 = slice_by_index(begin = var_7436_begin_0, end = var_7436_end_0, end_mask = var_7436_end_mask_0, squeeze_mask = var_7436_squeeze_mask_0, x = beta_17_cast_fp16)[name = string("op_7436_cast_fp16")];
            tensor<int32, [1]> var_7437_axes_0 = const()[name = string("op_7437_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_7437_cast_fp16 = expand_dims(axes = var_7437_axes_0, x = var_7436_cast_fp16)[name = string("op_7437_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_281_cast_fp16 = mul(x = var_7433_cast_fp16, y = var_7437_cast_fp16)[name = string("delta_281_cast_fp16")];
            tensor<int32, [1]> var_7440_axes_0 = const()[name = string("op_7440_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_7440_cast_fp16 = expand_dims(axes = var_7440_axes_0, x = delta_281_cast_fp16)[name = string("op_7440_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_7441_cast_fp16 = mul(x = var_7429_cast_fp16, y = var_7440_cast_fp16)[name = string("op_7441_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_563_cast_fp16 = add(x = state_561_cast_fp16, y = var_7441_cast_fp16)[name = string("state_563_cast_fp16")];
            tensor<int32, [4]> var_7445_begin_0 = const()[name = string("op_7445_begin_0"), val = tensor<int32, [4]>([0, 0, 12, 0])];
            tensor<int32, [4]> var_7445_end_0 = const()[name = string("op_7445_end_0"), val = tensor<int32, [4]>([1, 16, 13, 128])];
            tensor<bool, [4]> var_7445_end_mask_0 = const()[name = string("op_7445_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_7445_squeeze_mask_0 = const()[name = string("op_7445_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_7445_cast_fp16 = slice_by_index(begin = var_7445_begin_0, end = var_7445_end_0, end_mask = var_7445_end_mask_0, squeeze_mask = var_7445_squeeze_mask_0, x = _inversed_query_43_cast_fp16)[name = string("op_7445_cast_fp16")];
            tensor<int32, [1]> var_7446_axes_0 = const()[name = string("op_7446_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_7446_cast_fp16 = expand_dims(axes = var_7446_axes_0, x = var_7445_cast_fp16)[name = string("op_7446_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_7447_cast_fp16 = mul(x = state_563_cast_fp16, y = var_7446_cast_fp16)[name = string("op_7447_cast_fp16")];
            tensor<int32, [1]> var_7449_axes_0 = const()[name = string("op_7449_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_7449_keep_dims_0 = const()[name = string("op_7449_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_7449_cast_fp16 = reduce_sum(axes = var_7449_axes_0, keep_dims = var_7449_keep_dims_0, x = var_7447_cast_fp16)[name = string("op_7449_cast_fp16")];
            tensor<int32, [3]> var_7452_begin_0 = const()[name = string("op_7452_begin_0"), val = tensor<int32, [3]>([0, 0, 13])];
            tensor<int32, [3]> var_7452_end_0 = const()[name = string("op_7452_end_0"), val = tensor<int32, [3]>([1, 16, 14])];
            tensor<bool, [3]> var_7452_end_mask_0 = const()[name = string("op_7452_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_7452_squeeze_mask_0 = const()[name = string("op_7452_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_7452_cast_fp16 = slice_by_index(begin = var_7452_begin_0, end = var_7452_end_0, end_mask = var_7452_end_mask_0, squeeze_mask = var_7452_squeeze_mask_0, x = decay_17_cast_fp16)[name = string("op_7452_cast_fp16")];
            tensor<fp16, [1, 16]> var_7453_cast_fp16 = exp(x = var_7452_cast_fp16)[name = string("op_7453_cast_fp16")];
            tensor<int32, [1]> var_7454_axes_0 = const()[name = string("op_7454_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_7454_cast_fp16 = expand_dims(axes = var_7454_axes_0, x = var_7453_cast_fp16)[name = string("op_7454_cast_fp16")];
            tensor<int32, [1]> var_7455_axes_0 = const()[name = string("op_7455_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_7455_cast_fp16 = expand_dims(axes = var_7455_axes_0, x = var_7454_cast_fp16)[name = string("op_7455_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_565_cast_fp16 = mul(x = state_563_cast_fp16, y = var_7455_cast_fp16)[name = string("state_565_cast_fp16")];
            tensor<int32, [4]> key_token_283_begin_0 = const()[name = string("key_token_283_begin_0"), val = tensor<int32, [4]>([0, 0, 13, 0])];
            tensor<int32, [4]> key_token_283_end_0 = const()[name = string("key_token_283_end_0"), val = tensor<int32, [4]>([1, 16, 14, 128])];
            tensor<bool, [4]> key_token_283_end_mask_0 = const()[name = string("key_token_283_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_283_squeeze_mask_0 = const()[name = string("key_token_283_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_283_cast_fp16 = slice_by_index(begin = key_token_283_begin_0, end = key_token_283_end_0, end_mask = key_token_283_end_mask_0, squeeze_mask = key_token_283_squeeze_mask_0, x = key_43_cast_fp16)[name = string("key_token_283_cast_fp16")];
            tensor<int32, [4]> value_token_283_begin_0 = const()[name = string("value_token_283_begin_0"), val = tensor<int32, [4]>([0, 0, 13, 0])];
            tensor<int32, [4]> value_token_283_end_0 = const()[name = string("value_token_283_end_0"), val = tensor<int32, [4]>([1, 16, 14, 128])];
            tensor<bool, [4]> value_token_283_end_mask_0 = const()[name = string("value_token_283_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_283_squeeze_mask_0 = const()[name = string("value_token_283_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_283_cast_fp16 = slice_by_index(begin = value_token_283_begin_0, end = value_token_283_end_0, end_mask = value_token_283_end_mask_0, squeeze_mask = value_token_283_squeeze_mask_0, x = value_155_cast_fp16)[name = string("value_token_283_cast_fp16")];
            tensor<int32, [1]> var_7463_axes_0 = const()[name = string("op_7463_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_7463_cast_fp16 = expand_dims(axes = var_7463_axes_0, x = key_token_283_cast_fp16)[name = string("op_7463_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_7464_cast_fp16 = mul(x = state_565_cast_fp16, y = var_7463_cast_fp16)[name = string("op_7464_cast_fp16")];
            tensor<int32, [1]> memory_283_axes_0 = const()[name = string("memory_283_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_283_keep_dims_0 = const()[name = string("memory_283_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_283_cast_fp16 = reduce_sum(axes = memory_283_axes_0, keep_dims = memory_283_keep_dims_0, x = var_7464_cast_fp16)[name = string("memory_283_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_7467_cast_fp16 = sub(x = value_token_283_cast_fp16, y = memory_283_cast_fp16)[name = string("op_7467_cast_fp16")];
            tensor<int32, [3]> var_7470_begin_0 = const()[name = string("op_7470_begin_0"), val = tensor<int32, [3]>([0, 0, 13])];
            tensor<int32, [3]> var_7470_end_0 = const()[name = string("op_7470_end_0"), val = tensor<int32, [3]>([1, 16, 14])];
            tensor<bool, [3]> var_7470_end_mask_0 = const()[name = string("op_7470_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_7470_squeeze_mask_0 = const()[name = string("op_7470_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_7470_cast_fp16 = slice_by_index(begin = var_7470_begin_0, end = var_7470_end_0, end_mask = var_7470_end_mask_0, squeeze_mask = var_7470_squeeze_mask_0, x = beta_17_cast_fp16)[name = string("op_7470_cast_fp16")];
            tensor<int32, [1]> var_7471_axes_0 = const()[name = string("op_7471_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_7471_cast_fp16 = expand_dims(axes = var_7471_axes_0, x = var_7470_cast_fp16)[name = string("op_7471_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_283_cast_fp16 = mul(x = var_7467_cast_fp16, y = var_7471_cast_fp16)[name = string("delta_283_cast_fp16")];
            tensor<int32, [1]> var_7474_axes_0 = const()[name = string("op_7474_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_7474_cast_fp16 = expand_dims(axes = var_7474_axes_0, x = delta_283_cast_fp16)[name = string("op_7474_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_7475_cast_fp16 = mul(x = var_7463_cast_fp16, y = var_7474_cast_fp16)[name = string("op_7475_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_567_cast_fp16 = add(x = state_565_cast_fp16, y = var_7475_cast_fp16)[name = string("state_567_cast_fp16")];
            tensor<int32, [4]> var_7479_begin_0 = const()[name = string("op_7479_begin_0"), val = tensor<int32, [4]>([0, 0, 13, 0])];
            tensor<int32, [4]> var_7479_end_0 = const()[name = string("op_7479_end_0"), val = tensor<int32, [4]>([1, 16, 14, 128])];
            tensor<bool, [4]> var_7479_end_mask_0 = const()[name = string("op_7479_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_7479_squeeze_mask_0 = const()[name = string("op_7479_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_7479_cast_fp16 = slice_by_index(begin = var_7479_begin_0, end = var_7479_end_0, end_mask = var_7479_end_mask_0, squeeze_mask = var_7479_squeeze_mask_0, x = _inversed_query_43_cast_fp16)[name = string("op_7479_cast_fp16")];
            tensor<int32, [1]> var_7480_axes_0 = const()[name = string("op_7480_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_7480_cast_fp16 = expand_dims(axes = var_7480_axes_0, x = var_7479_cast_fp16)[name = string("op_7480_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_7481_cast_fp16 = mul(x = state_567_cast_fp16, y = var_7480_cast_fp16)[name = string("op_7481_cast_fp16")];
            tensor<int32, [1]> var_7483_axes_0 = const()[name = string("op_7483_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_7483_keep_dims_0 = const()[name = string("op_7483_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_7483_cast_fp16 = reduce_sum(axes = var_7483_axes_0, keep_dims = var_7483_keep_dims_0, x = var_7481_cast_fp16)[name = string("op_7483_cast_fp16")];
            tensor<int32, [3]> var_7486_begin_0 = const()[name = string("op_7486_begin_0"), val = tensor<int32, [3]>([0, 0, 14])];
            tensor<int32, [3]> var_7486_end_0 = const()[name = string("op_7486_end_0"), val = tensor<int32, [3]>([1, 16, 15])];
            tensor<bool, [3]> var_7486_end_mask_0 = const()[name = string("op_7486_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_7486_squeeze_mask_0 = const()[name = string("op_7486_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_7486_cast_fp16 = slice_by_index(begin = var_7486_begin_0, end = var_7486_end_0, end_mask = var_7486_end_mask_0, squeeze_mask = var_7486_squeeze_mask_0, x = decay_17_cast_fp16)[name = string("op_7486_cast_fp16")];
            tensor<fp16, [1, 16]> var_7487_cast_fp16 = exp(x = var_7486_cast_fp16)[name = string("op_7487_cast_fp16")];
            tensor<int32, [1]> var_7488_axes_0 = const()[name = string("op_7488_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_7488_cast_fp16 = expand_dims(axes = var_7488_axes_0, x = var_7487_cast_fp16)[name = string("op_7488_cast_fp16")];
            tensor<int32, [1]> var_7489_axes_0 = const()[name = string("op_7489_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_7489_cast_fp16 = expand_dims(axes = var_7489_axes_0, x = var_7488_cast_fp16)[name = string("op_7489_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_569_cast_fp16 = mul(x = state_567_cast_fp16, y = var_7489_cast_fp16)[name = string("state_569_cast_fp16")];
            tensor<int32, [4]> key_token_285_begin_0 = const()[name = string("key_token_285_begin_0"), val = tensor<int32, [4]>([0, 0, 14, 0])];
            tensor<int32, [4]> key_token_285_end_0 = const()[name = string("key_token_285_end_0"), val = tensor<int32, [4]>([1, 16, 15, 128])];
            tensor<bool, [4]> key_token_285_end_mask_0 = const()[name = string("key_token_285_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_285_squeeze_mask_0 = const()[name = string("key_token_285_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_285_cast_fp16 = slice_by_index(begin = key_token_285_begin_0, end = key_token_285_end_0, end_mask = key_token_285_end_mask_0, squeeze_mask = key_token_285_squeeze_mask_0, x = key_43_cast_fp16)[name = string("key_token_285_cast_fp16")];
            tensor<int32, [4]> value_token_285_begin_0 = const()[name = string("value_token_285_begin_0"), val = tensor<int32, [4]>([0, 0, 14, 0])];
            tensor<int32, [4]> value_token_285_end_0 = const()[name = string("value_token_285_end_0"), val = tensor<int32, [4]>([1, 16, 15, 128])];
            tensor<bool, [4]> value_token_285_end_mask_0 = const()[name = string("value_token_285_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_285_squeeze_mask_0 = const()[name = string("value_token_285_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_285_cast_fp16 = slice_by_index(begin = value_token_285_begin_0, end = value_token_285_end_0, end_mask = value_token_285_end_mask_0, squeeze_mask = value_token_285_squeeze_mask_0, x = value_155_cast_fp16)[name = string("value_token_285_cast_fp16")];
            tensor<int32, [1]> var_7497_axes_0 = const()[name = string("op_7497_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_7497_cast_fp16 = expand_dims(axes = var_7497_axes_0, x = key_token_285_cast_fp16)[name = string("op_7497_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_7498_cast_fp16 = mul(x = state_569_cast_fp16, y = var_7497_cast_fp16)[name = string("op_7498_cast_fp16")];
            tensor<int32, [1]> memory_285_axes_0 = const()[name = string("memory_285_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_285_keep_dims_0 = const()[name = string("memory_285_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_285_cast_fp16 = reduce_sum(axes = memory_285_axes_0, keep_dims = memory_285_keep_dims_0, x = var_7498_cast_fp16)[name = string("memory_285_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_7501_cast_fp16 = sub(x = value_token_285_cast_fp16, y = memory_285_cast_fp16)[name = string("op_7501_cast_fp16")];
            tensor<int32, [3]> var_7504_begin_0 = const()[name = string("op_7504_begin_0"), val = tensor<int32, [3]>([0, 0, 14])];
            tensor<int32, [3]> var_7504_end_0 = const()[name = string("op_7504_end_0"), val = tensor<int32, [3]>([1, 16, 15])];
            tensor<bool, [3]> var_7504_end_mask_0 = const()[name = string("op_7504_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_7504_squeeze_mask_0 = const()[name = string("op_7504_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_7504_cast_fp16 = slice_by_index(begin = var_7504_begin_0, end = var_7504_end_0, end_mask = var_7504_end_mask_0, squeeze_mask = var_7504_squeeze_mask_0, x = beta_17_cast_fp16)[name = string("op_7504_cast_fp16")];
            tensor<int32, [1]> var_7505_axes_0 = const()[name = string("op_7505_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_7505_cast_fp16 = expand_dims(axes = var_7505_axes_0, x = var_7504_cast_fp16)[name = string("op_7505_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_285_cast_fp16 = mul(x = var_7501_cast_fp16, y = var_7505_cast_fp16)[name = string("delta_285_cast_fp16")];
            tensor<int32, [1]> var_7508_axes_0 = const()[name = string("op_7508_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_7508_cast_fp16 = expand_dims(axes = var_7508_axes_0, x = delta_285_cast_fp16)[name = string("op_7508_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_7509_cast_fp16 = mul(x = var_7497_cast_fp16, y = var_7508_cast_fp16)[name = string("op_7509_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_571_cast_fp16 = add(x = state_569_cast_fp16, y = var_7509_cast_fp16)[name = string("state_571_cast_fp16")];
            tensor<int32, [4]> var_7513_begin_0 = const()[name = string("op_7513_begin_0"), val = tensor<int32, [4]>([0, 0, 14, 0])];
            tensor<int32, [4]> var_7513_end_0 = const()[name = string("op_7513_end_0"), val = tensor<int32, [4]>([1, 16, 15, 128])];
            tensor<bool, [4]> var_7513_end_mask_0 = const()[name = string("op_7513_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_7513_squeeze_mask_0 = const()[name = string("op_7513_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_7513_cast_fp16 = slice_by_index(begin = var_7513_begin_0, end = var_7513_end_0, end_mask = var_7513_end_mask_0, squeeze_mask = var_7513_squeeze_mask_0, x = _inversed_query_43_cast_fp16)[name = string("op_7513_cast_fp16")];
            tensor<int32, [1]> var_7514_axes_0 = const()[name = string("op_7514_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_7514_cast_fp16 = expand_dims(axes = var_7514_axes_0, x = var_7513_cast_fp16)[name = string("op_7514_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_7515_cast_fp16 = mul(x = state_571_cast_fp16, y = var_7514_cast_fp16)[name = string("op_7515_cast_fp16")];
            tensor<int32, [1]> var_7517_axes_0 = const()[name = string("op_7517_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_7517_keep_dims_0 = const()[name = string("op_7517_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_7517_cast_fp16 = reduce_sum(axes = var_7517_axes_0, keep_dims = var_7517_keep_dims_0, x = var_7515_cast_fp16)[name = string("op_7517_cast_fp16")];
            tensor<int32, [3]> var_7520_begin_0 = const()[name = string("op_7520_begin_0"), val = tensor<int32, [3]>([0, 0, 15])];
            tensor<int32, [3]> var_7520_end_0 = const()[name = string("op_7520_end_0"), val = tensor<int32, [3]>([1, 16, 16])];
            tensor<bool, [3]> var_7520_end_mask_0 = const()[name = string("op_7520_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_7520_squeeze_mask_0 = const()[name = string("op_7520_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_7520_cast_fp16 = slice_by_index(begin = var_7520_begin_0, end = var_7520_end_0, end_mask = var_7520_end_mask_0, squeeze_mask = var_7520_squeeze_mask_0, x = decay_17_cast_fp16)[name = string("op_7520_cast_fp16")];
            tensor<fp16, [1, 16]> var_7521_cast_fp16 = exp(x = var_7520_cast_fp16)[name = string("op_7521_cast_fp16")];
            tensor<int32, [1]> var_7522_axes_0 = const()[name = string("op_7522_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_7522_cast_fp16 = expand_dims(axes = var_7522_axes_0, x = var_7521_cast_fp16)[name = string("op_7522_cast_fp16")];
            tensor<int32, [1]> var_7523_axes_0 = const()[name = string("op_7523_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_7523_cast_fp16 = expand_dims(axes = var_7523_axes_0, x = var_7522_cast_fp16)[name = string("op_7523_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_573_cast_fp16 = mul(x = state_571_cast_fp16, y = var_7523_cast_fp16)[name = string("state_573_cast_fp16")];
            tensor<int32, [4]> key_token_287_begin_0 = const()[name = string("key_token_287_begin_0"), val = tensor<int32, [4]>([0, 0, 15, 0])];
            tensor<int32, [4]> key_token_287_end_0 = const()[name = string("key_token_287_end_0"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<bool, [4]> key_token_287_end_mask_0 = const()[name = string("key_token_287_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_287_squeeze_mask_0 = const()[name = string("key_token_287_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_287_cast_fp16 = slice_by_index(begin = key_token_287_begin_0, end = key_token_287_end_0, end_mask = key_token_287_end_mask_0, squeeze_mask = key_token_287_squeeze_mask_0, x = key_43_cast_fp16)[name = string("key_token_287_cast_fp16")];
            tensor<int32, [4]> value_token_287_begin_0 = const()[name = string("value_token_287_begin_0"), val = tensor<int32, [4]>([0, 0, 15, 0])];
            tensor<int32, [4]> value_token_287_end_0 = const()[name = string("value_token_287_end_0"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<bool, [4]> value_token_287_end_mask_0 = const()[name = string("value_token_287_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_287_squeeze_mask_0 = const()[name = string("value_token_287_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_287_cast_fp16 = slice_by_index(begin = value_token_287_begin_0, end = value_token_287_end_0, end_mask = value_token_287_end_mask_0, squeeze_mask = value_token_287_squeeze_mask_0, x = value_155_cast_fp16)[name = string("value_token_287_cast_fp16")];
            tensor<int32, [1]> var_7531_axes_0 = const()[name = string("op_7531_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_7531_cast_fp16 = expand_dims(axes = var_7531_axes_0, x = key_token_287_cast_fp16)[name = string("op_7531_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_7532_cast_fp16 = mul(x = state_573_cast_fp16, y = var_7531_cast_fp16)[name = string("op_7532_cast_fp16")];
            tensor<int32, [1]> memory_287_axes_0 = const()[name = string("memory_287_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_287_keep_dims_0 = const()[name = string("memory_287_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_287_cast_fp16 = reduce_sum(axes = memory_287_axes_0, keep_dims = memory_287_keep_dims_0, x = var_7532_cast_fp16)[name = string("memory_287_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_7535_cast_fp16 = sub(x = value_token_287_cast_fp16, y = memory_287_cast_fp16)[name = string("op_7535_cast_fp16")];
            tensor<int32, [3]> var_7538_begin_0 = const()[name = string("op_7538_begin_0"), val = tensor<int32, [3]>([0, 0, 15])];
            tensor<int32, [3]> var_7538_end_0 = const()[name = string("op_7538_end_0"), val = tensor<int32, [3]>([1, 16, 16])];
            tensor<bool, [3]> var_7538_end_mask_0 = const()[name = string("op_7538_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_7538_squeeze_mask_0 = const()[name = string("op_7538_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_7538_cast_fp16 = slice_by_index(begin = var_7538_begin_0, end = var_7538_end_0, end_mask = var_7538_end_mask_0, squeeze_mask = var_7538_squeeze_mask_0, x = beta_17_cast_fp16)[name = string("op_7538_cast_fp16")];
            tensor<int32, [1]> var_7539_axes_0 = const()[name = string("op_7539_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_7539_cast_fp16 = expand_dims(axes = var_7539_axes_0, x = var_7538_cast_fp16)[name = string("op_7539_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_287_cast_fp16 = mul(x = var_7535_cast_fp16, y = var_7539_cast_fp16)[name = string("delta_287_cast_fp16")];
            tensor<int32, [1]> var_7542_axes_0 = const()[name = string("op_7542_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_7542_cast_fp16 = expand_dims(axes = var_7542_axes_0, x = delta_287_cast_fp16)[name = string("op_7542_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_7543_cast_fp16 = mul(x = var_7531_cast_fp16, y = var_7542_cast_fp16)[name = string("op_7543_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> rec10_out = add(x = state_573_cast_fp16, y = var_7543_cast_fp16)[name = string("state_575_cast_fp16")];
            tensor<int32, [4]> var_7547_begin_0 = const()[name = string("op_7547_begin_0"), val = tensor<int32, [4]>([0, 0, 15, 0])];
            tensor<int32, [4]> var_7547_end_0 = const()[name = string("op_7547_end_0"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<bool, [4]> var_7547_end_mask_0 = const()[name = string("op_7547_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_7547_squeeze_mask_0 = const()[name = string("op_7547_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_7547_cast_fp16 = slice_by_index(begin = var_7547_begin_0, end = var_7547_end_0, end_mask = var_7547_end_mask_0, squeeze_mask = var_7547_squeeze_mask_0, x = _inversed_query_43_cast_fp16)[name = string("op_7547_cast_fp16")];
            tensor<int32, [1]> var_7548_axes_0 = const()[name = string("op_7548_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_7548_cast_fp16 = expand_dims(axes = var_7548_axes_0, x = var_7547_cast_fp16)[name = string("op_7548_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_7549_cast_fp16 = mul(x = rec10_out, y = var_7548_cast_fp16)[name = string("op_7549_cast_fp16")];
            tensor<int32, [1]> var_7551_axes_0 = const()[name = string("op_7551_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_7551_keep_dims_0 = const()[name = string("op_7551_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_7551_cast_fp16 = reduce_sum(axes = var_7551_axes_0, keep_dims = var_7551_keep_dims_0, x = var_7549_cast_fp16)[name = string("op_7551_cast_fp16")];
            int32 attention_81_axis_0 = const()[name = string("attention_81_axis_0"), val = int32(1)];
            tensor<fp16, [1, 16, 16, 128]> attention_81_cast_fp16 = stack(axis = attention_81_axis_0, values = (var_7041_cast_fp16, var_7075_cast_fp16, var_7109_cast_fp16, var_7143_cast_fp16, var_7177_cast_fp16, var_7211_cast_fp16, var_7245_cast_fp16, var_7279_cast_fp16, var_7313_cast_fp16, var_7347_cast_fp16, var_7381_cast_fp16, var_7415_cast_fp16, var_7449_cast_fp16, var_7483_cast_fp16, var_7517_cast_fp16, var_7551_cast_fp16))[name = string("attention_81_cast_fp16")];
            tensor<int32, [2]> var_7554 = const()[name = string("op_7554"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [256, 128]> attention_83_cast_fp16 = reshape(shape = var_7554, x = attention_81_cast_fp16)[name = string("attention_83_cast_fp16")];
            tensor<int32, [2]> var_7556 = const()[name = string("op_7556"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [256, 128]> input_137_cast_fp16 = reshape(shape = var_7556, x = linear_81_cast_fp16)[name = string("input_137_cast_fp16")];
            tensor<fp16, [256, 128]> var_7558_cast_fp16 = mul(x = attention_83_cast_fp16, y = attention_83_cast_fp16)[name = string("op_7558_cast_fp16")];
            tensor<int32, [1]> variance_67_axes_0 = const()[name = string("variance_67_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_67_keep_dims_0 = const()[name = string("variance_67_keep_dims_0"), val = bool(true)];
            tensor<fp16, [256, 1]> variance_67_cast_fp16 = reduce_mean(axes = variance_67_axes_0, keep_dims = variance_67_keep_dims_0, x = var_7558_cast_fp16)[name = string("variance_67_cast_fp16")];
            fp16 var_7561_to_fp16 = const()[name = string("op_7561_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [256, 1]> var_7562_cast_fp16 = add(x = variance_67_cast_fp16, y = var_7561_to_fp16)[name = string("op_7562_cast_fp16")];
            fp32 var_7563_epsilon_0 = const()[name = string("op_7563_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [256, 1]> var_7563_cast_fp16 = rsqrt(epsilon = var_7563_epsilon_0, x = var_7562_cast_fp16)[name = string("op_7563_cast_fp16")];
            tensor<fp16, [256, 128]> attention_85_cast_fp16 = mul(x = attention_83_cast_fp16, y = var_7563_cast_fp16)[name = string("attention_85_cast_fp16")];
            tensor<fp16, [128]> groups_2_2_gated_norm_promoted_to_fp16 = const()[name = string("groups_2_2_gated_norm_promoted_to_fp16"), val = tensor<fp16, [128]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(165130752)))];
            tensor<fp16, [256, 128]> attention_87_cast_fp16 = mul(x = attention_85_cast_fp16, y = groups_2_2_gated_norm_promoted_to_fp16)[name = string("attention_87_cast_fp16")];
            tensor<fp16, [256, 128]> var_7566_cast_fp16 = silu(x = input_137_cast_fp16)[name = string("op_7566_cast_fp16")];
            tensor<fp16, [256, 128]> attention_89_cast_fp16 = mul(x = attention_87_cast_fp16, y = var_7566_cast_fp16)[name = string("attention_89_cast_fp16")];
            tensor<int32, [2]> var_7568 = const()[name = string("op_7568"), val = tensor<int32, [2]>([16, 2048])];
            tensor<fp16, [16, 2048]> input_139_cast_fp16 = reshape(shape = var_7568, x = attention_89_cast_fp16)[name = string("input_139_cast_fp16")];
            tensor<fp16, [1024, 2048]> groups_2_2_out_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 2048]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(165131072))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(166704000))))[name = string("groups_2_2_out_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 1024]> linear_82_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = groups_2_2_out_proj_weight_promoted_to_fp16_palettized, x = input_139_cast_fp16)[name = string("linear_82_cast_fp16")];
            tensor<fp16, [16, 1024]> value_157_cast_fp16 = add(x = value_145_cast_fp16, y = linear_82_cast_fp16)[name = string("value_157_cast_fp16")];
            tensor<fp16, [16, 1024]> var_7573_cast_fp16 = mul(x = value_157_cast_fp16, y = value_157_cast_fp16)[name = string("op_7573_cast_fp16")];
            tensor<int32, [1]> variance_69_axes_0 = const()[name = string("variance_69_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_69_keep_dims_0 = const()[name = string("variance_69_keep_dims_0"), val = bool(true)];
            tensor<fp16, [16, 1]> variance_69_cast_fp16 = reduce_mean(axes = variance_69_axes_0, keep_dims = variance_69_keep_dims_0, x = var_7573_cast_fp16)[name = string("variance_69_cast_fp16")];
            fp16 var_7576_to_fp16 = const()[name = string("op_7576_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [16, 1]> var_7577_cast_fp16 = add(x = variance_69_cast_fp16, y = var_7576_to_fp16)[name = string("op_7577_cast_fp16")];
            fp32 var_7578_epsilon_0 = const()[name = string("op_7578_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [16, 1]> var_7578_cast_fp16 = rsqrt(epsilon = var_7578_epsilon_0, x = var_7577_cast_fp16)[name = string("op_7578_cast_fp16")];
            tensor<fp16, [16, 1024]> var_7579_cast_fp16 = mul(x = value_157_cast_fp16, y = var_7578_cast_fp16)[name = string("op_7579_cast_fp16")];
            tensor<fp16, [1024]> var_7581_to_fp16 = const()[name = string("op_7581_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(166712256)))];
            tensor<fp16, [16, 1024]> input_141_cast_fp16 = mul(x = var_7579_cast_fp16, y = var_7581_to_fp16)[name = string("input_141_cast_fp16")];
            tensor<fp16, [3584, 1024]> groups_2_2_gate_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(166714368))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(169466944))))[name = string("groups_2_2_gate_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 3584]> linear_83_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = groups_2_2_gate_proj_weight_promoted_to_fp16_palettized, x = input_141_cast_fp16)[name = string("linear_83_cast_fp16")];
            tensor<fp16, [16, 3584]> var_7585_cast_fp16 = silu(x = linear_83_cast_fp16)[name = string("op_7585_cast_fp16")];
            tensor<fp16, [3584, 1024]> groups_2_2_up_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(169495680))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(172248256))))[name = string("groups_2_2_up_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 3584]> linear_84_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = groups_2_2_up_proj_weight_promoted_to_fp16_palettized, x = input_141_cast_fp16)[name = string("linear_84_cast_fp16")];
            tensor<fp16, [16, 3584]> input_145_cast_fp16 = mul(x = var_7585_cast_fp16, y = linear_84_cast_fp16)[name = string("input_145_cast_fp16")];
            tensor<fp16, [1024, 3584]> groups_2_2_down_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 3584]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(172276992))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(175029568))))[name = string("groups_2_2_down_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 1024]> linear_85_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = groups_2_2_down_proj_weight_promoted_to_fp16_palettized, x = input_145_cast_fp16)[name = string("linear_85_cast_fp16")];
            tensor<fp16, [16, 1024]> value_159_cast_fp16 = add(x = value_157_cast_fp16, y = linear_85_cast_fp16)[name = string("value_159_cast_fp16")];
            int32 var_7610 = const()[name = string("op_7610"), val = int32(-1)];
            tensor<fp16, [16, 1024]> var_7617_cast_fp16 = mul(x = value_159_cast_fp16, y = value_159_cast_fp16)[name = string("op_7617_cast_fp16")];
            tensor<int32, [1]> variance_71_axes_0 = const()[name = string("variance_71_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_71_keep_dims_0 = const()[name = string("variance_71_keep_dims_0"), val = bool(true)];
            tensor<fp16, [16, 1]> variance_71_cast_fp16 = reduce_mean(axes = variance_71_axes_0, keep_dims = variance_71_keep_dims_0, x = var_7617_cast_fp16)[name = string("variance_71_cast_fp16")];
            fp16 var_7620_to_fp16 = const()[name = string("op_7620_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [16, 1]> var_7621_cast_fp16 = add(x = variance_71_cast_fp16, y = var_7620_to_fp16)[name = string("op_7621_cast_fp16")];
            fp32 var_7622_epsilon_0 = const()[name = string("op_7622_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [16, 1]> var_7622_cast_fp16 = rsqrt(epsilon = var_7622_epsilon_0, x = var_7621_cast_fp16)[name = string("op_7622_cast_fp16")];
            tensor<fp16, [16, 1024]> var_7623_cast_fp16 = mul(x = value_159_cast_fp16, y = var_7622_cast_fp16)[name = string("op_7623_cast_fp16")];
            tensor<fp16, [1024]> var_7625_to_fp16 = const()[name = string("op_7625_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(175037824)))];
            tensor<fp16, [16, 1024]> input_147_cast_fp16 = mul(x = var_7623_cast_fp16, y = var_7625_to_fp16)[name = string("input_147_cast_fp16")];
            tensor<fp16, [4096, 1024]> projections_2_q_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [4096, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(175039936))), lut = tensor<fp16, [256, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(178185728))))[name = string("projections_2_q_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 4096]> linear_86_cast_fp16 = linear(bias = linear_24_bias_0_to_fp16, weight = projections_2_q_proj_weight_promoted_to_fp16_palettized, x = input_147_cast_fp16)[name = string("linear_86_cast_fp16")];
            tensor<int32, [3]> var_7629 = const()[name = string("op_7629"), val = tensor<int32, [3]>([16, 8, 512])];
            tensor<fp16, [16, 8, 512]> query_and_gate_5_cast_fp16 = reshape(shape = var_7629, x = linear_86_cast_fp16)[name = string("query_and_gate_5_cast_fp16")];
            tensor<int32, [2]> var_7631_split_sizes_0 = const()[name = string("op_7631_split_sizes_0"), val = tensor<int32, [2]>([256, 256])];
            int32 var_7631_axis_0 = const()[name = string("op_7631_axis_0"), val = int32(-1)];
            tensor<fp16, [16, 8, 256]> var_7631_cast_fp16_0, tensor<fp16, [16, 8, 256]> var_7631_cast_fp16_1 = split(axis = var_7631_axis_0, split_sizes = var_7631_split_sizes_0, x = query_and_gate_5_cast_fp16)[name = string("op_7631_cast_fp16")];
            tensor<fp16, [512, 1024]> projections_2_k_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [512, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(178218560))), lut = tensor<fp16, [32, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(178611840))))[name = string("projections_2_k_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 512]> linear_87_cast_fp16 = linear(bias = linear_25_bias_0_to_fp16, weight = projections_2_k_proj_weight_promoted_to_fp16_palettized, x = input_147_cast_fp16)[name = string("linear_87_cast_fp16")];
            tensor<int32, [3]> var_7635 = const()[name = string("op_7635"), val = tensor<int32, [3]>([16, 2, 256])];
            tensor<fp16, [16, 2, 256]> value_163_cast_fp16 = reshape(shape = var_7635, x = linear_87_cast_fp16)[name = string("value_163_cast_fp16")];
            tensor<fp16, [512, 1024]> projections_2_v_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [512, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(178616000))), lut = tensor<fp16, [32, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(179009280))))[name = string("projections_2_v_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 512]> linear_88_cast_fp16 = linear(bias = linear_25_bias_0_to_fp16, weight = projections_2_v_proj_weight_promoted_to_fp16_palettized, x = input_147_cast_fp16)[name = string("linear_88_cast_fp16")];
            tensor<int32, [3]> var_7639 = const()[name = string("op_7639"), val = tensor<int32, [3]>([16, 2, 256])];
            tensor<fp16, [16, 2, 256]> value_169_cast_fp16 = reshape(shape = var_7639, x = linear_88_cast_fp16)[name = string("value_169_cast_fp16")];
            tensor<fp16, [16, 8, 256]> var_7641_cast_fp16 = mul(x = var_7631_cast_fp16_0, y = var_7631_cast_fp16_0)[name = string("op_7641_cast_fp16")];
            tensor<int32, [1]> variance_73_axes_0 = const()[name = string("variance_73_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_73_keep_dims_0 = const()[name = string("variance_73_keep_dims_0"), val = bool(true)];
            tensor<fp16, [16, 8, 1]> variance_73_cast_fp16 = reduce_mean(axes = variance_73_axes_0, keep_dims = variance_73_keep_dims_0, x = var_7641_cast_fp16)[name = string("variance_73_cast_fp16")];
            fp16 var_7644_to_fp16 = const()[name = string("op_7644_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [16, 8, 1]> var_7645_cast_fp16 = add(x = variance_73_cast_fp16, y = var_7644_to_fp16)[name = string("op_7645_cast_fp16")];
            fp32 var_7646_epsilon_0 = const()[name = string("op_7646_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [16, 8, 1]> var_7646_cast_fp16 = rsqrt(epsilon = var_7646_epsilon_0, x = var_7645_cast_fp16)[name = string("op_7646_cast_fp16")];
            tensor<fp16, [16, 8, 256]> var_7647_cast_fp16 = mul(x = var_7631_cast_fp16_0, y = var_7646_cast_fp16)[name = string("op_7647_cast_fp16")];
            tensor<fp16, [256]> var_7649_to_fp16 = const()[name = string("op_7649_to_fp16"), val = tensor<fp16, [256]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(179013440)))];
            tensor<fp16, [16, 8, 256]> var_7650_cast_fp16 = mul(x = var_7647_cast_fp16, y = var_7649_to_fp16)[name = string("op_7650_cast_fp16")];
            tensor<int32, [3]> var_7651 = const()[name = string("op_7651"), val = tensor<int32, [3]>([1, 0, 2])];
            tensor<int32, [1]> value_165_axes_0 = const()[name = string("value_165_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [8, 16, 256]> var_7652_cast_fp16 = transpose(perm = var_7651, x = var_7650_cast_fp16)[name = string("transpose_61")];
            tensor<fp16, [1, 8, 16, 256]> value_165_cast_fp16 = expand_dims(axes = value_165_axes_0, x = var_7652_cast_fp16)[name = string("value_165_cast_fp16")];
            tensor<fp16, [16, 2, 256]> var_7654_cast_fp16 = mul(x = value_163_cast_fp16, y = value_163_cast_fp16)[name = string("op_7654_cast_fp16")];
            tensor<int32, [1]> variance_75_axes_0 = const()[name = string("variance_75_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_75_keep_dims_0 = const()[name = string("variance_75_keep_dims_0"), val = bool(true)];
            tensor<fp16, [16, 2, 1]> variance_75_cast_fp16 = reduce_mean(axes = variance_75_axes_0, keep_dims = variance_75_keep_dims_0, x = var_7654_cast_fp16)[name = string("variance_75_cast_fp16")];
            fp16 var_7657_to_fp16 = const()[name = string("op_7657_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [16, 2, 1]> var_7658_cast_fp16 = add(x = variance_75_cast_fp16, y = var_7657_to_fp16)[name = string("op_7658_cast_fp16")];
            fp32 var_7659_epsilon_0 = const()[name = string("op_7659_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [16, 2, 1]> var_7659_cast_fp16 = rsqrt(epsilon = var_7659_epsilon_0, x = var_7658_cast_fp16)[name = string("op_7659_cast_fp16")];
            tensor<fp16, [16, 2, 256]> var_7660_cast_fp16 = mul(x = value_163_cast_fp16, y = var_7659_cast_fp16)[name = string("op_7660_cast_fp16")];
            tensor<fp16, [256]> var_7662_to_fp16 = const()[name = string("op_7662_to_fp16"), val = tensor<fp16, [256]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(179014016)))];
            tensor<fp16, [16, 2, 256]> var_7663_cast_fp16 = mul(x = var_7660_cast_fp16, y = var_7662_to_fp16)[name = string("op_7663_cast_fp16")];
            tensor<int32, [3]> var_7664 = const()[name = string("op_7664"), val = tensor<int32, [3]>([1, 0, 2])];
            tensor<int32, [1]> value_167_axes_0 = const()[name = string("value_167_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [2, 16, 256]> var_7665_cast_fp16 = transpose(perm = var_7664, x = var_7663_cast_fp16)[name = string("transpose_60")];
            tensor<fp16, [1, 2, 16, 256]> value_167_cast_fp16 = expand_dims(axes = value_167_axes_0, x = var_7665_cast_fp16)[name = string("value_167_cast_fp16")];
            tensor<int32, [4]> rotated_9_begin_0 = const()[name = string("rotated_9_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> rotated_9_end_0 = const()[name = string("rotated_9_end_0"), val = tensor<int32, [4]>([1, 8, 16, 64])];
            tensor<bool, [4]> rotated_9_end_mask_0 = const()[name = string("rotated_9_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
            tensor<fp16, [1, 8, 16, 64]> rotated_9_cast_fp16 = slice_by_index(begin = rotated_9_begin_0, end = rotated_9_end_0, end_mask = rotated_9_end_mask_0, x = value_165_cast_fp16)[name = string("rotated_9_cast_fp16")];
            tensor<int32, [4]> passthrough_9_begin_0 = const()[name = string("passthrough_9_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 64])];
            tensor<int32, [4]> passthrough_9_end_0 = const()[name = string("passthrough_9_end_0"), val = tensor<int32, [4]>([1, 8, 16, 256])];
            tensor<bool, [4]> passthrough_9_end_mask_0 = const()[name = string("passthrough_9_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
            tensor<fp16, [1, 8, 16, 192]> passthrough_9_cast_fp16 = slice_by_index(begin = passthrough_9_begin_0, end = passthrough_9_end_0, end_mask = passthrough_9_end_mask_0, x = value_165_cast_fp16)[name = string("passthrough_9_cast_fp16")];
            tensor<int32, [2]> var_7669_split_sizes_0 = const()[name = string("op_7669_split_sizes_0"), val = tensor<int32, [2]>([32, 32])];
            int32 var_7669_axis_0 = const()[name = string("op_7669_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 8, 16, 32]> var_7669_cast_fp16_0, tensor<fp16, [1, 8, 16, 32]> var_7669_cast_fp16_1 = split(axis = var_7669_axis_0, split_sizes = var_7669_split_sizes_0, x = rotated_9_cast_fp16)[name = string("op_7669_cast_fp16")];
            fp16 const_89_promoted_to_fp16 = const()[name = string("const_89_promoted_to_fp16"), val = fp16(-0x1p+0)];
            tensor<fp16, [1, 8, 16, 32]> var_7671_cast_fp16 = mul(x = var_7669_cast_fp16_1, y = const_89_promoted_to_fp16)[name = string("op_7671_cast_fp16")];
            bool rotated_half_9_interleave_0 = const()[name = string("rotated_half_9_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 8, 16, 64]> rotated_half_9_cast_fp16 = concat(axis = var_7610, interleave = rotated_half_9_interleave_0, values = (var_7671_cast_fp16, var_7669_cast_fp16_0))[name = string("rotated_half_9_cast_fp16")];
            tensor<fp16, [1, 8, 16, 64]> var_7674_cast_fp16 = mul(x = rotated_9_cast_fp16, y = cos)[name = string("op_7674_cast_fp16")];
            tensor<fp16, [1, 8, 16, 64]> var_7675_cast_fp16 = mul(x = rotated_half_9_cast_fp16, y = sin)[name = string("op_7675_cast_fp16")];
            tensor<fp16, [1, 8, 16, 64]> var_7676_cast_fp16 = add(x = var_7674_cast_fp16, y = var_7675_cast_fp16)[name = string("op_7676_cast_fp16")];
            bool query_45_interleave_0 = const()[name = string("query_45_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 8, 16, 256]> query_45_cast_fp16 = concat(axis = var_7610, interleave = query_45_interleave_0, values = (var_7676_cast_fp16, passthrough_9_cast_fp16))[name = string("query_45_cast_fp16")];
            tensor<int32, [4]> rotated_11_begin_0 = const()[name = string("rotated_11_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> rotated_11_end_0 = const()[name = string("rotated_11_end_0"), val = tensor<int32, [4]>([1, 2, 16, 64])];
            tensor<bool, [4]> rotated_11_end_mask_0 = const()[name = string("rotated_11_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
            tensor<fp16, [1, 2, 16, 64]> rotated_11_cast_fp16 = slice_by_index(begin = rotated_11_begin_0, end = rotated_11_end_0, end_mask = rotated_11_end_mask_0, x = value_167_cast_fp16)[name = string("rotated_11_cast_fp16")];
            tensor<int32, [4]> passthrough_11_begin_0 = const()[name = string("passthrough_11_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 64])];
            tensor<int32, [4]> passthrough_11_end_0 = const()[name = string("passthrough_11_end_0"), val = tensor<int32, [4]>([1, 2, 16, 256])];
            tensor<bool, [4]> passthrough_11_end_mask_0 = const()[name = string("passthrough_11_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
            tensor<fp16, [1, 2, 16, 192]> passthrough_11_cast_fp16 = slice_by_index(begin = passthrough_11_begin_0, end = passthrough_11_end_0, end_mask = passthrough_11_end_mask_0, x = value_167_cast_fp16)[name = string("passthrough_11_cast_fp16")];
            tensor<int32, [2]> var_7681_split_sizes_0 = const()[name = string("op_7681_split_sizes_0"), val = tensor<int32, [2]>([32, 32])];
            int32 var_7681_axis_0 = const()[name = string("op_7681_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 2, 16, 32]> var_7681_cast_fp16_0, tensor<fp16, [1, 2, 16, 32]> var_7681_cast_fp16_1 = split(axis = var_7681_axis_0, split_sizes = var_7681_split_sizes_0, x = rotated_11_cast_fp16)[name = string("op_7681_cast_fp16")];
            fp16 const_90_promoted_to_fp16 = const()[name = string("const_90_promoted_to_fp16"), val = fp16(-0x1p+0)];
            tensor<fp16, [1, 2, 16, 32]> var_7683_cast_fp16 = mul(x = var_7681_cast_fp16_1, y = const_90_promoted_to_fp16)[name = string("op_7683_cast_fp16")];
            bool rotated_half_11_interleave_0 = const()[name = string("rotated_half_11_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 2, 16, 64]> rotated_half_11_cast_fp16 = concat(axis = var_7610, interleave = rotated_half_11_interleave_0, values = (var_7683_cast_fp16, var_7681_cast_fp16_0))[name = string("rotated_half_11_cast_fp16")];
            tensor<fp16, [1, 2, 16, 64]> var_7686_cast_fp16 = mul(x = rotated_11_cast_fp16, y = cos)[name = string("op_7686_cast_fp16")];
            tensor<fp16, [1, 2, 16, 64]> var_7687_cast_fp16 = mul(x = rotated_half_11_cast_fp16, y = sin)[name = string("op_7687_cast_fp16")];
            tensor<fp16, [1, 2, 16, 64]> var_7688_cast_fp16 = add(x = var_7686_cast_fp16, y = var_7687_cast_fp16)[name = string("op_7688_cast_fp16")];
            bool key_45_interleave_0 = const()[name = string("key_45_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 2, 16, 256]> key_45_cast_fp16 = concat(axis = var_7610, interleave = key_45_interleave_0, values = (var_7688_cast_fp16, passthrough_11_cast_fp16))[name = string("key_45_cast_fp16")];
            tensor<int32, [4]> var_7691 = const()[name = string("op_7691"), val = tensor<int32, [4]>([2, 4, 16, 256])];
            tensor<fp16, [2, 4, 16, 256]> var_7692_cast_fp16 = reshape(shape = var_7691, x = query_45_cast_fp16)[name = string("op_7692_cast_fp16")];
            tensor<int32, [4]> transpose_4_perm_0 = const()[name = string("transpose_4_perm_0"), val = tensor<int32, [4]>([0, 1, 3, 2])];
            tensor<int32, [3]> var_7695 = const()[name = string("op_7695"), val = tensor<int32, [3]>([2, 16, 256])];
            tensor<fp16, [2, 16, 256]> key_47_cast_fp16 = reshape(shape = var_7695, x = key_45_cast_fp16)[name = string("key_47_cast_fp16")];
            tensor<int32, [3]> var_7697 = const()[name = string("op_7697"), val = tensor<int32, [3]>([1, 0, 2])];
            tensor<int32, [2]> var_7699 = const()[name = string("op_7699"), val = tensor<int32, [2]>([16, 2048])];
            tensor<fp16, [16, 2048]> gate_11_cast_fp16 = reshape(shape = var_7699, x = var_7631_cast_fp16_1)[name = string("gate_11_cast_fp16")];
            tensor<int32, [1]> var_7707_axes_0 = const()[name = string("op_7707_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [1, 2, 16, 256]> var_7707_cast_fp16 = expand_dims(axes = var_7707_axes_0, x = key_47_cast_fp16)[name = string("op_7707_cast_fp16")];
            tensor<int32, [1]> var_7709_axes_0 = const()[name = string("op_7709_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [1, 1, 2, 16, 256]> var_7709_cast_fp16 = expand_dims(axes = var_7709_axes_0, x = var_7707_cast_fp16)[name = string("op_7709_cast_fp16")];
            tensor<int32, [1]> expand_dims_32 = const()[name = string("expand_dims_32"), val = tensor<int32, [1]>([2])];
            tensor<int32, [1]> expand_dims_33 = const()[name = string("expand_dims_33"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_34 = const()[name = string("expand_dims_34"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_36 = const()[name = string("expand_dims_36"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_37 = const()[name = string("expand_dims_37"), val = tensor<int32, [1]>([3])];
            tensor<int32, [1]> expand_dims_38 = const()[name = string("expand_dims_38"), val = tensor<int32, [1]>([1])];
            int32 concat_18_axis_0 = const()[name = string("concat_18_axis_0"), val = int32(0)];
            bool concat_18_interleave_0 = const()[name = string("concat_18_interleave_0"), val = bool(false)];
            tensor<int32, [5]> concat_18 = concat(axis = concat_18_axis_0, interleave = concat_18_interleave_0, values = (expand_dims_32, expand_dims_33, expand_dims_34, current_pos, expand_dims_36))[name = string("concat_18")];
            tensor<int32, [1]> concat_19_values2_0 = const()[name = string("concat_19_values2_0"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> concat_19_values4_0 = const()[name = string("concat_19_values4_0"), val = tensor<int32, [1]>([0])];
            int32 concat_19_axis_0 = const()[name = string("concat_19_axis_0"), val = int32(0)];
            bool concat_19_interleave_0 = const()[name = string("concat_19_interleave_0"), val = bool(false)];
            tensor<int32, [5]> concat_19 = concat(axis = concat_19_axis_0, interleave = concat_19_interleave_0, values = (expand_dims_37, expand_dims_38, concat_19_values2_0, var_2332, concat_19_values4_0))[name = string("concat_19")];
            tensor<int32, [5]> kv_cache_internal_tensor_assign_5_stride_0 = const()[name = string("kv_cache_internal_tensor_assign_5_stride_0"), val = tensor<int32, [5]>([1, 1, 1, 1, 1])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_5_begin_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_5_begin_mask_0"), val = tensor<bool, [5]>([false, false, false, false, false])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_5_end_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_5_end_mask_0"), val = tensor<bool, [5]>([false, false, true, false, true])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_5_squeeze_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_5_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, false, false, false])];
            tensor<fp16, [4, 2, 2, 2048, 256]> kv_cache_internal_tensor_assign_5_cast_fp16 = slice_update(begin = concat_18, begin_mask = kv_cache_internal_tensor_assign_5_begin_mask_0, end = concat_19, end_mask = kv_cache_internal_tensor_assign_5_end_mask_0, squeeze_mask = kv_cache_internal_tensor_assign_5_squeeze_mask_0, stride = kv_cache_internal_tensor_assign_5_stride_0, update = var_7709_cast_fp16, x = coreml_update_state_11)[name = string("kv_cache_internal_tensor_assign_5_cast_fp16")];
            write_state(data = kv_cache_internal_tensor_assign_5_cast_fp16, input = kv_cache)[name = string("coreml_update_state_12_write_state")];
            tensor<fp16, [4, 2, 2, 2048, 256]> coreml_update_state_12 = read_state(input = kv_cache)[name = string("coreml_update_state_12")];
            tensor<int32, [1]> var_7740_axes_0 = const()[name = string("op_7740_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [2, 16, 256]> value_171_cast_fp16 = transpose(perm = var_7697, x = value_169_cast_fp16)[name = string("transpose_59")];
            tensor<fp16, [1, 2, 16, 256]> var_7740_cast_fp16 = expand_dims(axes = var_7740_axes_0, x = value_171_cast_fp16)[name = string("op_7740_cast_fp16")];
            tensor<int32, [1]> var_7742_axes_0 = const()[name = string("op_7742_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [1, 1, 2, 16, 256]> var_7742_cast_fp16 = expand_dims(axes = var_7742_axes_0, x = var_7740_cast_fp16)[name = string("op_7742_cast_fp16")];
            tensor<int32, [1]> expand_dims_40 = const()[name = string("expand_dims_40"), val = tensor<int32, [1]>([2])];
            tensor<int32, [1]> expand_dims_41 = const()[name = string("expand_dims_41"), val = tensor<int32, [1]>([1])];
            tensor<int32, [1]> expand_dims_42 = const()[name = string("expand_dims_42"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_44 = const()[name = string("expand_dims_44"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_45 = const()[name = string("expand_dims_45"), val = tensor<int32, [1]>([3])];
            tensor<int32, [1]> expand_dims_46 = const()[name = string("expand_dims_46"), val = tensor<int32, [1]>([2])];
            int32 concat_22_axis_0 = const()[name = string("concat_22_axis_0"), val = int32(0)];
            bool concat_22_interleave_0 = const()[name = string("concat_22_interleave_0"), val = bool(false)];
            tensor<int32, [5]> concat_22 = concat(axis = concat_22_axis_0, interleave = concat_22_interleave_0, values = (expand_dims_40, expand_dims_41, expand_dims_42, current_pos, expand_dims_44))[name = string("concat_22")];
            tensor<int32, [1]> concat_23_values2_0 = const()[name = string("concat_23_values2_0"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> concat_23_values4_0 = const()[name = string("concat_23_values4_0"), val = tensor<int32, [1]>([0])];
            int32 concat_23_axis_0 = const()[name = string("concat_23_axis_0"), val = int32(0)];
            bool concat_23_interleave_0 = const()[name = string("concat_23_interleave_0"), val = bool(false)];
            tensor<int32, [5]> concat_23 = concat(axis = concat_23_axis_0, interleave = concat_23_interleave_0, values = (expand_dims_45, expand_dims_46, concat_23_values2_0, var_2332, concat_23_values4_0))[name = string("concat_23")];
            tensor<int32, [5]> kv_cache_internal_tensor_assign_6_stride_0 = const()[name = string("kv_cache_internal_tensor_assign_6_stride_0"), val = tensor<int32, [5]>([1, 1, 1, 1, 1])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_6_begin_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_6_begin_mask_0"), val = tensor<bool, [5]>([false, false, false, false, false])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_6_end_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_6_end_mask_0"), val = tensor<bool, [5]>([false, false, true, false, true])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_6_squeeze_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_6_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, false, false, false])];
            tensor<fp16, [4, 2, 2, 2048, 256]> kv_cache_internal_tensor_assign_6_cast_fp16 = slice_update(begin = concat_22, begin_mask = kv_cache_internal_tensor_assign_6_begin_mask_0, end = concat_23, end_mask = kv_cache_internal_tensor_assign_6_end_mask_0, squeeze_mask = kv_cache_internal_tensor_assign_6_squeeze_mask_0, stride = kv_cache_internal_tensor_assign_6_stride_0, update = var_7742_cast_fp16, x = coreml_update_state_12)[name = string("kv_cache_internal_tensor_assign_6_cast_fp16")];
            write_state(data = kv_cache_internal_tensor_assign_6_cast_fp16, input = kv_cache)[name = string("coreml_update_state_13_write_state")];
            tensor<fp16, [4, 2, 2, 2048, 256]> coreml_update_state_13 = read_state(input = kv_cache)[name = string("coreml_update_state_13")];
            tensor<int32, [5]> var_7774_begin_0 = const()[name = string("op_7774_begin_0"), val = tensor<int32, [5]>([2, 0, 0, 0, 0])];
            tensor<int32, [5]> var_7774_end_0 = const()[name = string("op_7774_end_0"), val = tensor<int32, [5]>([3, 2, 2, 2048, 256])];
            tensor<bool, [5]> var_7774_end_mask_0 = const()[name = string("op_7774_end_mask_0"), val = tensor<bool, [5]>([false, true, true, true, true])];
            tensor<bool, [5]> var_7774_squeeze_mask_0 = const()[name = string("op_7774_squeeze_mask_0"), val = tensor<bool, [5]>([true, false, false, false, false])];
            tensor<fp16, [2, 2, 2048, 256]> var_7774_cast_fp16 = slice_by_index(begin = var_7774_begin_0, end = var_7774_end_0, end_mask = var_7774_end_mask_0, squeeze_mask = var_7774_squeeze_mask_0, x = coreml_update_state_13)[name = string("op_7774_cast_fp16")];
            tensor<int32, [4]> keys_5_begin_0 = const()[name = string("keys_5_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> keys_5_end_0 = const()[name = string("keys_5_end_0"), val = tensor<int32, [4]>([1, 2, 2048, 256])];
            tensor<bool, [4]> keys_5_end_mask_0 = const()[name = string("keys_5_end_mask_0"), val = tensor<bool, [4]>([false, true, true, true])];
            tensor<bool, [4]> keys_5_squeeze_mask_0 = const()[name = string("keys_5_squeeze_mask_0"), val = tensor<bool, [4]>([true, false, false, false])];
            tensor<fp16, [2, 2048, 256]> keys_5_cast_fp16 = slice_by_index(begin = keys_5_begin_0, end = keys_5_end_0, end_mask = keys_5_end_mask_0, squeeze_mask = keys_5_squeeze_mask_0, x = var_7774_cast_fp16)[name = string("keys_5_cast_fp16")];
            tensor<int32, [4]> values_5_begin_0 = const()[name = string("values_5_begin_0"), val = tensor<int32, [4]>([1, 0, 0, 0])];
            tensor<int32, [4]> values_5_end_0 = const()[name = string("values_5_end_0"), val = tensor<int32, [4]>([2, 2, 2048, 256])];
            tensor<bool, [4]> values_5_end_mask_0 = const()[name = string("values_5_end_mask_0"), val = tensor<bool, [4]>([false, true, true, true])];
            tensor<bool, [4]> values_5_squeeze_mask_0 = const()[name = string("values_5_squeeze_mask_0"), val = tensor<bool, [4]>([true, false, false, false])];
            tensor<fp16, [2, 2048, 256]> values_5_cast_fp16 = slice_by_index(begin = values_5_begin_0, end = values_5_end_0, end_mask = values_5_end_mask_0, squeeze_mask = values_5_squeeze_mask_0, x = var_7774_cast_fp16)[name = string("values_5_cast_fp16")];
            int32 var_7792 = const()[name = string("op_7792"), val = int32(1)];
            tensor<int32, [4]> var_7799 = const()[name = string("op_7799"), val = tensor<int32, [4]>([1, 2048, 1, 16])];
            tensor<fp16, [2, 4, 256, 16]> transpose_4_cast_fp16 = transpose(perm = transpose_4_perm_0, x = var_7692_cast_fp16)[name = string("transpose_58")];
            tensor<fp16, [1, 2048, 1, 16]> var_7800_cast_fp16 = reshape(shape = var_7799, x = transpose_4_cast_fp16)[name = string("op_7800_cast_fp16")];
            tensor<int32, [3]> var_7801 = const()[name = string("op_7801"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [4]> var_7804 = const()[name = string("op_7804"), val = tensor<int32, [4]>([1, 512, 1, 2048])];
            tensor<fp16, [2, 256, 2048]> var_7802_cast_fp16 = transpose(perm = var_7801, x = keys_5_cast_fp16)[name = string("transpose_57")];
            tensor<fp16, [1, 512, 1, 2048]> key_native_5_cast_fp16 = reshape(shape = var_7804, x = var_7802_cast_fp16)[name = string("key_native_5_cast_fp16")];
            tensor<int32, [3]> var_7806 = const()[name = string("op_7806"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [4]> var_7809 = const()[name = string("op_7809"), val = tensor<int32, [4]>([1, 512, 1, 2048])];
            tensor<fp16, [2, 256, 2048]> var_7807_cast_fp16 = transpose(perm = var_7806, x = values_5_cast_fp16)[name = string("transpose_56")];
            tensor<fp16, [1, 512, 1, 2048]> var_7810_cast_fp16 = reshape(shape = var_7809, x = var_7807_cast_fp16)[name = string("op_7810_cast_fp16")];
            tensor<int32, [8]> tile_38 = const()[name = string("tile_38"), val = tensor<int32, [8]>([256, 256, 256, 256, 256, 256, 256, 256])];
            int32 var_7814_axis_0 = const()[name = string("op_7814_axis_0"), val = int32(1)];
            tensor<fp16, [1, 256, 1, 16]> var_7814_cast_fp16_0, tensor<fp16, [1, 256, 1, 16]> var_7814_cast_fp16_1, tensor<fp16, [1, 256, 1, 16]> var_7814_cast_fp16_2, tensor<fp16, [1, 256, 1, 16]> var_7814_cast_fp16_3, tensor<fp16, [1, 256, 1, 16]> var_7814_cast_fp16_4, tensor<fp16, [1, 256, 1, 16]> var_7814_cast_fp16_5, tensor<fp16, [1, 256, 1, 16]> var_7814_cast_fp16_6, tensor<fp16, [1, 256, 1, 16]> var_7814_cast_fp16_7 = split(axis = var_7814_axis_0, split_sizes = tile_38, x = var_7800_cast_fp16)[name = string("op_7814_cast_fp16")];
            tensor<int32, [4]> var_7823_perm_0 = const()[name = string("op_7823_perm_0"), val = tensor<int32, [4]>([0, 3, 2, 1])];
            tensor<int32, [2]> tile_39 = const()[name = string("tile_39"), val = tensor<int32, [2]>([256, 256])];
            int32 var_7824_axis_0 = const()[name = string("op_7824_axis_0"), val = int32(3)];
            tensor<fp16, [1, 2048, 1, 512]> var_7823_cast_fp16 = transpose(perm = var_7823_perm_0, x = key_native_5_cast_fp16)[name = string("transpose_55")];
            tensor<fp16, [1, 2048, 1, 256]> var_7824_cast_fp16_0, tensor<fp16, [1, 2048, 1, 256]> var_7824_cast_fp16_1 = split(axis = var_7824_axis_0, split_sizes = tile_39, x = var_7823_cast_fp16)[name = string("op_7824_cast_fp16")];
            tensor<int32, [2]> tile_40 = const()[name = string("tile_40"), val = tensor<int32, [2]>([256, 256])];
            int32 var_7827_axis_0 = const()[name = string("op_7827_axis_0"), val = int32(1)];
            tensor<fp16, [1, 256, 1, 2048]> var_7827_cast_fp16_0, tensor<fp16, [1, 256, 1, 2048]> var_7827_cast_fp16_1 = split(axis = var_7827_axis_0, split_sizes = tile_40, x = var_7810_cast_fp16)[name = string("op_7827_cast_fp16")];
            string scores_33_equation_0 = const()[name = string("scores_33_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 16]> scores_33_cast_fp16 = einsum(equation = scores_33_equation_0, values = (var_7824_cast_fp16_0, var_7814_cast_fp16_0))[name = string("scores_33_cast_fp16")];
            fp16 var_7832_to_fp16 = const()[name = string("op_7832_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 16]> var_7833_cast_fp16 = mul(x = scores_33_cast_fp16, y = var_7832_to_fp16)[name = string("op_7833_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_7834_cast_fp16 = add(x = var_7833_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_7834_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_7835_cast_fp16 = softmax(axis = var_7792, x = var_7834_cast_fp16)[name = string("op_7835_cast_fp16")];
            tensor<int32, [4]> tile_41 = const()[name = string("tile_41"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_7836_axis_0 = const()[name = string("op_7836_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 16]> var_7836_cast_fp16_0, tensor<fp16, [1, 512, 1, 16]> var_7836_cast_fp16_1, tensor<fp16, [1, 512, 1, 16]> var_7836_cast_fp16_2, tensor<fp16, [1, 512, 1, 16]> var_7836_cast_fp16_3 = split(axis = var_7836_axis_0, split_sizes = tile_41, x = var_7835_cast_fp16)[name = string("op_7836_cast_fp16")];
            tensor<int32, [4]> tile_42 = const()[name = string("tile_42"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_7841_axis_0 = const()[name = string("op_7841_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_7841_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_7841_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_7841_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_7841_cast_fp16_3 = split(axis = var_7841_axis_0, split_sizes = tile_42, x = var_7827_cast_fp16_0)[name = string("op_7841_cast_fp16")];
            fp16 var_7846_to_fp16 = const()[name = string("op_7846_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_7847_cast_fp16 = mul(x = var_7836_cast_fp16_0, y = var_7846_to_fp16)[name = string("op_7847_cast_fp16")];
            string var_7849_equation_0 = const()[name = string("op_7849_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_7849_cast_fp16 = einsum(equation = var_7849_equation_0, values = (var_7841_cast_fp16_0, var_7847_cast_fp16))[name = string("op_7849_cast_fp16")];
            fp16 var_7850_to_fp16 = const()[name = string("op_7850_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_7851_cast_fp16 = mul(x = var_7836_cast_fp16_1, y = var_7850_to_fp16)[name = string("op_7851_cast_fp16")];
            string var_7853_equation_0 = const()[name = string("op_7853_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_7853_cast_fp16 = einsum(equation = var_7853_equation_0, values = (var_7841_cast_fp16_1, var_7851_cast_fp16))[name = string("op_7853_cast_fp16")];
            fp16 var_7854_to_fp16 = const()[name = string("op_7854_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_7855_cast_fp16 = mul(x = var_7836_cast_fp16_2, y = var_7854_to_fp16)[name = string("op_7855_cast_fp16")];
            string var_7857_equation_0 = const()[name = string("op_7857_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_7857_cast_fp16 = einsum(equation = var_7857_equation_0, values = (var_7841_cast_fp16_2, var_7855_cast_fp16))[name = string("op_7857_cast_fp16")];
            fp16 var_7858_to_fp16 = const()[name = string("op_7858_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_7859_cast_fp16 = mul(x = var_7836_cast_fp16_3, y = var_7858_to_fp16)[name = string("op_7859_cast_fp16")];
            string var_7861_equation_0 = const()[name = string("op_7861_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_7861_cast_fp16 = einsum(equation = var_7861_equation_0, values = (var_7841_cast_fp16_3, var_7859_cast_fp16))[name = string("op_7861_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_7862_cast_fp16 = add(x = var_7849_cast_fp16, y = var_7853_cast_fp16)[name = string("op_7862_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_7863_cast_fp16 = add(x = var_7857_cast_fp16, y = var_7861_cast_fp16)[name = string("op_7863_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_7864_cast_fp16 = add(x = var_7862_cast_fp16, y = var_7863_cast_fp16)[name = string("op_7864_cast_fp16")];
            fp16 _inversed_7866_y_0_to_fp16 = const()[name = string("_inversed_7866_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 16]> _inversed_7866_cast_fp16 = mul(x = var_7864_cast_fp16, y = _inversed_7866_y_0_to_fp16)[name = string("_inversed_7866_cast_fp16")];
            string scores_35_equation_0 = const()[name = string("scores_35_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 16]> scores_35_cast_fp16 = einsum(equation = scores_35_equation_0, values = (var_7824_cast_fp16_0, var_7814_cast_fp16_1))[name = string("scores_35_cast_fp16")];
            fp16 var_7869_to_fp16 = const()[name = string("op_7869_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 16]> var_7870_cast_fp16 = mul(x = scores_35_cast_fp16, y = var_7869_to_fp16)[name = string("op_7870_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_7871_cast_fp16 = add(x = var_7870_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_7871_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_7872_cast_fp16 = softmax(axis = var_7792, x = var_7871_cast_fp16)[name = string("op_7872_cast_fp16")];
            tensor<int32, [4]> tile_43 = const()[name = string("tile_43"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_7873_axis_0 = const()[name = string("op_7873_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 16]> var_7873_cast_fp16_0, tensor<fp16, [1, 512, 1, 16]> var_7873_cast_fp16_1, tensor<fp16, [1, 512, 1, 16]> var_7873_cast_fp16_2, tensor<fp16, [1, 512, 1, 16]> var_7873_cast_fp16_3 = split(axis = var_7873_axis_0, split_sizes = tile_43, x = var_7872_cast_fp16)[name = string("op_7873_cast_fp16")];
            tensor<int32, [4]> tile_44 = const()[name = string("tile_44"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_7878_axis_0 = const()[name = string("op_7878_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_7878_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_7878_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_7878_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_7878_cast_fp16_3 = split(axis = var_7878_axis_0, split_sizes = tile_44, x = var_7827_cast_fp16_0)[name = string("op_7878_cast_fp16")];
            fp16 var_7883_to_fp16 = const()[name = string("op_7883_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_7884_cast_fp16 = mul(x = var_7873_cast_fp16_0, y = var_7883_to_fp16)[name = string("op_7884_cast_fp16")];
            string var_7886_equation_0 = const()[name = string("op_7886_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_7886_cast_fp16 = einsum(equation = var_7886_equation_0, values = (var_7878_cast_fp16_0, var_7884_cast_fp16))[name = string("op_7886_cast_fp16")];
            fp16 var_7887_to_fp16 = const()[name = string("op_7887_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_7888_cast_fp16 = mul(x = var_7873_cast_fp16_1, y = var_7887_to_fp16)[name = string("op_7888_cast_fp16")];
            string var_7890_equation_0 = const()[name = string("op_7890_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_7890_cast_fp16 = einsum(equation = var_7890_equation_0, values = (var_7878_cast_fp16_1, var_7888_cast_fp16))[name = string("op_7890_cast_fp16")];
            fp16 var_7891_to_fp16 = const()[name = string("op_7891_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_7892_cast_fp16 = mul(x = var_7873_cast_fp16_2, y = var_7891_to_fp16)[name = string("op_7892_cast_fp16")];
            string var_7894_equation_0 = const()[name = string("op_7894_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_7894_cast_fp16 = einsum(equation = var_7894_equation_0, values = (var_7878_cast_fp16_2, var_7892_cast_fp16))[name = string("op_7894_cast_fp16")];
            fp16 var_7895_to_fp16 = const()[name = string("op_7895_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_7896_cast_fp16 = mul(x = var_7873_cast_fp16_3, y = var_7895_to_fp16)[name = string("op_7896_cast_fp16")];
            string var_7898_equation_0 = const()[name = string("op_7898_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_7898_cast_fp16 = einsum(equation = var_7898_equation_0, values = (var_7878_cast_fp16_3, var_7896_cast_fp16))[name = string("op_7898_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_7899_cast_fp16 = add(x = var_7886_cast_fp16, y = var_7890_cast_fp16)[name = string("op_7899_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_7900_cast_fp16 = add(x = var_7894_cast_fp16, y = var_7898_cast_fp16)[name = string("op_7900_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_7901_cast_fp16 = add(x = var_7899_cast_fp16, y = var_7900_cast_fp16)[name = string("op_7901_cast_fp16")];
            fp16 _inversed_7903_y_0_to_fp16 = const()[name = string("_inversed_7903_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 16]> _inversed_7903_cast_fp16 = mul(x = var_7901_cast_fp16, y = _inversed_7903_y_0_to_fp16)[name = string("_inversed_7903_cast_fp16")];
            string scores_37_equation_0 = const()[name = string("scores_37_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 16]> scores_37_cast_fp16 = einsum(equation = scores_37_equation_0, values = (var_7824_cast_fp16_0, var_7814_cast_fp16_2))[name = string("scores_37_cast_fp16")];
            fp16 var_7906_to_fp16 = const()[name = string("op_7906_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 16]> var_7907_cast_fp16 = mul(x = scores_37_cast_fp16, y = var_7906_to_fp16)[name = string("op_7907_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_7908_cast_fp16 = add(x = var_7907_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_7908_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_7909_cast_fp16 = softmax(axis = var_7792, x = var_7908_cast_fp16)[name = string("op_7909_cast_fp16")];
            tensor<int32, [4]> tile_45 = const()[name = string("tile_45"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_7910_axis_0 = const()[name = string("op_7910_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 16]> var_7910_cast_fp16_0, tensor<fp16, [1, 512, 1, 16]> var_7910_cast_fp16_1, tensor<fp16, [1, 512, 1, 16]> var_7910_cast_fp16_2, tensor<fp16, [1, 512, 1, 16]> var_7910_cast_fp16_3 = split(axis = var_7910_axis_0, split_sizes = tile_45, x = var_7909_cast_fp16)[name = string("op_7910_cast_fp16")];
            tensor<int32, [4]> tile_46 = const()[name = string("tile_46"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_7915_axis_0 = const()[name = string("op_7915_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_7915_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_7915_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_7915_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_7915_cast_fp16_3 = split(axis = var_7915_axis_0, split_sizes = tile_46, x = var_7827_cast_fp16_0)[name = string("op_7915_cast_fp16")];
            fp16 var_7920_to_fp16 = const()[name = string("op_7920_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_7921_cast_fp16 = mul(x = var_7910_cast_fp16_0, y = var_7920_to_fp16)[name = string("op_7921_cast_fp16")];
            string var_7923_equation_0 = const()[name = string("op_7923_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_7923_cast_fp16 = einsum(equation = var_7923_equation_0, values = (var_7915_cast_fp16_0, var_7921_cast_fp16))[name = string("op_7923_cast_fp16")];
            fp16 var_7924_to_fp16 = const()[name = string("op_7924_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_7925_cast_fp16 = mul(x = var_7910_cast_fp16_1, y = var_7924_to_fp16)[name = string("op_7925_cast_fp16")];
            string var_7927_equation_0 = const()[name = string("op_7927_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_7927_cast_fp16 = einsum(equation = var_7927_equation_0, values = (var_7915_cast_fp16_1, var_7925_cast_fp16))[name = string("op_7927_cast_fp16")];
            fp16 var_7928_to_fp16 = const()[name = string("op_7928_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_7929_cast_fp16 = mul(x = var_7910_cast_fp16_2, y = var_7928_to_fp16)[name = string("op_7929_cast_fp16")];
            string var_7931_equation_0 = const()[name = string("op_7931_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_7931_cast_fp16 = einsum(equation = var_7931_equation_0, values = (var_7915_cast_fp16_2, var_7929_cast_fp16))[name = string("op_7931_cast_fp16")];
            fp16 var_7932_to_fp16 = const()[name = string("op_7932_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_7933_cast_fp16 = mul(x = var_7910_cast_fp16_3, y = var_7932_to_fp16)[name = string("op_7933_cast_fp16")];
            string var_7935_equation_0 = const()[name = string("op_7935_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_7935_cast_fp16 = einsum(equation = var_7935_equation_0, values = (var_7915_cast_fp16_3, var_7933_cast_fp16))[name = string("op_7935_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_7936_cast_fp16 = add(x = var_7923_cast_fp16, y = var_7927_cast_fp16)[name = string("op_7936_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_7937_cast_fp16 = add(x = var_7931_cast_fp16, y = var_7935_cast_fp16)[name = string("op_7937_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_7938_cast_fp16 = add(x = var_7936_cast_fp16, y = var_7937_cast_fp16)[name = string("op_7938_cast_fp16")];
            fp16 _inversed_7940_y_0_to_fp16 = const()[name = string("_inversed_7940_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 16]> _inversed_7940_cast_fp16 = mul(x = var_7938_cast_fp16, y = _inversed_7940_y_0_to_fp16)[name = string("_inversed_7940_cast_fp16")];
            string scores_39_equation_0 = const()[name = string("scores_39_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 16]> scores_39_cast_fp16 = einsum(equation = scores_39_equation_0, values = (var_7824_cast_fp16_0, var_7814_cast_fp16_3))[name = string("scores_39_cast_fp16")];
            fp16 var_7943_to_fp16 = const()[name = string("op_7943_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 16]> var_7944_cast_fp16 = mul(x = scores_39_cast_fp16, y = var_7943_to_fp16)[name = string("op_7944_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_7945_cast_fp16 = add(x = var_7944_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_7945_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_7946_cast_fp16 = softmax(axis = var_7792, x = var_7945_cast_fp16)[name = string("op_7946_cast_fp16")];
            tensor<int32, [4]> tile_47 = const()[name = string("tile_47"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_7947_axis_0 = const()[name = string("op_7947_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 16]> var_7947_cast_fp16_0, tensor<fp16, [1, 512, 1, 16]> var_7947_cast_fp16_1, tensor<fp16, [1, 512, 1, 16]> var_7947_cast_fp16_2, tensor<fp16, [1, 512, 1, 16]> var_7947_cast_fp16_3 = split(axis = var_7947_axis_0, split_sizes = tile_47, x = var_7946_cast_fp16)[name = string("op_7947_cast_fp16")];
            tensor<int32, [4]> tile_48 = const()[name = string("tile_48"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_7952_axis_0 = const()[name = string("op_7952_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_7952_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_7952_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_7952_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_7952_cast_fp16_3 = split(axis = var_7952_axis_0, split_sizes = tile_48, x = var_7827_cast_fp16_0)[name = string("op_7952_cast_fp16")];
            fp16 var_7957_to_fp16 = const()[name = string("op_7957_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_7958_cast_fp16 = mul(x = var_7947_cast_fp16_0, y = var_7957_to_fp16)[name = string("op_7958_cast_fp16")];
            string var_7960_equation_0 = const()[name = string("op_7960_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_7960_cast_fp16 = einsum(equation = var_7960_equation_0, values = (var_7952_cast_fp16_0, var_7958_cast_fp16))[name = string("op_7960_cast_fp16")];
            fp16 var_7961_to_fp16 = const()[name = string("op_7961_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_7962_cast_fp16 = mul(x = var_7947_cast_fp16_1, y = var_7961_to_fp16)[name = string("op_7962_cast_fp16")];
            string var_7964_equation_0 = const()[name = string("op_7964_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_7964_cast_fp16 = einsum(equation = var_7964_equation_0, values = (var_7952_cast_fp16_1, var_7962_cast_fp16))[name = string("op_7964_cast_fp16")];
            fp16 var_7965_to_fp16 = const()[name = string("op_7965_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_7966_cast_fp16 = mul(x = var_7947_cast_fp16_2, y = var_7965_to_fp16)[name = string("op_7966_cast_fp16")];
            string var_7968_equation_0 = const()[name = string("op_7968_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_7968_cast_fp16 = einsum(equation = var_7968_equation_0, values = (var_7952_cast_fp16_2, var_7966_cast_fp16))[name = string("op_7968_cast_fp16")];
            fp16 var_7969_to_fp16 = const()[name = string("op_7969_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_7970_cast_fp16 = mul(x = var_7947_cast_fp16_3, y = var_7969_to_fp16)[name = string("op_7970_cast_fp16")];
            string var_7972_equation_0 = const()[name = string("op_7972_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_7972_cast_fp16 = einsum(equation = var_7972_equation_0, values = (var_7952_cast_fp16_3, var_7970_cast_fp16))[name = string("op_7972_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_7973_cast_fp16 = add(x = var_7960_cast_fp16, y = var_7964_cast_fp16)[name = string("op_7973_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_7974_cast_fp16 = add(x = var_7968_cast_fp16, y = var_7972_cast_fp16)[name = string("op_7974_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_7975_cast_fp16 = add(x = var_7973_cast_fp16, y = var_7974_cast_fp16)[name = string("op_7975_cast_fp16")];
            fp16 _inversed_7977_y_0_to_fp16 = const()[name = string("_inversed_7977_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 16]> _inversed_7977_cast_fp16 = mul(x = var_7975_cast_fp16, y = _inversed_7977_y_0_to_fp16)[name = string("_inversed_7977_cast_fp16")];
            string scores_41_equation_0 = const()[name = string("scores_41_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 16]> scores_41_cast_fp16 = einsum(equation = scores_41_equation_0, values = (var_7824_cast_fp16_1, var_7814_cast_fp16_4))[name = string("scores_41_cast_fp16")];
            fp16 var_7980_to_fp16 = const()[name = string("op_7980_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 16]> var_7981_cast_fp16 = mul(x = scores_41_cast_fp16, y = var_7980_to_fp16)[name = string("op_7981_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_7982_cast_fp16 = add(x = var_7981_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_7982_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_7983_cast_fp16 = softmax(axis = var_7792, x = var_7982_cast_fp16)[name = string("op_7983_cast_fp16")];
            tensor<int32, [4]> tile_49 = const()[name = string("tile_49"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_7984_axis_0 = const()[name = string("op_7984_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 16]> var_7984_cast_fp16_0, tensor<fp16, [1, 512, 1, 16]> var_7984_cast_fp16_1, tensor<fp16, [1, 512, 1, 16]> var_7984_cast_fp16_2, tensor<fp16, [1, 512, 1, 16]> var_7984_cast_fp16_3 = split(axis = var_7984_axis_0, split_sizes = tile_49, x = var_7983_cast_fp16)[name = string("op_7984_cast_fp16")];
            tensor<int32, [4]> tile_50 = const()[name = string("tile_50"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_7989_axis_0 = const()[name = string("op_7989_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_7989_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_7989_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_7989_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_7989_cast_fp16_3 = split(axis = var_7989_axis_0, split_sizes = tile_50, x = var_7827_cast_fp16_1)[name = string("op_7989_cast_fp16")];
            fp16 var_7994_to_fp16 = const()[name = string("op_7994_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_7995_cast_fp16 = mul(x = var_7984_cast_fp16_0, y = var_7994_to_fp16)[name = string("op_7995_cast_fp16")];
            string var_7997_equation_0 = const()[name = string("op_7997_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_7997_cast_fp16 = einsum(equation = var_7997_equation_0, values = (var_7989_cast_fp16_0, var_7995_cast_fp16))[name = string("op_7997_cast_fp16")];
            fp16 var_7998_to_fp16 = const()[name = string("op_7998_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_7999_cast_fp16 = mul(x = var_7984_cast_fp16_1, y = var_7998_to_fp16)[name = string("op_7999_cast_fp16")];
            string var_8001_equation_0 = const()[name = string("op_8001_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_8001_cast_fp16 = einsum(equation = var_8001_equation_0, values = (var_7989_cast_fp16_1, var_7999_cast_fp16))[name = string("op_8001_cast_fp16")];
            fp16 var_8002_to_fp16 = const()[name = string("op_8002_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_8003_cast_fp16 = mul(x = var_7984_cast_fp16_2, y = var_8002_to_fp16)[name = string("op_8003_cast_fp16")];
            string var_8005_equation_0 = const()[name = string("op_8005_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_8005_cast_fp16 = einsum(equation = var_8005_equation_0, values = (var_7989_cast_fp16_2, var_8003_cast_fp16))[name = string("op_8005_cast_fp16")];
            fp16 var_8006_to_fp16 = const()[name = string("op_8006_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_8007_cast_fp16 = mul(x = var_7984_cast_fp16_3, y = var_8006_to_fp16)[name = string("op_8007_cast_fp16")];
            string var_8009_equation_0 = const()[name = string("op_8009_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_8009_cast_fp16 = einsum(equation = var_8009_equation_0, values = (var_7989_cast_fp16_3, var_8007_cast_fp16))[name = string("op_8009_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_8010_cast_fp16 = add(x = var_7997_cast_fp16, y = var_8001_cast_fp16)[name = string("op_8010_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_8011_cast_fp16 = add(x = var_8005_cast_fp16, y = var_8009_cast_fp16)[name = string("op_8011_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_8012_cast_fp16 = add(x = var_8010_cast_fp16, y = var_8011_cast_fp16)[name = string("op_8012_cast_fp16")];
            fp16 _inversed_8014_y_0_to_fp16 = const()[name = string("_inversed_8014_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 16]> _inversed_8014_cast_fp16 = mul(x = var_8012_cast_fp16, y = _inversed_8014_y_0_to_fp16)[name = string("_inversed_8014_cast_fp16")];
            string scores_43_equation_0 = const()[name = string("scores_43_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 16]> scores_43_cast_fp16 = einsum(equation = scores_43_equation_0, values = (var_7824_cast_fp16_1, var_7814_cast_fp16_5))[name = string("scores_43_cast_fp16")];
            fp16 var_8017_to_fp16 = const()[name = string("op_8017_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 16]> var_8018_cast_fp16 = mul(x = scores_43_cast_fp16, y = var_8017_to_fp16)[name = string("op_8018_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_8019_cast_fp16 = add(x = var_8018_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_8019_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_8020_cast_fp16 = softmax(axis = var_7792, x = var_8019_cast_fp16)[name = string("op_8020_cast_fp16")];
            tensor<int32, [4]> tile_51 = const()[name = string("tile_51"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_8021_axis_0 = const()[name = string("op_8021_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 16]> var_8021_cast_fp16_0, tensor<fp16, [1, 512, 1, 16]> var_8021_cast_fp16_1, tensor<fp16, [1, 512, 1, 16]> var_8021_cast_fp16_2, tensor<fp16, [1, 512, 1, 16]> var_8021_cast_fp16_3 = split(axis = var_8021_axis_0, split_sizes = tile_51, x = var_8020_cast_fp16)[name = string("op_8021_cast_fp16")];
            tensor<int32, [4]> tile_52 = const()[name = string("tile_52"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_8026_axis_0 = const()[name = string("op_8026_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_8026_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_8026_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_8026_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_8026_cast_fp16_3 = split(axis = var_8026_axis_0, split_sizes = tile_52, x = var_7827_cast_fp16_1)[name = string("op_8026_cast_fp16")];
            fp16 var_8031_to_fp16 = const()[name = string("op_8031_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_8032_cast_fp16 = mul(x = var_8021_cast_fp16_0, y = var_8031_to_fp16)[name = string("op_8032_cast_fp16")];
            string var_8034_equation_0 = const()[name = string("op_8034_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_8034_cast_fp16 = einsum(equation = var_8034_equation_0, values = (var_8026_cast_fp16_0, var_8032_cast_fp16))[name = string("op_8034_cast_fp16")];
            fp16 var_8035_to_fp16 = const()[name = string("op_8035_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_8036_cast_fp16 = mul(x = var_8021_cast_fp16_1, y = var_8035_to_fp16)[name = string("op_8036_cast_fp16")];
            string var_8038_equation_0 = const()[name = string("op_8038_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_8038_cast_fp16 = einsum(equation = var_8038_equation_0, values = (var_8026_cast_fp16_1, var_8036_cast_fp16))[name = string("op_8038_cast_fp16")];
            fp16 var_8039_to_fp16 = const()[name = string("op_8039_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_8040_cast_fp16 = mul(x = var_8021_cast_fp16_2, y = var_8039_to_fp16)[name = string("op_8040_cast_fp16")];
            string var_8042_equation_0 = const()[name = string("op_8042_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_8042_cast_fp16 = einsum(equation = var_8042_equation_0, values = (var_8026_cast_fp16_2, var_8040_cast_fp16))[name = string("op_8042_cast_fp16")];
            fp16 var_8043_to_fp16 = const()[name = string("op_8043_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_8044_cast_fp16 = mul(x = var_8021_cast_fp16_3, y = var_8043_to_fp16)[name = string("op_8044_cast_fp16")];
            string var_8046_equation_0 = const()[name = string("op_8046_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_8046_cast_fp16 = einsum(equation = var_8046_equation_0, values = (var_8026_cast_fp16_3, var_8044_cast_fp16))[name = string("op_8046_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_8047_cast_fp16 = add(x = var_8034_cast_fp16, y = var_8038_cast_fp16)[name = string("op_8047_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_8048_cast_fp16 = add(x = var_8042_cast_fp16, y = var_8046_cast_fp16)[name = string("op_8048_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_8049_cast_fp16 = add(x = var_8047_cast_fp16, y = var_8048_cast_fp16)[name = string("op_8049_cast_fp16")];
            fp16 _inversed_8051_y_0_to_fp16 = const()[name = string("_inversed_8051_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 16]> _inversed_8051_cast_fp16 = mul(x = var_8049_cast_fp16, y = _inversed_8051_y_0_to_fp16)[name = string("_inversed_8051_cast_fp16")];
            string scores_45_equation_0 = const()[name = string("scores_45_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 16]> scores_45_cast_fp16 = einsum(equation = scores_45_equation_0, values = (var_7824_cast_fp16_1, var_7814_cast_fp16_6))[name = string("scores_45_cast_fp16")];
            fp16 var_8054_to_fp16 = const()[name = string("op_8054_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 16]> var_8055_cast_fp16 = mul(x = scores_45_cast_fp16, y = var_8054_to_fp16)[name = string("op_8055_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_8056_cast_fp16 = add(x = var_8055_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_8056_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_8057_cast_fp16 = softmax(axis = var_7792, x = var_8056_cast_fp16)[name = string("op_8057_cast_fp16")];
            tensor<int32, [4]> tile_53 = const()[name = string("tile_53"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_8058_axis_0 = const()[name = string("op_8058_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 16]> var_8058_cast_fp16_0, tensor<fp16, [1, 512, 1, 16]> var_8058_cast_fp16_1, tensor<fp16, [1, 512, 1, 16]> var_8058_cast_fp16_2, tensor<fp16, [1, 512, 1, 16]> var_8058_cast_fp16_3 = split(axis = var_8058_axis_0, split_sizes = tile_53, x = var_8057_cast_fp16)[name = string("op_8058_cast_fp16")];
            tensor<int32, [4]> tile_54 = const()[name = string("tile_54"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_8063_axis_0 = const()[name = string("op_8063_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_8063_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_8063_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_8063_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_8063_cast_fp16_3 = split(axis = var_8063_axis_0, split_sizes = tile_54, x = var_7827_cast_fp16_1)[name = string("op_8063_cast_fp16")];
            fp16 var_8068_to_fp16 = const()[name = string("op_8068_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_8069_cast_fp16 = mul(x = var_8058_cast_fp16_0, y = var_8068_to_fp16)[name = string("op_8069_cast_fp16")];
            string var_8071_equation_0 = const()[name = string("op_8071_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_8071_cast_fp16 = einsum(equation = var_8071_equation_0, values = (var_8063_cast_fp16_0, var_8069_cast_fp16))[name = string("op_8071_cast_fp16")];
            fp16 var_8072_to_fp16 = const()[name = string("op_8072_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_8073_cast_fp16 = mul(x = var_8058_cast_fp16_1, y = var_8072_to_fp16)[name = string("op_8073_cast_fp16")];
            string var_8075_equation_0 = const()[name = string("op_8075_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_8075_cast_fp16 = einsum(equation = var_8075_equation_0, values = (var_8063_cast_fp16_1, var_8073_cast_fp16))[name = string("op_8075_cast_fp16")];
            fp16 var_8076_to_fp16 = const()[name = string("op_8076_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_8077_cast_fp16 = mul(x = var_8058_cast_fp16_2, y = var_8076_to_fp16)[name = string("op_8077_cast_fp16")];
            string var_8079_equation_0 = const()[name = string("op_8079_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_8079_cast_fp16 = einsum(equation = var_8079_equation_0, values = (var_8063_cast_fp16_2, var_8077_cast_fp16))[name = string("op_8079_cast_fp16")];
            fp16 var_8080_to_fp16 = const()[name = string("op_8080_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_8081_cast_fp16 = mul(x = var_8058_cast_fp16_3, y = var_8080_to_fp16)[name = string("op_8081_cast_fp16")];
            string var_8083_equation_0 = const()[name = string("op_8083_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_8083_cast_fp16 = einsum(equation = var_8083_equation_0, values = (var_8063_cast_fp16_3, var_8081_cast_fp16))[name = string("op_8083_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_8084_cast_fp16 = add(x = var_8071_cast_fp16, y = var_8075_cast_fp16)[name = string("op_8084_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_8085_cast_fp16 = add(x = var_8079_cast_fp16, y = var_8083_cast_fp16)[name = string("op_8085_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_8086_cast_fp16 = add(x = var_8084_cast_fp16, y = var_8085_cast_fp16)[name = string("op_8086_cast_fp16")];
            fp16 _inversed_8088_y_0_to_fp16 = const()[name = string("_inversed_8088_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 16]> _inversed_8088_cast_fp16 = mul(x = var_8086_cast_fp16, y = _inversed_8088_y_0_to_fp16)[name = string("_inversed_8088_cast_fp16")];
            string scores_47_equation_0 = const()[name = string("scores_47_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 16]> scores_47_cast_fp16 = einsum(equation = scores_47_equation_0, values = (var_7824_cast_fp16_1, var_7814_cast_fp16_7))[name = string("scores_47_cast_fp16")];
            fp16 var_8091_to_fp16 = const()[name = string("op_8091_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 16]> var_8092_cast_fp16 = mul(x = scores_47_cast_fp16, y = var_8091_to_fp16)[name = string("op_8092_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_8093_cast_fp16 = add(x = var_8092_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_8093_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_8094_cast_fp16 = softmax(axis = var_7792, x = var_8093_cast_fp16)[name = string("op_8094_cast_fp16")];
            tensor<int32, [4]> tile_55 = const()[name = string("tile_55"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_8095_axis_0 = const()[name = string("op_8095_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 16]> var_8095_cast_fp16_0, tensor<fp16, [1, 512, 1, 16]> var_8095_cast_fp16_1, tensor<fp16, [1, 512, 1, 16]> var_8095_cast_fp16_2, tensor<fp16, [1, 512, 1, 16]> var_8095_cast_fp16_3 = split(axis = var_8095_axis_0, split_sizes = tile_55, x = var_8094_cast_fp16)[name = string("op_8095_cast_fp16")];
            tensor<int32, [4]> tile_56 = const()[name = string("tile_56"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_8100_axis_0 = const()[name = string("op_8100_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_8100_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_8100_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_8100_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_8100_cast_fp16_3 = split(axis = var_8100_axis_0, split_sizes = tile_56, x = var_7827_cast_fp16_1)[name = string("op_8100_cast_fp16")];
            fp16 var_8105_to_fp16 = const()[name = string("op_8105_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_8106_cast_fp16 = mul(x = var_8095_cast_fp16_0, y = var_8105_to_fp16)[name = string("op_8106_cast_fp16")];
            string var_8108_equation_0 = const()[name = string("op_8108_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_8108_cast_fp16 = einsum(equation = var_8108_equation_0, values = (var_8100_cast_fp16_0, var_8106_cast_fp16))[name = string("op_8108_cast_fp16")];
            fp16 var_8109_to_fp16 = const()[name = string("op_8109_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_8110_cast_fp16 = mul(x = var_8095_cast_fp16_1, y = var_8109_to_fp16)[name = string("op_8110_cast_fp16")];
            string var_8112_equation_0 = const()[name = string("op_8112_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_8112_cast_fp16 = einsum(equation = var_8112_equation_0, values = (var_8100_cast_fp16_1, var_8110_cast_fp16))[name = string("op_8112_cast_fp16")];
            fp16 var_8113_to_fp16 = const()[name = string("op_8113_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_8114_cast_fp16 = mul(x = var_8095_cast_fp16_2, y = var_8113_to_fp16)[name = string("op_8114_cast_fp16")];
            string var_8116_equation_0 = const()[name = string("op_8116_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_8116_cast_fp16 = einsum(equation = var_8116_equation_0, values = (var_8100_cast_fp16_2, var_8114_cast_fp16))[name = string("op_8116_cast_fp16")];
            fp16 var_8117_to_fp16 = const()[name = string("op_8117_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_8118_cast_fp16 = mul(x = var_8095_cast_fp16_3, y = var_8117_to_fp16)[name = string("op_8118_cast_fp16")];
            string var_8120_equation_0 = const()[name = string("op_8120_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_8120_cast_fp16 = einsum(equation = var_8120_equation_0, values = (var_8100_cast_fp16_3, var_8118_cast_fp16))[name = string("op_8120_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_8121_cast_fp16 = add(x = var_8108_cast_fp16, y = var_8112_cast_fp16)[name = string("op_8121_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_8122_cast_fp16 = add(x = var_8116_cast_fp16, y = var_8120_cast_fp16)[name = string("op_8122_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_8123_cast_fp16 = add(x = var_8121_cast_fp16, y = var_8122_cast_fp16)[name = string("op_8123_cast_fp16")];
            fp16 _inversed_8125_y_0_to_fp16 = const()[name = string("_inversed_8125_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 16]> _inversed_8125_cast_fp16 = mul(x = var_8123_cast_fp16, y = _inversed_8125_y_0_to_fp16)[name = string("_inversed_8125_cast_fp16")];
            bool var_8127_interleave_0 = const()[name = string("op_8127_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 2048, 1, 16]> var_8127_cast_fp16 = concat(axis = var_7792, interleave = var_8127_interleave_0, values = (_inversed_7866_cast_fp16, _inversed_7903_cast_fp16, _inversed_7940_cast_fp16, _inversed_7977_cast_fp16, _inversed_8014_cast_fp16, _inversed_8051_cast_fp16, _inversed_8088_cast_fp16, _inversed_8125_cast_fp16))[name = string("op_8127_cast_fp16")];
            tensor<int32, [4]> var_8128 = const()[name = string("op_8128"), val = tensor<int32, [4]>([2, 4, 256, 16])];
            tensor<fp16, [2, 4, 256, 16]> var_8129_cast_fp16 = reshape(shape = var_8128, x = var_8127_cast_fp16)[name = string("op_8129_cast_fp16")];
            tensor<int32, [4]> transpose_5_perm_0 = const()[name = string("transpose_5_perm_0"), val = tensor<int32, [4]>([3, 0, 1, 2])];
            tensor<int32, [2]> var_8148 = const()[name = string("op_8148"), val = tensor<int32, [2]>([16, 2048])];
            tensor<fp16, [16, 2, 4, 256]> transpose_5_cast_fp16 = transpose(perm = transpose_5_perm_0, x = var_8129_cast_fp16)[name = string("transpose_54")];
            tensor<fp16, [16, 2048]> attention_output_11_cast_fp16 = reshape(shape = var_8148, x = transpose_5_cast_fp16)[name = string("attention_output_11_cast_fp16")];
            tensor<fp16, [16, 2048]> var_8150_cast_fp16 = sigmoid(x = gate_11_cast_fp16)[name = string("op_8150_cast_fp16")];
            tensor<fp16, [16, 2048]> input_149_cast_fp16 = mul(x = attention_output_11_cast_fp16, y = var_8150_cast_fp16)[name = string("input_149_cast_fp16")];
            tensor<fp16, [1024, 2048]> completions_2_o_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 2048]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(179014592))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(180587520))))[name = string("completions_2_o_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 1024]> linear_89_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = completions_2_o_proj_weight_promoted_to_fp16_palettized, x = input_149_cast_fp16)[name = string("linear_89_cast_fp16")];
            tensor<fp16, [16, 1024]> value_173_cast_fp16 = add(x = value_159_cast_fp16, y = linear_89_cast_fp16)[name = string("value_173_cast_fp16")];
            tensor<fp16, [16, 1024]> var_8155_cast_fp16 = mul(x = value_173_cast_fp16, y = value_173_cast_fp16)[name = string("op_8155_cast_fp16")];
            tensor<int32, [1]> variance_77_axes_0 = const()[name = string("variance_77_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_77_keep_dims_0 = const()[name = string("variance_77_keep_dims_0"), val = bool(true)];
            tensor<fp16, [16, 1]> variance_77_cast_fp16 = reduce_mean(axes = variance_77_axes_0, keep_dims = variance_77_keep_dims_0, x = var_8155_cast_fp16)[name = string("variance_77_cast_fp16")];
            fp16 var_8158_to_fp16 = const()[name = string("op_8158_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [16, 1]> var_8159_cast_fp16 = add(x = variance_77_cast_fp16, y = var_8158_to_fp16)[name = string("op_8159_cast_fp16")];
            fp32 var_8160_epsilon_0 = const()[name = string("op_8160_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [16, 1]> var_8160_cast_fp16 = rsqrt(epsilon = var_8160_epsilon_0, x = var_8159_cast_fp16)[name = string("op_8160_cast_fp16")];
            tensor<fp16, [16, 1024]> var_8161_cast_fp16 = mul(x = value_173_cast_fp16, y = var_8160_cast_fp16)[name = string("op_8161_cast_fp16")];
            tensor<fp16, [1024]> var_8163_to_fp16 = const()[name = string("op_8163_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(180595776)))];
            tensor<fp16, [16, 1024]> input_151_cast_fp16 = mul(x = var_8161_cast_fp16, y = var_8163_to_fp16)[name = string("input_151_cast_fp16")];
            tensor<fp16, [3584, 1024]> completions_2_gate_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(180597888))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(183350464))))[name = string("completions_2_gate_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 3584]> linear_90_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = completions_2_gate_proj_weight_promoted_to_fp16_palettized, x = input_151_cast_fp16)[name = string("linear_90_cast_fp16")];
            tensor<fp16, [16, 3584]> var_8167_cast_fp16 = silu(x = linear_90_cast_fp16)[name = string("op_8167_cast_fp16")];
            tensor<fp16, [3584, 1024]> completions_2_up_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(183379200))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(186131776))))[name = string("completions_2_up_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 3584]> linear_91_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = completions_2_up_proj_weight_promoted_to_fp16_palettized, x = input_151_cast_fp16)[name = string("linear_91_cast_fp16")];
            tensor<fp16, [16, 3584]> input_155_cast_fp16 = mul(x = var_8167_cast_fp16, y = linear_91_cast_fp16)[name = string("input_155_cast_fp16")];
            tensor<fp16, [1024, 3584]> completions_2_down_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 3584]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(186160512))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(188913088))))[name = string("completions_2_down_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 1024]> linear_92_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = completions_2_down_proj_weight_promoted_to_fp16_palettized, x = input_155_cast_fp16)[name = string("linear_92_cast_fp16")];
            tensor<fp16, [16, 1024]> value_175_cast_fp16 = add(x = value_173_cast_fp16, y = linear_92_cast_fp16)[name = string("value_175_cast_fp16")];
            int32 var_8201 = const()[name = string("op_8201"), val = int32(-1)];
            tensor<fp16, [16, 1024]> var_8216_cast_fp16 = mul(x = value_175_cast_fp16, y = value_175_cast_fp16)[name = string("op_8216_cast_fp16")];
            tensor<int32, [1]> variance_79_axes_0 = const()[name = string("variance_79_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_79_keep_dims_0 = const()[name = string("variance_79_keep_dims_0"), val = bool(true)];
            tensor<fp16, [16, 1]> variance_79_cast_fp16 = reduce_mean(axes = variance_79_axes_0, keep_dims = variance_79_keep_dims_0, x = var_8216_cast_fp16)[name = string("variance_79_cast_fp16")];
            fp16 var_8219_to_fp16 = const()[name = string("op_8219_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [16, 1]> var_8220_cast_fp16 = add(x = variance_79_cast_fp16, y = var_8219_to_fp16)[name = string("op_8220_cast_fp16")];
            fp32 var_8221_epsilon_0 = const()[name = string("op_8221_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [16, 1]> var_8221_cast_fp16 = rsqrt(epsilon = var_8221_epsilon_0, x = var_8220_cast_fp16)[name = string("op_8221_cast_fp16")];
            tensor<fp16, [16, 1024]> var_8222_cast_fp16 = mul(x = value_175_cast_fp16, y = var_8221_cast_fp16)[name = string("op_8222_cast_fp16")];
            tensor<fp16, [1024]> var_8224_to_fp16 = const()[name = string("op_8224_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(188921344)))];
            tensor<fp16, [16, 1024]> input_157_cast_fp16 = mul(x = var_8222_cast_fp16, y = var_8224_to_fp16)[name = string("input_157_cast_fp16")];
            tensor<fp16, [6144, 1024]> groups_3_0_in_proj_qkv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(188923456))), lut = tensor<fp16, [384, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(193642112))))[name = string("groups_3_0_in_proj_qkv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 6144]> linear_93_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = groups_3_0_in_proj_qkv_weight_promoted_to_fp16_palettized, x = input_157_cast_fp16)[name = string("linear_93_cast_fp16")];
            tensor<int32, [2]> var_8228_perm_0 = const()[name = string("op_8228_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> mixed_19_axes_0 = const()[name = string("mixed_19_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [6144, 16]> var_8228_cast_fp16 = transpose(perm = var_8228_perm_0, x = linear_93_cast_fp16)[name = string("transpose_53")];
            tensor<fp16, [1, 6144, 16]> mixed_19_cast_fp16 = expand_dims(axes = mixed_19_axes_0, x = var_8228_cast_fp16)[name = string("mixed_19_cast_fp16")];
            bool convolution_input_19_interleave_0 = const()[name = string("convolution_input_19_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 6144, 19]> convolution_input_19_cast_fp16 = concat(axis = var_8201, interleave = convolution_input_19_interleave_0, values = (conv12, mixed_19_cast_fp16))[name = string("convolution_input_19_cast_fp16")];
            string input_159_pad_type_0 = const()[name = string("input_159_pad_type_0"), val = string("valid")];
            int32 input_159_groups_0 = const()[name = string("input_159_groups_0"), val = int32(6144)];
            tensor<int32, [1]> input_159_strides_0 = const()[name = string("input_159_strides_0"), val = tensor<int32, [1]>([1])];
            tensor<int32, [2]> input_159_pad_0 = const()[name = string("input_159_pad_0"), val = tensor<int32, [2]>([0, 0])];
            tensor<int32, [1]> input_159_dilations_0 = const()[name = string("input_159_dilations_0"), val = tensor<int32, [1]>([1])];
            tensor<fp16, [6144, 1, 4]> groups_3_0_conv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1, 4]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(193691328))), lut = tensor<fp16, [384, 1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(193709824))))[name = string("groups_3_0_conv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 6144, 16]> input_159_cast_fp16 = conv(dilations = input_159_dilations_0, groups = input_159_groups_0, pad = input_159_pad_0, pad_type = input_159_pad_type_0, strides = input_159_strides_0, weight = groups_3_0_conv_weight_promoted_to_fp16_palettized, x = convolution_input_19_cast_fp16)[name = string("input_159_cast_fp16")];
            tensor<fp16, [1, 6144, 16]> var_8237_cast_fp16 = silu(x = input_159_cast_fp16)[name = string("op_8237_cast_fp16")];
            tensor<int32, [3]> var_8238_perm_0 = const()[name = string("op_8238_perm_0"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [3]> var_8241_begin_0 = const()[name = string("op_8241_begin_0"), val = tensor<int32, [3]>([0, 0, 16])];
            tensor<int32, [3]> var_8241_end_0 = const()[name = string("op_8241_end_0"), val = tensor<int32, [3]>([1, 6144, 19])];
            tensor<bool, [3]> var_8241_end_mask_0 = const()[name = string("op_8241_end_mask_0"), val = tensor<bool, [3]>([true, true, true])];
            tensor<fp16, [1, 6144, 3]> conv12_out = slice_by_index(begin = var_8241_begin_0, end = var_8241_end_0, end_mask = var_8241_end_mask_0, x = convolution_input_19_cast_fp16)[name = string("op_8241_cast_fp16")];
            tensor<int32, [3]> var_8242 = const()[name = string("op_8242"), val = tensor<int32, [3]>([2048, 2048, 2048])];
            int32 var_8243_axis_0 = const()[name = string("op_8243_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 16, 6144]> var_8238_cast_fp16 = transpose(perm = var_8238_perm_0, x = var_8237_cast_fp16)[name = string("transpose_52")];
            tensor<fp16, [1, 16, 2048]> var_8243_cast_fp16_0, tensor<fp16, [1, 16, 2048]> var_8243_cast_fp16_1, tensor<fp16, [1, 16, 2048]> var_8243_cast_fp16_2 = split(axis = var_8243_axis_0, split_sizes = var_8242, x = var_8238_cast_fp16)[name = string("op_8243_cast_fp16")];
            tensor<int32, [4]> var_8247 = const()[name = string("op_8247"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<fp16, [1, 16, 16, 128]> value_179_cast_fp16 = reshape(shape = var_8247, x = var_8243_cast_fp16_0)[name = string("value_179_cast_fp16")];
            tensor<int32, [4]> var_8249 = const()[name = string("op_8249"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<fp16, [1, 16, 16, 128]> value_181_cast_fp16 = reshape(shape = var_8249, x = var_8243_cast_fp16_1)[name = string("value_181_cast_fp16")];
            tensor<int32, [4]> var_8251 = const()[name = string("op_8251"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<fp16, [1, 16, 16, 128]> value_183_cast_fp16 = reshape(shape = var_8251, x = var_8243_cast_fp16_2)[name = string("value_183_cast_fp16")];
            tensor<fp16, [1, 16, 16, 128]> var_8253_cast_fp16 = mul(x = value_179_cast_fp16, y = value_179_cast_fp16)[name = string("op_8253_cast_fp16")];
            tensor<int32, [1]> var_8255_axes_0 = const()[name = string("op_8255_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_8255_keep_dims_0 = const()[name = string("op_8255_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 16, 16, 1]> var_8255_cast_fp16 = reduce_sum(axes = var_8255_axes_0, keep_dims = var_8255_keep_dims_0, x = var_8253_cast_fp16)[name = string("op_8255_cast_fp16")];
            fp16 var_8256_to_fp16 = const()[name = string("op_8256_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 16, 16, 1]> var_8257_cast_fp16 = add(x = var_8255_cast_fp16, y = var_8256_to_fp16)[name = string("op_8257_cast_fp16")];
            fp32 var_8258_epsilon_0 = const()[name = string("op_8258_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 16, 16, 1]> var_8258_cast_fp16 = rsqrt(epsilon = var_8258_epsilon_0, x = var_8257_cast_fp16)[name = string("op_8258_cast_fp16")];
            tensor<fp16, [1, 16, 16, 128]> var_8259_cast_fp16 = mul(x = value_179_cast_fp16, y = var_8258_cast_fp16)[name = string("op_8259_cast_fp16")];
            tensor<int32, [4]> var_8260_perm_0 = const()[name = string("op_8260_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            fp16 _inversed_query_51_y_0_to_fp16 = const()[name = string("_inversed_query_51_y_0_to_fp16"), val = fp16(0x1.6ap-4)];
            tensor<fp16, [1, 16, 16, 128]> var_8260_cast_fp16 = transpose(perm = var_8260_perm_0, x = var_8259_cast_fp16)[name = string("transpose_51")];
            tensor<fp16, [1, 16, 16, 128]> _inversed_query_51_cast_fp16 = mul(x = var_8260_cast_fp16, y = _inversed_query_51_y_0_to_fp16)[name = string("_inversed_query_51_cast_fp16")];
            tensor<fp16, [1, 16, 16, 128]> var_8263_cast_fp16 = mul(x = value_181_cast_fp16, y = value_181_cast_fp16)[name = string("op_8263_cast_fp16")];
            tensor<int32, [1]> var_8265_axes_0 = const()[name = string("op_8265_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_8265_keep_dims_0 = const()[name = string("op_8265_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 16, 16, 1]> var_8265_cast_fp16 = reduce_sum(axes = var_8265_axes_0, keep_dims = var_8265_keep_dims_0, x = var_8263_cast_fp16)[name = string("op_8265_cast_fp16")];
            fp16 var_8266_to_fp16 = const()[name = string("op_8266_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 16, 16, 1]> var_8267_cast_fp16 = add(x = var_8265_cast_fp16, y = var_8266_to_fp16)[name = string("op_8267_cast_fp16")];
            fp32 var_8268_epsilon_0 = const()[name = string("op_8268_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 16, 16, 1]> var_8268_cast_fp16 = rsqrt(epsilon = var_8268_epsilon_0, x = var_8267_cast_fp16)[name = string("op_8268_cast_fp16")];
            tensor<fp16, [1, 16, 16, 128]> var_8269_cast_fp16 = mul(x = value_181_cast_fp16, y = var_8268_cast_fp16)[name = string("op_8269_cast_fp16")];
            tensor<int32, [4]> key_51_perm_0 = const()[name = string("key_51_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<int32, [4]> value_185_perm_0 = const()[name = string("value_185_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<fp16, [16, 1024]> groups_3_0_in_proj_b_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(193759040))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(193771392))))[name = string("groups_3_0_in_proj_b_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16]> linear_94_bias_0_to_fp16 = const()[name = string("linear_94_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(4862656)))];
            tensor<fp16, [16, 16]> linear_94_cast_fp16 = linear(bias = linear_94_bias_0_to_fp16, weight = groups_3_0_in_proj_b_weight_promoted_to_fp16_palettized, x = input_157_cast_fp16)[name = string("linear_94_cast_fp16")];
            tensor<fp16, [16, 16]> var_8274_cast_fp16 = sigmoid(x = linear_94_cast_fp16)[name = string("op_8274_cast_fp16")];
            tensor<int32, [2]> var_8275_perm_0 = const()[name = string("op_8275_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> beta_19_axes_0 = const()[name = string("beta_19_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 16]> var_8275_cast_fp16 = transpose(perm = var_8275_perm_0, x = var_8274_cast_fp16)[name = string("transpose_50")];
            tensor<fp16, [1, 16, 16]> beta_19_cast_fp16 = expand_dims(axes = beta_19_axes_0, x = var_8275_cast_fp16)[name = string("beta_19_cast_fp16")];
            tensor<fp16, [16, 1024]> op_8281_weight_0_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(193771584))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(193783936))))[name = string("op_8281_weight_0_to_fp16_palettized")];
            tensor<fp16, [16]> var_8281_bias_0_to_fp16 = const()[name = string("op_8281_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(193784128)))];
            tensor<fp16, [16, 16]> var_8281_cast_fp16 = linear(bias = var_8281_bias_0_to_fp16, weight = op_8281_weight_0_to_fp16_palettized, x = input_157_cast_fp16)[name = string("op_8281_cast_fp16")];
            tensor<fp16, [16, 16]> var_8282_cast_fp16 = softplus(x = var_8281_cast_fp16)[name = string("op_8282_cast_fp16")];
            tensor<fp16, [16]> var_8278_promoted_to_fp16 = const()[name = string("op_8278_promoted_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(193784256)))];
            tensor<fp16, [16, 16]> var_8283_cast_fp16 = mul(x = var_8278_promoted_to_fp16, y = var_8282_cast_fp16)[name = string("op_8283_cast_fp16")];
            tensor<int32, [2]> var_8284_perm_0 = const()[name = string("op_8284_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> decay_19_axes_0 = const()[name = string("decay_19_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 16]> var_8284_cast_fp16 = transpose(perm = var_8284_perm_0, x = var_8283_cast_fp16)[name = string("transpose_49")];
            tensor<fp16, [1, 16, 16]> decay_19_cast_fp16 = expand_dims(axes = decay_19_axes_0, x = var_8284_cast_fp16)[name = string("decay_19_cast_fp16")];
            tensor<fp16, [2048, 1024]> groups_3_0_in_proj_z_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [2048, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(193784384))), lut = tensor<fp16, [128, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(195357312))))[name = string("groups_3_0_in_proj_z_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 2048]> linear_96_cast_fp16 = linear(bias = linear_3_bias_0_to_fp16, weight = groups_3_0_in_proj_z_weight_promoted_to_fp16_palettized, x = input_157_cast_fp16)[name = string("linear_96_cast_fp16")];
            tensor<int32, [3]> var_8292_begin_0 = const()[name = string("op_8292_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_8292_end_0 = const()[name = string("op_8292_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_8292_end_mask_0 = const()[name = string("op_8292_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_8292_squeeze_mask_0 = const()[name = string("op_8292_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_8292_cast_fp16 = slice_by_index(begin = var_8292_begin_0, end = var_8292_end_0, end_mask = var_8292_end_mask_0, squeeze_mask = var_8292_squeeze_mask_0, x = decay_19_cast_fp16)[name = string("op_8292_cast_fp16")];
            tensor<fp16, [1, 16]> var_8293_cast_fp16 = exp(x = var_8292_cast_fp16)[name = string("op_8293_cast_fp16")];
            tensor<int32, [1]> var_8294_axes_0 = const()[name = string("op_8294_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_8294_cast_fp16 = expand_dims(axes = var_8294_axes_0, x = var_8293_cast_fp16)[name = string("op_8294_cast_fp16")];
            tensor<int32, [1]> var_8295_axes_0 = const()[name = string("op_8295_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_8295_cast_fp16 = expand_dims(axes = var_8295_axes_0, x = var_8294_cast_fp16)[name = string("op_8295_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_577_cast_fp16 = mul(x = rec12, y = var_8295_cast_fp16)[name = string("state_577_cast_fp16")];
            tensor<int32, [4]> key_token_289_begin_0 = const()[name = string("key_token_289_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> key_token_289_end_0 = const()[name = string("key_token_289_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> key_token_289_end_mask_0 = const()[name = string("key_token_289_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_289_squeeze_mask_0 = const()[name = string("key_token_289_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 16, 128]> key_51_cast_fp16 = transpose(perm = key_51_perm_0, x = var_8269_cast_fp16)[name = string("transpose_48")];
            tensor<fp16, [1, 16, 128]> key_token_289_cast_fp16 = slice_by_index(begin = key_token_289_begin_0, end = key_token_289_end_0, end_mask = key_token_289_end_mask_0, squeeze_mask = key_token_289_squeeze_mask_0, x = key_51_cast_fp16)[name = string("key_token_289_cast_fp16")];
            tensor<int32, [4]> value_token_289_begin_0 = const()[name = string("value_token_289_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> value_token_289_end_0 = const()[name = string("value_token_289_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> value_token_289_end_mask_0 = const()[name = string("value_token_289_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_289_squeeze_mask_0 = const()[name = string("value_token_289_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 16, 128]> value_185_cast_fp16 = transpose(perm = value_185_perm_0, x = value_183_cast_fp16)[name = string("transpose_47")];
            tensor<fp16, [1, 16, 128]> value_token_289_cast_fp16 = slice_by_index(begin = value_token_289_begin_0, end = value_token_289_end_0, end_mask = value_token_289_end_mask_0, squeeze_mask = value_token_289_squeeze_mask_0, x = value_185_cast_fp16)[name = string("value_token_289_cast_fp16")];
            tensor<int32, [1]> var_8303_axes_0 = const()[name = string("op_8303_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_8303_cast_fp16 = expand_dims(axes = var_8303_axes_0, x = key_token_289_cast_fp16)[name = string("op_8303_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_8304_cast_fp16 = mul(x = state_577_cast_fp16, y = var_8303_cast_fp16)[name = string("op_8304_cast_fp16")];
            tensor<int32, [1]> memory_289_axes_0 = const()[name = string("memory_289_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_289_keep_dims_0 = const()[name = string("memory_289_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_289_cast_fp16 = reduce_sum(axes = memory_289_axes_0, keep_dims = memory_289_keep_dims_0, x = var_8304_cast_fp16)[name = string("memory_289_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_8307_cast_fp16 = sub(x = value_token_289_cast_fp16, y = memory_289_cast_fp16)[name = string("op_8307_cast_fp16")];
            tensor<int32, [3]> var_8310_begin_0 = const()[name = string("op_8310_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_8310_end_0 = const()[name = string("op_8310_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_8310_end_mask_0 = const()[name = string("op_8310_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_8310_squeeze_mask_0 = const()[name = string("op_8310_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_8310_cast_fp16 = slice_by_index(begin = var_8310_begin_0, end = var_8310_end_0, end_mask = var_8310_end_mask_0, squeeze_mask = var_8310_squeeze_mask_0, x = beta_19_cast_fp16)[name = string("op_8310_cast_fp16")];
            tensor<int32, [1]> var_8311_axes_0 = const()[name = string("op_8311_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_8311_cast_fp16 = expand_dims(axes = var_8311_axes_0, x = var_8310_cast_fp16)[name = string("op_8311_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_289_cast_fp16 = mul(x = var_8307_cast_fp16, y = var_8311_cast_fp16)[name = string("delta_289_cast_fp16")];
            tensor<int32, [1]> var_8314_axes_0 = const()[name = string("op_8314_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_8314_cast_fp16 = expand_dims(axes = var_8314_axes_0, x = delta_289_cast_fp16)[name = string("op_8314_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_8315_cast_fp16 = mul(x = var_8303_cast_fp16, y = var_8314_cast_fp16)[name = string("op_8315_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_579_cast_fp16 = add(x = state_577_cast_fp16, y = var_8315_cast_fp16)[name = string("state_579_cast_fp16")];
            tensor<int32, [4]> var_8319_begin_0 = const()[name = string("op_8319_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> var_8319_end_0 = const()[name = string("op_8319_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> var_8319_end_mask_0 = const()[name = string("op_8319_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_8319_squeeze_mask_0 = const()[name = string("op_8319_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_8319_cast_fp16 = slice_by_index(begin = var_8319_begin_0, end = var_8319_end_0, end_mask = var_8319_end_mask_0, squeeze_mask = var_8319_squeeze_mask_0, x = _inversed_query_51_cast_fp16)[name = string("op_8319_cast_fp16")];
            tensor<int32, [1]> var_8320_axes_0 = const()[name = string("op_8320_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_8320_cast_fp16 = expand_dims(axes = var_8320_axes_0, x = var_8319_cast_fp16)[name = string("op_8320_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_8321_cast_fp16 = mul(x = state_579_cast_fp16, y = var_8320_cast_fp16)[name = string("op_8321_cast_fp16")];
            tensor<int32, [1]> var_8323_axes_0 = const()[name = string("op_8323_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_8323_keep_dims_0 = const()[name = string("op_8323_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_8323_cast_fp16 = reduce_sum(axes = var_8323_axes_0, keep_dims = var_8323_keep_dims_0, x = var_8321_cast_fp16)[name = string("op_8323_cast_fp16")];
            tensor<int32, [3]> var_8326_begin_0 = const()[name = string("op_8326_begin_0"), val = tensor<int32, [3]>([0, 0, 1])];
            tensor<int32, [3]> var_8326_end_0 = const()[name = string("op_8326_end_0"), val = tensor<int32, [3]>([1, 16, 2])];
            tensor<bool, [3]> var_8326_end_mask_0 = const()[name = string("op_8326_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_8326_squeeze_mask_0 = const()[name = string("op_8326_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_8326_cast_fp16 = slice_by_index(begin = var_8326_begin_0, end = var_8326_end_0, end_mask = var_8326_end_mask_0, squeeze_mask = var_8326_squeeze_mask_0, x = decay_19_cast_fp16)[name = string("op_8326_cast_fp16")];
            tensor<fp16, [1, 16]> var_8327_cast_fp16 = exp(x = var_8326_cast_fp16)[name = string("op_8327_cast_fp16")];
            tensor<int32, [1]> var_8328_axes_0 = const()[name = string("op_8328_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_8328_cast_fp16 = expand_dims(axes = var_8328_axes_0, x = var_8327_cast_fp16)[name = string("op_8328_cast_fp16")];
            tensor<int32, [1]> var_8329_axes_0 = const()[name = string("op_8329_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_8329_cast_fp16 = expand_dims(axes = var_8329_axes_0, x = var_8328_cast_fp16)[name = string("op_8329_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_581_cast_fp16 = mul(x = state_579_cast_fp16, y = var_8329_cast_fp16)[name = string("state_581_cast_fp16")];
            tensor<int32, [4]> key_token_291_begin_0 = const()[name = string("key_token_291_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> key_token_291_end_0 = const()[name = string("key_token_291_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> key_token_291_end_mask_0 = const()[name = string("key_token_291_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_291_squeeze_mask_0 = const()[name = string("key_token_291_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_291_cast_fp16 = slice_by_index(begin = key_token_291_begin_0, end = key_token_291_end_0, end_mask = key_token_291_end_mask_0, squeeze_mask = key_token_291_squeeze_mask_0, x = key_51_cast_fp16)[name = string("key_token_291_cast_fp16")];
            tensor<int32, [4]> value_token_291_begin_0 = const()[name = string("value_token_291_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> value_token_291_end_0 = const()[name = string("value_token_291_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> value_token_291_end_mask_0 = const()[name = string("value_token_291_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_291_squeeze_mask_0 = const()[name = string("value_token_291_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_291_cast_fp16 = slice_by_index(begin = value_token_291_begin_0, end = value_token_291_end_0, end_mask = value_token_291_end_mask_0, squeeze_mask = value_token_291_squeeze_mask_0, x = value_185_cast_fp16)[name = string("value_token_291_cast_fp16")];
            tensor<int32, [1]> var_8337_axes_0 = const()[name = string("op_8337_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_8337_cast_fp16 = expand_dims(axes = var_8337_axes_0, x = key_token_291_cast_fp16)[name = string("op_8337_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_8338_cast_fp16 = mul(x = state_581_cast_fp16, y = var_8337_cast_fp16)[name = string("op_8338_cast_fp16")];
            tensor<int32, [1]> memory_291_axes_0 = const()[name = string("memory_291_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_291_keep_dims_0 = const()[name = string("memory_291_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_291_cast_fp16 = reduce_sum(axes = memory_291_axes_0, keep_dims = memory_291_keep_dims_0, x = var_8338_cast_fp16)[name = string("memory_291_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_8341_cast_fp16 = sub(x = value_token_291_cast_fp16, y = memory_291_cast_fp16)[name = string("op_8341_cast_fp16")];
            tensor<int32, [3]> var_8344_begin_0 = const()[name = string("op_8344_begin_0"), val = tensor<int32, [3]>([0, 0, 1])];
            tensor<int32, [3]> var_8344_end_0 = const()[name = string("op_8344_end_0"), val = tensor<int32, [3]>([1, 16, 2])];
            tensor<bool, [3]> var_8344_end_mask_0 = const()[name = string("op_8344_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_8344_squeeze_mask_0 = const()[name = string("op_8344_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_8344_cast_fp16 = slice_by_index(begin = var_8344_begin_0, end = var_8344_end_0, end_mask = var_8344_end_mask_0, squeeze_mask = var_8344_squeeze_mask_0, x = beta_19_cast_fp16)[name = string("op_8344_cast_fp16")];
            tensor<int32, [1]> var_8345_axes_0 = const()[name = string("op_8345_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_8345_cast_fp16 = expand_dims(axes = var_8345_axes_0, x = var_8344_cast_fp16)[name = string("op_8345_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_291_cast_fp16 = mul(x = var_8341_cast_fp16, y = var_8345_cast_fp16)[name = string("delta_291_cast_fp16")];
            tensor<int32, [1]> var_8348_axes_0 = const()[name = string("op_8348_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_8348_cast_fp16 = expand_dims(axes = var_8348_axes_0, x = delta_291_cast_fp16)[name = string("op_8348_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_8349_cast_fp16 = mul(x = var_8337_cast_fp16, y = var_8348_cast_fp16)[name = string("op_8349_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_583_cast_fp16 = add(x = state_581_cast_fp16, y = var_8349_cast_fp16)[name = string("state_583_cast_fp16")];
            tensor<int32, [4]> var_8353_begin_0 = const()[name = string("op_8353_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> var_8353_end_0 = const()[name = string("op_8353_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> var_8353_end_mask_0 = const()[name = string("op_8353_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_8353_squeeze_mask_0 = const()[name = string("op_8353_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_8353_cast_fp16 = slice_by_index(begin = var_8353_begin_0, end = var_8353_end_0, end_mask = var_8353_end_mask_0, squeeze_mask = var_8353_squeeze_mask_0, x = _inversed_query_51_cast_fp16)[name = string("op_8353_cast_fp16")];
            tensor<int32, [1]> var_8354_axes_0 = const()[name = string("op_8354_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_8354_cast_fp16 = expand_dims(axes = var_8354_axes_0, x = var_8353_cast_fp16)[name = string("op_8354_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_8355_cast_fp16 = mul(x = state_583_cast_fp16, y = var_8354_cast_fp16)[name = string("op_8355_cast_fp16")];
            tensor<int32, [1]> var_8357_axes_0 = const()[name = string("op_8357_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_8357_keep_dims_0 = const()[name = string("op_8357_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_8357_cast_fp16 = reduce_sum(axes = var_8357_axes_0, keep_dims = var_8357_keep_dims_0, x = var_8355_cast_fp16)[name = string("op_8357_cast_fp16")];
            tensor<int32, [3]> var_8360_begin_0 = const()[name = string("op_8360_begin_0"), val = tensor<int32, [3]>([0, 0, 2])];
            tensor<int32, [3]> var_8360_end_0 = const()[name = string("op_8360_end_0"), val = tensor<int32, [3]>([1, 16, 3])];
            tensor<bool, [3]> var_8360_end_mask_0 = const()[name = string("op_8360_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_8360_squeeze_mask_0 = const()[name = string("op_8360_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_8360_cast_fp16 = slice_by_index(begin = var_8360_begin_0, end = var_8360_end_0, end_mask = var_8360_end_mask_0, squeeze_mask = var_8360_squeeze_mask_0, x = decay_19_cast_fp16)[name = string("op_8360_cast_fp16")];
            tensor<fp16, [1, 16]> var_8361_cast_fp16 = exp(x = var_8360_cast_fp16)[name = string("op_8361_cast_fp16")];
            tensor<int32, [1]> var_8362_axes_0 = const()[name = string("op_8362_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_8362_cast_fp16 = expand_dims(axes = var_8362_axes_0, x = var_8361_cast_fp16)[name = string("op_8362_cast_fp16")];
            tensor<int32, [1]> var_8363_axes_0 = const()[name = string("op_8363_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_8363_cast_fp16 = expand_dims(axes = var_8363_axes_0, x = var_8362_cast_fp16)[name = string("op_8363_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_585_cast_fp16 = mul(x = state_583_cast_fp16, y = var_8363_cast_fp16)[name = string("state_585_cast_fp16")];
            tensor<int32, [4]> key_token_293_begin_0 = const()[name = string("key_token_293_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> key_token_293_end_0 = const()[name = string("key_token_293_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> key_token_293_end_mask_0 = const()[name = string("key_token_293_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_293_squeeze_mask_0 = const()[name = string("key_token_293_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_293_cast_fp16 = slice_by_index(begin = key_token_293_begin_0, end = key_token_293_end_0, end_mask = key_token_293_end_mask_0, squeeze_mask = key_token_293_squeeze_mask_0, x = key_51_cast_fp16)[name = string("key_token_293_cast_fp16")];
            tensor<int32, [4]> value_token_293_begin_0 = const()[name = string("value_token_293_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> value_token_293_end_0 = const()[name = string("value_token_293_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> value_token_293_end_mask_0 = const()[name = string("value_token_293_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_293_squeeze_mask_0 = const()[name = string("value_token_293_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_293_cast_fp16 = slice_by_index(begin = value_token_293_begin_0, end = value_token_293_end_0, end_mask = value_token_293_end_mask_0, squeeze_mask = value_token_293_squeeze_mask_0, x = value_185_cast_fp16)[name = string("value_token_293_cast_fp16")];
            tensor<int32, [1]> var_8371_axes_0 = const()[name = string("op_8371_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_8371_cast_fp16 = expand_dims(axes = var_8371_axes_0, x = key_token_293_cast_fp16)[name = string("op_8371_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_8372_cast_fp16 = mul(x = state_585_cast_fp16, y = var_8371_cast_fp16)[name = string("op_8372_cast_fp16")];
            tensor<int32, [1]> memory_293_axes_0 = const()[name = string("memory_293_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_293_keep_dims_0 = const()[name = string("memory_293_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_293_cast_fp16 = reduce_sum(axes = memory_293_axes_0, keep_dims = memory_293_keep_dims_0, x = var_8372_cast_fp16)[name = string("memory_293_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_8375_cast_fp16 = sub(x = value_token_293_cast_fp16, y = memory_293_cast_fp16)[name = string("op_8375_cast_fp16")];
            tensor<int32, [3]> var_8378_begin_0 = const()[name = string("op_8378_begin_0"), val = tensor<int32, [3]>([0, 0, 2])];
            tensor<int32, [3]> var_8378_end_0 = const()[name = string("op_8378_end_0"), val = tensor<int32, [3]>([1, 16, 3])];
            tensor<bool, [3]> var_8378_end_mask_0 = const()[name = string("op_8378_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_8378_squeeze_mask_0 = const()[name = string("op_8378_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_8378_cast_fp16 = slice_by_index(begin = var_8378_begin_0, end = var_8378_end_0, end_mask = var_8378_end_mask_0, squeeze_mask = var_8378_squeeze_mask_0, x = beta_19_cast_fp16)[name = string("op_8378_cast_fp16")];
            tensor<int32, [1]> var_8379_axes_0 = const()[name = string("op_8379_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_8379_cast_fp16 = expand_dims(axes = var_8379_axes_0, x = var_8378_cast_fp16)[name = string("op_8379_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_293_cast_fp16 = mul(x = var_8375_cast_fp16, y = var_8379_cast_fp16)[name = string("delta_293_cast_fp16")];
            tensor<int32, [1]> var_8382_axes_0 = const()[name = string("op_8382_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_8382_cast_fp16 = expand_dims(axes = var_8382_axes_0, x = delta_293_cast_fp16)[name = string("op_8382_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_8383_cast_fp16 = mul(x = var_8371_cast_fp16, y = var_8382_cast_fp16)[name = string("op_8383_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_587_cast_fp16 = add(x = state_585_cast_fp16, y = var_8383_cast_fp16)[name = string("state_587_cast_fp16")];
            tensor<int32, [4]> var_8387_begin_0 = const()[name = string("op_8387_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> var_8387_end_0 = const()[name = string("op_8387_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> var_8387_end_mask_0 = const()[name = string("op_8387_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_8387_squeeze_mask_0 = const()[name = string("op_8387_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_8387_cast_fp16 = slice_by_index(begin = var_8387_begin_0, end = var_8387_end_0, end_mask = var_8387_end_mask_0, squeeze_mask = var_8387_squeeze_mask_0, x = _inversed_query_51_cast_fp16)[name = string("op_8387_cast_fp16")];
            tensor<int32, [1]> var_8388_axes_0 = const()[name = string("op_8388_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_8388_cast_fp16 = expand_dims(axes = var_8388_axes_0, x = var_8387_cast_fp16)[name = string("op_8388_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_8389_cast_fp16 = mul(x = state_587_cast_fp16, y = var_8388_cast_fp16)[name = string("op_8389_cast_fp16")];
            tensor<int32, [1]> var_8391_axes_0 = const()[name = string("op_8391_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_8391_keep_dims_0 = const()[name = string("op_8391_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_8391_cast_fp16 = reduce_sum(axes = var_8391_axes_0, keep_dims = var_8391_keep_dims_0, x = var_8389_cast_fp16)[name = string("op_8391_cast_fp16")];
            tensor<int32, [3]> var_8394_begin_0 = const()[name = string("op_8394_begin_0"), val = tensor<int32, [3]>([0, 0, 3])];
            tensor<int32, [3]> var_8394_end_0 = const()[name = string("op_8394_end_0"), val = tensor<int32, [3]>([1, 16, 4])];
            tensor<bool, [3]> var_8394_end_mask_0 = const()[name = string("op_8394_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_8394_squeeze_mask_0 = const()[name = string("op_8394_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_8394_cast_fp16 = slice_by_index(begin = var_8394_begin_0, end = var_8394_end_0, end_mask = var_8394_end_mask_0, squeeze_mask = var_8394_squeeze_mask_0, x = decay_19_cast_fp16)[name = string("op_8394_cast_fp16")];
            tensor<fp16, [1, 16]> var_8395_cast_fp16 = exp(x = var_8394_cast_fp16)[name = string("op_8395_cast_fp16")];
            tensor<int32, [1]> var_8396_axes_0 = const()[name = string("op_8396_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_8396_cast_fp16 = expand_dims(axes = var_8396_axes_0, x = var_8395_cast_fp16)[name = string("op_8396_cast_fp16")];
            tensor<int32, [1]> var_8397_axes_0 = const()[name = string("op_8397_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_8397_cast_fp16 = expand_dims(axes = var_8397_axes_0, x = var_8396_cast_fp16)[name = string("op_8397_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_589_cast_fp16 = mul(x = state_587_cast_fp16, y = var_8397_cast_fp16)[name = string("state_589_cast_fp16")];
            tensor<int32, [4]> key_token_295_begin_0 = const()[name = string("key_token_295_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> key_token_295_end_0 = const()[name = string("key_token_295_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> key_token_295_end_mask_0 = const()[name = string("key_token_295_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_295_squeeze_mask_0 = const()[name = string("key_token_295_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_295_cast_fp16 = slice_by_index(begin = key_token_295_begin_0, end = key_token_295_end_0, end_mask = key_token_295_end_mask_0, squeeze_mask = key_token_295_squeeze_mask_0, x = key_51_cast_fp16)[name = string("key_token_295_cast_fp16")];
            tensor<int32, [4]> value_token_295_begin_0 = const()[name = string("value_token_295_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> value_token_295_end_0 = const()[name = string("value_token_295_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> value_token_295_end_mask_0 = const()[name = string("value_token_295_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_295_squeeze_mask_0 = const()[name = string("value_token_295_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_295_cast_fp16 = slice_by_index(begin = value_token_295_begin_0, end = value_token_295_end_0, end_mask = value_token_295_end_mask_0, squeeze_mask = value_token_295_squeeze_mask_0, x = value_185_cast_fp16)[name = string("value_token_295_cast_fp16")];
            tensor<int32, [1]> var_8405_axes_0 = const()[name = string("op_8405_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_8405_cast_fp16 = expand_dims(axes = var_8405_axes_0, x = key_token_295_cast_fp16)[name = string("op_8405_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_8406_cast_fp16 = mul(x = state_589_cast_fp16, y = var_8405_cast_fp16)[name = string("op_8406_cast_fp16")];
            tensor<int32, [1]> memory_295_axes_0 = const()[name = string("memory_295_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_295_keep_dims_0 = const()[name = string("memory_295_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_295_cast_fp16 = reduce_sum(axes = memory_295_axes_0, keep_dims = memory_295_keep_dims_0, x = var_8406_cast_fp16)[name = string("memory_295_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_8409_cast_fp16 = sub(x = value_token_295_cast_fp16, y = memory_295_cast_fp16)[name = string("op_8409_cast_fp16")];
            tensor<int32, [3]> var_8412_begin_0 = const()[name = string("op_8412_begin_0"), val = tensor<int32, [3]>([0, 0, 3])];
            tensor<int32, [3]> var_8412_end_0 = const()[name = string("op_8412_end_0"), val = tensor<int32, [3]>([1, 16, 4])];
            tensor<bool, [3]> var_8412_end_mask_0 = const()[name = string("op_8412_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_8412_squeeze_mask_0 = const()[name = string("op_8412_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_8412_cast_fp16 = slice_by_index(begin = var_8412_begin_0, end = var_8412_end_0, end_mask = var_8412_end_mask_0, squeeze_mask = var_8412_squeeze_mask_0, x = beta_19_cast_fp16)[name = string("op_8412_cast_fp16")];
            tensor<int32, [1]> var_8413_axes_0 = const()[name = string("op_8413_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_8413_cast_fp16 = expand_dims(axes = var_8413_axes_0, x = var_8412_cast_fp16)[name = string("op_8413_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_295_cast_fp16 = mul(x = var_8409_cast_fp16, y = var_8413_cast_fp16)[name = string("delta_295_cast_fp16")];
            tensor<int32, [1]> var_8416_axes_0 = const()[name = string("op_8416_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_8416_cast_fp16 = expand_dims(axes = var_8416_axes_0, x = delta_295_cast_fp16)[name = string("op_8416_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_8417_cast_fp16 = mul(x = var_8405_cast_fp16, y = var_8416_cast_fp16)[name = string("op_8417_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_591_cast_fp16 = add(x = state_589_cast_fp16, y = var_8417_cast_fp16)[name = string("state_591_cast_fp16")];
            tensor<int32, [4]> var_8421_begin_0 = const()[name = string("op_8421_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> var_8421_end_0 = const()[name = string("op_8421_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> var_8421_end_mask_0 = const()[name = string("op_8421_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_8421_squeeze_mask_0 = const()[name = string("op_8421_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_8421_cast_fp16 = slice_by_index(begin = var_8421_begin_0, end = var_8421_end_0, end_mask = var_8421_end_mask_0, squeeze_mask = var_8421_squeeze_mask_0, x = _inversed_query_51_cast_fp16)[name = string("op_8421_cast_fp16")];
            tensor<int32, [1]> var_8422_axes_0 = const()[name = string("op_8422_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_8422_cast_fp16 = expand_dims(axes = var_8422_axes_0, x = var_8421_cast_fp16)[name = string("op_8422_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_8423_cast_fp16 = mul(x = state_591_cast_fp16, y = var_8422_cast_fp16)[name = string("op_8423_cast_fp16")];
            tensor<int32, [1]> var_8425_axes_0 = const()[name = string("op_8425_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_8425_keep_dims_0 = const()[name = string("op_8425_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_8425_cast_fp16 = reduce_sum(axes = var_8425_axes_0, keep_dims = var_8425_keep_dims_0, x = var_8423_cast_fp16)[name = string("op_8425_cast_fp16")];
            tensor<int32, [3]> var_8428_begin_0 = const()[name = string("op_8428_begin_0"), val = tensor<int32, [3]>([0, 0, 4])];
            tensor<int32, [3]> var_8428_end_0 = const()[name = string("op_8428_end_0"), val = tensor<int32, [3]>([1, 16, 5])];
            tensor<bool, [3]> var_8428_end_mask_0 = const()[name = string("op_8428_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_8428_squeeze_mask_0 = const()[name = string("op_8428_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_8428_cast_fp16 = slice_by_index(begin = var_8428_begin_0, end = var_8428_end_0, end_mask = var_8428_end_mask_0, squeeze_mask = var_8428_squeeze_mask_0, x = decay_19_cast_fp16)[name = string("op_8428_cast_fp16")];
            tensor<fp16, [1, 16]> var_8429_cast_fp16 = exp(x = var_8428_cast_fp16)[name = string("op_8429_cast_fp16")];
            tensor<int32, [1]> var_8430_axes_0 = const()[name = string("op_8430_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_8430_cast_fp16 = expand_dims(axes = var_8430_axes_0, x = var_8429_cast_fp16)[name = string("op_8430_cast_fp16")];
            tensor<int32, [1]> var_8431_axes_0 = const()[name = string("op_8431_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_8431_cast_fp16 = expand_dims(axes = var_8431_axes_0, x = var_8430_cast_fp16)[name = string("op_8431_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_593_cast_fp16 = mul(x = state_591_cast_fp16, y = var_8431_cast_fp16)[name = string("state_593_cast_fp16")];
            tensor<int32, [4]> key_token_297_begin_0 = const()[name = string("key_token_297_begin_0"), val = tensor<int32, [4]>([0, 0, 4, 0])];
            tensor<int32, [4]> key_token_297_end_0 = const()[name = string("key_token_297_end_0"), val = tensor<int32, [4]>([1, 16, 5, 128])];
            tensor<bool, [4]> key_token_297_end_mask_0 = const()[name = string("key_token_297_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_297_squeeze_mask_0 = const()[name = string("key_token_297_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_297_cast_fp16 = slice_by_index(begin = key_token_297_begin_0, end = key_token_297_end_0, end_mask = key_token_297_end_mask_0, squeeze_mask = key_token_297_squeeze_mask_0, x = key_51_cast_fp16)[name = string("key_token_297_cast_fp16")];
            tensor<int32, [4]> value_token_297_begin_0 = const()[name = string("value_token_297_begin_0"), val = tensor<int32, [4]>([0, 0, 4, 0])];
            tensor<int32, [4]> value_token_297_end_0 = const()[name = string("value_token_297_end_0"), val = tensor<int32, [4]>([1, 16, 5, 128])];
            tensor<bool, [4]> value_token_297_end_mask_0 = const()[name = string("value_token_297_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_297_squeeze_mask_0 = const()[name = string("value_token_297_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_297_cast_fp16 = slice_by_index(begin = value_token_297_begin_0, end = value_token_297_end_0, end_mask = value_token_297_end_mask_0, squeeze_mask = value_token_297_squeeze_mask_0, x = value_185_cast_fp16)[name = string("value_token_297_cast_fp16")];
            tensor<int32, [1]> var_8439_axes_0 = const()[name = string("op_8439_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_8439_cast_fp16 = expand_dims(axes = var_8439_axes_0, x = key_token_297_cast_fp16)[name = string("op_8439_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_8440_cast_fp16 = mul(x = state_593_cast_fp16, y = var_8439_cast_fp16)[name = string("op_8440_cast_fp16")];
            tensor<int32, [1]> memory_297_axes_0 = const()[name = string("memory_297_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_297_keep_dims_0 = const()[name = string("memory_297_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_297_cast_fp16 = reduce_sum(axes = memory_297_axes_0, keep_dims = memory_297_keep_dims_0, x = var_8440_cast_fp16)[name = string("memory_297_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_8443_cast_fp16 = sub(x = value_token_297_cast_fp16, y = memory_297_cast_fp16)[name = string("op_8443_cast_fp16")];
            tensor<int32, [3]> var_8446_begin_0 = const()[name = string("op_8446_begin_0"), val = tensor<int32, [3]>([0, 0, 4])];
            tensor<int32, [3]> var_8446_end_0 = const()[name = string("op_8446_end_0"), val = tensor<int32, [3]>([1, 16, 5])];
            tensor<bool, [3]> var_8446_end_mask_0 = const()[name = string("op_8446_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_8446_squeeze_mask_0 = const()[name = string("op_8446_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_8446_cast_fp16 = slice_by_index(begin = var_8446_begin_0, end = var_8446_end_0, end_mask = var_8446_end_mask_0, squeeze_mask = var_8446_squeeze_mask_0, x = beta_19_cast_fp16)[name = string("op_8446_cast_fp16")];
            tensor<int32, [1]> var_8447_axes_0 = const()[name = string("op_8447_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_8447_cast_fp16 = expand_dims(axes = var_8447_axes_0, x = var_8446_cast_fp16)[name = string("op_8447_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_297_cast_fp16 = mul(x = var_8443_cast_fp16, y = var_8447_cast_fp16)[name = string("delta_297_cast_fp16")];
            tensor<int32, [1]> var_8450_axes_0 = const()[name = string("op_8450_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_8450_cast_fp16 = expand_dims(axes = var_8450_axes_0, x = delta_297_cast_fp16)[name = string("op_8450_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_8451_cast_fp16 = mul(x = var_8439_cast_fp16, y = var_8450_cast_fp16)[name = string("op_8451_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_595_cast_fp16 = add(x = state_593_cast_fp16, y = var_8451_cast_fp16)[name = string("state_595_cast_fp16")];
            tensor<int32, [4]> var_8455_begin_0 = const()[name = string("op_8455_begin_0"), val = tensor<int32, [4]>([0, 0, 4, 0])];
            tensor<int32, [4]> var_8455_end_0 = const()[name = string("op_8455_end_0"), val = tensor<int32, [4]>([1, 16, 5, 128])];
            tensor<bool, [4]> var_8455_end_mask_0 = const()[name = string("op_8455_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_8455_squeeze_mask_0 = const()[name = string("op_8455_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_8455_cast_fp16 = slice_by_index(begin = var_8455_begin_0, end = var_8455_end_0, end_mask = var_8455_end_mask_0, squeeze_mask = var_8455_squeeze_mask_0, x = _inversed_query_51_cast_fp16)[name = string("op_8455_cast_fp16")];
            tensor<int32, [1]> var_8456_axes_0 = const()[name = string("op_8456_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_8456_cast_fp16 = expand_dims(axes = var_8456_axes_0, x = var_8455_cast_fp16)[name = string("op_8456_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_8457_cast_fp16 = mul(x = state_595_cast_fp16, y = var_8456_cast_fp16)[name = string("op_8457_cast_fp16")];
            tensor<int32, [1]> var_8459_axes_0 = const()[name = string("op_8459_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_8459_keep_dims_0 = const()[name = string("op_8459_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_8459_cast_fp16 = reduce_sum(axes = var_8459_axes_0, keep_dims = var_8459_keep_dims_0, x = var_8457_cast_fp16)[name = string("op_8459_cast_fp16")];
            tensor<int32, [3]> var_8462_begin_0 = const()[name = string("op_8462_begin_0"), val = tensor<int32, [3]>([0, 0, 5])];
            tensor<int32, [3]> var_8462_end_0 = const()[name = string("op_8462_end_0"), val = tensor<int32, [3]>([1, 16, 6])];
            tensor<bool, [3]> var_8462_end_mask_0 = const()[name = string("op_8462_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_8462_squeeze_mask_0 = const()[name = string("op_8462_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_8462_cast_fp16 = slice_by_index(begin = var_8462_begin_0, end = var_8462_end_0, end_mask = var_8462_end_mask_0, squeeze_mask = var_8462_squeeze_mask_0, x = decay_19_cast_fp16)[name = string("op_8462_cast_fp16")];
            tensor<fp16, [1, 16]> var_8463_cast_fp16 = exp(x = var_8462_cast_fp16)[name = string("op_8463_cast_fp16")];
            tensor<int32, [1]> var_8464_axes_0 = const()[name = string("op_8464_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_8464_cast_fp16 = expand_dims(axes = var_8464_axes_0, x = var_8463_cast_fp16)[name = string("op_8464_cast_fp16")];
            tensor<int32, [1]> var_8465_axes_0 = const()[name = string("op_8465_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_8465_cast_fp16 = expand_dims(axes = var_8465_axes_0, x = var_8464_cast_fp16)[name = string("op_8465_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_597_cast_fp16 = mul(x = state_595_cast_fp16, y = var_8465_cast_fp16)[name = string("state_597_cast_fp16")];
            tensor<int32, [4]> key_token_299_begin_0 = const()[name = string("key_token_299_begin_0"), val = tensor<int32, [4]>([0, 0, 5, 0])];
            tensor<int32, [4]> key_token_299_end_0 = const()[name = string("key_token_299_end_0"), val = tensor<int32, [4]>([1, 16, 6, 128])];
            tensor<bool, [4]> key_token_299_end_mask_0 = const()[name = string("key_token_299_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_299_squeeze_mask_0 = const()[name = string("key_token_299_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_299_cast_fp16 = slice_by_index(begin = key_token_299_begin_0, end = key_token_299_end_0, end_mask = key_token_299_end_mask_0, squeeze_mask = key_token_299_squeeze_mask_0, x = key_51_cast_fp16)[name = string("key_token_299_cast_fp16")];
            tensor<int32, [4]> value_token_299_begin_0 = const()[name = string("value_token_299_begin_0"), val = tensor<int32, [4]>([0, 0, 5, 0])];
            tensor<int32, [4]> value_token_299_end_0 = const()[name = string("value_token_299_end_0"), val = tensor<int32, [4]>([1, 16, 6, 128])];
            tensor<bool, [4]> value_token_299_end_mask_0 = const()[name = string("value_token_299_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_299_squeeze_mask_0 = const()[name = string("value_token_299_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_299_cast_fp16 = slice_by_index(begin = value_token_299_begin_0, end = value_token_299_end_0, end_mask = value_token_299_end_mask_0, squeeze_mask = value_token_299_squeeze_mask_0, x = value_185_cast_fp16)[name = string("value_token_299_cast_fp16")];
            tensor<int32, [1]> var_8473_axes_0 = const()[name = string("op_8473_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_8473_cast_fp16 = expand_dims(axes = var_8473_axes_0, x = key_token_299_cast_fp16)[name = string("op_8473_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_8474_cast_fp16 = mul(x = state_597_cast_fp16, y = var_8473_cast_fp16)[name = string("op_8474_cast_fp16")];
            tensor<int32, [1]> memory_299_axes_0 = const()[name = string("memory_299_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_299_keep_dims_0 = const()[name = string("memory_299_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_299_cast_fp16 = reduce_sum(axes = memory_299_axes_0, keep_dims = memory_299_keep_dims_0, x = var_8474_cast_fp16)[name = string("memory_299_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_8477_cast_fp16 = sub(x = value_token_299_cast_fp16, y = memory_299_cast_fp16)[name = string("op_8477_cast_fp16")];
            tensor<int32, [3]> var_8480_begin_0 = const()[name = string("op_8480_begin_0"), val = tensor<int32, [3]>([0, 0, 5])];
            tensor<int32, [3]> var_8480_end_0 = const()[name = string("op_8480_end_0"), val = tensor<int32, [3]>([1, 16, 6])];
            tensor<bool, [3]> var_8480_end_mask_0 = const()[name = string("op_8480_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_8480_squeeze_mask_0 = const()[name = string("op_8480_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_8480_cast_fp16 = slice_by_index(begin = var_8480_begin_0, end = var_8480_end_0, end_mask = var_8480_end_mask_0, squeeze_mask = var_8480_squeeze_mask_0, x = beta_19_cast_fp16)[name = string("op_8480_cast_fp16")];
            tensor<int32, [1]> var_8481_axes_0 = const()[name = string("op_8481_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_8481_cast_fp16 = expand_dims(axes = var_8481_axes_0, x = var_8480_cast_fp16)[name = string("op_8481_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_299_cast_fp16 = mul(x = var_8477_cast_fp16, y = var_8481_cast_fp16)[name = string("delta_299_cast_fp16")];
            tensor<int32, [1]> var_8484_axes_0 = const()[name = string("op_8484_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_8484_cast_fp16 = expand_dims(axes = var_8484_axes_0, x = delta_299_cast_fp16)[name = string("op_8484_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_8485_cast_fp16 = mul(x = var_8473_cast_fp16, y = var_8484_cast_fp16)[name = string("op_8485_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_599_cast_fp16 = add(x = state_597_cast_fp16, y = var_8485_cast_fp16)[name = string("state_599_cast_fp16")];
            tensor<int32, [4]> var_8489_begin_0 = const()[name = string("op_8489_begin_0"), val = tensor<int32, [4]>([0, 0, 5, 0])];
            tensor<int32, [4]> var_8489_end_0 = const()[name = string("op_8489_end_0"), val = tensor<int32, [4]>([1, 16, 6, 128])];
            tensor<bool, [4]> var_8489_end_mask_0 = const()[name = string("op_8489_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_8489_squeeze_mask_0 = const()[name = string("op_8489_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_8489_cast_fp16 = slice_by_index(begin = var_8489_begin_0, end = var_8489_end_0, end_mask = var_8489_end_mask_0, squeeze_mask = var_8489_squeeze_mask_0, x = _inversed_query_51_cast_fp16)[name = string("op_8489_cast_fp16")];
            tensor<int32, [1]> var_8490_axes_0 = const()[name = string("op_8490_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_8490_cast_fp16 = expand_dims(axes = var_8490_axes_0, x = var_8489_cast_fp16)[name = string("op_8490_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_8491_cast_fp16 = mul(x = state_599_cast_fp16, y = var_8490_cast_fp16)[name = string("op_8491_cast_fp16")];
            tensor<int32, [1]> var_8493_axes_0 = const()[name = string("op_8493_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_8493_keep_dims_0 = const()[name = string("op_8493_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_8493_cast_fp16 = reduce_sum(axes = var_8493_axes_0, keep_dims = var_8493_keep_dims_0, x = var_8491_cast_fp16)[name = string("op_8493_cast_fp16")];
            tensor<int32, [3]> var_8496_begin_0 = const()[name = string("op_8496_begin_0"), val = tensor<int32, [3]>([0, 0, 6])];
            tensor<int32, [3]> var_8496_end_0 = const()[name = string("op_8496_end_0"), val = tensor<int32, [3]>([1, 16, 7])];
            tensor<bool, [3]> var_8496_end_mask_0 = const()[name = string("op_8496_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_8496_squeeze_mask_0 = const()[name = string("op_8496_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_8496_cast_fp16 = slice_by_index(begin = var_8496_begin_0, end = var_8496_end_0, end_mask = var_8496_end_mask_0, squeeze_mask = var_8496_squeeze_mask_0, x = decay_19_cast_fp16)[name = string("op_8496_cast_fp16")];
            tensor<fp16, [1, 16]> var_8497_cast_fp16 = exp(x = var_8496_cast_fp16)[name = string("op_8497_cast_fp16")];
            tensor<int32, [1]> var_8498_axes_0 = const()[name = string("op_8498_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_8498_cast_fp16 = expand_dims(axes = var_8498_axes_0, x = var_8497_cast_fp16)[name = string("op_8498_cast_fp16")];
            tensor<int32, [1]> var_8499_axes_0 = const()[name = string("op_8499_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_8499_cast_fp16 = expand_dims(axes = var_8499_axes_0, x = var_8498_cast_fp16)[name = string("op_8499_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_601_cast_fp16 = mul(x = state_599_cast_fp16, y = var_8499_cast_fp16)[name = string("state_601_cast_fp16")];
            tensor<int32, [4]> key_token_301_begin_0 = const()[name = string("key_token_301_begin_0"), val = tensor<int32, [4]>([0, 0, 6, 0])];
            tensor<int32, [4]> key_token_301_end_0 = const()[name = string("key_token_301_end_0"), val = tensor<int32, [4]>([1, 16, 7, 128])];
            tensor<bool, [4]> key_token_301_end_mask_0 = const()[name = string("key_token_301_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_301_squeeze_mask_0 = const()[name = string("key_token_301_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_301_cast_fp16 = slice_by_index(begin = key_token_301_begin_0, end = key_token_301_end_0, end_mask = key_token_301_end_mask_0, squeeze_mask = key_token_301_squeeze_mask_0, x = key_51_cast_fp16)[name = string("key_token_301_cast_fp16")];
            tensor<int32, [4]> value_token_301_begin_0 = const()[name = string("value_token_301_begin_0"), val = tensor<int32, [4]>([0, 0, 6, 0])];
            tensor<int32, [4]> value_token_301_end_0 = const()[name = string("value_token_301_end_0"), val = tensor<int32, [4]>([1, 16, 7, 128])];
            tensor<bool, [4]> value_token_301_end_mask_0 = const()[name = string("value_token_301_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_301_squeeze_mask_0 = const()[name = string("value_token_301_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_301_cast_fp16 = slice_by_index(begin = value_token_301_begin_0, end = value_token_301_end_0, end_mask = value_token_301_end_mask_0, squeeze_mask = value_token_301_squeeze_mask_0, x = value_185_cast_fp16)[name = string("value_token_301_cast_fp16")];
            tensor<int32, [1]> var_8507_axes_0 = const()[name = string("op_8507_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_8507_cast_fp16 = expand_dims(axes = var_8507_axes_0, x = key_token_301_cast_fp16)[name = string("op_8507_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_8508_cast_fp16 = mul(x = state_601_cast_fp16, y = var_8507_cast_fp16)[name = string("op_8508_cast_fp16")];
            tensor<int32, [1]> memory_301_axes_0 = const()[name = string("memory_301_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_301_keep_dims_0 = const()[name = string("memory_301_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_301_cast_fp16 = reduce_sum(axes = memory_301_axes_0, keep_dims = memory_301_keep_dims_0, x = var_8508_cast_fp16)[name = string("memory_301_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_8511_cast_fp16 = sub(x = value_token_301_cast_fp16, y = memory_301_cast_fp16)[name = string("op_8511_cast_fp16")];
            tensor<int32, [3]> var_8514_begin_0 = const()[name = string("op_8514_begin_0"), val = tensor<int32, [3]>([0, 0, 6])];
            tensor<int32, [3]> var_8514_end_0 = const()[name = string("op_8514_end_0"), val = tensor<int32, [3]>([1, 16, 7])];
            tensor<bool, [3]> var_8514_end_mask_0 = const()[name = string("op_8514_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_8514_squeeze_mask_0 = const()[name = string("op_8514_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_8514_cast_fp16 = slice_by_index(begin = var_8514_begin_0, end = var_8514_end_0, end_mask = var_8514_end_mask_0, squeeze_mask = var_8514_squeeze_mask_0, x = beta_19_cast_fp16)[name = string("op_8514_cast_fp16")];
            tensor<int32, [1]> var_8515_axes_0 = const()[name = string("op_8515_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_8515_cast_fp16 = expand_dims(axes = var_8515_axes_0, x = var_8514_cast_fp16)[name = string("op_8515_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_301_cast_fp16 = mul(x = var_8511_cast_fp16, y = var_8515_cast_fp16)[name = string("delta_301_cast_fp16")];
            tensor<int32, [1]> var_8518_axes_0 = const()[name = string("op_8518_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_8518_cast_fp16 = expand_dims(axes = var_8518_axes_0, x = delta_301_cast_fp16)[name = string("op_8518_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_8519_cast_fp16 = mul(x = var_8507_cast_fp16, y = var_8518_cast_fp16)[name = string("op_8519_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_603_cast_fp16 = add(x = state_601_cast_fp16, y = var_8519_cast_fp16)[name = string("state_603_cast_fp16")];
            tensor<int32, [4]> var_8523_begin_0 = const()[name = string("op_8523_begin_0"), val = tensor<int32, [4]>([0, 0, 6, 0])];
            tensor<int32, [4]> var_8523_end_0 = const()[name = string("op_8523_end_0"), val = tensor<int32, [4]>([1, 16, 7, 128])];
            tensor<bool, [4]> var_8523_end_mask_0 = const()[name = string("op_8523_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_8523_squeeze_mask_0 = const()[name = string("op_8523_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_8523_cast_fp16 = slice_by_index(begin = var_8523_begin_0, end = var_8523_end_0, end_mask = var_8523_end_mask_0, squeeze_mask = var_8523_squeeze_mask_0, x = _inversed_query_51_cast_fp16)[name = string("op_8523_cast_fp16")];
            tensor<int32, [1]> var_8524_axes_0 = const()[name = string("op_8524_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_8524_cast_fp16 = expand_dims(axes = var_8524_axes_0, x = var_8523_cast_fp16)[name = string("op_8524_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_8525_cast_fp16 = mul(x = state_603_cast_fp16, y = var_8524_cast_fp16)[name = string("op_8525_cast_fp16")];
            tensor<int32, [1]> var_8527_axes_0 = const()[name = string("op_8527_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_8527_keep_dims_0 = const()[name = string("op_8527_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_8527_cast_fp16 = reduce_sum(axes = var_8527_axes_0, keep_dims = var_8527_keep_dims_0, x = var_8525_cast_fp16)[name = string("op_8527_cast_fp16")];
            tensor<int32, [3]> var_8530_begin_0 = const()[name = string("op_8530_begin_0"), val = tensor<int32, [3]>([0, 0, 7])];
            tensor<int32, [3]> var_8530_end_0 = const()[name = string("op_8530_end_0"), val = tensor<int32, [3]>([1, 16, 8])];
            tensor<bool, [3]> var_8530_end_mask_0 = const()[name = string("op_8530_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_8530_squeeze_mask_0 = const()[name = string("op_8530_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_8530_cast_fp16 = slice_by_index(begin = var_8530_begin_0, end = var_8530_end_0, end_mask = var_8530_end_mask_0, squeeze_mask = var_8530_squeeze_mask_0, x = decay_19_cast_fp16)[name = string("op_8530_cast_fp16")];
            tensor<fp16, [1, 16]> var_8531_cast_fp16 = exp(x = var_8530_cast_fp16)[name = string("op_8531_cast_fp16")];
            tensor<int32, [1]> var_8532_axes_0 = const()[name = string("op_8532_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_8532_cast_fp16 = expand_dims(axes = var_8532_axes_0, x = var_8531_cast_fp16)[name = string("op_8532_cast_fp16")];
            tensor<int32, [1]> var_8533_axes_0 = const()[name = string("op_8533_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_8533_cast_fp16 = expand_dims(axes = var_8533_axes_0, x = var_8532_cast_fp16)[name = string("op_8533_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_605_cast_fp16 = mul(x = state_603_cast_fp16, y = var_8533_cast_fp16)[name = string("state_605_cast_fp16")];
            tensor<int32, [4]> key_token_303_begin_0 = const()[name = string("key_token_303_begin_0"), val = tensor<int32, [4]>([0, 0, 7, 0])];
            tensor<int32, [4]> key_token_303_end_0 = const()[name = string("key_token_303_end_0"), val = tensor<int32, [4]>([1, 16, 8, 128])];
            tensor<bool, [4]> key_token_303_end_mask_0 = const()[name = string("key_token_303_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_303_squeeze_mask_0 = const()[name = string("key_token_303_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_303_cast_fp16 = slice_by_index(begin = key_token_303_begin_0, end = key_token_303_end_0, end_mask = key_token_303_end_mask_0, squeeze_mask = key_token_303_squeeze_mask_0, x = key_51_cast_fp16)[name = string("key_token_303_cast_fp16")];
            tensor<int32, [4]> value_token_303_begin_0 = const()[name = string("value_token_303_begin_0"), val = tensor<int32, [4]>([0, 0, 7, 0])];
            tensor<int32, [4]> value_token_303_end_0 = const()[name = string("value_token_303_end_0"), val = tensor<int32, [4]>([1, 16, 8, 128])];
            tensor<bool, [4]> value_token_303_end_mask_0 = const()[name = string("value_token_303_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_303_squeeze_mask_0 = const()[name = string("value_token_303_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_303_cast_fp16 = slice_by_index(begin = value_token_303_begin_0, end = value_token_303_end_0, end_mask = value_token_303_end_mask_0, squeeze_mask = value_token_303_squeeze_mask_0, x = value_185_cast_fp16)[name = string("value_token_303_cast_fp16")];
            tensor<int32, [1]> var_8541_axes_0 = const()[name = string("op_8541_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_8541_cast_fp16 = expand_dims(axes = var_8541_axes_0, x = key_token_303_cast_fp16)[name = string("op_8541_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_8542_cast_fp16 = mul(x = state_605_cast_fp16, y = var_8541_cast_fp16)[name = string("op_8542_cast_fp16")];
            tensor<int32, [1]> memory_303_axes_0 = const()[name = string("memory_303_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_303_keep_dims_0 = const()[name = string("memory_303_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_303_cast_fp16 = reduce_sum(axes = memory_303_axes_0, keep_dims = memory_303_keep_dims_0, x = var_8542_cast_fp16)[name = string("memory_303_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_8545_cast_fp16 = sub(x = value_token_303_cast_fp16, y = memory_303_cast_fp16)[name = string("op_8545_cast_fp16")];
            tensor<int32, [3]> var_8548_begin_0 = const()[name = string("op_8548_begin_0"), val = tensor<int32, [3]>([0, 0, 7])];
            tensor<int32, [3]> var_8548_end_0 = const()[name = string("op_8548_end_0"), val = tensor<int32, [3]>([1, 16, 8])];
            tensor<bool, [3]> var_8548_end_mask_0 = const()[name = string("op_8548_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_8548_squeeze_mask_0 = const()[name = string("op_8548_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_8548_cast_fp16 = slice_by_index(begin = var_8548_begin_0, end = var_8548_end_0, end_mask = var_8548_end_mask_0, squeeze_mask = var_8548_squeeze_mask_0, x = beta_19_cast_fp16)[name = string("op_8548_cast_fp16")];
            tensor<int32, [1]> var_8549_axes_0 = const()[name = string("op_8549_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_8549_cast_fp16 = expand_dims(axes = var_8549_axes_0, x = var_8548_cast_fp16)[name = string("op_8549_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_303_cast_fp16 = mul(x = var_8545_cast_fp16, y = var_8549_cast_fp16)[name = string("delta_303_cast_fp16")];
            tensor<int32, [1]> var_8552_axes_0 = const()[name = string("op_8552_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_8552_cast_fp16 = expand_dims(axes = var_8552_axes_0, x = delta_303_cast_fp16)[name = string("op_8552_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_8553_cast_fp16 = mul(x = var_8541_cast_fp16, y = var_8552_cast_fp16)[name = string("op_8553_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_607_cast_fp16 = add(x = state_605_cast_fp16, y = var_8553_cast_fp16)[name = string("state_607_cast_fp16")];
            tensor<int32, [4]> var_8557_begin_0 = const()[name = string("op_8557_begin_0"), val = tensor<int32, [4]>([0, 0, 7, 0])];
            tensor<int32, [4]> var_8557_end_0 = const()[name = string("op_8557_end_0"), val = tensor<int32, [4]>([1, 16, 8, 128])];
            tensor<bool, [4]> var_8557_end_mask_0 = const()[name = string("op_8557_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_8557_squeeze_mask_0 = const()[name = string("op_8557_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_8557_cast_fp16 = slice_by_index(begin = var_8557_begin_0, end = var_8557_end_0, end_mask = var_8557_end_mask_0, squeeze_mask = var_8557_squeeze_mask_0, x = _inversed_query_51_cast_fp16)[name = string("op_8557_cast_fp16")];
            tensor<int32, [1]> var_8558_axes_0 = const()[name = string("op_8558_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_8558_cast_fp16 = expand_dims(axes = var_8558_axes_0, x = var_8557_cast_fp16)[name = string("op_8558_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_8559_cast_fp16 = mul(x = state_607_cast_fp16, y = var_8558_cast_fp16)[name = string("op_8559_cast_fp16")];
            tensor<int32, [1]> var_8561_axes_0 = const()[name = string("op_8561_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_8561_keep_dims_0 = const()[name = string("op_8561_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_8561_cast_fp16 = reduce_sum(axes = var_8561_axes_0, keep_dims = var_8561_keep_dims_0, x = var_8559_cast_fp16)[name = string("op_8561_cast_fp16")];
            tensor<int32, [3]> var_8564_begin_0 = const()[name = string("op_8564_begin_0"), val = tensor<int32, [3]>([0, 0, 8])];
            tensor<int32, [3]> var_8564_end_0 = const()[name = string("op_8564_end_0"), val = tensor<int32, [3]>([1, 16, 9])];
            tensor<bool, [3]> var_8564_end_mask_0 = const()[name = string("op_8564_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_8564_squeeze_mask_0 = const()[name = string("op_8564_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_8564_cast_fp16 = slice_by_index(begin = var_8564_begin_0, end = var_8564_end_0, end_mask = var_8564_end_mask_0, squeeze_mask = var_8564_squeeze_mask_0, x = decay_19_cast_fp16)[name = string("op_8564_cast_fp16")];
            tensor<fp16, [1, 16]> var_8565_cast_fp16 = exp(x = var_8564_cast_fp16)[name = string("op_8565_cast_fp16")];
            tensor<int32, [1]> var_8566_axes_0 = const()[name = string("op_8566_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_8566_cast_fp16 = expand_dims(axes = var_8566_axes_0, x = var_8565_cast_fp16)[name = string("op_8566_cast_fp16")];
            tensor<int32, [1]> var_8567_axes_0 = const()[name = string("op_8567_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_8567_cast_fp16 = expand_dims(axes = var_8567_axes_0, x = var_8566_cast_fp16)[name = string("op_8567_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_609_cast_fp16 = mul(x = state_607_cast_fp16, y = var_8567_cast_fp16)[name = string("state_609_cast_fp16")];
            tensor<int32, [4]> key_token_305_begin_0 = const()[name = string("key_token_305_begin_0"), val = tensor<int32, [4]>([0, 0, 8, 0])];
            tensor<int32, [4]> key_token_305_end_0 = const()[name = string("key_token_305_end_0"), val = tensor<int32, [4]>([1, 16, 9, 128])];
            tensor<bool, [4]> key_token_305_end_mask_0 = const()[name = string("key_token_305_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_305_squeeze_mask_0 = const()[name = string("key_token_305_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_305_cast_fp16 = slice_by_index(begin = key_token_305_begin_0, end = key_token_305_end_0, end_mask = key_token_305_end_mask_0, squeeze_mask = key_token_305_squeeze_mask_0, x = key_51_cast_fp16)[name = string("key_token_305_cast_fp16")];
            tensor<int32, [4]> value_token_305_begin_0 = const()[name = string("value_token_305_begin_0"), val = tensor<int32, [4]>([0, 0, 8, 0])];
            tensor<int32, [4]> value_token_305_end_0 = const()[name = string("value_token_305_end_0"), val = tensor<int32, [4]>([1, 16, 9, 128])];
            tensor<bool, [4]> value_token_305_end_mask_0 = const()[name = string("value_token_305_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_305_squeeze_mask_0 = const()[name = string("value_token_305_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_305_cast_fp16 = slice_by_index(begin = value_token_305_begin_0, end = value_token_305_end_0, end_mask = value_token_305_end_mask_0, squeeze_mask = value_token_305_squeeze_mask_0, x = value_185_cast_fp16)[name = string("value_token_305_cast_fp16")];
            tensor<int32, [1]> var_8575_axes_0 = const()[name = string("op_8575_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_8575_cast_fp16 = expand_dims(axes = var_8575_axes_0, x = key_token_305_cast_fp16)[name = string("op_8575_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_8576_cast_fp16 = mul(x = state_609_cast_fp16, y = var_8575_cast_fp16)[name = string("op_8576_cast_fp16")];
            tensor<int32, [1]> memory_305_axes_0 = const()[name = string("memory_305_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_305_keep_dims_0 = const()[name = string("memory_305_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_305_cast_fp16 = reduce_sum(axes = memory_305_axes_0, keep_dims = memory_305_keep_dims_0, x = var_8576_cast_fp16)[name = string("memory_305_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_8579_cast_fp16 = sub(x = value_token_305_cast_fp16, y = memory_305_cast_fp16)[name = string("op_8579_cast_fp16")];
            tensor<int32, [3]> var_8582_begin_0 = const()[name = string("op_8582_begin_0"), val = tensor<int32, [3]>([0, 0, 8])];
            tensor<int32, [3]> var_8582_end_0 = const()[name = string("op_8582_end_0"), val = tensor<int32, [3]>([1, 16, 9])];
            tensor<bool, [3]> var_8582_end_mask_0 = const()[name = string("op_8582_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_8582_squeeze_mask_0 = const()[name = string("op_8582_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_8582_cast_fp16 = slice_by_index(begin = var_8582_begin_0, end = var_8582_end_0, end_mask = var_8582_end_mask_0, squeeze_mask = var_8582_squeeze_mask_0, x = beta_19_cast_fp16)[name = string("op_8582_cast_fp16")];
            tensor<int32, [1]> var_8583_axes_0 = const()[name = string("op_8583_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_8583_cast_fp16 = expand_dims(axes = var_8583_axes_0, x = var_8582_cast_fp16)[name = string("op_8583_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_305_cast_fp16 = mul(x = var_8579_cast_fp16, y = var_8583_cast_fp16)[name = string("delta_305_cast_fp16")];
            tensor<int32, [1]> var_8586_axes_0 = const()[name = string("op_8586_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_8586_cast_fp16 = expand_dims(axes = var_8586_axes_0, x = delta_305_cast_fp16)[name = string("op_8586_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_8587_cast_fp16 = mul(x = var_8575_cast_fp16, y = var_8586_cast_fp16)[name = string("op_8587_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_611_cast_fp16 = add(x = state_609_cast_fp16, y = var_8587_cast_fp16)[name = string("state_611_cast_fp16")];
            tensor<int32, [4]> var_8591_begin_0 = const()[name = string("op_8591_begin_0"), val = tensor<int32, [4]>([0, 0, 8, 0])];
            tensor<int32, [4]> var_8591_end_0 = const()[name = string("op_8591_end_0"), val = tensor<int32, [4]>([1, 16, 9, 128])];
            tensor<bool, [4]> var_8591_end_mask_0 = const()[name = string("op_8591_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_8591_squeeze_mask_0 = const()[name = string("op_8591_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_8591_cast_fp16 = slice_by_index(begin = var_8591_begin_0, end = var_8591_end_0, end_mask = var_8591_end_mask_0, squeeze_mask = var_8591_squeeze_mask_0, x = _inversed_query_51_cast_fp16)[name = string("op_8591_cast_fp16")];
            tensor<int32, [1]> var_8592_axes_0 = const()[name = string("op_8592_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_8592_cast_fp16 = expand_dims(axes = var_8592_axes_0, x = var_8591_cast_fp16)[name = string("op_8592_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_8593_cast_fp16 = mul(x = state_611_cast_fp16, y = var_8592_cast_fp16)[name = string("op_8593_cast_fp16")];
            tensor<int32, [1]> var_8595_axes_0 = const()[name = string("op_8595_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_8595_keep_dims_0 = const()[name = string("op_8595_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_8595_cast_fp16 = reduce_sum(axes = var_8595_axes_0, keep_dims = var_8595_keep_dims_0, x = var_8593_cast_fp16)[name = string("op_8595_cast_fp16")];
            tensor<int32, [3]> var_8598_begin_0 = const()[name = string("op_8598_begin_0"), val = tensor<int32, [3]>([0, 0, 9])];
            tensor<int32, [3]> var_8598_end_0 = const()[name = string("op_8598_end_0"), val = tensor<int32, [3]>([1, 16, 10])];
            tensor<bool, [3]> var_8598_end_mask_0 = const()[name = string("op_8598_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_8598_squeeze_mask_0 = const()[name = string("op_8598_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_8598_cast_fp16 = slice_by_index(begin = var_8598_begin_0, end = var_8598_end_0, end_mask = var_8598_end_mask_0, squeeze_mask = var_8598_squeeze_mask_0, x = decay_19_cast_fp16)[name = string("op_8598_cast_fp16")];
            tensor<fp16, [1, 16]> var_8599_cast_fp16 = exp(x = var_8598_cast_fp16)[name = string("op_8599_cast_fp16")];
            tensor<int32, [1]> var_8600_axes_0 = const()[name = string("op_8600_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_8600_cast_fp16 = expand_dims(axes = var_8600_axes_0, x = var_8599_cast_fp16)[name = string("op_8600_cast_fp16")];
            tensor<int32, [1]> var_8601_axes_0 = const()[name = string("op_8601_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_8601_cast_fp16 = expand_dims(axes = var_8601_axes_0, x = var_8600_cast_fp16)[name = string("op_8601_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_613_cast_fp16 = mul(x = state_611_cast_fp16, y = var_8601_cast_fp16)[name = string("state_613_cast_fp16")];
            tensor<int32, [4]> key_token_307_begin_0 = const()[name = string("key_token_307_begin_0"), val = tensor<int32, [4]>([0, 0, 9, 0])];
            tensor<int32, [4]> key_token_307_end_0 = const()[name = string("key_token_307_end_0"), val = tensor<int32, [4]>([1, 16, 10, 128])];
            tensor<bool, [4]> key_token_307_end_mask_0 = const()[name = string("key_token_307_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_307_squeeze_mask_0 = const()[name = string("key_token_307_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_307_cast_fp16 = slice_by_index(begin = key_token_307_begin_0, end = key_token_307_end_0, end_mask = key_token_307_end_mask_0, squeeze_mask = key_token_307_squeeze_mask_0, x = key_51_cast_fp16)[name = string("key_token_307_cast_fp16")];
            tensor<int32, [4]> value_token_307_begin_0 = const()[name = string("value_token_307_begin_0"), val = tensor<int32, [4]>([0, 0, 9, 0])];
            tensor<int32, [4]> value_token_307_end_0 = const()[name = string("value_token_307_end_0"), val = tensor<int32, [4]>([1, 16, 10, 128])];
            tensor<bool, [4]> value_token_307_end_mask_0 = const()[name = string("value_token_307_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_307_squeeze_mask_0 = const()[name = string("value_token_307_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_307_cast_fp16 = slice_by_index(begin = value_token_307_begin_0, end = value_token_307_end_0, end_mask = value_token_307_end_mask_0, squeeze_mask = value_token_307_squeeze_mask_0, x = value_185_cast_fp16)[name = string("value_token_307_cast_fp16")];
            tensor<int32, [1]> var_8609_axes_0 = const()[name = string("op_8609_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_8609_cast_fp16 = expand_dims(axes = var_8609_axes_0, x = key_token_307_cast_fp16)[name = string("op_8609_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_8610_cast_fp16 = mul(x = state_613_cast_fp16, y = var_8609_cast_fp16)[name = string("op_8610_cast_fp16")];
            tensor<int32, [1]> memory_307_axes_0 = const()[name = string("memory_307_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_307_keep_dims_0 = const()[name = string("memory_307_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_307_cast_fp16 = reduce_sum(axes = memory_307_axes_0, keep_dims = memory_307_keep_dims_0, x = var_8610_cast_fp16)[name = string("memory_307_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_8613_cast_fp16 = sub(x = value_token_307_cast_fp16, y = memory_307_cast_fp16)[name = string("op_8613_cast_fp16")];
            tensor<int32, [3]> var_8616_begin_0 = const()[name = string("op_8616_begin_0"), val = tensor<int32, [3]>([0, 0, 9])];
            tensor<int32, [3]> var_8616_end_0 = const()[name = string("op_8616_end_0"), val = tensor<int32, [3]>([1, 16, 10])];
            tensor<bool, [3]> var_8616_end_mask_0 = const()[name = string("op_8616_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_8616_squeeze_mask_0 = const()[name = string("op_8616_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_8616_cast_fp16 = slice_by_index(begin = var_8616_begin_0, end = var_8616_end_0, end_mask = var_8616_end_mask_0, squeeze_mask = var_8616_squeeze_mask_0, x = beta_19_cast_fp16)[name = string("op_8616_cast_fp16")];
            tensor<int32, [1]> var_8617_axes_0 = const()[name = string("op_8617_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_8617_cast_fp16 = expand_dims(axes = var_8617_axes_0, x = var_8616_cast_fp16)[name = string("op_8617_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_307_cast_fp16 = mul(x = var_8613_cast_fp16, y = var_8617_cast_fp16)[name = string("delta_307_cast_fp16")];
            tensor<int32, [1]> var_8620_axes_0 = const()[name = string("op_8620_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_8620_cast_fp16 = expand_dims(axes = var_8620_axes_0, x = delta_307_cast_fp16)[name = string("op_8620_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_8621_cast_fp16 = mul(x = var_8609_cast_fp16, y = var_8620_cast_fp16)[name = string("op_8621_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_615_cast_fp16 = add(x = state_613_cast_fp16, y = var_8621_cast_fp16)[name = string("state_615_cast_fp16")];
            tensor<int32, [4]> var_8625_begin_0 = const()[name = string("op_8625_begin_0"), val = tensor<int32, [4]>([0, 0, 9, 0])];
            tensor<int32, [4]> var_8625_end_0 = const()[name = string("op_8625_end_0"), val = tensor<int32, [4]>([1, 16, 10, 128])];
            tensor<bool, [4]> var_8625_end_mask_0 = const()[name = string("op_8625_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_8625_squeeze_mask_0 = const()[name = string("op_8625_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_8625_cast_fp16 = slice_by_index(begin = var_8625_begin_0, end = var_8625_end_0, end_mask = var_8625_end_mask_0, squeeze_mask = var_8625_squeeze_mask_0, x = _inversed_query_51_cast_fp16)[name = string("op_8625_cast_fp16")];
            tensor<int32, [1]> var_8626_axes_0 = const()[name = string("op_8626_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_8626_cast_fp16 = expand_dims(axes = var_8626_axes_0, x = var_8625_cast_fp16)[name = string("op_8626_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_8627_cast_fp16 = mul(x = state_615_cast_fp16, y = var_8626_cast_fp16)[name = string("op_8627_cast_fp16")];
            tensor<int32, [1]> var_8629_axes_0 = const()[name = string("op_8629_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_8629_keep_dims_0 = const()[name = string("op_8629_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_8629_cast_fp16 = reduce_sum(axes = var_8629_axes_0, keep_dims = var_8629_keep_dims_0, x = var_8627_cast_fp16)[name = string("op_8629_cast_fp16")];
            tensor<int32, [3]> var_8632_begin_0 = const()[name = string("op_8632_begin_0"), val = tensor<int32, [3]>([0, 0, 10])];
            tensor<int32, [3]> var_8632_end_0 = const()[name = string("op_8632_end_0"), val = tensor<int32, [3]>([1, 16, 11])];
            tensor<bool, [3]> var_8632_end_mask_0 = const()[name = string("op_8632_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_8632_squeeze_mask_0 = const()[name = string("op_8632_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_8632_cast_fp16 = slice_by_index(begin = var_8632_begin_0, end = var_8632_end_0, end_mask = var_8632_end_mask_0, squeeze_mask = var_8632_squeeze_mask_0, x = decay_19_cast_fp16)[name = string("op_8632_cast_fp16")];
            tensor<fp16, [1, 16]> var_8633_cast_fp16 = exp(x = var_8632_cast_fp16)[name = string("op_8633_cast_fp16")];
            tensor<int32, [1]> var_8634_axes_0 = const()[name = string("op_8634_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_8634_cast_fp16 = expand_dims(axes = var_8634_axes_0, x = var_8633_cast_fp16)[name = string("op_8634_cast_fp16")];
            tensor<int32, [1]> var_8635_axes_0 = const()[name = string("op_8635_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_8635_cast_fp16 = expand_dims(axes = var_8635_axes_0, x = var_8634_cast_fp16)[name = string("op_8635_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_617_cast_fp16 = mul(x = state_615_cast_fp16, y = var_8635_cast_fp16)[name = string("state_617_cast_fp16")];
            tensor<int32, [4]> key_token_309_begin_0 = const()[name = string("key_token_309_begin_0"), val = tensor<int32, [4]>([0, 0, 10, 0])];
            tensor<int32, [4]> key_token_309_end_0 = const()[name = string("key_token_309_end_0"), val = tensor<int32, [4]>([1, 16, 11, 128])];
            tensor<bool, [4]> key_token_309_end_mask_0 = const()[name = string("key_token_309_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_309_squeeze_mask_0 = const()[name = string("key_token_309_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_309_cast_fp16 = slice_by_index(begin = key_token_309_begin_0, end = key_token_309_end_0, end_mask = key_token_309_end_mask_0, squeeze_mask = key_token_309_squeeze_mask_0, x = key_51_cast_fp16)[name = string("key_token_309_cast_fp16")];
            tensor<int32, [4]> value_token_309_begin_0 = const()[name = string("value_token_309_begin_0"), val = tensor<int32, [4]>([0, 0, 10, 0])];
            tensor<int32, [4]> value_token_309_end_0 = const()[name = string("value_token_309_end_0"), val = tensor<int32, [4]>([1, 16, 11, 128])];
            tensor<bool, [4]> value_token_309_end_mask_0 = const()[name = string("value_token_309_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_309_squeeze_mask_0 = const()[name = string("value_token_309_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_309_cast_fp16 = slice_by_index(begin = value_token_309_begin_0, end = value_token_309_end_0, end_mask = value_token_309_end_mask_0, squeeze_mask = value_token_309_squeeze_mask_0, x = value_185_cast_fp16)[name = string("value_token_309_cast_fp16")];
            tensor<int32, [1]> var_8643_axes_0 = const()[name = string("op_8643_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_8643_cast_fp16 = expand_dims(axes = var_8643_axes_0, x = key_token_309_cast_fp16)[name = string("op_8643_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_8644_cast_fp16 = mul(x = state_617_cast_fp16, y = var_8643_cast_fp16)[name = string("op_8644_cast_fp16")];
            tensor<int32, [1]> memory_309_axes_0 = const()[name = string("memory_309_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_309_keep_dims_0 = const()[name = string("memory_309_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_309_cast_fp16 = reduce_sum(axes = memory_309_axes_0, keep_dims = memory_309_keep_dims_0, x = var_8644_cast_fp16)[name = string("memory_309_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_8647_cast_fp16 = sub(x = value_token_309_cast_fp16, y = memory_309_cast_fp16)[name = string("op_8647_cast_fp16")];
            tensor<int32, [3]> var_8650_begin_0 = const()[name = string("op_8650_begin_0"), val = tensor<int32, [3]>([0, 0, 10])];
            tensor<int32, [3]> var_8650_end_0 = const()[name = string("op_8650_end_0"), val = tensor<int32, [3]>([1, 16, 11])];
            tensor<bool, [3]> var_8650_end_mask_0 = const()[name = string("op_8650_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_8650_squeeze_mask_0 = const()[name = string("op_8650_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_8650_cast_fp16 = slice_by_index(begin = var_8650_begin_0, end = var_8650_end_0, end_mask = var_8650_end_mask_0, squeeze_mask = var_8650_squeeze_mask_0, x = beta_19_cast_fp16)[name = string("op_8650_cast_fp16")];
            tensor<int32, [1]> var_8651_axes_0 = const()[name = string("op_8651_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_8651_cast_fp16 = expand_dims(axes = var_8651_axes_0, x = var_8650_cast_fp16)[name = string("op_8651_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_309_cast_fp16 = mul(x = var_8647_cast_fp16, y = var_8651_cast_fp16)[name = string("delta_309_cast_fp16")];
            tensor<int32, [1]> var_8654_axes_0 = const()[name = string("op_8654_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_8654_cast_fp16 = expand_dims(axes = var_8654_axes_0, x = delta_309_cast_fp16)[name = string("op_8654_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_8655_cast_fp16 = mul(x = var_8643_cast_fp16, y = var_8654_cast_fp16)[name = string("op_8655_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_619_cast_fp16 = add(x = state_617_cast_fp16, y = var_8655_cast_fp16)[name = string("state_619_cast_fp16")];
            tensor<int32, [4]> var_8659_begin_0 = const()[name = string("op_8659_begin_0"), val = tensor<int32, [4]>([0, 0, 10, 0])];
            tensor<int32, [4]> var_8659_end_0 = const()[name = string("op_8659_end_0"), val = tensor<int32, [4]>([1, 16, 11, 128])];
            tensor<bool, [4]> var_8659_end_mask_0 = const()[name = string("op_8659_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_8659_squeeze_mask_0 = const()[name = string("op_8659_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_8659_cast_fp16 = slice_by_index(begin = var_8659_begin_0, end = var_8659_end_0, end_mask = var_8659_end_mask_0, squeeze_mask = var_8659_squeeze_mask_0, x = _inversed_query_51_cast_fp16)[name = string("op_8659_cast_fp16")];
            tensor<int32, [1]> var_8660_axes_0 = const()[name = string("op_8660_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_8660_cast_fp16 = expand_dims(axes = var_8660_axes_0, x = var_8659_cast_fp16)[name = string("op_8660_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_8661_cast_fp16 = mul(x = state_619_cast_fp16, y = var_8660_cast_fp16)[name = string("op_8661_cast_fp16")];
            tensor<int32, [1]> var_8663_axes_0 = const()[name = string("op_8663_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_8663_keep_dims_0 = const()[name = string("op_8663_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_8663_cast_fp16 = reduce_sum(axes = var_8663_axes_0, keep_dims = var_8663_keep_dims_0, x = var_8661_cast_fp16)[name = string("op_8663_cast_fp16")];
            tensor<int32, [3]> var_8666_begin_0 = const()[name = string("op_8666_begin_0"), val = tensor<int32, [3]>([0, 0, 11])];
            tensor<int32, [3]> var_8666_end_0 = const()[name = string("op_8666_end_0"), val = tensor<int32, [3]>([1, 16, 12])];
            tensor<bool, [3]> var_8666_end_mask_0 = const()[name = string("op_8666_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_8666_squeeze_mask_0 = const()[name = string("op_8666_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_8666_cast_fp16 = slice_by_index(begin = var_8666_begin_0, end = var_8666_end_0, end_mask = var_8666_end_mask_0, squeeze_mask = var_8666_squeeze_mask_0, x = decay_19_cast_fp16)[name = string("op_8666_cast_fp16")];
            tensor<fp16, [1, 16]> var_8667_cast_fp16 = exp(x = var_8666_cast_fp16)[name = string("op_8667_cast_fp16")];
            tensor<int32, [1]> var_8668_axes_0 = const()[name = string("op_8668_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_8668_cast_fp16 = expand_dims(axes = var_8668_axes_0, x = var_8667_cast_fp16)[name = string("op_8668_cast_fp16")];
            tensor<int32, [1]> var_8669_axes_0 = const()[name = string("op_8669_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_8669_cast_fp16 = expand_dims(axes = var_8669_axes_0, x = var_8668_cast_fp16)[name = string("op_8669_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_621_cast_fp16 = mul(x = state_619_cast_fp16, y = var_8669_cast_fp16)[name = string("state_621_cast_fp16")];
            tensor<int32, [4]> key_token_311_begin_0 = const()[name = string("key_token_311_begin_0"), val = tensor<int32, [4]>([0, 0, 11, 0])];
            tensor<int32, [4]> key_token_311_end_0 = const()[name = string("key_token_311_end_0"), val = tensor<int32, [4]>([1, 16, 12, 128])];
            tensor<bool, [4]> key_token_311_end_mask_0 = const()[name = string("key_token_311_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_311_squeeze_mask_0 = const()[name = string("key_token_311_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_311_cast_fp16 = slice_by_index(begin = key_token_311_begin_0, end = key_token_311_end_0, end_mask = key_token_311_end_mask_0, squeeze_mask = key_token_311_squeeze_mask_0, x = key_51_cast_fp16)[name = string("key_token_311_cast_fp16")];
            tensor<int32, [4]> value_token_311_begin_0 = const()[name = string("value_token_311_begin_0"), val = tensor<int32, [4]>([0, 0, 11, 0])];
            tensor<int32, [4]> value_token_311_end_0 = const()[name = string("value_token_311_end_0"), val = tensor<int32, [4]>([1, 16, 12, 128])];
            tensor<bool, [4]> value_token_311_end_mask_0 = const()[name = string("value_token_311_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_311_squeeze_mask_0 = const()[name = string("value_token_311_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_311_cast_fp16 = slice_by_index(begin = value_token_311_begin_0, end = value_token_311_end_0, end_mask = value_token_311_end_mask_0, squeeze_mask = value_token_311_squeeze_mask_0, x = value_185_cast_fp16)[name = string("value_token_311_cast_fp16")];
            tensor<int32, [1]> var_8677_axes_0 = const()[name = string("op_8677_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_8677_cast_fp16 = expand_dims(axes = var_8677_axes_0, x = key_token_311_cast_fp16)[name = string("op_8677_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_8678_cast_fp16 = mul(x = state_621_cast_fp16, y = var_8677_cast_fp16)[name = string("op_8678_cast_fp16")];
            tensor<int32, [1]> memory_311_axes_0 = const()[name = string("memory_311_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_311_keep_dims_0 = const()[name = string("memory_311_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_311_cast_fp16 = reduce_sum(axes = memory_311_axes_0, keep_dims = memory_311_keep_dims_0, x = var_8678_cast_fp16)[name = string("memory_311_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_8681_cast_fp16 = sub(x = value_token_311_cast_fp16, y = memory_311_cast_fp16)[name = string("op_8681_cast_fp16")];
            tensor<int32, [3]> var_8684_begin_0 = const()[name = string("op_8684_begin_0"), val = tensor<int32, [3]>([0, 0, 11])];
            tensor<int32, [3]> var_8684_end_0 = const()[name = string("op_8684_end_0"), val = tensor<int32, [3]>([1, 16, 12])];
            tensor<bool, [3]> var_8684_end_mask_0 = const()[name = string("op_8684_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_8684_squeeze_mask_0 = const()[name = string("op_8684_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_8684_cast_fp16 = slice_by_index(begin = var_8684_begin_0, end = var_8684_end_0, end_mask = var_8684_end_mask_0, squeeze_mask = var_8684_squeeze_mask_0, x = beta_19_cast_fp16)[name = string("op_8684_cast_fp16")];
            tensor<int32, [1]> var_8685_axes_0 = const()[name = string("op_8685_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_8685_cast_fp16 = expand_dims(axes = var_8685_axes_0, x = var_8684_cast_fp16)[name = string("op_8685_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_311_cast_fp16 = mul(x = var_8681_cast_fp16, y = var_8685_cast_fp16)[name = string("delta_311_cast_fp16")];
            tensor<int32, [1]> var_8688_axes_0 = const()[name = string("op_8688_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_8688_cast_fp16 = expand_dims(axes = var_8688_axes_0, x = delta_311_cast_fp16)[name = string("op_8688_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_8689_cast_fp16 = mul(x = var_8677_cast_fp16, y = var_8688_cast_fp16)[name = string("op_8689_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_623_cast_fp16 = add(x = state_621_cast_fp16, y = var_8689_cast_fp16)[name = string("state_623_cast_fp16")];
            tensor<int32, [4]> var_8693_begin_0 = const()[name = string("op_8693_begin_0"), val = tensor<int32, [4]>([0, 0, 11, 0])];
            tensor<int32, [4]> var_8693_end_0 = const()[name = string("op_8693_end_0"), val = tensor<int32, [4]>([1, 16, 12, 128])];
            tensor<bool, [4]> var_8693_end_mask_0 = const()[name = string("op_8693_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_8693_squeeze_mask_0 = const()[name = string("op_8693_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_8693_cast_fp16 = slice_by_index(begin = var_8693_begin_0, end = var_8693_end_0, end_mask = var_8693_end_mask_0, squeeze_mask = var_8693_squeeze_mask_0, x = _inversed_query_51_cast_fp16)[name = string("op_8693_cast_fp16")];
            tensor<int32, [1]> var_8694_axes_0 = const()[name = string("op_8694_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_8694_cast_fp16 = expand_dims(axes = var_8694_axes_0, x = var_8693_cast_fp16)[name = string("op_8694_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_8695_cast_fp16 = mul(x = state_623_cast_fp16, y = var_8694_cast_fp16)[name = string("op_8695_cast_fp16")];
            tensor<int32, [1]> var_8697_axes_0 = const()[name = string("op_8697_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_8697_keep_dims_0 = const()[name = string("op_8697_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_8697_cast_fp16 = reduce_sum(axes = var_8697_axes_0, keep_dims = var_8697_keep_dims_0, x = var_8695_cast_fp16)[name = string("op_8697_cast_fp16")];
            tensor<int32, [3]> var_8700_begin_0 = const()[name = string("op_8700_begin_0"), val = tensor<int32, [3]>([0, 0, 12])];
            tensor<int32, [3]> var_8700_end_0 = const()[name = string("op_8700_end_0"), val = tensor<int32, [3]>([1, 16, 13])];
            tensor<bool, [3]> var_8700_end_mask_0 = const()[name = string("op_8700_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_8700_squeeze_mask_0 = const()[name = string("op_8700_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_8700_cast_fp16 = slice_by_index(begin = var_8700_begin_0, end = var_8700_end_0, end_mask = var_8700_end_mask_0, squeeze_mask = var_8700_squeeze_mask_0, x = decay_19_cast_fp16)[name = string("op_8700_cast_fp16")];
            tensor<fp16, [1, 16]> var_8701_cast_fp16 = exp(x = var_8700_cast_fp16)[name = string("op_8701_cast_fp16")];
            tensor<int32, [1]> var_8702_axes_0 = const()[name = string("op_8702_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_8702_cast_fp16 = expand_dims(axes = var_8702_axes_0, x = var_8701_cast_fp16)[name = string("op_8702_cast_fp16")];
            tensor<int32, [1]> var_8703_axes_0 = const()[name = string("op_8703_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_8703_cast_fp16 = expand_dims(axes = var_8703_axes_0, x = var_8702_cast_fp16)[name = string("op_8703_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_625_cast_fp16 = mul(x = state_623_cast_fp16, y = var_8703_cast_fp16)[name = string("state_625_cast_fp16")];
            tensor<int32, [4]> key_token_313_begin_0 = const()[name = string("key_token_313_begin_0"), val = tensor<int32, [4]>([0, 0, 12, 0])];
            tensor<int32, [4]> key_token_313_end_0 = const()[name = string("key_token_313_end_0"), val = tensor<int32, [4]>([1, 16, 13, 128])];
            tensor<bool, [4]> key_token_313_end_mask_0 = const()[name = string("key_token_313_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_313_squeeze_mask_0 = const()[name = string("key_token_313_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_313_cast_fp16 = slice_by_index(begin = key_token_313_begin_0, end = key_token_313_end_0, end_mask = key_token_313_end_mask_0, squeeze_mask = key_token_313_squeeze_mask_0, x = key_51_cast_fp16)[name = string("key_token_313_cast_fp16")];
            tensor<int32, [4]> value_token_313_begin_0 = const()[name = string("value_token_313_begin_0"), val = tensor<int32, [4]>([0, 0, 12, 0])];
            tensor<int32, [4]> value_token_313_end_0 = const()[name = string("value_token_313_end_0"), val = tensor<int32, [4]>([1, 16, 13, 128])];
            tensor<bool, [4]> value_token_313_end_mask_0 = const()[name = string("value_token_313_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_313_squeeze_mask_0 = const()[name = string("value_token_313_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_313_cast_fp16 = slice_by_index(begin = value_token_313_begin_0, end = value_token_313_end_0, end_mask = value_token_313_end_mask_0, squeeze_mask = value_token_313_squeeze_mask_0, x = value_185_cast_fp16)[name = string("value_token_313_cast_fp16")];
            tensor<int32, [1]> var_8711_axes_0 = const()[name = string("op_8711_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_8711_cast_fp16 = expand_dims(axes = var_8711_axes_0, x = key_token_313_cast_fp16)[name = string("op_8711_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_8712_cast_fp16 = mul(x = state_625_cast_fp16, y = var_8711_cast_fp16)[name = string("op_8712_cast_fp16")];
            tensor<int32, [1]> memory_313_axes_0 = const()[name = string("memory_313_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_313_keep_dims_0 = const()[name = string("memory_313_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_313_cast_fp16 = reduce_sum(axes = memory_313_axes_0, keep_dims = memory_313_keep_dims_0, x = var_8712_cast_fp16)[name = string("memory_313_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_8715_cast_fp16 = sub(x = value_token_313_cast_fp16, y = memory_313_cast_fp16)[name = string("op_8715_cast_fp16")];
            tensor<int32, [3]> var_8718_begin_0 = const()[name = string("op_8718_begin_0"), val = tensor<int32, [3]>([0, 0, 12])];
            tensor<int32, [3]> var_8718_end_0 = const()[name = string("op_8718_end_0"), val = tensor<int32, [3]>([1, 16, 13])];
            tensor<bool, [3]> var_8718_end_mask_0 = const()[name = string("op_8718_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_8718_squeeze_mask_0 = const()[name = string("op_8718_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_8718_cast_fp16 = slice_by_index(begin = var_8718_begin_0, end = var_8718_end_0, end_mask = var_8718_end_mask_0, squeeze_mask = var_8718_squeeze_mask_0, x = beta_19_cast_fp16)[name = string("op_8718_cast_fp16")];
            tensor<int32, [1]> var_8719_axes_0 = const()[name = string("op_8719_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_8719_cast_fp16 = expand_dims(axes = var_8719_axes_0, x = var_8718_cast_fp16)[name = string("op_8719_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_313_cast_fp16 = mul(x = var_8715_cast_fp16, y = var_8719_cast_fp16)[name = string("delta_313_cast_fp16")];
            tensor<int32, [1]> var_8722_axes_0 = const()[name = string("op_8722_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_8722_cast_fp16 = expand_dims(axes = var_8722_axes_0, x = delta_313_cast_fp16)[name = string("op_8722_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_8723_cast_fp16 = mul(x = var_8711_cast_fp16, y = var_8722_cast_fp16)[name = string("op_8723_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_627_cast_fp16 = add(x = state_625_cast_fp16, y = var_8723_cast_fp16)[name = string("state_627_cast_fp16")];
            tensor<int32, [4]> var_8727_begin_0 = const()[name = string("op_8727_begin_0"), val = tensor<int32, [4]>([0, 0, 12, 0])];
            tensor<int32, [4]> var_8727_end_0 = const()[name = string("op_8727_end_0"), val = tensor<int32, [4]>([1, 16, 13, 128])];
            tensor<bool, [4]> var_8727_end_mask_0 = const()[name = string("op_8727_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_8727_squeeze_mask_0 = const()[name = string("op_8727_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_8727_cast_fp16 = slice_by_index(begin = var_8727_begin_0, end = var_8727_end_0, end_mask = var_8727_end_mask_0, squeeze_mask = var_8727_squeeze_mask_0, x = _inversed_query_51_cast_fp16)[name = string("op_8727_cast_fp16")];
            tensor<int32, [1]> var_8728_axes_0 = const()[name = string("op_8728_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_8728_cast_fp16 = expand_dims(axes = var_8728_axes_0, x = var_8727_cast_fp16)[name = string("op_8728_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_8729_cast_fp16 = mul(x = state_627_cast_fp16, y = var_8728_cast_fp16)[name = string("op_8729_cast_fp16")];
            tensor<int32, [1]> var_8731_axes_0 = const()[name = string("op_8731_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_8731_keep_dims_0 = const()[name = string("op_8731_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_8731_cast_fp16 = reduce_sum(axes = var_8731_axes_0, keep_dims = var_8731_keep_dims_0, x = var_8729_cast_fp16)[name = string("op_8731_cast_fp16")];
            tensor<int32, [3]> var_8734_begin_0 = const()[name = string("op_8734_begin_0"), val = tensor<int32, [3]>([0, 0, 13])];
            tensor<int32, [3]> var_8734_end_0 = const()[name = string("op_8734_end_0"), val = tensor<int32, [3]>([1, 16, 14])];
            tensor<bool, [3]> var_8734_end_mask_0 = const()[name = string("op_8734_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_8734_squeeze_mask_0 = const()[name = string("op_8734_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_8734_cast_fp16 = slice_by_index(begin = var_8734_begin_0, end = var_8734_end_0, end_mask = var_8734_end_mask_0, squeeze_mask = var_8734_squeeze_mask_0, x = decay_19_cast_fp16)[name = string("op_8734_cast_fp16")];
            tensor<fp16, [1, 16]> var_8735_cast_fp16 = exp(x = var_8734_cast_fp16)[name = string("op_8735_cast_fp16")];
            tensor<int32, [1]> var_8736_axes_0 = const()[name = string("op_8736_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_8736_cast_fp16 = expand_dims(axes = var_8736_axes_0, x = var_8735_cast_fp16)[name = string("op_8736_cast_fp16")];
            tensor<int32, [1]> var_8737_axes_0 = const()[name = string("op_8737_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_8737_cast_fp16 = expand_dims(axes = var_8737_axes_0, x = var_8736_cast_fp16)[name = string("op_8737_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_629_cast_fp16 = mul(x = state_627_cast_fp16, y = var_8737_cast_fp16)[name = string("state_629_cast_fp16")];
            tensor<int32, [4]> key_token_315_begin_0 = const()[name = string("key_token_315_begin_0"), val = tensor<int32, [4]>([0, 0, 13, 0])];
            tensor<int32, [4]> key_token_315_end_0 = const()[name = string("key_token_315_end_0"), val = tensor<int32, [4]>([1, 16, 14, 128])];
            tensor<bool, [4]> key_token_315_end_mask_0 = const()[name = string("key_token_315_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_315_squeeze_mask_0 = const()[name = string("key_token_315_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_315_cast_fp16 = slice_by_index(begin = key_token_315_begin_0, end = key_token_315_end_0, end_mask = key_token_315_end_mask_0, squeeze_mask = key_token_315_squeeze_mask_0, x = key_51_cast_fp16)[name = string("key_token_315_cast_fp16")];
            tensor<int32, [4]> value_token_315_begin_0 = const()[name = string("value_token_315_begin_0"), val = tensor<int32, [4]>([0, 0, 13, 0])];
            tensor<int32, [4]> value_token_315_end_0 = const()[name = string("value_token_315_end_0"), val = tensor<int32, [4]>([1, 16, 14, 128])];
            tensor<bool, [4]> value_token_315_end_mask_0 = const()[name = string("value_token_315_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_315_squeeze_mask_0 = const()[name = string("value_token_315_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_315_cast_fp16 = slice_by_index(begin = value_token_315_begin_0, end = value_token_315_end_0, end_mask = value_token_315_end_mask_0, squeeze_mask = value_token_315_squeeze_mask_0, x = value_185_cast_fp16)[name = string("value_token_315_cast_fp16")];
            tensor<int32, [1]> var_8745_axes_0 = const()[name = string("op_8745_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_8745_cast_fp16 = expand_dims(axes = var_8745_axes_0, x = key_token_315_cast_fp16)[name = string("op_8745_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_8746_cast_fp16 = mul(x = state_629_cast_fp16, y = var_8745_cast_fp16)[name = string("op_8746_cast_fp16")];
            tensor<int32, [1]> memory_315_axes_0 = const()[name = string("memory_315_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_315_keep_dims_0 = const()[name = string("memory_315_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_315_cast_fp16 = reduce_sum(axes = memory_315_axes_0, keep_dims = memory_315_keep_dims_0, x = var_8746_cast_fp16)[name = string("memory_315_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_8749_cast_fp16 = sub(x = value_token_315_cast_fp16, y = memory_315_cast_fp16)[name = string("op_8749_cast_fp16")];
            tensor<int32, [3]> var_8752_begin_0 = const()[name = string("op_8752_begin_0"), val = tensor<int32, [3]>([0, 0, 13])];
            tensor<int32, [3]> var_8752_end_0 = const()[name = string("op_8752_end_0"), val = tensor<int32, [3]>([1, 16, 14])];
            tensor<bool, [3]> var_8752_end_mask_0 = const()[name = string("op_8752_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_8752_squeeze_mask_0 = const()[name = string("op_8752_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_8752_cast_fp16 = slice_by_index(begin = var_8752_begin_0, end = var_8752_end_0, end_mask = var_8752_end_mask_0, squeeze_mask = var_8752_squeeze_mask_0, x = beta_19_cast_fp16)[name = string("op_8752_cast_fp16")];
            tensor<int32, [1]> var_8753_axes_0 = const()[name = string("op_8753_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_8753_cast_fp16 = expand_dims(axes = var_8753_axes_0, x = var_8752_cast_fp16)[name = string("op_8753_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_315_cast_fp16 = mul(x = var_8749_cast_fp16, y = var_8753_cast_fp16)[name = string("delta_315_cast_fp16")];
            tensor<int32, [1]> var_8756_axes_0 = const()[name = string("op_8756_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_8756_cast_fp16 = expand_dims(axes = var_8756_axes_0, x = delta_315_cast_fp16)[name = string("op_8756_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_8757_cast_fp16 = mul(x = var_8745_cast_fp16, y = var_8756_cast_fp16)[name = string("op_8757_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_631_cast_fp16 = add(x = state_629_cast_fp16, y = var_8757_cast_fp16)[name = string("state_631_cast_fp16")];
            tensor<int32, [4]> var_8761_begin_0 = const()[name = string("op_8761_begin_0"), val = tensor<int32, [4]>([0, 0, 13, 0])];
            tensor<int32, [4]> var_8761_end_0 = const()[name = string("op_8761_end_0"), val = tensor<int32, [4]>([1, 16, 14, 128])];
            tensor<bool, [4]> var_8761_end_mask_0 = const()[name = string("op_8761_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_8761_squeeze_mask_0 = const()[name = string("op_8761_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_8761_cast_fp16 = slice_by_index(begin = var_8761_begin_0, end = var_8761_end_0, end_mask = var_8761_end_mask_0, squeeze_mask = var_8761_squeeze_mask_0, x = _inversed_query_51_cast_fp16)[name = string("op_8761_cast_fp16")];
            tensor<int32, [1]> var_8762_axes_0 = const()[name = string("op_8762_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_8762_cast_fp16 = expand_dims(axes = var_8762_axes_0, x = var_8761_cast_fp16)[name = string("op_8762_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_8763_cast_fp16 = mul(x = state_631_cast_fp16, y = var_8762_cast_fp16)[name = string("op_8763_cast_fp16")];
            tensor<int32, [1]> var_8765_axes_0 = const()[name = string("op_8765_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_8765_keep_dims_0 = const()[name = string("op_8765_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_8765_cast_fp16 = reduce_sum(axes = var_8765_axes_0, keep_dims = var_8765_keep_dims_0, x = var_8763_cast_fp16)[name = string("op_8765_cast_fp16")];
            tensor<int32, [3]> var_8768_begin_0 = const()[name = string("op_8768_begin_0"), val = tensor<int32, [3]>([0, 0, 14])];
            tensor<int32, [3]> var_8768_end_0 = const()[name = string("op_8768_end_0"), val = tensor<int32, [3]>([1, 16, 15])];
            tensor<bool, [3]> var_8768_end_mask_0 = const()[name = string("op_8768_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_8768_squeeze_mask_0 = const()[name = string("op_8768_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_8768_cast_fp16 = slice_by_index(begin = var_8768_begin_0, end = var_8768_end_0, end_mask = var_8768_end_mask_0, squeeze_mask = var_8768_squeeze_mask_0, x = decay_19_cast_fp16)[name = string("op_8768_cast_fp16")];
            tensor<fp16, [1, 16]> var_8769_cast_fp16 = exp(x = var_8768_cast_fp16)[name = string("op_8769_cast_fp16")];
            tensor<int32, [1]> var_8770_axes_0 = const()[name = string("op_8770_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_8770_cast_fp16 = expand_dims(axes = var_8770_axes_0, x = var_8769_cast_fp16)[name = string("op_8770_cast_fp16")];
            tensor<int32, [1]> var_8771_axes_0 = const()[name = string("op_8771_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_8771_cast_fp16 = expand_dims(axes = var_8771_axes_0, x = var_8770_cast_fp16)[name = string("op_8771_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_633_cast_fp16 = mul(x = state_631_cast_fp16, y = var_8771_cast_fp16)[name = string("state_633_cast_fp16")];
            tensor<int32, [4]> key_token_317_begin_0 = const()[name = string("key_token_317_begin_0"), val = tensor<int32, [4]>([0, 0, 14, 0])];
            tensor<int32, [4]> key_token_317_end_0 = const()[name = string("key_token_317_end_0"), val = tensor<int32, [4]>([1, 16, 15, 128])];
            tensor<bool, [4]> key_token_317_end_mask_0 = const()[name = string("key_token_317_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_317_squeeze_mask_0 = const()[name = string("key_token_317_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_317_cast_fp16 = slice_by_index(begin = key_token_317_begin_0, end = key_token_317_end_0, end_mask = key_token_317_end_mask_0, squeeze_mask = key_token_317_squeeze_mask_0, x = key_51_cast_fp16)[name = string("key_token_317_cast_fp16")];
            tensor<int32, [4]> value_token_317_begin_0 = const()[name = string("value_token_317_begin_0"), val = tensor<int32, [4]>([0, 0, 14, 0])];
            tensor<int32, [4]> value_token_317_end_0 = const()[name = string("value_token_317_end_0"), val = tensor<int32, [4]>([1, 16, 15, 128])];
            tensor<bool, [4]> value_token_317_end_mask_0 = const()[name = string("value_token_317_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_317_squeeze_mask_0 = const()[name = string("value_token_317_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_317_cast_fp16 = slice_by_index(begin = value_token_317_begin_0, end = value_token_317_end_0, end_mask = value_token_317_end_mask_0, squeeze_mask = value_token_317_squeeze_mask_0, x = value_185_cast_fp16)[name = string("value_token_317_cast_fp16")];
            tensor<int32, [1]> var_8779_axes_0 = const()[name = string("op_8779_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_8779_cast_fp16 = expand_dims(axes = var_8779_axes_0, x = key_token_317_cast_fp16)[name = string("op_8779_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_8780_cast_fp16 = mul(x = state_633_cast_fp16, y = var_8779_cast_fp16)[name = string("op_8780_cast_fp16")];
            tensor<int32, [1]> memory_317_axes_0 = const()[name = string("memory_317_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_317_keep_dims_0 = const()[name = string("memory_317_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_317_cast_fp16 = reduce_sum(axes = memory_317_axes_0, keep_dims = memory_317_keep_dims_0, x = var_8780_cast_fp16)[name = string("memory_317_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_8783_cast_fp16 = sub(x = value_token_317_cast_fp16, y = memory_317_cast_fp16)[name = string("op_8783_cast_fp16")];
            tensor<int32, [3]> var_8786_begin_0 = const()[name = string("op_8786_begin_0"), val = tensor<int32, [3]>([0, 0, 14])];
            tensor<int32, [3]> var_8786_end_0 = const()[name = string("op_8786_end_0"), val = tensor<int32, [3]>([1, 16, 15])];
            tensor<bool, [3]> var_8786_end_mask_0 = const()[name = string("op_8786_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_8786_squeeze_mask_0 = const()[name = string("op_8786_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_8786_cast_fp16 = slice_by_index(begin = var_8786_begin_0, end = var_8786_end_0, end_mask = var_8786_end_mask_0, squeeze_mask = var_8786_squeeze_mask_0, x = beta_19_cast_fp16)[name = string("op_8786_cast_fp16")];
            tensor<int32, [1]> var_8787_axes_0 = const()[name = string("op_8787_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_8787_cast_fp16 = expand_dims(axes = var_8787_axes_0, x = var_8786_cast_fp16)[name = string("op_8787_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_317_cast_fp16 = mul(x = var_8783_cast_fp16, y = var_8787_cast_fp16)[name = string("delta_317_cast_fp16")];
            tensor<int32, [1]> var_8790_axes_0 = const()[name = string("op_8790_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_8790_cast_fp16 = expand_dims(axes = var_8790_axes_0, x = delta_317_cast_fp16)[name = string("op_8790_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_8791_cast_fp16 = mul(x = var_8779_cast_fp16, y = var_8790_cast_fp16)[name = string("op_8791_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_635_cast_fp16 = add(x = state_633_cast_fp16, y = var_8791_cast_fp16)[name = string("state_635_cast_fp16")];
            tensor<int32, [4]> var_8795_begin_0 = const()[name = string("op_8795_begin_0"), val = tensor<int32, [4]>([0, 0, 14, 0])];
            tensor<int32, [4]> var_8795_end_0 = const()[name = string("op_8795_end_0"), val = tensor<int32, [4]>([1, 16, 15, 128])];
            tensor<bool, [4]> var_8795_end_mask_0 = const()[name = string("op_8795_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_8795_squeeze_mask_0 = const()[name = string("op_8795_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_8795_cast_fp16 = slice_by_index(begin = var_8795_begin_0, end = var_8795_end_0, end_mask = var_8795_end_mask_0, squeeze_mask = var_8795_squeeze_mask_0, x = _inversed_query_51_cast_fp16)[name = string("op_8795_cast_fp16")];
            tensor<int32, [1]> var_8796_axes_0 = const()[name = string("op_8796_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_8796_cast_fp16 = expand_dims(axes = var_8796_axes_0, x = var_8795_cast_fp16)[name = string("op_8796_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_8797_cast_fp16 = mul(x = state_635_cast_fp16, y = var_8796_cast_fp16)[name = string("op_8797_cast_fp16")];
            tensor<int32, [1]> var_8799_axes_0 = const()[name = string("op_8799_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_8799_keep_dims_0 = const()[name = string("op_8799_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_8799_cast_fp16 = reduce_sum(axes = var_8799_axes_0, keep_dims = var_8799_keep_dims_0, x = var_8797_cast_fp16)[name = string("op_8799_cast_fp16")];
            tensor<int32, [3]> var_8802_begin_0 = const()[name = string("op_8802_begin_0"), val = tensor<int32, [3]>([0, 0, 15])];
            tensor<int32, [3]> var_8802_end_0 = const()[name = string("op_8802_end_0"), val = tensor<int32, [3]>([1, 16, 16])];
            tensor<bool, [3]> var_8802_end_mask_0 = const()[name = string("op_8802_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_8802_squeeze_mask_0 = const()[name = string("op_8802_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_8802_cast_fp16 = slice_by_index(begin = var_8802_begin_0, end = var_8802_end_0, end_mask = var_8802_end_mask_0, squeeze_mask = var_8802_squeeze_mask_0, x = decay_19_cast_fp16)[name = string("op_8802_cast_fp16")];
            tensor<fp16, [1, 16]> var_8803_cast_fp16 = exp(x = var_8802_cast_fp16)[name = string("op_8803_cast_fp16")];
            tensor<int32, [1]> var_8804_axes_0 = const()[name = string("op_8804_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_8804_cast_fp16 = expand_dims(axes = var_8804_axes_0, x = var_8803_cast_fp16)[name = string("op_8804_cast_fp16")];
            tensor<int32, [1]> var_8805_axes_0 = const()[name = string("op_8805_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_8805_cast_fp16 = expand_dims(axes = var_8805_axes_0, x = var_8804_cast_fp16)[name = string("op_8805_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_637_cast_fp16 = mul(x = state_635_cast_fp16, y = var_8805_cast_fp16)[name = string("state_637_cast_fp16")];
            tensor<int32, [4]> key_token_319_begin_0 = const()[name = string("key_token_319_begin_0"), val = tensor<int32, [4]>([0, 0, 15, 0])];
            tensor<int32, [4]> key_token_319_end_0 = const()[name = string("key_token_319_end_0"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<bool, [4]> key_token_319_end_mask_0 = const()[name = string("key_token_319_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_319_squeeze_mask_0 = const()[name = string("key_token_319_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_319_cast_fp16 = slice_by_index(begin = key_token_319_begin_0, end = key_token_319_end_0, end_mask = key_token_319_end_mask_0, squeeze_mask = key_token_319_squeeze_mask_0, x = key_51_cast_fp16)[name = string("key_token_319_cast_fp16")];
            tensor<int32, [4]> value_token_319_begin_0 = const()[name = string("value_token_319_begin_0"), val = tensor<int32, [4]>([0, 0, 15, 0])];
            tensor<int32, [4]> value_token_319_end_0 = const()[name = string("value_token_319_end_0"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<bool, [4]> value_token_319_end_mask_0 = const()[name = string("value_token_319_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_319_squeeze_mask_0 = const()[name = string("value_token_319_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_319_cast_fp16 = slice_by_index(begin = value_token_319_begin_0, end = value_token_319_end_0, end_mask = value_token_319_end_mask_0, squeeze_mask = value_token_319_squeeze_mask_0, x = value_185_cast_fp16)[name = string("value_token_319_cast_fp16")];
            tensor<int32, [1]> var_8813_axes_0 = const()[name = string("op_8813_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_8813_cast_fp16 = expand_dims(axes = var_8813_axes_0, x = key_token_319_cast_fp16)[name = string("op_8813_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_8814_cast_fp16 = mul(x = state_637_cast_fp16, y = var_8813_cast_fp16)[name = string("op_8814_cast_fp16")];
            tensor<int32, [1]> memory_319_axes_0 = const()[name = string("memory_319_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_319_keep_dims_0 = const()[name = string("memory_319_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_319_cast_fp16 = reduce_sum(axes = memory_319_axes_0, keep_dims = memory_319_keep_dims_0, x = var_8814_cast_fp16)[name = string("memory_319_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_8817_cast_fp16 = sub(x = value_token_319_cast_fp16, y = memory_319_cast_fp16)[name = string("op_8817_cast_fp16")];
            tensor<int32, [3]> var_8820_begin_0 = const()[name = string("op_8820_begin_0"), val = tensor<int32, [3]>([0, 0, 15])];
            tensor<int32, [3]> var_8820_end_0 = const()[name = string("op_8820_end_0"), val = tensor<int32, [3]>([1, 16, 16])];
            tensor<bool, [3]> var_8820_end_mask_0 = const()[name = string("op_8820_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_8820_squeeze_mask_0 = const()[name = string("op_8820_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_8820_cast_fp16 = slice_by_index(begin = var_8820_begin_0, end = var_8820_end_0, end_mask = var_8820_end_mask_0, squeeze_mask = var_8820_squeeze_mask_0, x = beta_19_cast_fp16)[name = string("op_8820_cast_fp16")];
            tensor<int32, [1]> var_8821_axes_0 = const()[name = string("op_8821_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_8821_cast_fp16 = expand_dims(axes = var_8821_axes_0, x = var_8820_cast_fp16)[name = string("op_8821_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_319_cast_fp16 = mul(x = var_8817_cast_fp16, y = var_8821_cast_fp16)[name = string("delta_319_cast_fp16")];
            tensor<int32, [1]> var_8824_axes_0 = const()[name = string("op_8824_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_8824_cast_fp16 = expand_dims(axes = var_8824_axes_0, x = delta_319_cast_fp16)[name = string("op_8824_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_8825_cast_fp16 = mul(x = var_8813_cast_fp16, y = var_8824_cast_fp16)[name = string("op_8825_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> rec12_out = add(x = state_637_cast_fp16, y = var_8825_cast_fp16)[name = string("state_639_cast_fp16")];
            tensor<int32, [4]> var_8829_begin_0 = const()[name = string("op_8829_begin_0"), val = tensor<int32, [4]>([0, 0, 15, 0])];
            tensor<int32, [4]> var_8829_end_0 = const()[name = string("op_8829_end_0"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<bool, [4]> var_8829_end_mask_0 = const()[name = string("op_8829_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_8829_squeeze_mask_0 = const()[name = string("op_8829_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_8829_cast_fp16 = slice_by_index(begin = var_8829_begin_0, end = var_8829_end_0, end_mask = var_8829_end_mask_0, squeeze_mask = var_8829_squeeze_mask_0, x = _inversed_query_51_cast_fp16)[name = string("op_8829_cast_fp16")];
            tensor<int32, [1]> var_8830_axes_0 = const()[name = string("op_8830_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_8830_cast_fp16 = expand_dims(axes = var_8830_axes_0, x = var_8829_cast_fp16)[name = string("op_8830_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_8831_cast_fp16 = mul(x = rec12_out, y = var_8830_cast_fp16)[name = string("op_8831_cast_fp16")];
            tensor<int32, [1]> var_8833_axes_0 = const()[name = string("op_8833_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_8833_keep_dims_0 = const()[name = string("op_8833_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_8833_cast_fp16 = reduce_sum(axes = var_8833_axes_0, keep_dims = var_8833_keep_dims_0, x = var_8831_cast_fp16)[name = string("op_8833_cast_fp16")];
            int32 attention_91_axis_0 = const()[name = string("attention_91_axis_0"), val = int32(1)];
            tensor<fp16, [1, 16, 16, 128]> attention_91_cast_fp16 = stack(axis = attention_91_axis_0, values = (var_8323_cast_fp16, var_8357_cast_fp16, var_8391_cast_fp16, var_8425_cast_fp16, var_8459_cast_fp16, var_8493_cast_fp16, var_8527_cast_fp16, var_8561_cast_fp16, var_8595_cast_fp16, var_8629_cast_fp16, var_8663_cast_fp16, var_8697_cast_fp16, var_8731_cast_fp16, var_8765_cast_fp16, var_8799_cast_fp16, var_8833_cast_fp16))[name = string("attention_91_cast_fp16")];
            tensor<int32, [2]> var_8836 = const()[name = string("op_8836"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [256, 128]> attention_93_cast_fp16 = reshape(shape = var_8836, x = attention_91_cast_fp16)[name = string("attention_93_cast_fp16")];
            tensor<int32, [2]> var_8838 = const()[name = string("op_8838"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [256, 128]> input_161_cast_fp16 = reshape(shape = var_8838, x = linear_96_cast_fp16)[name = string("input_161_cast_fp16")];
            tensor<fp16, [256, 128]> var_8840_cast_fp16 = mul(x = attention_93_cast_fp16, y = attention_93_cast_fp16)[name = string("op_8840_cast_fp16")];
            tensor<int32, [1]> variance_81_axes_0 = const()[name = string("variance_81_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_81_keep_dims_0 = const()[name = string("variance_81_keep_dims_0"), val = bool(true)];
            tensor<fp16, [256, 1]> variance_81_cast_fp16 = reduce_mean(axes = variance_81_axes_0, keep_dims = variance_81_keep_dims_0, x = var_8840_cast_fp16)[name = string("variance_81_cast_fp16")];
            fp16 var_8843_to_fp16 = const()[name = string("op_8843_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [256, 1]> var_8844_cast_fp16 = add(x = variance_81_cast_fp16, y = var_8843_to_fp16)[name = string("op_8844_cast_fp16")];
            fp32 var_8845_epsilon_0 = const()[name = string("op_8845_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [256, 1]> var_8845_cast_fp16 = rsqrt(epsilon = var_8845_epsilon_0, x = var_8844_cast_fp16)[name = string("op_8845_cast_fp16")];
            tensor<fp16, [256, 128]> attention_95_cast_fp16 = mul(x = attention_93_cast_fp16, y = var_8845_cast_fp16)[name = string("attention_95_cast_fp16")];
            tensor<fp16, [128]> groups_3_0_gated_norm_promoted_to_fp16 = const()[name = string("groups_3_0_gated_norm_promoted_to_fp16"), val = tensor<fp16, [128]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(195373760)))];
            tensor<fp16, [256, 128]> attention_97_cast_fp16 = mul(x = attention_95_cast_fp16, y = groups_3_0_gated_norm_promoted_to_fp16)[name = string("attention_97_cast_fp16")];
            tensor<fp16, [256, 128]> var_8848_cast_fp16 = silu(x = input_161_cast_fp16)[name = string("op_8848_cast_fp16")];
            tensor<fp16, [256, 128]> attention_99_cast_fp16 = mul(x = attention_97_cast_fp16, y = var_8848_cast_fp16)[name = string("attention_99_cast_fp16")];
            tensor<int32, [2]> var_8850 = const()[name = string("op_8850"), val = tensor<int32, [2]>([16, 2048])];
            tensor<fp16, [16, 2048]> input_163_cast_fp16 = reshape(shape = var_8850, x = attention_99_cast_fp16)[name = string("input_163_cast_fp16")];
            tensor<fp16, [1024, 2048]> groups_3_0_out_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 2048]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(195374080))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(196947008))))[name = string("groups_3_0_out_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 1024]> linear_97_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = groups_3_0_out_proj_weight_promoted_to_fp16_palettized, x = input_163_cast_fp16)[name = string("linear_97_cast_fp16")];
            tensor<fp16, [16, 1024]> value_187_cast_fp16 = add(x = value_175_cast_fp16, y = linear_97_cast_fp16)[name = string("value_187_cast_fp16")];
            tensor<fp16, [16, 1024]> var_8855_cast_fp16 = mul(x = value_187_cast_fp16, y = value_187_cast_fp16)[name = string("op_8855_cast_fp16")];
            tensor<int32, [1]> variance_83_axes_0 = const()[name = string("variance_83_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_83_keep_dims_0 = const()[name = string("variance_83_keep_dims_0"), val = bool(true)];
            tensor<fp16, [16, 1]> variance_83_cast_fp16 = reduce_mean(axes = variance_83_axes_0, keep_dims = variance_83_keep_dims_0, x = var_8855_cast_fp16)[name = string("variance_83_cast_fp16")];
            fp16 var_8858_to_fp16 = const()[name = string("op_8858_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [16, 1]> var_8859_cast_fp16 = add(x = variance_83_cast_fp16, y = var_8858_to_fp16)[name = string("op_8859_cast_fp16")];
            fp32 var_8860_epsilon_0 = const()[name = string("op_8860_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [16, 1]> var_8860_cast_fp16 = rsqrt(epsilon = var_8860_epsilon_0, x = var_8859_cast_fp16)[name = string("op_8860_cast_fp16")];
            tensor<fp16, [16, 1024]> var_8861_cast_fp16 = mul(x = value_187_cast_fp16, y = var_8860_cast_fp16)[name = string("op_8861_cast_fp16")];
            tensor<fp16, [1024]> var_8863_to_fp16 = const()[name = string("op_8863_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(196955264)))];
            tensor<fp16, [16, 1024]> input_165_cast_fp16 = mul(x = var_8861_cast_fp16, y = var_8863_to_fp16)[name = string("input_165_cast_fp16")];
            tensor<fp16, [3584, 1024]> groups_3_0_gate_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(196957376))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(199709952))))[name = string("groups_3_0_gate_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 3584]> linear_98_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = groups_3_0_gate_proj_weight_promoted_to_fp16_palettized, x = input_165_cast_fp16)[name = string("linear_98_cast_fp16")];
            tensor<fp16, [16, 3584]> var_8867_cast_fp16 = silu(x = linear_98_cast_fp16)[name = string("op_8867_cast_fp16")];
            tensor<fp16, [3584, 1024]> groups_3_0_up_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(199738688))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(202491264))))[name = string("groups_3_0_up_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 3584]> linear_99_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = groups_3_0_up_proj_weight_promoted_to_fp16_palettized, x = input_165_cast_fp16)[name = string("linear_99_cast_fp16")];
            tensor<fp16, [16, 3584]> input_169_cast_fp16 = mul(x = var_8867_cast_fp16, y = linear_99_cast_fp16)[name = string("input_169_cast_fp16")];
            tensor<fp16, [1024, 3584]> groups_3_0_down_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 3584]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(202520000))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(205272576))))[name = string("groups_3_0_down_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 1024]> linear_100_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = groups_3_0_down_proj_weight_promoted_to_fp16_palettized, x = input_169_cast_fp16)[name = string("linear_100_cast_fp16")];
            tensor<fp16, [16, 1024]> value_189_cast_fp16 = add(x = value_187_cast_fp16, y = linear_100_cast_fp16)[name = string("value_189_cast_fp16")];
            int32 var_8905 = const()[name = string("op_8905"), val = int32(-1)];
            tensor<fp16, [16, 1024]> var_8920_cast_fp16 = mul(x = value_189_cast_fp16, y = value_189_cast_fp16)[name = string("op_8920_cast_fp16")];
            tensor<int32, [1]> variance_85_axes_0 = const()[name = string("variance_85_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_85_keep_dims_0 = const()[name = string("variance_85_keep_dims_0"), val = bool(true)];
            tensor<fp16, [16, 1]> variance_85_cast_fp16 = reduce_mean(axes = variance_85_axes_0, keep_dims = variance_85_keep_dims_0, x = var_8920_cast_fp16)[name = string("variance_85_cast_fp16")];
            fp16 var_8923_to_fp16 = const()[name = string("op_8923_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [16, 1]> var_8924_cast_fp16 = add(x = variance_85_cast_fp16, y = var_8923_to_fp16)[name = string("op_8924_cast_fp16")];
            fp32 var_8925_epsilon_0 = const()[name = string("op_8925_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [16, 1]> var_8925_cast_fp16 = rsqrt(epsilon = var_8925_epsilon_0, x = var_8924_cast_fp16)[name = string("op_8925_cast_fp16")];
            tensor<fp16, [16, 1024]> var_8926_cast_fp16 = mul(x = value_189_cast_fp16, y = var_8925_cast_fp16)[name = string("op_8926_cast_fp16")];
            tensor<fp16, [1024]> var_8928_to_fp16 = const()[name = string("op_8928_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(205280832)))];
            tensor<fp16, [16, 1024]> input_171_cast_fp16 = mul(x = var_8926_cast_fp16, y = var_8928_to_fp16)[name = string("input_171_cast_fp16")];
            tensor<fp16, [6144, 1024]> groups_3_1_in_proj_qkv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(205282944))), lut = tensor<fp16, [384, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(210001600))))[name = string("groups_3_1_in_proj_qkv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 6144]> linear_101_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = groups_3_1_in_proj_qkv_weight_promoted_to_fp16_palettized, x = input_171_cast_fp16)[name = string("linear_101_cast_fp16")];
            tensor<int32, [2]> var_8932_perm_0 = const()[name = string("op_8932_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> mixed_21_axes_0 = const()[name = string("mixed_21_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [6144, 16]> var_8932_cast_fp16 = transpose(perm = var_8932_perm_0, x = linear_101_cast_fp16)[name = string("transpose_46")];
            tensor<fp16, [1, 6144, 16]> mixed_21_cast_fp16 = expand_dims(axes = mixed_21_axes_0, x = var_8932_cast_fp16)[name = string("mixed_21_cast_fp16")];
            bool convolution_input_21_interleave_0 = const()[name = string("convolution_input_21_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 6144, 19]> convolution_input_21_cast_fp16 = concat(axis = var_8905, interleave = convolution_input_21_interleave_0, values = (conv13, mixed_21_cast_fp16))[name = string("convolution_input_21_cast_fp16")];
            string input_173_pad_type_0 = const()[name = string("input_173_pad_type_0"), val = string("valid")];
            int32 input_173_groups_0 = const()[name = string("input_173_groups_0"), val = int32(6144)];
            tensor<int32, [1]> input_173_strides_0 = const()[name = string("input_173_strides_0"), val = tensor<int32, [1]>([1])];
            tensor<int32, [2]> input_173_pad_0 = const()[name = string("input_173_pad_0"), val = tensor<int32, [2]>([0, 0])];
            tensor<int32, [1]> input_173_dilations_0 = const()[name = string("input_173_dilations_0"), val = tensor<int32, [1]>([1])];
            tensor<fp16, [6144, 1, 4]> groups_3_1_conv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1, 4]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(210050816))), lut = tensor<fp16, [384, 1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(210069312))))[name = string("groups_3_1_conv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 6144, 16]> input_173_cast_fp16 = conv(dilations = input_173_dilations_0, groups = input_173_groups_0, pad = input_173_pad_0, pad_type = input_173_pad_type_0, strides = input_173_strides_0, weight = groups_3_1_conv_weight_promoted_to_fp16_palettized, x = convolution_input_21_cast_fp16)[name = string("input_173_cast_fp16")];
            tensor<fp16, [1, 6144, 16]> var_8941_cast_fp16 = silu(x = input_173_cast_fp16)[name = string("op_8941_cast_fp16")];
            tensor<int32, [3]> var_8942_perm_0 = const()[name = string("op_8942_perm_0"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [3]> var_8945_begin_0 = const()[name = string("op_8945_begin_0"), val = tensor<int32, [3]>([0, 0, 16])];
            tensor<int32, [3]> var_8945_end_0 = const()[name = string("op_8945_end_0"), val = tensor<int32, [3]>([1, 6144, 19])];
            tensor<bool, [3]> var_8945_end_mask_0 = const()[name = string("op_8945_end_mask_0"), val = tensor<bool, [3]>([true, true, true])];
            tensor<fp16, [1, 6144, 3]> conv13_out = slice_by_index(begin = var_8945_begin_0, end = var_8945_end_0, end_mask = var_8945_end_mask_0, x = convolution_input_21_cast_fp16)[name = string("op_8945_cast_fp16")];
            tensor<int32, [3]> var_8946 = const()[name = string("op_8946"), val = tensor<int32, [3]>([2048, 2048, 2048])];
            int32 var_8947_axis_0 = const()[name = string("op_8947_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 16, 6144]> var_8942_cast_fp16 = transpose(perm = var_8942_perm_0, x = var_8941_cast_fp16)[name = string("transpose_45")];
            tensor<fp16, [1, 16, 2048]> var_8947_cast_fp16_0, tensor<fp16, [1, 16, 2048]> var_8947_cast_fp16_1, tensor<fp16, [1, 16, 2048]> var_8947_cast_fp16_2 = split(axis = var_8947_axis_0, split_sizes = var_8946, x = var_8942_cast_fp16)[name = string("op_8947_cast_fp16")];
            tensor<int32, [4]> var_8951 = const()[name = string("op_8951"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<fp16, [1, 16, 16, 128]> value_193_cast_fp16 = reshape(shape = var_8951, x = var_8947_cast_fp16_0)[name = string("value_193_cast_fp16")];
            tensor<int32, [4]> var_8953 = const()[name = string("op_8953"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<fp16, [1, 16, 16, 128]> value_195_cast_fp16 = reshape(shape = var_8953, x = var_8947_cast_fp16_1)[name = string("value_195_cast_fp16")];
            tensor<int32, [4]> var_8955 = const()[name = string("op_8955"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<fp16, [1, 16, 16, 128]> value_197_cast_fp16 = reshape(shape = var_8955, x = var_8947_cast_fp16_2)[name = string("value_197_cast_fp16")];
            tensor<fp16, [1, 16, 16, 128]> var_8957_cast_fp16 = mul(x = value_193_cast_fp16, y = value_193_cast_fp16)[name = string("op_8957_cast_fp16")];
            tensor<int32, [1]> var_8959_axes_0 = const()[name = string("op_8959_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_8959_keep_dims_0 = const()[name = string("op_8959_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 16, 16, 1]> var_8959_cast_fp16 = reduce_sum(axes = var_8959_axes_0, keep_dims = var_8959_keep_dims_0, x = var_8957_cast_fp16)[name = string("op_8959_cast_fp16")];
            fp16 var_8960_to_fp16 = const()[name = string("op_8960_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 16, 16, 1]> var_8961_cast_fp16 = add(x = var_8959_cast_fp16, y = var_8960_to_fp16)[name = string("op_8961_cast_fp16")];
            fp32 var_8962_epsilon_0 = const()[name = string("op_8962_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 16, 16, 1]> var_8962_cast_fp16 = rsqrt(epsilon = var_8962_epsilon_0, x = var_8961_cast_fp16)[name = string("op_8962_cast_fp16")];
            tensor<fp16, [1, 16, 16, 128]> var_8963_cast_fp16 = mul(x = value_193_cast_fp16, y = var_8962_cast_fp16)[name = string("op_8963_cast_fp16")];
            tensor<int32, [4]> var_8964_perm_0 = const()[name = string("op_8964_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            fp16 _inversed_query_55_y_0_to_fp16 = const()[name = string("_inversed_query_55_y_0_to_fp16"), val = fp16(0x1.6ap-4)];
            tensor<fp16, [1, 16, 16, 128]> var_8964_cast_fp16 = transpose(perm = var_8964_perm_0, x = var_8963_cast_fp16)[name = string("transpose_44")];
            tensor<fp16, [1, 16, 16, 128]> _inversed_query_55_cast_fp16 = mul(x = var_8964_cast_fp16, y = _inversed_query_55_y_0_to_fp16)[name = string("_inversed_query_55_cast_fp16")];
            tensor<fp16, [1, 16, 16, 128]> var_8967_cast_fp16 = mul(x = value_195_cast_fp16, y = value_195_cast_fp16)[name = string("op_8967_cast_fp16")];
            tensor<int32, [1]> var_8969_axes_0 = const()[name = string("op_8969_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_8969_keep_dims_0 = const()[name = string("op_8969_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 16, 16, 1]> var_8969_cast_fp16 = reduce_sum(axes = var_8969_axes_0, keep_dims = var_8969_keep_dims_0, x = var_8967_cast_fp16)[name = string("op_8969_cast_fp16")];
            fp16 var_8970_to_fp16 = const()[name = string("op_8970_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 16, 16, 1]> var_8971_cast_fp16 = add(x = var_8969_cast_fp16, y = var_8970_to_fp16)[name = string("op_8971_cast_fp16")];
            fp32 var_8972_epsilon_0 = const()[name = string("op_8972_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 16, 16, 1]> var_8972_cast_fp16 = rsqrt(epsilon = var_8972_epsilon_0, x = var_8971_cast_fp16)[name = string("op_8972_cast_fp16")];
            tensor<fp16, [1, 16, 16, 128]> var_8973_cast_fp16 = mul(x = value_195_cast_fp16, y = var_8972_cast_fp16)[name = string("op_8973_cast_fp16")];
            tensor<int32, [4]> key_55_perm_0 = const()[name = string("key_55_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<int32, [4]> value_199_perm_0 = const()[name = string("value_199_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<fp16, [16, 1024]> groups_3_1_in_proj_b_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(210118528))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(210130880))))[name = string("groups_3_1_in_proj_b_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16]> linear_102_bias_0_to_fp16 = const()[name = string("linear_102_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(4862656)))];
            tensor<fp16, [16, 16]> linear_102_cast_fp16 = linear(bias = linear_102_bias_0_to_fp16, weight = groups_3_1_in_proj_b_weight_promoted_to_fp16_palettized, x = input_171_cast_fp16)[name = string("linear_102_cast_fp16")];
            tensor<fp16, [16, 16]> var_8978_cast_fp16 = sigmoid(x = linear_102_cast_fp16)[name = string("op_8978_cast_fp16")];
            tensor<int32, [2]> var_8979_perm_0 = const()[name = string("op_8979_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> beta_21_axes_0 = const()[name = string("beta_21_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 16]> var_8979_cast_fp16 = transpose(perm = var_8979_perm_0, x = var_8978_cast_fp16)[name = string("transpose_43")];
            tensor<fp16, [1, 16, 16]> beta_21_cast_fp16 = expand_dims(axes = beta_21_axes_0, x = var_8979_cast_fp16)[name = string("beta_21_cast_fp16")];
            tensor<fp16, [16, 1024]> op_8985_weight_0_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(210131072))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(210143424))))[name = string("op_8985_weight_0_to_fp16_palettized")];
            tensor<fp16, [16]> var_8985_bias_0_to_fp16 = const()[name = string("op_8985_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(210143616)))];
            tensor<fp16, [16, 16]> var_8985_cast_fp16 = linear(bias = var_8985_bias_0_to_fp16, weight = op_8985_weight_0_to_fp16_palettized, x = input_171_cast_fp16)[name = string("op_8985_cast_fp16")];
            tensor<fp16, [16, 16]> var_8986_cast_fp16 = softplus(x = var_8985_cast_fp16)[name = string("op_8986_cast_fp16")];
            tensor<fp16, [16]> var_8982_promoted_to_fp16 = const()[name = string("op_8982_promoted_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(210143744)))];
            tensor<fp16, [16, 16]> var_8987_cast_fp16 = mul(x = var_8982_promoted_to_fp16, y = var_8986_cast_fp16)[name = string("op_8987_cast_fp16")];
            tensor<int32, [2]> var_8988_perm_0 = const()[name = string("op_8988_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> decay_21_axes_0 = const()[name = string("decay_21_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 16]> var_8988_cast_fp16 = transpose(perm = var_8988_perm_0, x = var_8987_cast_fp16)[name = string("transpose_42")];
            tensor<fp16, [1, 16, 16]> decay_21_cast_fp16 = expand_dims(axes = decay_21_axes_0, x = var_8988_cast_fp16)[name = string("decay_21_cast_fp16")];
            tensor<fp16, [2048, 1024]> groups_3_1_in_proj_z_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [2048, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(210143872))), lut = tensor<fp16, [128, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(211716800))))[name = string("groups_3_1_in_proj_z_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 2048]> linear_104_cast_fp16 = linear(bias = linear_3_bias_0_to_fp16, weight = groups_3_1_in_proj_z_weight_promoted_to_fp16_palettized, x = input_171_cast_fp16)[name = string("linear_104_cast_fp16")];
            tensor<int32, [3]> var_8996_begin_0 = const()[name = string("op_8996_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_8996_end_0 = const()[name = string("op_8996_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_8996_end_mask_0 = const()[name = string("op_8996_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_8996_squeeze_mask_0 = const()[name = string("op_8996_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_8996_cast_fp16 = slice_by_index(begin = var_8996_begin_0, end = var_8996_end_0, end_mask = var_8996_end_mask_0, squeeze_mask = var_8996_squeeze_mask_0, x = decay_21_cast_fp16)[name = string("op_8996_cast_fp16")];
            tensor<fp16, [1, 16]> var_8997_cast_fp16 = exp(x = var_8996_cast_fp16)[name = string("op_8997_cast_fp16")];
            tensor<int32, [1]> var_8998_axes_0 = const()[name = string("op_8998_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_8998_cast_fp16 = expand_dims(axes = var_8998_axes_0, x = var_8997_cast_fp16)[name = string("op_8998_cast_fp16")];
            tensor<int32, [1]> var_8999_axes_0 = const()[name = string("op_8999_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_8999_cast_fp16 = expand_dims(axes = var_8999_axes_0, x = var_8998_cast_fp16)[name = string("op_8999_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_641_cast_fp16 = mul(x = rec13, y = var_8999_cast_fp16)[name = string("state_641_cast_fp16")];
            tensor<int32, [4]> key_token_321_begin_0 = const()[name = string("key_token_321_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> key_token_321_end_0 = const()[name = string("key_token_321_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> key_token_321_end_mask_0 = const()[name = string("key_token_321_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_321_squeeze_mask_0 = const()[name = string("key_token_321_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 16, 128]> key_55_cast_fp16 = transpose(perm = key_55_perm_0, x = var_8973_cast_fp16)[name = string("transpose_41")];
            tensor<fp16, [1, 16, 128]> key_token_321_cast_fp16 = slice_by_index(begin = key_token_321_begin_0, end = key_token_321_end_0, end_mask = key_token_321_end_mask_0, squeeze_mask = key_token_321_squeeze_mask_0, x = key_55_cast_fp16)[name = string("key_token_321_cast_fp16")];
            tensor<int32, [4]> value_token_321_begin_0 = const()[name = string("value_token_321_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> value_token_321_end_0 = const()[name = string("value_token_321_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> value_token_321_end_mask_0 = const()[name = string("value_token_321_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_321_squeeze_mask_0 = const()[name = string("value_token_321_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 16, 128]> value_199_cast_fp16 = transpose(perm = value_199_perm_0, x = value_197_cast_fp16)[name = string("transpose_40")];
            tensor<fp16, [1, 16, 128]> value_token_321_cast_fp16 = slice_by_index(begin = value_token_321_begin_0, end = value_token_321_end_0, end_mask = value_token_321_end_mask_0, squeeze_mask = value_token_321_squeeze_mask_0, x = value_199_cast_fp16)[name = string("value_token_321_cast_fp16")];
            tensor<int32, [1]> var_9007_axes_0 = const()[name = string("op_9007_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_9007_cast_fp16 = expand_dims(axes = var_9007_axes_0, x = key_token_321_cast_fp16)[name = string("op_9007_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_9008_cast_fp16 = mul(x = state_641_cast_fp16, y = var_9007_cast_fp16)[name = string("op_9008_cast_fp16")];
            tensor<int32, [1]> memory_321_axes_0 = const()[name = string("memory_321_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_321_keep_dims_0 = const()[name = string("memory_321_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_321_cast_fp16 = reduce_sum(axes = memory_321_axes_0, keep_dims = memory_321_keep_dims_0, x = var_9008_cast_fp16)[name = string("memory_321_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_9011_cast_fp16 = sub(x = value_token_321_cast_fp16, y = memory_321_cast_fp16)[name = string("op_9011_cast_fp16")];
            tensor<int32, [3]> var_9014_begin_0 = const()[name = string("op_9014_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_9014_end_0 = const()[name = string("op_9014_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_9014_end_mask_0 = const()[name = string("op_9014_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_9014_squeeze_mask_0 = const()[name = string("op_9014_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_9014_cast_fp16 = slice_by_index(begin = var_9014_begin_0, end = var_9014_end_0, end_mask = var_9014_end_mask_0, squeeze_mask = var_9014_squeeze_mask_0, x = beta_21_cast_fp16)[name = string("op_9014_cast_fp16")];
            tensor<int32, [1]> var_9015_axes_0 = const()[name = string("op_9015_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_9015_cast_fp16 = expand_dims(axes = var_9015_axes_0, x = var_9014_cast_fp16)[name = string("op_9015_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_321_cast_fp16 = mul(x = var_9011_cast_fp16, y = var_9015_cast_fp16)[name = string("delta_321_cast_fp16")];
            tensor<int32, [1]> var_9018_axes_0 = const()[name = string("op_9018_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_9018_cast_fp16 = expand_dims(axes = var_9018_axes_0, x = delta_321_cast_fp16)[name = string("op_9018_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_9019_cast_fp16 = mul(x = var_9007_cast_fp16, y = var_9018_cast_fp16)[name = string("op_9019_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_643_cast_fp16 = add(x = state_641_cast_fp16, y = var_9019_cast_fp16)[name = string("state_643_cast_fp16")];
            tensor<int32, [4]> var_9023_begin_0 = const()[name = string("op_9023_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> var_9023_end_0 = const()[name = string("op_9023_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> var_9023_end_mask_0 = const()[name = string("op_9023_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_9023_squeeze_mask_0 = const()[name = string("op_9023_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_9023_cast_fp16 = slice_by_index(begin = var_9023_begin_0, end = var_9023_end_0, end_mask = var_9023_end_mask_0, squeeze_mask = var_9023_squeeze_mask_0, x = _inversed_query_55_cast_fp16)[name = string("op_9023_cast_fp16")];
            tensor<int32, [1]> var_9024_axes_0 = const()[name = string("op_9024_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_9024_cast_fp16 = expand_dims(axes = var_9024_axes_0, x = var_9023_cast_fp16)[name = string("op_9024_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_9025_cast_fp16 = mul(x = state_643_cast_fp16, y = var_9024_cast_fp16)[name = string("op_9025_cast_fp16")];
            tensor<int32, [1]> var_9027_axes_0 = const()[name = string("op_9027_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_9027_keep_dims_0 = const()[name = string("op_9027_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_9027_cast_fp16 = reduce_sum(axes = var_9027_axes_0, keep_dims = var_9027_keep_dims_0, x = var_9025_cast_fp16)[name = string("op_9027_cast_fp16")];
            tensor<int32, [3]> var_9030_begin_0 = const()[name = string("op_9030_begin_0"), val = tensor<int32, [3]>([0, 0, 1])];
            tensor<int32, [3]> var_9030_end_0 = const()[name = string("op_9030_end_0"), val = tensor<int32, [3]>([1, 16, 2])];
            tensor<bool, [3]> var_9030_end_mask_0 = const()[name = string("op_9030_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_9030_squeeze_mask_0 = const()[name = string("op_9030_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_9030_cast_fp16 = slice_by_index(begin = var_9030_begin_0, end = var_9030_end_0, end_mask = var_9030_end_mask_0, squeeze_mask = var_9030_squeeze_mask_0, x = decay_21_cast_fp16)[name = string("op_9030_cast_fp16")];
            tensor<fp16, [1, 16]> var_9031_cast_fp16 = exp(x = var_9030_cast_fp16)[name = string("op_9031_cast_fp16")];
            tensor<int32, [1]> var_9032_axes_0 = const()[name = string("op_9032_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_9032_cast_fp16 = expand_dims(axes = var_9032_axes_0, x = var_9031_cast_fp16)[name = string("op_9032_cast_fp16")];
            tensor<int32, [1]> var_9033_axes_0 = const()[name = string("op_9033_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_9033_cast_fp16 = expand_dims(axes = var_9033_axes_0, x = var_9032_cast_fp16)[name = string("op_9033_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_645_cast_fp16 = mul(x = state_643_cast_fp16, y = var_9033_cast_fp16)[name = string("state_645_cast_fp16")];
            tensor<int32, [4]> key_token_323_begin_0 = const()[name = string("key_token_323_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> key_token_323_end_0 = const()[name = string("key_token_323_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> key_token_323_end_mask_0 = const()[name = string("key_token_323_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_323_squeeze_mask_0 = const()[name = string("key_token_323_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_323_cast_fp16 = slice_by_index(begin = key_token_323_begin_0, end = key_token_323_end_0, end_mask = key_token_323_end_mask_0, squeeze_mask = key_token_323_squeeze_mask_0, x = key_55_cast_fp16)[name = string("key_token_323_cast_fp16")];
            tensor<int32, [4]> value_token_323_begin_0 = const()[name = string("value_token_323_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> value_token_323_end_0 = const()[name = string("value_token_323_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> value_token_323_end_mask_0 = const()[name = string("value_token_323_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_323_squeeze_mask_0 = const()[name = string("value_token_323_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_323_cast_fp16 = slice_by_index(begin = value_token_323_begin_0, end = value_token_323_end_0, end_mask = value_token_323_end_mask_0, squeeze_mask = value_token_323_squeeze_mask_0, x = value_199_cast_fp16)[name = string("value_token_323_cast_fp16")];
            tensor<int32, [1]> var_9041_axes_0 = const()[name = string("op_9041_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_9041_cast_fp16 = expand_dims(axes = var_9041_axes_0, x = key_token_323_cast_fp16)[name = string("op_9041_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_9042_cast_fp16 = mul(x = state_645_cast_fp16, y = var_9041_cast_fp16)[name = string("op_9042_cast_fp16")];
            tensor<int32, [1]> memory_323_axes_0 = const()[name = string("memory_323_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_323_keep_dims_0 = const()[name = string("memory_323_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_323_cast_fp16 = reduce_sum(axes = memory_323_axes_0, keep_dims = memory_323_keep_dims_0, x = var_9042_cast_fp16)[name = string("memory_323_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_9045_cast_fp16 = sub(x = value_token_323_cast_fp16, y = memory_323_cast_fp16)[name = string("op_9045_cast_fp16")];
            tensor<int32, [3]> var_9048_begin_0 = const()[name = string("op_9048_begin_0"), val = tensor<int32, [3]>([0, 0, 1])];
            tensor<int32, [3]> var_9048_end_0 = const()[name = string("op_9048_end_0"), val = tensor<int32, [3]>([1, 16, 2])];
            tensor<bool, [3]> var_9048_end_mask_0 = const()[name = string("op_9048_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_9048_squeeze_mask_0 = const()[name = string("op_9048_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_9048_cast_fp16 = slice_by_index(begin = var_9048_begin_0, end = var_9048_end_0, end_mask = var_9048_end_mask_0, squeeze_mask = var_9048_squeeze_mask_0, x = beta_21_cast_fp16)[name = string("op_9048_cast_fp16")];
            tensor<int32, [1]> var_9049_axes_0 = const()[name = string("op_9049_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_9049_cast_fp16 = expand_dims(axes = var_9049_axes_0, x = var_9048_cast_fp16)[name = string("op_9049_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_323_cast_fp16 = mul(x = var_9045_cast_fp16, y = var_9049_cast_fp16)[name = string("delta_323_cast_fp16")];
            tensor<int32, [1]> var_9052_axes_0 = const()[name = string("op_9052_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_9052_cast_fp16 = expand_dims(axes = var_9052_axes_0, x = delta_323_cast_fp16)[name = string("op_9052_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_9053_cast_fp16 = mul(x = var_9041_cast_fp16, y = var_9052_cast_fp16)[name = string("op_9053_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_647_cast_fp16 = add(x = state_645_cast_fp16, y = var_9053_cast_fp16)[name = string("state_647_cast_fp16")];
            tensor<int32, [4]> var_9057_begin_0 = const()[name = string("op_9057_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> var_9057_end_0 = const()[name = string("op_9057_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> var_9057_end_mask_0 = const()[name = string("op_9057_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_9057_squeeze_mask_0 = const()[name = string("op_9057_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_9057_cast_fp16 = slice_by_index(begin = var_9057_begin_0, end = var_9057_end_0, end_mask = var_9057_end_mask_0, squeeze_mask = var_9057_squeeze_mask_0, x = _inversed_query_55_cast_fp16)[name = string("op_9057_cast_fp16")];
            tensor<int32, [1]> var_9058_axes_0 = const()[name = string("op_9058_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_9058_cast_fp16 = expand_dims(axes = var_9058_axes_0, x = var_9057_cast_fp16)[name = string("op_9058_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_9059_cast_fp16 = mul(x = state_647_cast_fp16, y = var_9058_cast_fp16)[name = string("op_9059_cast_fp16")];
            tensor<int32, [1]> var_9061_axes_0 = const()[name = string("op_9061_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_9061_keep_dims_0 = const()[name = string("op_9061_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_9061_cast_fp16 = reduce_sum(axes = var_9061_axes_0, keep_dims = var_9061_keep_dims_0, x = var_9059_cast_fp16)[name = string("op_9061_cast_fp16")];
            tensor<int32, [3]> var_9064_begin_0 = const()[name = string("op_9064_begin_0"), val = tensor<int32, [3]>([0, 0, 2])];
            tensor<int32, [3]> var_9064_end_0 = const()[name = string("op_9064_end_0"), val = tensor<int32, [3]>([1, 16, 3])];
            tensor<bool, [3]> var_9064_end_mask_0 = const()[name = string("op_9064_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_9064_squeeze_mask_0 = const()[name = string("op_9064_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_9064_cast_fp16 = slice_by_index(begin = var_9064_begin_0, end = var_9064_end_0, end_mask = var_9064_end_mask_0, squeeze_mask = var_9064_squeeze_mask_0, x = decay_21_cast_fp16)[name = string("op_9064_cast_fp16")];
            tensor<fp16, [1, 16]> var_9065_cast_fp16 = exp(x = var_9064_cast_fp16)[name = string("op_9065_cast_fp16")];
            tensor<int32, [1]> var_9066_axes_0 = const()[name = string("op_9066_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_9066_cast_fp16 = expand_dims(axes = var_9066_axes_0, x = var_9065_cast_fp16)[name = string("op_9066_cast_fp16")];
            tensor<int32, [1]> var_9067_axes_0 = const()[name = string("op_9067_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_9067_cast_fp16 = expand_dims(axes = var_9067_axes_0, x = var_9066_cast_fp16)[name = string("op_9067_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_649_cast_fp16 = mul(x = state_647_cast_fp16, y = var_9067_cast_fp16)[name = string("state_649_cast_fp16")];
            tensor<int32, [4]> key_token_325_begin_0 = const()[name = string("key_token_325_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> key_token_325_end_0 = const()[name = string("key_token_325_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> key_token_325_end_mask_0 = const()[name = string("key_token_325_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_325_squeeze_mask_0 = const()[name = string("key_token_325_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_325_cast_fp16 = slice_by_index(begin = key_token_325_begin_0, end = key_token_325_end_0, end_mask = key_token_325_end_mask_0, squeeze_mask = key_token_325_squeeze_mask_0, x = key_55_cast_fp16)[name = string("key_token_325_cast_fp16")];
            tensor<int32, [4]> value_token_325_begin_0 = const()[name = string("value_token_325_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> value_token_325_end_0 = const()[name = string("value_token_325_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> value_token_325_end_mask_0 = const()[name = string("value_token_325_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_325_squeeze_mask_0 = const()[name = string("value_token_325_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_325_cast_fp16 = slice_by_index(begin = value_token_325_begin_0, end = value_token_325_end_0, end_mask = value_token_325_end_mask_0, squeeze_mask = value_token_325_squeeze_mask_0, x = value_199_cast_fp16)[name = string("value_token_325_cast_fp16")];
            tensor<int32, [1]> var_9075_axes_0 = const()[name = string("op_9075_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_9075_cast_fp16 = expand_dims(axes = var_9075_axes_0, x = key_token_325_cast_fp16)[name = string("op_9075_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_9076_cast_fp16 = mul(x = state_649_cast_fp16, y = var_9075_cast_fp16)[name = string("op_9076_cast_fp16")];
            tensor<int32, [1]> memory_325_axes_0 = const()[name = string("memory_325_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_325_keep_dims_0 = const()[name = string("memory_325_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_325_cast_fp16 = reduce_sum(axes = memory_325_axes_0, keep_dims = memory_325_keep_dims_0, x = var_9076_cast_fp16)[name = string("memory_325_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_9079_cast_fp16 = sub(x = value_token_325_cast_fp16, y = memory_325_cast_fp16)[name = string("op_9079_cast_fp16")];
            tensor<int32, [3]> var_9082_begin_0 = const()[name = string("op_9082_begin_0"), val = tensor<int32, [3]>([0, 0, 2])];
            tensor<int32, [3]> var_9082_end_0 = const()[name = string("op_9082_end_0"), val = tensor<int32, [3]>([1, 16, 3])];
            tensor<bool, [3]> var_9082_end_mask_0 = const()[name = string("op_9082_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_9082_squeeze_mask_0 = const()[name = string("op_9082_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_9082_cast_fp16 = slice_by_index(begin = var_9082_begin_0, end = var_9082_end_0, end_mask = var_9082_end_mask_0, squeeze_mask = var_9082_squeeze_mask_0, x = beta_21_cast_fp16)[name = string("op_9082_cast_fp16")];
            tensor<int32, [1]> var_9083_axes_0 = const()[name = string("op_9083_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_9083_cast_fp16 = expand_dims(axes = var_9083_axes_0, x = var_9082_cast_fp16)[name = string("op_9083_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_325_cast_fp16 = mul(x = var_9079_cast_fp16, y = var_9083_cast_fp16)[name = string("delta_325_cast_fp16")];
            tensor<int32, [1]> var_9086_axes_0 = const()[name = string("op_9086_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_9086_cast_fp16 = expand_dims(axes = var_9086_axes_0, x = delta_325_cast_fp16)[name = string("op_9086_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_9087_cast_fp16 = mul(x = var_9075_cast_fp16, y = var_9086_cast_fp16)[name = string("op_9087_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_651_cast_fp16 = add(x = state_649_cast_fp16, y = var_9087_cast_fp16)[name = string("state_651_cast_fp16")];
            tensor<int32, [4]> var_9091_begin_0 = const()[name = string("op_9091_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> var_9091_end_0 = const()[name = string("op_9091_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> var_9091_end_mask_0 = const()[name = string("op_9091_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_9091_squeeze_mask_0 = const()[name = string("op_9091_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_9091_cast_fp16 = slice_by_index(begin = var_9091_begin_0, end = var_9091_end_0, end_mask = var_9091_end_mask_0, squeeze_mask = var_9091_squeeze_mask_0, x = _inversed_query_55_cast_fp16)[name = string("op_9091_cast_fp16")];
            tensor<int32, [1]> var_9092_axes_0 = const()[name = string("op_9092_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_9092_cast_fp16 = expand_dims(axes = var_9092_axes_0, x = var_9091_cast_fp16)[name = string("op_9092_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_9093_cast_fp16 = mul(x = state_651_cast_fp16, y = var_9092_cast_fp16)[name = string("op_9093_cast_fp16")];
            tensor<int32, [1]> var_9095_axes_0 = const()[name = string("op_9095_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_9095_keep_dims_0 = const()[name = string("op_9095_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_9095_cast_fp16 = reduce_sum(axes = var_9095_axes_0, keep_dims = var_9095_keep_dims_0, x = var_9093_cast_fp16)[name = string("op_9095_cast_fp16")];
            tensor<int32, [3]> var_9098_begin_0 = const()[name = string("op_9098_begin_0"), val = tensor<int32, [3]>([0, 0, 3])];
            tensor<int32, [3]> var_9098_end_0 = const()[name = string("op_9098_end_0"), val = tensor<int32, [3]>([1, 16, 4])];
            tensor<bool, [3]> var_9098_end_mask_0 = const()[name = string("op_9098_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_9098_squeeze_mask_0 = const()[name = string("op_9098_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_9098_cast_fp16 = slice_by_index(begin = var_9098_begin_0, end = var_9098_end_0, end_mask = var_9098_end_mask_0, squeeze_mask = var_9098_squeeze_mask_0, x = decay_21_cast_fp16)[name = string("op_9098_cast_fp16")];
            tensor<fp16, [1, 16]> var_9099_cast_fp16 = exp(x = var_9098_cast_fp16)[name = string("op_9099_cast_fp16")];
            tensor<int32, [1]> var_9100_axes_0 = const()[name = string("op_9100_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_9100_cast_fp16 = expand_dims(axes = var_9100_axes_0, x = var_9099_cast_fp16)[name = string("op_9100_cast_fp16")];
            tensor<int32, [1]> var_9101_axes_0 = const()[name = string("op_9101_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_9101_cast_fp16 = expand_dims(axes = var_9101_axes_0, x = var_9100_cast_fp16)[name = string("op_9101_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_653_cast_fp16 = mul(x = state_651_cast_fp16, y = var_9101_cast_fp16)[name = string("state_653_cast_fp16")];
            tensor<int32, [4]> key_token_327_begin_0 = const()[name = string("key_token_327_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> key_token_327_end_0 = const()[name = string("key_token_327_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> key_token_327_end_mask_0 = const()[name = string("key_token_327_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_327_squeeze_mask_0 = const()[name = string("key_token_327_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_327_cast_fp16 = slice_by_index(begin = key_token_327_begin_0, end = key_token_327_end_0, end_mask = key_token_327_end_mask_0, squeeze_mask = key_token_327_squeeze_mask_0, x = key_55_cast_fp16)[name = string("key_token_327_cast_fp16")];
            tensor<int32, [4]> value_token_327_begin_0 = const()[name = string("value_token_327_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> value_token_327_end_0 = const()[name = string("value_token_327_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> value_token_327_end_mask_0 = const()[name = string("value_token_327_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_327_squeeze_mask_0 = const()[name = string("value_token_327_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_327_cast_fp16 = slice_by_index(begin = value_token_327_begin_0, end = value_token_327_end_0, end_mask = value_token_327_end_mask_0, squeeze_mask = value_token_327_squeeze_mask_0, x = value_199_cast_fp16)[name = string("value_token_327_cast_fp16")];
            tensor<int32, [1]> var_9109_axes_0 = const()[name = string("op_9109_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_9109_cast_fp16 = expand_dims(axes = var_9109_axes_0, x = key_token_327_cast_fp16)[name = string("op_9109_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_9110_cast_fp16 = mul(x = state_653_cast_fp16, y = var_9109_cast_fp16)[name = string("op_9110_cast_fp16")];
            tensor<int32, [1]> memory_327_axes_0 = const()[name = string("memory_327_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_327_keep_dims_0 = const()[name = string("memory_327_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_327_cast_fp16 = reduce_sum(axes = memory_327_axes_0, keep_dims = memory_327_keep_dims_0, x = var_9110_cast_fp16)[name = string("memory_327_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_9113_cast_fp16 = sub(x = value_token_327_cast_fp16, y = memory_327_cast_fp16)[name = string("op_9113_cast_fp16")];
            tensor<int32, [3]> var_9116_begin_0 = const()[name = string("op_9116_begin_0"), val = tensor<int32, [3]>([0, 0, 3])];
            tensor<int32, [3]> var_9116_end_0 = const()[name = string("op_9116_end_0"), val = tensor<int32, [3]>([1, 16, 4])];
            tensor<bool, [3]> var_9116_end_mask_0 = const()[name = string("op_9116_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_9116_squeeze_mask_0 = const()[name = string("op_9116_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_9116_cast_fp16 = slice_by_index(begin = var_9116_begin_0, end = var_9116_end_0, end_mask = var_9116_end_mask_0, squeeze_mask = var_9116_squeeze_mask_0, x = beta_21_cast_fp16)[name = string("op_9116_cast_fp16")];
            tensor<int32, [1]> var_9117_axes_0 = const()[name = string("op_9117_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_9117_cast_fp16 = expand_dims(axes = var_9117_axes_0, x = var_9116_cast_fp16)[name = string("op_9117_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_327_cast_fp16 = mul(x = var_9113_cast_fp16, y = var_9117_cast_fp16)[name = string("delta_327_cast_fp16")];
            tensor<int32, [1]> var_9120_axes_0 = const()[name = string("op_9120_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_9120_cast_fp16 = expand_dims(axes = var_9120_axes_0, x = delta_327_cast_fp16)[name = string("op_9120_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_9121_cast_fp16 = mul(x = var_9109_cast_fp16, y = var_9120_cast_fp16)[name = string("op_9121_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_655_cast_fp16 = add(x = state_653_cast_fp16, y = var_9121_cast_fp16)[name = string("state_655_cast_fp16")];
            tensor<int32, [4]> var_9125_begin_0 = const()[name = string("op_9125_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> var_9125_end_0 = const()[name = string("op_9125_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> var_9125_end_mask_0 = const()[name = string("op_9125_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_9125_squeeze_mask_0 = const()[name = string("op_9125_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_9125_cast_fp16 = slice_by_index(begin = var_9125_begin_0, end = var_9125_end_0, end_mask = var_9125_end_mask_0, squeeze_mask = var_9125_squeeze_mask_0, x = _inversed_query_55_cast_fp16)[name = string("op_9125_cast_fp16")];
            tensor<int32, [1]> var_9126_axes_0 = const()[name = string("op_9126_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_9126_cast_fp16 = expand_dims(axes = var_9126_axes_0, x = var_9125_cast_fp16)[name = string("op_9126_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_9127_cast_fp16 = mul(x = state_655_cast_fp16, y = var_9126_cast_fp16)[name = string("op_9127_cast_fp16")];
            tensor<int32, [1]> var_9129_axes_0 = const()[name = string("op_9129_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_9129_keep_dims_0 = const()[name = string("op_9129_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_9129_cast_fp16 = reduce_sum(axes = var_9129_axes_0, keep_dims = var_9129_keep_dims_0, x = var_9127_cast_fp16)[name = string("op_9129_cast_fp16")];
            tensor<int32, [3]> var_9132_begin_0 = const()[name = string("op_9132_begin_0"), val = tensor<int32, [3]>([0, 0, 4])];
            tensor<int32, [3]> var_9132_end_0 = const()[name = string("op_9132_end_0"), val = tensor<int32, [3]>([1, 16, 5])];
            tensor<bool, [3]> var_9132_end_mask_0 = const()[name = string("op_9132_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_9132_squeeze_mask_0 = const()[name = string("op_9132_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_9132_cast_fp16 = slice_by_index(begin = var_9132_begin_0, end = var_9132_end_0, end_mask = var_9132_end_mask_0, squeeze_mask = var_9132_squeeze_mask_0, x = decay_21_cast_fp16)[name = string("op_9132_cast_fp16")];
            tensor<fp16, [1, 16]> var_9133_cast_fp16 = exp(x = var_9132_cast_fp16)[name = string("op_9133_cast_fp16")];
            tensor<int32, [1]> var_9134_axes_0 = const()[name = string("op_9134_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_9134_cast_fp16 = expand_dims(axes = var_9134_axes_0, x = var_9133_cast_fp16)[name = string("op_9134_cast_fp16")];
            tensor<int32, [1]> var_9135_axes_0 = const()[name = string("op_9135_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_9135_cast_fp16 = expand_dims(axes = var_9135_axes_0, x = var_9134_cast_fp16)[name = string("op_9135_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_657_cast_fp16 = mul(x = state_655_cast_fp16, y = var_9135_cast_fp16)[name = string("state_657_cast_fp16")];
            tensor<int32, [4]> key_token_329_begin_0 = const()[name = string("key_token_329_begin_0"), val = tensor<int32, [4]>([0, 0, 4, 0])];
            tensor<int32, [4]> key_token_329_end_0 = const()[name = string("key_token_329_end_0"), val = tensor<int32, [4]>([1, 16, 5, 128])];
            tensor<bool, [4]> key_token_329_end_mask_0 = const()[name = string("key_token_329_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_329_squeeze_mask_0 = const()[name = string("key_token_329_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_329_cast_fp16 = slice_by_index(begin = key_token_329_begin_0, end = key_token_329_end_0, end_mask = key_token_329_end_mask_0, squeeze_mask = key_token_329_squeeze_mask_0, x = key_55_cast_fp16)[name = string("key_token_329_cast_fp16")];
            tensor<int32, [4]> value_token_329_begin_0 = const()[name = string("value_token_329_begin_0"), val = tensor<int32, [4]>([0, 0, 4, 0])];
            tensor<int32, [4]> value_token_329_end_0 = const()[name = string("value_token_329_end_0"), val = tensor<int32, [4]>([1, 16, 5, 128])];
            tensor<bool, [4]> value_token_329_end_mask_0 = const()[name = string("value_token_329_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_329_squeeze_mask_0 = const()[name = string("value_token_329_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_329_cast_fp16 = slice_by_index(begin = value_token_329_begin_0, end = value_token_329_end_0, end_mask = value_token_329_end_mask_0, squeeze_mask = value_token_329_squeeze_mask_0, x = value_199_cast_fp16)[name = string("value_token_329_cast_fp16")];
            tensor<int32, [1]> var_9143_axes_0 = const()[name = string("op_9143_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_9143_cast_fp16 = expand_dims(axes = var_9143_axes_0, x = key_token_329_cast_fp16)[name = string("op_9143_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_9144_cast_fp16 = mul(x = state_657_cast_fp16, y = var_9143_cast_fp16)[name = string("op_9144_cast_fp16")];
            tensor<int32, [1]> memory_329_axes_0 = const()[name = string("memory_329_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_329_keep_dims_0 = const()[name = string("memory_329_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_329_cast_fp16 = reduce_sum(axes = memory_329_axes_0, keep_dims = memory_329_keep_dims_0, x = var_9144_cast_fp16)[name = string("memory_329_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_9147_cast_fp16 = sub(x = value_token_329_cast_fp16, y = memory_329_cast_fp16)[name = string("op_9147_cast_fp16")];
            tensor<int32, [3]> var_9150_begin_0 = const()[name = string("op_9150_begin_0"), val = tensor<int32, [3]>([0, 0, 4])];
            tensor<int32, [3]> var_9150_end_0 = const()[name = string("op_9150_end_0"), val = tensor<int32, [3]>([1, 16, 5])];
            tensor<bool, [3]> var_9150_end_mask_0 = const()[name = string("op_9150_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_9150_squeeze_mask_0 = const()[name = string("op_9150_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_9150_cast_fp16 = slice_by_index(begin = var_9150_begin_0, end = var_9150_end_0, end_mask = var_9150_end_mask_0, squeeze_mask = var_9150_squeeze_mask_0, x = beta_21_cast_fp16)[name = string("op_9150_cast_fp16")];
            tensor<int32, [1]> var_9151_axes_0 = const()[name = string("op_9151_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_9151_cast_fp16 = expand_dims(axes = var_9151_axes_0, x = var_9150_cast_fp16)[name = string("op_9151_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_329_cast_fp16 = mul(x = var_9147_cast_fp16, y = var_9151_cast_fp16)[name = string("delta_329_cast_fp16")];
            tensor<int32, [1]> var_9154_axes_0 = const()[name = string("op_9154_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_9154_cast_fp16 = expand_dims(axes = var_9154_axes_0, x = delta_329_cast_fp16)[name = string("op_9154_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_9155_cast_fp16 = mul(x = var_9143_cast_fp16, y = var_9154_cast_fp16)[name = string("op_9155_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_659_cast_fp16 = add(x = state_657_cast_fp16, y = var_9155_cast_fp16)[name = string("state_659_cast_fp16")];
            tensor<int32, [4]> var_9159_begin_0 = const()[name = string("op_9159_begin_0"), val = tensor<int32, [4]>([0, 0, 4, 0])];
            tensor<int32, [4]> var_9159_end_0 = const()[name = string("op_9159_end_0"), val = tensor<int32, [4]>([1, 16, 5, 128])];
            tensor<bool, [4]> var_9159_end_mask_0 = const()[name = string("op_9159_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_9159_squeeze_mask_0 = const()[name = string("op_9159_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_9159_cast_fp16 = slice_by_index(begin = var_9159_begin_0, end = var_9159_end_0, end_mask = var_9159_end_mask_0, squeeze_mask = var_9159_squeeze_mask_0, x = _inversed_query_55_cast_fp16)[name = string("op_9159_cast_fp16")];
            tensor<int32, [1]> var_9160_axes_0 = const()[name = string("op_9160_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_9160_cast_fp16 = expand_dims(axes = var_9160_axes_0, x = var_9159_cast_fp16)[name = string("op_9160_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_9161_cast_fp16 = mul(x = state_659_cast_fp16, y = var_9160_cast_fp16)[name = string("op_9161_cast_fp16")];
            tensor<int32, [1]> var_9163_axes_0 = const()[name = string("op_9163_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_9163_keep_dims_0 = const()[name = string("op_9163_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_9163_cast_fp16 = reduce_sum(axes = var_9163_axes_0, keep_dims = var_9163_keep_dims_0, x = var_9161_cast_fp16)[name = string("op_9163_cast_fp16")];
            tensor<int32, [3]> var_9166_begin_0 = const()[name = string("op_9166_begin_0"), val = tensor<int32, [3]>([0, 0, 5])];
            tensor<int32, [3]> var_9166_end_0 = const()[name = string("op_9166_end_0"), val = tensor<int32, [3]>([1, 16, 6])];
            tensor<bool, [3]> var_9166_end_mask_0 = const()[name = string("op_9166_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_9166_squeeze_mask_0 = const()[name = string("op_9166_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_9166_cast_fp16 = slice_by_index(begin = var_9166_begin_0, end = var_9166_end_0, end_mask = var_9166_end_mask_0, squeeze_mask = var_9166_squeeze_mask_0, x = decay_21_cast_fp16)[name = string("op_9166_cast_fp16")];
            tensor<fp16, [1, 16]> var_9167_cast_fp16 = exp(x = var_9166_cast_fp16)[name = string("op_9167_cast_fp16")];
            tensor<int32, [1]> var_9168_axes_0 = const()[name = string("op_9168_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_9168_cast_fp16 = expand_dims(axes = var_9168_axes_0, x = var_9167_cast_fp16)[name = string("op_9168_cast_fp16")];
            tensor<int32, [1]> var_9169_axes_0 = const()[name = string("op_9169_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_9169_cast_fp16 = expand_dims(axes = var_9169_axes_0, x = var_9168_cast_fp16)[name = string("op_9169_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_661_cast_fp16 = mul(x = state_659_cast_fp16, y = var_9169_cast_fp16)[name = string("state_661_cast_fp16")];
            tensor<int32, [4]> key_token_331_begin_0 = const()[name = string("key_token_331_begin_0"), val = tensor<int32, [4]>([0, 0, 5, 0])];
            tensor<int32, [4]> key_token_331_end_0 = const()[name = string("key_token_331_end_0"), val = tensor<int32, [4]>([1, 16, 6, 128])];
            tensor<bool, [4]> key_token_331_end_mask_0 = const()[name = string("key_token_331_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_331_squeeze_mask_0 = const()[name = string("key_token_331_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_331_cast_fp16 = slice_by_index(begin = key_token_331_begin_0, end = key_token_331_end_0, end_mask = key_token_331_end_mask_0, squeeze_mask = key_token_331_squeeze_mask_0, x = key_55_cast_fp16)[name = string("key_token_331_cast_fp16")];
            tensor<int32, [4]> value_token_331_begin_0 = const()[name = string("value_token_331_begin_0"), val = tensor<int32, [4]>([0, 0, 5, 0])];
            tensor<int32, [4]> value_token_331_end_0 = const()[name = string("value_token_331_end_0"), val = tensor<int32, [4]>([1, 16, 6, 128])];
            tensor<bool, [4]> value_token_331_end_mask_0 = const()[name = string("value_token_331_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_331_squeeze_mask_0 = const()[name = string("value_token_331_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_331_cast_fp16 = slice_by_index(begin = value_token_331_begin_0, end = value_token_331_end_0, end_mask = value_token_331_end_mask_0, squeeze_mask = value_token_331_squeeze_mask_0, x = value_199_cast_fp16)[name = string("value_token_331_cast_fp16")];
            tensor<int32, [1]> var_9177_axes_0 = const()[name = string("op_9177_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_9177_cast_fp16 = expand_dims(axes = var_9177_axes_0, x = key_token_331_cast_fp16)[name = string("op_9177_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_9178_cast_fp16 = mul(x = state_661_cast_fp16, y = var_9177_cast_fp16)[name = string("op_9178_cast_fp16")];
            tensor<int32, [1]> memory_331_axes_0 = const()[name = string("memory_331_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_331_keep_dims_0 = const()[name = string("memory_331_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_331_cast_fp16 = reduce_sum(axes = memory_331_axes_0, keep_dims = memory_331_keep_dims_0, x = var_9178_cast_fp16)[name = string("memory_331_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_9181_cast_fp16 = sub(x = value_token_331_cast_fp16, y = memory_331_cast_fp16)[name = string("op_9181_cast_fp16")];
            tensor<int32, [3]> var_9184_begin_0 = const()[name = string("op_9184_begin_0"), val = tensor<int32, [3]>([0, 0, 5])];
            tensor<int32, [3]> var_9184_end_0 = const()[name = string("op_9184_end_0"), val = tensor<int32, [3]>([1, 16, 6])];
            tensor<bool, [3]> var_9184_end_mask_0 = const()[name = string("op_9184_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_9184_squeeze_mask_0 = const()[name = string("op_9184_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_9184_cast_fp16 = slice_by_index(begin = var_9184_begin_0, end = var_9184_end_0, end_mask = var_9184_end_mask_0, squeeze_mask = var_9184_squeeze_mask_0, x = beta_21_cast_fp16)[name = string("op_9184_cast_fp16")];
            tensor<int32, [1]> var_9185_axes_0 = const()[name = string("op_9185_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_9185_cast_fp16 = expand_dims(axes = var_9185_axes_0, x = var_9184_cast_fp16)[name = string("op_9185_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_331_cast_fp16 = mul(x = var_9181_cast_fp16, y = var_9185_cast_fp16)[name = string("delta_331_cast_fp16")];
            tensor<int32, [1]> var_9188_axes_0 = const()[name = string("op_9188_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_9188_cast_fp16 = expand_dims(axes = var_9188_axes_0, x = delta_331_cast_fp16)[name = string("op_9188_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_9189_cast_fp16 = mul(x = var_9177_cast_fp16, y = var_9188_cast_fp16)[name = string("op_9189_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_663_cast_fp16 = add(x = state_661_cast_fp16, y = var_9189_cast_fp16)[name = string("state_663_cast_fp16")];
            tensor<int32, [4]> var_9193_begin_0 = const()[name = string("op_9193_begin_0"), val = tensor<int32, [4]>([0, 0, 5, 0])];
            tensor<int32, [4]> var_9193_end_0 = const()[name = string("op_9193_end_0"), val = tensor<int32, [4]>([1, 16, 6, 128])];
            tensor<bool, [4]> var_9193_end_mask_0 = const()[name = string("op_9193_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_9193_squeeze_mask_0 = const()[name = string("op_9193_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_9193_cast_fp16 = slice_by_index(begin = var_9193_begin_0, end = var_9193_end_0, end_mask = var_9193_end_mask_0, squeeze_mask = var_9193_squeeze_mask_0, x = _inversed_query_55_cast_fp16)[name = string("op_9193_cast_fp16")];
            tensor<int32, [1]> var_9194_axes_0 = const()[name = string("op_9194_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_9194_cast_fp16 = expand_dims(axes = var_9194_axes_0, x = var_9193_cast_fp16)[name = string("op_9194_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_9195_cast_fp16 = mul(x = state_663_cast_fp16, y = var_9194_cast_fp16)[name = string("op_9195_cast_fp16")];
            tensor<int32, [1]> var_9197_axes_0 = const()[name = string("op_9197_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_9197_keep_dims_0 = const()[name = string("op_9197_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_9197_cast_fp16 = reduce_sum(axes = var_9197_axes_0, keep_dims = var_9197_keep_dims_0, x = var_9195_cast_fp16)[name = string("op_9197_cast_fp16")];
            tensor<int32, [3]> var_9200_begin_0 = const()[name = string("op_9200_begin_0"), val = tensor<int32, [3]>([0, 0, 6])];
            tensor<int32, [3]> var_9200_end_0 = const()[name = string("op_9200_end_0"), val = tensor<int32, [3]>([1, 16, 7])];
            tensor<bool, [3]> var_9200_end_mask_0 = const()[name = string("op_9200_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_9200_squeeze_mask_0 = const()[name = string("op_9200_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_9200_cast_fp16 = slice_by_index(begin = var_9200_begin_0, end = var_9200_end_0, end_mask = var_9200_end_mask_0, squeeze_mask = var_9200_squeeze_mask_0, x = decay_21_cast_fp16)[name = string("op_9200_cast_fp16")];
            tensor<fp16, [1, 16]> var_9201_cast_fp16 = exp(x = var_9200_cast_fp16)[name = string("op_9201_cast_fp16")];
            tensor<int32, [1]> var_9202_axes_0 = const()[name = string("op_9202_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_9202_cast_fp16 = expand_dims(axes = var_9202_axes_0, x = var_9201_cast_fp16)[name = string("op_9202_cast_fp16")];
            tensor<int32, [1]> var_9203_axes_0 = const()[name = string("op_9203_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_9203_cast_fp16 = expand_dims(axes = var_9203_axes_0, x = var_9202_cast_fp16)[name = string("op_9203_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_665_cast_fp16 = mul(x = state_663_cast_fp16, y = var_9203_cast_fp16)[name = string("state_665_cast_fp16")];
            tensor<int32, [4]> key_token_333_begin_0 = const()[name = string("key_token_333_begin_0"), val = tensor<int32, [4]>([0, 0, 6, 0])];
            tensor<int32, [4]> key_token_333_end_0 = const()[name = string("key_token_333_end_0"), val = tensor<int32, [4]>([1, 16, 7, 128])];
            tensor<bool, [4]> key_token_333_end_mask_0 = const()[name = string("key_token_333_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_333_squeeze_mask_0 = const()[name = string("key_token_333_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_333_cast_fp16 = slice_by_index(begin = key_token_333_begin_0, end = key_token_333_end_0, end_mask = key_token_333_end_mask_0, squeeze_mask = key_token_333_squeeze_mask_0, x = key_55_cast_fp16)[name = string("key_token_333_cast_fp16")];
            tensor<int32, [4]> value_token_333_begin_0 = const()[name = string("value_token_333_begin_0"), val = tensor<int32, [4]>([0, 0, 6, 0])];
            tensor<int32, [4]> value_token_333_end_0 = const()[name = string("value_token_333_end_0"), val = tensor<int32, [4]>([1, 16, 7, 128])];
            tensor<bool, [4]> value_token_333_end_mask_0 = const()[name = string("value_token_333_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_333_squeeze_mask_0 = const()[name = string("value_token_333_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_333_cast_fp16 = slice_by_index(begin = value_token_333_begin_0, end = value_token_333_end_0, end_mask = value_token_333_end_mask_0, squeeze_mask = value_token_333_squeeze_mask_0, x = value_199_cast_fp16)[name = string("value_token_333_cast_fp16")];
            tensor<int32, [1]> var_9211_axes_0 = const()[name = string("op_9211_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_9211_cast_fp16 = expand_dims(axes = var_9211_axes_0, x = key_token_333_cast_fp16)[name = string("op_9211_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_9212_cast_fp16 = mul(x = state_665_cast_fp16, y = var_9211_cast_fp16)[name = string("op_9212_cast_fp16")];
            tensor<int32, [1]> memory_333_axes_0 = const()[name = string("memory_333_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_333_keep_dims_0 = const()[name = string("memory_333_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_333_cast_fp16 = reduce_sum(axes = memory_333_axes_0, keep_dims = memory_333_keep_dims_0, x = var_9212_cast_fp16)[name = string("memory_333_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_9215_cast_fp16 = sub(x = value_token_333_cast_fp16, y = memory_333_cast_fp16)[name = string("op_9215_cast_fp16")];
            tensor<int32, [3]> var_9218_begin_0 = const()[name = string("op_9218_begin_0"), val = tensor<int32, [3]>([0, 0, 6])];
            tensor<int32, [3]> var_9218_end_0 = const()[name = string("op_9218_end_0"), val = tensor<int32, [3]>([1, 16, 7])];
            tensor<bool, [3]> var_9218_end_mask_0 = const()[name = string("op_9218_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_9218_squeeze_mask_0 = const()[name = string("op_9218_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_9218_cast_fp16 = slice_by_index(begin = var_9218_begin_0, end = var_9218_end_0, end_mask = var_9218_end_mask_0, squeeze_mask = var_9218_squeeze_mask_0, x = beta_21_cast_fp16)[name = string("op_9218_cast_fp16")];
            tensor<int32, [1]> var_9219_axes_0 = const()[name = string("op_9219_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_9219_cast_fp16 = expand_dims(axes = var_9219_axes_0, x = var_9218_cast_fp16)[name = string("op_9219_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_333_cast_fp16 = mul(x = var_9215_cast_fp16, y = var_9219_cast_fp16)[name = string("delta_333_cast_fp16")];
            tensor<int32, [1]> var_9222_axes_0 = const()[name = string("op_9222_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_9222_cast_fp16 = expand_dims(axes = var_9222_axes_0, x = delta_333_cast_fp16)[name = string("op_9222_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_9223_cast_fp16 = mul(x = var_9211_cast_fp16, y = var_9222_cast_fp16)[name = string("op_9223_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_667_cast_fp16 = add(x = state_665_cast_fp16, y = var_9223_cast_fp16)[name = string("state_667_cast_fp16")];
            tensor<int32, [4]> var_9227_begin_0 = const()[name = string("op_9227_begin_0"), val = tensor<int32, [4]>([0, 0, 6, 0])];
            tensor<int32, [4]> var_9227_end_0 = const()[name = string("op_9227_end_0"), val = tensor<int32, [4]>([1, 16, 7, 128])];
            tensor<bool, [4]> var_9227_end_mask_0 = const()[name = string("op_9227_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_9227_squeeze_mask_0 = const()[name = string("op_9227_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_9227_cast_fp16 = slice_by_index(begin = var_9227_begin_0, end = var_9227_end_0, end_mask = var_9227_end_mask_0, squeeze_mask = var_9227_squeeze_mask_0, x = _inversed_query_55_cast_fp16)[name = string("op_9227_cast_fp16")];
            tensor<int32, [1]> var_9228_axes_0 = const()[name = string("op_9228_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_9228_cast_fp16 = expand_dims(axes = var_9228_axes_0, x = var_9227_cast_fp16)[name = string("op_9228_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_9229_cast_fp16 = mul(x = state_667_cast_fp16, y = var_9228_cast_fp16)[name = string("op_9229_cast_fp16")];
            tensor<int32, [1]> var_9231_axes_0 = const()[name = string("op_9231_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_9231_keep_dims_0 = const()[name = string("op_9231_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_9231_cast_fp16 = reduce_sum(axes = var_9231_axes_0, keep_dims = var_9231_keep_dims_0, x = var_9229_cast_fp16)[name = string("op_9231_cast_fp16")];
            tensor<int32, [3]> var_9234_begin_0 = const()[name = string("op_9234_begin_0"), val = tensor<int32, [3]>([0, 0, 7])];
            tensor<int32, [3]> var_9234_end_0 = const()[name = string("op_9234_end_0"), val = tensor<int32, [3]>([1, 16, 8])];
            tensor<bool, [3]> var_9234_end_mask_0 = const()[name = string("op_9234_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_9234_squeeze_mask_0 = const()[name = string("op_9234_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_9234_cast_fp16 = slice_by_index(begin = var_9234_begin_0, end = var_9234_end_0, end_mask = var_9234_end_mask_0, squeeze_mask = var_9234_squeeze_mask_0, x = decay_21_cast_fp16)[name = string("op_9234_cast_fp16")];
            tensor<fp16, [1, 16]> var_9235_cast_fp16 = exp(x = var_9234_cast_fp16)[name = string("op_9235_cast_fp16")];
            tensor<int32, [1]> var_9236_axes_0 = const()[name = string("op_9236_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_9236_cast_fp16 = expand_dims(axes = var_9236_axes_0, x = var_9235_cast_fp16)[name = string("op_9236_cast_fp16")];
            tensor<int32, [1]> var_9237_axes_0 = const()[name = string("op_9237_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_9237_cast_fp16 = expand_dims(axes = var_9237_axes_0, x = var_9236_cast_fp16)[name = string("op_9237_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_669_cast_fp16 = mul(x = state_667_cast_fp16, y = var_9237_cast_fp16)[name = string("state_669_cast_fp16")];
            tensor<int32, [4]> key_token_335_begin_0 = const()[name = string("key_token_335_begin_0"), val = tensor<int32, [4]>([0, 0, 7, 0])];
            tensor<int32, [4]> key_token_335_end_0 = const()[name = string("key_token_335_end_0"), val = tensor<int32, [4]>([1, 16, 8, 128])];
            tensor<bool, [4]> key_token_335_end_mask_0 = const()[name = string("key_token_335_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_335_squeeze_mask_0 = const()[name = string("key_token_335_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_335_cast_fp16 = slice_by_index(begin = key_token_335_begin_0, end = key_token_335_end_0, end_mask = key_token_335_end_mask_0, squeeze_mask = key_token_335_squeeze_mask_0, x = key_55_cast_fp16)[name = string("key_token_335_cast_fp16")];
            tensor<int32, [4]> value_token_335_begin_0 = const()[name = string("value_token_335_begin_0"), val = tensor<int32, [4]>([0, 0, 7, 0])];
            tensor<int32, [4]> value_token_335_end_0 = const()[name = string("value_token_335_end_0"), val = tensor<int32, [4]>([1, 16, 8, 128])];
            tensor<bool, [4]> value_token_335_end_mask_0 = const()[name = string("value_token_335_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_335_squeeze_mask_0 = const()[name = string("value_token_335_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_335_cast_fp16 = slice_by_index(begin = value_token_335_begin_0, end = value_token_335_end_0, end_mask = value_token_335_end_mask_0, squeeze_mask = value_token_335_squeeze_mask_0, x = value_199_cast_fp16)[name = string("value_token_335_cast_fp16")];
            tensor<int32, [1]> var_9245_axes_0 = const()[name = string("op_9245_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_9245_cast_fp16 = expand_dims(axes = var_9245_axes_0, x = key_token_335_cast_fp16)[name = string("op_9245_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_9246_cast_fp16 = mul(x = state_669_cast_fp16, y = var_9245_cast_fp16)[name = string("op_9246_cast_fp16")];
            tensor<int32, [1]> memory_335_axes_0 = const()[name = string("memory_335_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_335_keep_dims_0 = const()[name = string("memory_335_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_335_cast_fp16 = reduce_sum(axes = memory_335_axes_0, keep_dims = memory_335_keep_dims_0, x = var_9246_cast_fp16)[name = string("memory_335_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_9249_cast_fp16 = sub(x = value_token_335_cast_fp16, y = memory_335_cast_fp16)[name = string("op_9249_cast_fp16")];
            tensor<int32, [3]> var_9252_begin_0 = const()[name = string("op_9252_begin_0"), val = tensor<int32, [3]>([0, 0, 7])];
            tensor<int32, [3]> var_9252_end_0 = const()[name = string("op_9252_end_0"), val = tensor<int32, [3]>([1, 16, 8])];
            tensor<bool, [3]> var_9252_end_mask_0 = const()[name = string("op_9252_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_9252_squeeze_mask_0 = const()[name = string("op_9252_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_9252_cast_fp16 = slice_by_index(begin = var_9252_begin_0, end = var_9252_end_0, end_mask = var_9252_end_mask_0, squeeze_mask = var_9252_squeeze_mask_0, x = beta_21_cast_fp16)[name = string("op_9252_cast_fp16")];
            tensor<int32, [1]> var_9253_axes_0 = const()[name = string("op_9253_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_9253_cast_fp16 = expand_dims(axes = var_9253_axes_0, x = var_9252_cast_fp16)[name = string("op_9253_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_335_cast_fp16 = mul(x = var_9249_cast_fp16, y = var_9253_cast_fp16)[name = string("delta_335_cast_fp16")];
            tensor<int32, [1]> var_9256_axes_0 = const()[name = string("op_9256_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_9256_cast_fp16 = expand_dims(axes = var_9256_axes_0, x = delta_335_cast_fp16)[name = string("op_9256_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_9257_cast_fp16 = mul(x = var_9245_cast_fp16, y = var_9256_cast_fp16)[name = string("op_9257_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_671_cast_fp16 = add(x = state_669_cast_fp16, y = var_9257_cast_fp16)[name = string("state_671_cast_fp16")];
            tensor<int32, [4]> var_9261_begin_0 = const()[name = string("op_9261_begin_0"), val = tensor<int32, [4]>([0, 0, 7, 0])];
            tensor<int32, [4]> var_9261_end_0 = const()[name = string("op_9261_end_0"), val = tensor<int32, [4]>([1, 16, 8, 128])];
            tensor<bool, [4]> var_9261_end_mask_0 = const()[name = string("op_9261_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_9261_squeeze_mask_0 = const()[name = string("op_9261_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_9261_cast_fp16 = slice_by_index(begin = var_9261_begin_0, end = var_9261_end_0, end_mask = var_9261_end_mask_0, squeeze_mask = var_9261_squeeze_mask_0, x = _inversed_query_55_cast_fp16)[name = string("op_9261_cast_fp16")];
            tensor<int32, [1]> var_9262_axes_0 = const()[name = string("op_9262_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_9262_cast_fp16 = expand_dims(axes = var_9262_axes_0, x = var_9261_cast_fp16)[name = string("op_9262_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_9263_cast_fp16 = mul(x = state_671_cast_fp16, y = var_9262_cast_fp16)[name = string("op_9263_cast_fp16")];
            tensor<int32, [1]> var_9265_axes_0 = const()[name = string("op_9265_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_9265_keep_dims_0 = const()[name = string("op_9265_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_9265_cast_fp16 = reduce_sum(axes = var_9265_axes_0, keep_dims = var_9265_keep_dims_0, x = var_9263_cast_fp16)[name = string("op_9265_cast_fp16")];
            tensor<int32, [3]> var_9268_begin_0 = const()[name = string("op_9268_begin_0"), val = tensor<int32, [3]>([0, 0, 8])];
            tensor<int32, [3]> var_9268_end_0 = const()[name = string("op_9268_end_0"), val = tensor<int32, [3]>([1, 16, 9])];
            tensor<bool, [3]> var_9268_end_mask_0 = const()[name = string("op_9268_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_9268_squeeze_mask_0 = const()[name = string("op_9268_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_9268_cast_fp16 = slice_by_index(begin = var_9268_begin_0, end = var_9268_end_0, end_mask = var_9268_end_mask_0, squeeze_mask = var_9268_squeeze_mask_0, x = decay_21_cast_fp16)[name = string("op_9268_cast_fp16")];
            tensor<fp16, [1, 16]> var_9269_cast_fp16 = exp(x = var_9268_cast_fp16)[name = string("op_9269_cast_fp16")];
            tensor<int32, [1]> var_9270_axes_0 = const()[name = string("op_9270_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_9270_cast_fp16 = expand_dims(axes = var_9270_axes_0, x = var_9269_cast_fp16)[name = string("op_9270_cast_fp16")];
            tensor<int32, [1]> var_9271_axes_0 = const()[name = string("op_9271_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_9271_cast_fp16 = expand_dims(axes = var_9271_axes_0, x = var_9270_cast_fp16)[name = string("op_9271_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_673_cast_fp16 = mul(x = state_671_cast_fp16, y = var_9271_cast_fp16)[name = string("state_673_cast_fp16")];
            tensor<int32, [4]> key_token_337_begin_0 = const()[name = string("key_token_337_begin_0"), val = tensor<int32, [4]>([0, 0, 8, 0])];
            tensor<int32, [4]> key_token_337_end_0 = const()[name = string("key_token_337_end_0"), val = tensor<int32, [4]>([1, 16, 9, 128])];
            tensor<bool, [4]> key_token_337_end_mask_0 = const()[name = string("key_token_337_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_337_squeeze_mask_0 = const()[name = string("key_token_337_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_337_cast_fp16 = slice_by_index(begin = key_token_337_begin_0, end = key_token_337_end_0, end_mask = key_token_337_end_mask_0, squeeze_mask = key_token_337_squeeze_mask_0, x = key_55_cast_fp16)[name = string("key_token_337_cast_fp16")];
            tensor<int32, [4]> value_token_337_begin_0 = const()[name = string("value_token_337_begin_0"), val = tensor<int32, [4]>([0, 0, 8, 0])];
            tensor<int32, [4]> value_token_337_end_0 = const()[name = string("value_token_337_end_0"), val = tensor<int32, [4]>([1, 16, 9, 128])];
            tensor<bool, [4]> value_token_337_end_mask_0 = const()[name = string("value_token_337_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_337_squeeze_mask_0 = const()[name = string("value_token_337_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_337_cast_fp16 = slice_by_index(begin = value_token_337_begin_0, end = value_token_337_end_0, end_mask = value_token_337_end_mask_0, squeeze_mask = value_token_337_squeeze_mask_0, x = value_199_cast_fp16)[name = string("value_token_337_cast_fp16")];
            tensor<int32, [1]> var_9279_axes_0 = const()[name = string("op_9279_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_9279_cast_fp16 = expand_dims(axes = var_9279_axes_0, x = key_token_337_cast_fp16)[name = string("op_9279_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_9280_cast_fp16 = mul(x = state_673_cast_fp16, y = var_9279_cast_fp16)[name = string("op_9280_cast_fp16")];
            tensor<int32, [1]> memory_337_axes_0 = const()[name = string("memory_337_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_337_keep_dims_0 = const()[name = string("memory_337_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_337_cast_fp16 = reduce_sum(axes = memory_337_axes_0, keep_dims = memory_337_keep_dims_0, x = var_9280_cast_fp16)[name = string("memory_337_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_9283_cast_fp16 = sub(x = value_token_337_cast_fp16, y = memory_337_cast_fp16)[name = string("op_9283_cast_fp16")];
            tensor<int32, [3]> var_9286_begin_0 = const()[name = string("op_9286_begin_0"), val = tensor<int32, [3]>([0, 0, 8])];
            tensor<int32, [3]> var_9286_end_0 = const()[name = string("op_9286_end_0"), val = tensor<int32, [3]>([1, 16, 9])];
            tensor<bool, [3]> var_9286_end_mask_0 = const()[name = string("op_9286_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_9286_squeeze_mask_0 = const()[name = string("op_9286_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_9286_cast_fp16 = slice_by_index(begin = var_9286_begin_0, end = var_9286_end_0, end_mask = var_9286_end_mask_0, squeeze_mask = var_9286_squeeze_mask_0, x = beta_21_cast_fp16)[name = string("op_9286_cast_fp16")];
            tensor<int32, [1]> var_9287_axes_0 = const()[name = string("op_9287_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_9287_cast_fp16 = expand_dims(axes = var_9287_axes_0, x = var_9286_cast_fp16)[name = string("op_9287_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_337_cast_fp16 = mul(x = var_9283_cast_fp16, y = var_9287_cast_fp16)[name = string("delta_337_cast_fp16")];
            tensor<int32, [1]> var_9290_axes_0 = const()[name = string("op_9290_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_9290_cast_fp16 = expand_dims(axes = var_9290_axes_0, x = delta_337_cast_fp16)[name = string("op_9290_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_9291_cast_fp16 = mul(x = var_9279_cast_fp16, y = var_9290_cast_fp16)[name = string("op_9291_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_675_cast_fp16 = add(x = state_673_cast_fp16, y = var_9291_cast_fp16)[name = string("state_675_cast_fp16")];
            tensor<int32, [4]> var_9295_begin_0 = const()[name = string("op_9295_begin_0"), val = tensor<int32, [4]>([0, 0, 8, 0])];
            tensor<int32, [4]> var_9295_end_0 = const()[name = string("op_9295_end_0"), val = tensor<int32, [4]>([1, 16, 9, 128])];
            tensor<bool, [4]> var_9295_end_mask_0 = const()[name = string("op_9295_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_9295_squeeze_mask_0 = const()[name = string("op_9295_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_9295_cast_fp16 = slice_by_index(begin = var_9295_begin_0, end = var_9295_end_0, end_mask = var_9295_end_mask_0, squeeze_mask = var_9295_squeeze_mask_0, x = _inversed_query_55_cast_fp16)[name = string("op_9295_cast_fp16")];
            tensor<int32, [1]> var_9296_axes_0 = const()[name = string("op_9296_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_9296_cast_fp16 = expand_dims(axes = var_9296_axes_0, x = var_9295_cast_fp16)[name = string("op_9296_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_9297_cast_fp16 = mul(x = state_675_cast_fp16, y = var_9296_cast_fp16)[name = string("op_9297_cast_fp16")];
            tensor<int32, [1]> var_9299_axes_0 = const()[name = string("op_9299_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_9299_keep_dims_0 = const()[name = string("op_9299_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_9299_cast_fp16 = reduce_sum(axes = var_9299_axes_0, keep_dims = var_9299_keep_dims_0, x = var_9297_cast_fp16)[name = string("op_9299_cast_fp16")];
            tensor<int32, [3]> var_9302_begin_0 = const()[name = string("op_9302_begin_0"), val = tensor<int32, [3]>([0, 0, 9])];
            tensor<int32, [3]> var_9302_end_0 = const()[name = string("op_9302_end_0"), val = tensor<int32, [3]>([1, 16, 10])];
            tensor<bool, [3]> var_9302_end_mask_0 = const()[name = string("op_9302_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_9302_squeeze_mask_0 = const()[name = string("op_9302_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_9302_cast_fp16 = slice_by_index(begin = var_9302_begin_0, end = var_9302_end_0, end_mask = var_9302_end_mask_0, squeeze_mask = var_9302_squeeze_mask_0, x = decay_21_cast_fp16)[name = string("op_9302_cast_fp16")];
            tensor<fp16, [1, 16]> var_9303_cast_fp16 = exp(x = var_9302_cast_fp16)[name = string("op_9303_cast_fp16")];
            tensor<int32, [1]> var_9304_axes_0 = const()[name = string("op_9304_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_9304_cast_fp16 = expand_dims(axes = var_9304_axes_0, x = var_9303_cast_fp16)[name = string("op_9304_cast_fp16")];
            tensor<int32, [1]> var_9305_axes_0 = const()[name = string("op_9305_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_9305_cast_fp16 = expand_dims(axes = var_9305_axes_0, x = var_9304_cast_fp16)[name = string("op_9305_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_677_cast_fp16 = mul(x = state_675_cast_fp16, y = var_9305_cast_fp16)[name = string("state_677_cast_fp16")];
            tensor<int32, [4]> key_token_339_begin_0 = const()[name = string("key_token_339_begin_0"), val = tensor<int32, [4]>([0, 0, 9, 0])];
            tensor<int32, [4]> key_token_339_end_0 = const()[name = string("key_token_339_end_0"), val = tensor<int32, [4]>([1, 16, 10, 128])];
            tensor<bool, [4]> key_token_339_end_mask_0 = const()[name = string("key_token_339_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_339_squeeze_mask_0 = const()[name = string("key_token_339_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_339_cast_fp16 = slice_by_index(begin = key_token_339_begin_0, end = key_token_339_end_0, end_mask = key_token_339_end_mask_0, squeeze_mask = key_token_339_squeeze_mask_0, x = key_55_cast_fp16)[name = string("key_token_339_cast_fp16")];
            tensor<int32, [4]> value_token_339_begin_0 = const()[name = string("value_token_339_begin_0"), val = tensor<int32, [4]>([0, 0, 9, 0])];
            tensor<int32, [4]> value_token_339_end_0 = const()[name = string("value_token_339_end_0"), val = tensor<int32, [4]>([1, 16, 10, 128])];
            tensor<bool, [4]> value_token_339_end_mask_0 = const()[name = string("value_token_339_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_339_squeeze_mask_0 = const()[name = string("value_token_339_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_339_cast_fp16 = slice_by_index(begin = value_token_339_begin_0, end = value_token_339_end_0, end_mask = value_token_339_end_mask_0, squeeze_mask = value_token_339_squeeze_mask_0, x = value_199_cast_fp16)[name = string("value_token_339_cast_fp16")];
            tensor<int32, [1]> var_9313_axes_0 = const()[name = string("op_9313_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_9313_cast_fp16 = expand_dims(axes = var_9313_axes_0, x = key_token_339_cast_fp16)[name = string("op_9313_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_9314_cast_fp16 = mul(x = state_677_cast_fp16, y = var_9313_cast_fp16)[name = string("op_9314_cast_fp16")];
            tensor<int32, [1]> memory_339_axes_0 = const()[name = string("memory_339_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_339_keep_dims_0 = const()[name = string("memory_339_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_339_cast_fp16 = reduce_sum(axes = memory_339_axes_0, keep_dims = memory_339_keep_dims_0, x = var_9314_cast_fp16)[name = string("memory_339_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_9317_cast_fp16 = sub(x = value_token_339_cast_fp16, y = memory_339_cast_fp16)[name = string("op_9317_cast_fp16")];
            tensor<int32, [3]> var_9320_begin_0 = const()[name = string("op_9320_begin_0"), val = tensor<int32, [3]>([0, 0, 9])];
            tensor<int32, [3]> var_9320_end_0 = const()[name = string("op_9320_end_0"), val = tensor<int32, [3]>([1, 16, 10])];
            tensor<bool, [3]> var_9320_end_mask_0 = const()[name = string("op_9320_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_9320_squeeze_mask_0 = const()[name = string("op_9320_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_9320_cast_fp16 = slice_by_index(begin = var_9320_begin_0, end = var_9320_end_0, end_mask = var_9320_end_mask_0, squeeze_mask = var_9320_squeeze_mask_0, x = beta_21_cast_fp16)[name = string("op_9320_cast_fp16")];
            tensor<int32, [1]> var_9321_axes_0 = const()[name = string("op_9321_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_9321_cast_fp16 = expand_dims(axes = var_9321_axes_0, x = var_9320_cast_fp16)[name = string("op_9321_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_339_cast_fp16 = mul(x = var_9317_cast_fp16, y = var_9321_cast_fp16)[name = string("delta_339_cast_fp16")];
            tensor<int32, [1]> var_9324_axes_0 = const()[name = string("op_9324_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_9324_cast_fp16 = expand_dims(axes = var_9324_axes_0, x = delta_339_cast_fp16)[name = string("op_9324_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_9325_cast_fp16 = mul(x = var_9313_cast_fp16, y = var_9324_cast_fp16)[name = string("op_9325_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_679_cast_fp16 = add(x = state_677_cast_fp16, y = var_9325_cast_fp16)[name = string("state_679_cast_fp16")];
            tensor<int32, [4]> var_9329_begin_0 = const()[name = string("op_9329_begin_0"), val = tensor<int32, [4]>([0, 0, 9, 0])];
            tensor<int32, [4]> var_9329_end_0 = const()[name = string("op_9329_end_0"), val = tensor<int32, [4]>([1, 16, 10, 128])];
            tensor<bool, [4]> var_9329_end_mask_0 = const()[name = string("op_9329_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_9329_squeeze_mask_0 = const()[name = string("op_9329_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_9329_cast_fp16 = slice_by_index(begin = var_9329_begin_0, end = var_9329_end_0, end_mask = var_9329_end_mask_0, squeeze_mask = var_9329_squeeze_mask_0, x = _inversed_query_55_cast_fp16)[name = string("op_9329_cast_fp16")];
            tensor<int32, [1]> var_9330_axes_0 = const()[name = string("op_9330_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_9330_cast_fp16 = expand_dims(axes = var_9330_axes_0, x = var_9329_cast_fp16)[name = string("op_9330_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_9331_cast_fp16 = mul(x = state_679_cast_fp16, y = var_9330_cast_fp16)[name = string("op_9331_cast_fp16")];
            tensor<int32, [1]> var_9333_axes_0 = const()[name = string("op_9333_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_9333_keep_dims_0 = const()[name = string("op_9333_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_9333_cast_fp16 = reduce_sum(axes = var_9333_axes_0, keep_dims = var_9333_keep_dims_0, x = var_9331_cast_fp16)[name = string("op_9333_cast_fp16")];
            tensor<int32, [3]> var_9336_begin_0 = const()[name = string("op_9336_begin_0"), val = tensor<int32, [3]>([0, 0, 10])];
            tensor<int32, [3]> var_9336_end_0 = const()[name = string("op_9336_end_0"), val = tensor<int32, [3]>([1, 16, 11])];
            tensor<bool, [3]> var_9336_end_mask_0 = const()[name = string("op_9336_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_9336_squeeze_mask_0 = const()[name = string("op_9336_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_9336_cast_fp16 = slice_by_index(begin = var_9336_begin_0, end = var_9336_end_0, end_mask = var_9336_end_mask_0, squeeze_mask = var_9336_squeeze_mask_0, x = decay_21_cast_fp16)[name = string("op_9336_cast_fp16")];
            tensor<fp16, [1, 16]> var_9337_cast_fp16 = exp(x = var_9336_cast_fp16)[name = string("op_9337_cast_fp16")];
            tensor<int32, [1]> var_9338_axes_0 = const()[name = string("op_9338_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_9338_cast_fp16 = expand_dims(axes = var_9338_axes_0, x = var_9337_cast_fp16)[name = string("op_9338_cast_fp16")];
            tensor<int32, [1]> var_9339_axes_0 = const()[name = string("op_9339_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_9339_cast_fp16 = expand_dims(axes = var_9339_axes_0, x = var_9338_cast_fp16)[name = string("op_9339_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_681_cast_fp16 = mul(x = state_679_cast_fp16, y = var_9339_cast_fp16)[name = string("state_681_cast_fp16")];
            tensor<int32, [4]> key_token_341_begin_0 = const()[name = string("key_token_341_begin_0"), val = tensor<int32, [4]>([0, 0, 10, 0])];
            tensor<int32, [4]> key_token_341_end_0 = const()[name = string("key_token_341_end_0"), val = tensor<int32, [4]>([1, 16, 11, 128])];
            tensor<bool, [4]> key_token_341_end_mask_0 = const()[name = string("key_token_341_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_341_squeeze_mask_0 = const()[name = string("key_token_341_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_341_cast_fp16 = slice_by_index(begin = key_token_341_begin_0, end = key_token_341_end_0, end_mask = key_token_341_end_mask_0, squeeze_mask = key_token_341_squeeze_mask_0, x = key_55_cast_fp16)[name = string("key_token_341_cast_fp16")];
            tensor<int32, [4]> value_token_341_begin_0 = const()[name = string("value_token_341_begin_0"), val = tensor<int32, [4]>([0, 0, 10, 0])];
            tensor<int32, [4]> value_token_341_end_0 = const()[name = string("value_token_341_end_0"), val = tensor<int32, [4]>([1, 16, 11, 128])];
            tensor<bool, [4]> value_token_341_end_mask_0 = const()[name = string("value_token_341_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_341_squeeze_mask_0 = const()[name = string("value_token_341_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_341_cast_fp16 = slice_by_index(begin = value_token_341_begin_0, end = value_token_341_end_0, end_mask = value_token_341_end_mask_0, squeeze_mask = value_token_341_squeeze_mask_0, x = value_199_cast_fp16)[name = string("value_token_341_cast_fp16")];
            tensor<int32, [1]> var_9347_axes_0 = const()[name = string("op_9347_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_9347_cast_fp16 = expand_dims(axes = var_9347_axes_0, x = key_token_341_cast_fp16)[name = string("op_9347_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_9348_cast_fp16 = mul(x = state_681_cast_fp16, y = var_9347_cast_fp16)[name = string("op_9348_cast_fp16")];
            tensor<int32, [1]> memory_341_axes_0 = const()[name = string("memory_341_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_341_keep_dims_0 = const()[name = string("memory_341_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_341_cast_fp16 = reduce_sum(axes = memory_341_axes_0, keep_dims = memory_341_keep_dims_0, x = var_9348_cast_fp16)[name = string("memory_341_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_9351_cast_fp16 = sub(x = value_token_341_cast_fp16, y = memory_341_cast_fp16)[name = string("op_9351_cast_fp16")];
            tensor<int32, [3]> var_9354_begin_0 = const()[name = string("op_9354_begin_0"), val = tensor<int32, [3]>([0, 0, 10])];
            tensor<int32, [3]> var_9354_end_0 = const()[name = string("op_9354_end_0"), val = tensor<int32, [3]>([1, 16, 11])];
            tensor<bool, [3]> var_9354_end_mask_0 = const()[name = string("op_9354_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_9354_squeeze_mask_0 = const()[name = string("op_9354_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_9354_cast_fp16 = slice_by_index(begin = var_9354_begin_0, end = var_9354_end_0, end_mask = var_9354_end_mask_0, squeeze_mask = var_9354_squeeze_mask_0, x = beta_21_cast_fp16)[name = string("op_9354_cast_fp16")];
            tensor<int32, [1]> var_9355_axes_0 = const()[name = string("op_9355_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_9355_cast_fp16 = expand_dims(axes = var_9355_axes_0, x = var_9354_cast_fp16)[name = string("op_9355_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_341_cast_fp16 = mul(x = var_9351_cast_fp16, y = var_9355_cast_fp16)[name = string("delta_341_cast_fp16")];
            tensor<int32, [1]> var_9358_axes_0 = const()[name = string("op_9358_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_9358_cast_fp16 = expand_dims(axes = var_9358_axes_0, x = delta_341_cast_fp16)[name = string("op_9358_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_9359_cast_fp16 = mul(x = var_9347_cast_fp16, y = var_9358_cast_fp16)[name = string("op_9359_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_683_cast_fp16 = add(x = state_681_cast_fp16, y = var_9359_cast_fp16)[name = string("state_683_cast_fp16")];
            tensor<int32, [4]> var_9363_begin_0 = const()[name = string("op_9363_begin_0"), val = tensor<int32, [4]>([0, 0, 10, 0])];
            tensor<int32, [4]> var_9363_end_0 = const()[name = string("op_9363_end_0"), val = tensor<int32, [4]>([1, 16, 11, 128])];
            tensor<bool, [4]> var_9363_end_mask_0 = const()[name = string("op_9363_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_9363_squeeze_mask_0 = const()[name = string("op_9363_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_9363_cast_fp16 = slice_by_index(begin = var_9363_begin_0, end = var_9363_end_0, end_mask = var_9363_end_mask_0, squeeze_mask = var_9363_squeeze_mask_0, x = _inversed_query_55_cast_fp16)[name = string("op_9363_cast_fp16")];
            tensor<int32, [1]> var_9364_axes_0 = const()[name = string("op_9364_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_9364_cast_fp16 = expand_dims(axes = var_9364_axes_0, x = var_9363_cast_fp16)[name = string("op_9364_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_9365_cast_fp16 = mul(x = state_683_cast_fp16, y = var_9364_cast_fp16)[name = string("op_9365_cast_fp16")];
            tensor<int32, [1]> var_9367_axes_0 = const()[name = string("op_9367_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_9367_keep_dims_0 = const()[name = string("op_9367_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_9367_cast_fp16 = reduce_sum(axes = var_9367_axes_0, keep_dims = var_9367_keep_dims_0, x = var_9365_cast_fp16)[name = string("op_9367_cast_fp16")];
            tensor<int32, [3]> var_9370_begin_0 = const()[name = string("op_9370_begin_0"), val = tensor<int32, [3]>([0, 0, 11])];
            tensor<int32, [3]> var_9370_end_0 = const()[name = string("op_9370_end_0"), val = tensor<int32, [3]>([1, 16, 12])];
            tensor<bool, [3]> var_9370_end_mask_0 = const()[name = string("op_9370_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_9370_squeeze_mask_0 = const()[name = string("op_9370_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_9370_cast_fp16 = slice_by_index(begin = var_9370_begin_0, end = var_9370_end_0, end_mask = var_9370_end_mask_0, squeeze_mask = var_9370_squeeze_mask_0, x = decay_21_cast_fp16)[name = string("op_9370_cast_fp16")];
            tensor<fp16, [1, 16]> var_9371_cast_fp16 = exp(x = var_9370_cast_fp16)[name = string("op_9371_cast_fp16")];
            tensor<int32, [1]> var_9372_axes_0 = const()[name = string("op_9372_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_9372_cast_fp16 = expand_dims(axes = var_9372_axes_0, x = var_9371_cast_fp16)[name = string("op_9372_cast_fp16")];
            tensor<int32, [1]> var_9373_axes_0 = const()[name = string("op_9373_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_9373_cast_fp16 = expand_dims(axes = var_9373_axes_0, x = var_9372_cast_fp16)[name = string("op_9373_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_685_cast_fp16 = mul(x = state_683_cast_fp16, y = var_9373_cast_fp16)[name = string("state_685_cast_fp16")];
            tensor<int32, [4]> key_token_343_begin_0 = const()[name = string("key_token_343_begin_0"), val = tensor<int32, [4]>([0, 0, 11, 0])];
            tensor<int32, [4]> key_token_343_end_0 = const()[name = string("key_token_343_end_0"), val = tensor<int32, [4]>([1, 16, 12, 128])];
            tensor<bool, [4]> key_token_343_end_mask_0 = const()[name = string("key_token_343_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_343_squeeze_mask_0 = const()[name = string("key_token_343_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_343_cast_fp16 = slice_by_index(begin = key_token_343_begin_0, end = key_token_343_end_0, end_mask = key_token_343_end_mask_0, squeeze_mask = key_token_343_squeeze_mask_0, x = key_55_cast_fp16)[name = string("key_token_343_cast_fp16")];
            tensor<int32, [4]> value_token_343_begin_0 = const()[name = string("value_token_343_begin_0"), val = tensor<int32, [4]>([0, 0, 11, 0])];
            tensor<int32, [4]> value_token_343_end_0 = const()[name = string("value_token_343_end_0"), val = tensor<int32, [4]>([1, 16, 12, 128])];
            tensor<bool, [4]> value_token_343_end_mask_0 = const()[name = string("value_token_343_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_343_squeeze_mask_0 = const()[name = string("value_token_343_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_343_cast_fp16 = slice_by_index(begin = value_token_343_begin_0, end = value_token_343_end_0, end_mask = value_token_343_end_mask_0, squeeze_mask = value_token_343_squeeze_mask_0, x = value_199_cast_fp16)[name = string("value_token_343_cast_fp16")];
            tensor<int32, [1]> var_9381_axes_0 = const()[name = string("op_9381_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_9381_cast_fp16 = expand_dims(axes = var_9381_axes_0, x = key_token_343_cast_fp16)[name = string("op_9381_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_9382_cast_fp16 = mul(x = state_685_cast_fp16, y = var_9381_cast_fp16)[name = string("op_9382_cast_fp16")];
            tensor<int32, [1]> memory_343_axes_0 = const()[name = string("memory_343_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_343_keep_dims_0 = const()[name = string("memory_343_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_343_cast_fp16 = reduce_sum(axes = memory_343_axes_0, keep_dims = memory_343_keep_dims_0, x = var_9382_cast_fp16)[name = string("memory_343_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_9385_cast_fp16 = sub(x = value_token_343_cast_fp16, y = memory_343_cast_fp16)[name = string("op_9385_cast_fp16")];
            tensor<int32, [3]> var_9388_begin_0 = const()[name = string("op_9388_begin_0"), val = tensor<int32, [3]>([0, 0, 11])];
            tensor<int32, [3]> var_9388_end_0 = const()[name = string("op_9388_end_0"), val = tensor<int32, [3]>([1, 16, 12])];
            tensor<bool, [3]> var_9388_end_mask_0 = const()[name = string("op_9388_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_9388_squeeze_mask_0 = const()[name = string("op_9388_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_9388_cast_fp16 = slice_by_index(begin = var_9388_begin_0, end = var_9388_end_0, end_mask = var_9388_end_mask_0, squeeze_mask = var_9388_squeeze_mask_0, x = beta_21_cast_fp16)[name = string("op_9388_cast_fp16")];
            tensor<int32, [1]> var_9389_axes_0 = const()[name = string("op_9389_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_9389_cast_fp16 = expand_dims(axes = var_9389_axes_0, x = var_9388_cast_fp16)[name = string("op_9389_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_343_cast_fp16 = mul(x = var_9385_cast_fp16, y = var_9389_cast_fp16)[name = string("delta_343_cast_fp16")];
            tensor<int32, [1]> var_9392_axes_0 = const()[name = string("op_9392_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_9392_cast_fp16 = expand_dims(axes = var_9392_axes_0, x = delta_343_cast_fp16)[name = string("op_9392_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_9393_cast_fp16 = mul(x = var_9381_cast_fp16, y = var_9392_cast_fp16)[name = string("op_9393_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_687_cast_fp16 = add(x = state_685_cast_fp16, y = var_9393_cast_fp16)[name = string("state_687_cast_fp16")];
            tensor<int32, [4]> var_9397_begin_0 = const()[name = string("op_9397_begin_0"), val = tensor<int32, [4]>([0, 0, 11, 0])];
            tensor<int32, [4]> var_9397_end_0 = const()[name = string("op_9397_end_0"), val = tensor<int32, [4]>([1, 16, 12, 128])];
            tensor<bool, [4]> var_9397_end_mask_0 = const()[name = string("op_9397_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_9397_squeeze_mask_0 = const()[name = string("op_9397_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_9397_cast_fp16 = slice_by_index(begin = var_9397_begin_0, end = var_9397_end_0, end_mask = var_9397_end_mask_0, squeeze_mask = var_9397_squeeze_mask_0, x = _inversed_query_55_cast_fp16)[name = string("op_9397_cast_fp16")];
            tensor<int32, [1]> var_9398_axes_0 = const()[name = string("op_9398_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_9398_cast_fp16 = expand_dims(axes = var_9398_axes_0, x = var_9397_cast_fp16)[name = string("op_9398_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_9399_cast_fp16 = mul(x = state_687_cast_fp16, y = var_9398_cast_fp16)[name = string("op_9399_cast_fp16")];
            tensor<int32, [1]> var_9401_axes_0 = const()[name = string("op_9401_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_9401_keep_dims_0 = const()[name = string("op_9401_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_9401_cast_fp16 = reduce_sum(axes = var_9401_axes_0, keep_dims = var_9401_keep_dims_0, x = var_9399_cast_fp16)[name = string("op_9401_cast_fp16")];
            tensor<int32, [3]> var_9404_begin_0 = const()[name = string("op_9404_begin_0"), val = tensor<int32, [3]>([0, 0, 12])];
            tensor<int32, [3]> var_9404_end_0 = const()[name = string("op_9404_end_0"), val = tensor<int32, [3]>([1, 16, 13])];
            tensor<bool, [3]> var_9404_end_mask_0 = const()[name = string("op_9404_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_9404_squeeze_mask_0 = const()[name = string("op_9404_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_9404_cast_fp16 = slice_by_index(begin = var_9404_begin_0, end = var_9404_end_0, end_mask = var_9404_end_mask_0, squeeze_mask = var_9404_squeeze_mask_0, x = decay_21_cast_fp16)[name = string("op_9404_cast_fp16")];
            tensor<fp16, [1, 16]> var_9405_cast_fp16 = exp(x = var_9404_cast_fp16)[name = string("op_9405_cast_fp16")];
            tensor<int32, [1]> var_9406_axes_0 = const()[name = string("op_9406_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_9406_cast_fp16 = expand_dims(axes = var_9406_axes_0, x = var_9405_cast_fp16)[name = string("op_9406_cast_fp16")];
            tensor<int32, [1]> var_9407_axes_0 = const()[name = string("op_9407_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_9407_cast_fp16 = expand_dims(axes = var_9407_axes_0, x = var_9406_cast_fp16)[name = string("op_9407_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_689_cast_fp16 = mul(x = state_687_cast_fp16, y = var_9407_cast_fp16)[name = string("state_689_cast_fp16")];
            tensor<int32, [4]> key_token_345_begin_0 = const()[name = string("key_token_345_begin_0"), val = tensor<int32, [4]>([0, 0, 12, 0])];
            tensor<int32, [4]> key_token_345_end_0 = const()[name = string("key_token_345_end_0"), val = tensor<int32, [4]>([1, 16, 13, 128])];
            tensor<bool, [4]> key_token_345_end_mask_0 = const()[name = string("key_token_345_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_345_squeeze_mask_0 = const()[name = string("key_token_345_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_345_cast_fp16 = slice_by_index(begin = key_token_345_begin_0, end = key_token_345_end_0, end_mask = key_token_345_end_mask_0, squeeze_mask = key_token_345_squeeze_mask_0, x = key_55_cast_fp16)[name = string("key_token_345_cast_fp16")];
            tensor<int32, [4]> value_token_345_begin_0 = const()[name = string("value_token_345_begin_0"), val = tensor<int32, [4]>([0, 0, 12, 0])];
            tensor<int32, [4]> value_token_345_end_0 = const()[name = string("value_token_345_end_0"), val = tensor<int32, [4]>([1, 16, 13, 128])];
            tensor<bool, [4]> value_token_345_end_mask_0 = const()[name = string("value_token_345_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_345_squeeze_mask_0 = const()[name = string("value_token_345_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_345_cast_fp16 = slice_by_index(begin = value_token_345_begin_0, end = value_token_345_end_0, end_mask = value_token_345_end_mask_0, squeeze_mask = value_token_345_squeeze_mask_0, x = value_199_cast_fp16)[name = string("value_token_345_cast_fp16")];
            tensor<int32, [1]> var_9415_axes_0 = const()[name = string("op_9415_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_9415_cast_fp16 = expand_dims(axes = var_9415_axes_0, x = key_token_345_cast_fp16)[name = string("op_9415_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_9416_cast_fp16 = mul(x = state_689_cast_fp16, y = var_9415_cast_fp16)[name = string("op_9416_cast_fp16")];
            tensor<int32, [1]> memory_345_axes_0 = const()[name = string("memory_345_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_345_keep_dims_0 = const()[name = string("memory_345_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_345_cast_fp16 = reduce_sum(axes = memory_345_axes_0, keep_dims = memory_345_keep_dims_0, x = var_9416_cast_fp16)[name = string("memory_345_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_9419_cast_fp16 = sub(x = value_token_345_cast_fp16, y = memory_345_cast_fp16)[name = string("op_9419_cast_fp16")];
            tensor<int32, [3]> var_9422_begin_0 = const()[name = string("op_9422_begin_0"), val = tensor<int32, [3]>([0, 0, 12])];
            tensor<int32, [3]> var_9422_end_0 = const()[name = string("op_9422_end_0"), val = tensor<int32, [3]>([1, 16, 13])];
            tensor<bool, [3]> var_9422_end_mask_0 = const()[name = string("op_9422_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_9422_squeeze_mask_0 = const()[name = string("op_9422_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_9422_cast_fp16 = slice_by_index(begin = var_9422_begin_0, end = var_9422_end_0, end_mask = var_9422_end_mask_0, squeeze_mask = var_9422_squeeze_mask_0, x = beta_21_cast_fp16)[name = string("op_9422_cast_fp16")];
            tensor<int32, [1]> var_9423_axes_0 = const()[name = string("op_9423_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_9423_cast_fp16 = expand_dims(axes = var_9423_axes_0, x = var_9422_cast_fp16)[name = string("op_9423_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_345_cast_fp16 = mul(x = var_9419_cast_fp16, y = var_9423_cast_fp16)[name = string("delta_345_cast_fp16")];
            tensor<int32, [1]> var_9426_axes_0 = const()[name = string("op_9426_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_9426_cast_fp16 = expand_dims(axes = var_9426_axes_0, x = delta_345_cast_fp16)[name = string("op_9426_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_9427_cast_fp16 = mul(x = var_9415_cast_fp16, y = var_9426_cast_fp16)[name = string("op_9427_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_691_cast_fp16 = add(x = state_689_cast_fp16, y = var_9427_cast_fp16)[name = string("state_691_cast_fp16")];
            tensor<int32, [4]> var_9431_begin_0 = const()[name = string("op_9431_begin_0"), val = tensor<int32, [4]>([0, 0, 12, 0])];
            tensor<int32, [4]> var_9431_end_0 = const()[name = string("op_9431_end_0"), val = tensor<int32, [4]>([1, 16, 13, 128])];
            tensor<bool, [4]> var_9431_end_mask_0 = const()[name = string("op_9431_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_9431_squeeze_mask_0 = const()[name = string("op_9431_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_9431_cast_fp16 = slice_by_index(begin = var_9431_begin_0, end = var_9431_end_0, end_mask = var_9431_end_mask_0, squeeze_mask = var_9431_squeeze_mask_0, x = _inversed_query_55_cast_fp16)[name = string("op_9431_cast_fp16")];
            tensor<int32, [1]> var_9432_axes_0 = const()[name = string("op_9432_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_9432_cast_fp16 = expand_dims(axes = var_9432_axes_0, x = var_9431_cast_fp16)[name = string("op_9432_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_9433_cast_fp16 = mul(x = state_691_cast_fp16, y = var_9432_cast_fp16)[name = string("op_9433_cast_fp16")];
            tensor<int32, [1]> var_9435_axes_0 = const()[name = string("op_9435_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_9435_keep_dims_0 = const()[name = string("op_9435_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_9435_cast_fp16 = reduce_sum(axes = var_9435_axes_0, keep_dims = var_9435_keep_dims_0, x = var_9433_cast_fp16)[name = string("op_9435_cast_fp16")];
            tensor<int32, [3]> var_9438_begin_0 = const()[name = string("op_9438_begin_0"), val = tensor<int32, [3]>([0, 0, 13])];
            tensor<int32, [3]> var_9438_end_0 = const()[name = string("op_9438_end_0"), val = tensor<int32, [3]>([1, 16, 14])];
            tensor<bool, [3]> var_9438_end_mask_0 = const()[name = string("op_9438_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_9438_squeeze_mask_0 = const()[name = string("op_9438_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_9438_cast_fp16 = slice_by_index(begin = var_9438_begin_0, end = var_9438_end_0, end_mask = var_9438_end_mask_0, squeeze_mask = var_9438_squeeze_mask_0, x = decay_21_cast_fp16)[name = string("op_9438_cast_fp16")];
            tensor<fp16, [1, 16]> var_9439_cast_fp16 = exp(x = var_9438_cast_fp16)[name = string("op_9439_cast_fp16")];
            tensor<int32, [1]> var_9440_axes_0 = const()[name = string("op_9440_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_9440_cast_fp16 = expand_dims(axes = var_9440_axes_0, x = var_9439_cast_fp16)[name = string("op_9440_cast_fp16")];
            tensor<int32, [1]> var_9441_axes_0 = const()[name = string("op_9441_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_9441_cast_fp16 = expand_dims(axes = var_9441_axes_0, x = var_9440_cast_fp16)[name = string("op_9441_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_693_cast_fp16 = mul(x = state_691_cast_fp16, y = var_9441_cast_fp16)[name = string("state_693_cast_fp16")];
            tensor<int32, [4]> key_token_347_begin_0 = const()[name = string("key_token_347_begin_0"), val = tensor<int32, [4]>([0, 0, 13, 0])];
            tensor<int32, [4]> key_token_347_end_0 = const()[name = string("key_token_347_end_0"), val = tensor<int32, [4]>([1, 16, 14, 128])];
            tensor<bool, [4]> key_token_347_end_mask_0 = const()[name = string("key_token_347_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_347_squeeze_mask_0 = const()[name = string("key_token_347_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_347_cast_fp16 = slice_by_index(begin = key_token_347_begin_0, end = key_token_347_end_0, end_mask = key_token_347_end_mask_0, squeeze_mask = key_token_347_squeeze_mask_0, x = key_55_cast_fp16)[name = string("key_token_347_cast_fp16")];
            tensor<int32, [4]> value_token_347_begin_0 = const()[name = string("value_token_347_begin_0"), val = tensor<int32, [4]>([0, 0, 13, 0])];
            tensor<int32, [4]> value_token_347_end_0 = const()[name = string("value_token_347_end_0"), val = tensor<int32, [4]>([1, 16, 14, 128])];
            tensor<bool, [4]> value_token_347_end_mask_0 = const()[name = string("value_token_347_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_347_squeeze_mask_0 = const()[name = string("value_token_347_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_347_cast_fp16 = slice_by_index(begin = value_token_347_begin_0, end = value_token_347_end_0, end_mask = value_token_347_end_mask_0, squeeze_mask = value_token_347_squeeze_mask_0, x = value_199_cast_fp16)[name = string("value_token_347_cast_fp16")];
            tensor<int32, [1]> var_9449_axes_0 = const()[name = string("op_9449_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_9449_cast_fp16 = expand_dims(axes = var_9449_axes_0, x = key_token_347_cast_fp16)[name = string("op_9449_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_9450_cast_fp16 = mul(x = state_693_cast_fp16, y = var_9449_cast_fp16)[name = string("op_9450_cast_fp16")];
            tensor<int32, [1]> memory_347_axes_0 = const()[name = string("memory_347_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_347_keep_dims_0 = const()[name = string("memory_347_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_347_cast_fp16 = reduce_sum(axes = memory_347_axes_0, keep_dims = memory_347_keep_dims_0, x = var_9450_cast_fp16)[name = string("memory_347_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_9453_cast_fp16 = sub(x = value_token_347_cast_fp16, y = memory_347_cast_fp16)[name = string("op_9453_cast_fp16")];
            tensor<int32, [3]> var_9456_begin_0 = const()[name = string("op_9456_begin_0"), val = tensor<int32, [3]>([0, 0, 13])];
            tensor<int32, [3]> var_9456_end_0 = const()[name = string("op_9456_end_0"), val = tensor<int32, [3]>([1, 16, 14])];
            tensor<bool, [3]> var_9456_end_mask_0 = const()[name = string("op_9456_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_9456_squeeze_mask_0 = const()[name = string("op_9456_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_9456_cast_fp16 = slice_by_index(begin = var_9456_begin_0, end = var_9456_end_0, end_mask = var_9456_end_mask_0, squeeze_mask = var_9456_squeeze_mask_0, x = beta_21_cast_fp16)[name = string("op_9456_cast_fp16")];
            tensor<int32, [1]> var_9457_axes_0 = const()[name = string("op_9457_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_9457_cast_fp16 = expand_dims(axes = var_9457_axes_0, x = var_9456_cast_fp16)[name = string("op_9457_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_347_cast_fp16 = mul(x = var_9453_cast_fp16, y = var_9457_cast_fp16)[name = string("delta_347_cast_fp16")];
            tensor<int32, [1]> var_9460_axes_0 = const()[name = string("op_9460_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_9460_cast_fp16 = expand_dims(axes = var_9460_axes_0, x = delta_347_cast_fp16)[name = string("op_9460_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_9461_cast_fp16 = mul(x = var_9449_cast_fp16, y = var_9460_cast_fp16)[name = string("op_9461_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_695_cast_fp16 = add(x = state_693_cast_fp16, y = var_9461_cast_fp16)[name = string("state_695_cast_fp16")];
            tensor<int32, [4]> var_9465_begin_0 = const()[name = string("op_9465_begin_0"), val = tensor<int32, [4]>([0, 0, 13, 0])];
            tensor<int32, [4]> var_9465_end_0 = const()[name = string("op_9465_end_0"), val = tensor<int32, [4]>([1, 16, 14, 128])];
            tensor<bool, [4]> var_9465_end_mask_0 = const()[name = string("op_9465_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_9465_squeeze_mask_0 = const()[name = string("op_9465_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_9465_cast_fp16 = slice_by_index(begin = var_9465_begin_0, end = var_9465_end_0, end_mask = var_9465_end_mask_0, squeeze_mask = var_9465_squeeze_mask_0, x = _inversed_query_55_cast_fp16)[name = string("op_9465_cast_fp16")];
            tensor<int32, [1]> var_9466_axes_0 = const()[name = string("op_9466_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_9466_cast_fp16 = expand_dims(axes = var_9466_axes_0, x = var_9465_cast_fp16)[name = string("op_9466_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_9467_cast_fp16 = mul(x = state_695_cast_fp16, y = var_9466_cast_fp16)[name = string("op_9467_cast_fp16")];
            tensor<int32, [1]> var_9469_axes_0 = const()[name = string("op_9469_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_9469_keep_dims_0 = const()[name = string("op_9469_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_9469_cast_fp16 = reduce_sum(axes = var_9469_axes_0, keep_dims = var_9469_keep_dims_0, x = var_9467_cast_fp16)[name = string("op_9469_cast_fp16")];
            tensor<int32, [3]> var_9472_begin_0 = const()[name = string("op_9472_begin_0"), val = tensor<int32, [3]>([0, 0, 14])];
            tensor<int32, [3]> var_9472_end_0 = const()[name = string("op_9472_end_0"), val = tensor<int32, [3]>([1, 16, 15])];
            tensor<bool, [3]> var_9472_end_mask_0 = const()[name = string("op_9472_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_9472_squeeze_mask_0 = const()[name = string("op_9472_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_9472_cast_fp16 = slice_by_index(begin = var_9472_begin_0, end = var_9472_end_0, end_mask = var_9472_end_mask_0, squeeze_mask = var_9472_squeeze_mask_0, x = decay_21_cast_fp16)[name = string("op_9472_cast_fp16")];
            tensor<fp16, [1, 16]> var_9473_cast_fp16 = exp(x = var_9472_cast_fp16)[name = string("op_9473_cast_fp16")];
            tensor<int32, [1]> var_9474_axes_0 = const()[name = string("op_9474_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_9474_cast_fp16 = expand_dims(axes = var_9474_axes_0, x = var_9473_cast_fp16)[name = string("op_9474_cast_fp16")];
            tensor<int32, [1]> var_9475_axes_0 = const()[name = string("op_9475_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_9475_cast_fp16 = expand_dims(axes = var_9475_axes_0, x = var_9474_cast_fp16)[name = string("op_9475_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_697_cast_fp16 = mul(x = state_695_cast_fp16, y = var_9475_cast_fp16)[name = string("state_697_cast_fp16")];
            tensor<int32, [4]> key_token_349_begin_0 = const()[name = string("key_token_349_begin_0"), val = tensor<int32, [4]>([0, 0, 14, 0])];
            tensor<int32, [4]> key_token_349_end_0 = const()[name = string("key_token_349_end_0"), val = tensor<int32, [4]>([1, 16, 15, 128])];
            tensor<bool, [4]> key_token_349_end_mask_0 = const()[name = string("key_token_349_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_349_squeeze_mask_0 = const()[name = string("key_token_349_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_349_cast_fp16 = slice_by_index(begin = key_token_349_begin_0, end = key_token_349_end_0, end_mask = key_token_349_end_mask_0, squeeze_mask = key_token_349_squeeze_mask_0, x = key_55_cast_fp16)[name = string("key_token_349_cast_fp16")];
            tensor<int32, [4]> value_token_349_begin_0 = const()[name = string("value_token_349_begin_0"), val = tensor<int32, [4]>([0, 0, 14, 0])];
            tensor<int32, [4]> value_token_349_end_0 = const()[name = string("value_token_349_end_0"), val = tensor<int32, [4]>([1, 16, 15, 128])];
            tensor<bool, [4]> value_token_349_end_mask_0 = const()[name = string("value_token_349_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_349_squeeze_mask_0 = const()[name = string("value_token_349_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_349_cast_fp16 = slice_by_index(begin = value_token_349_begin_0, end = value_token_349_end_0, end_mask = value_token_349_end_mask_0, squeeze_mask = value_token_349_squeeze_mask_0, x = value_199_cast_fp16)[name = string("value_token_349_cast_fp16")];
            tensor<int32, [1]> var_9483_axes_0 = const()[name = string("op_9483_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_9483_cast_fp16 = expand_dims(axes = var_9483_axes_0, x = key_token_349_cast_fp16)[name = string("op_9483_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_9484_cast_fp16 = mul(x = state_697_cast_fp16, y = var_9483_cast_fp16)[name = string("op_9484_cast_fp16")];
            tensor<int32, [1]> memory_349_axes_0 = const()[name = string("memory_349_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_349_keep_dims_0 = const()[name = string("memory_349_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_349_cast_fp16 = reduce_sum(axes = memory_349_axes_0, keep_dims = memory_349_keep_dims_0, x = var_9484_cast_fp16)[name = string("memory_349_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_9487_cast_fp16 = sub(x = value_token_349_cast_fp16, y = memory_349_cast_fp16)[name = string("op_9487_cast_fp16")];
            tensor<int32, [3]> var_9490_begin_0 = const()[name = string("op_9490_begin_0"), val = tensor<int32, [3]>([0, 0, 14])];
            tensor<int32, [3]> var_9490_end_0 = const()[name = string("op_9490_end_0"), val = tensor<int32, [3]>([1, 16, 15])];
            tensor<bool, [3]> var_9490_end_mask_0 = const()[name = string("op_9490_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_9490_squeeze_mask_0 = const()[name = string("op_9490_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_9490_cast_fp16 = slice_by_index(begin = var_9490_begin_0, end = var_9490_end_0, end_mask = var_9490_end_mask_0, squeeze_mask = var_9490_squeeze_mask_0, x = beta_21_cast_fp16)[name = string("op_9490_cast_fp16")];
            tensor<int32, [1]> var_9491_axes_0 = const()[name = string("op_9491_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_9491_cast_fp16 = expand_dims(axes = var_9491_axes_0, x = var_9490_cast_fp16)[name = string("op_9491_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_349_cast_fp16 = mul(x = var_9487_cast_fp16, y = var_9491_cast_fp16)[name = string("delta_349_cast_fp16")];
            tensor<int32, [1]> var_9494_axes_0 = const()[name = string("op_9494_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_9494_cast_fp16 = expand_dims(axes = var_9494_axes_0, x = delta_349_cast_fp16)[name = string("op_9494_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_9495_cast_fp16 = mul(x = var_9483_cast_fp16, y = var_9494_cast_fp16)[name = string("op_9495_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_699_cast_fp16 = add(x = state_697_cast_fp16, y = var_9495_cast_fp16)[name = string("state_699_cast_fp16")];
            tensor<int32, [4]> var_9499_begin_0 = const()[name = string("op_9499_begin_0"), val = tensor<int32, [4]>([0, 0, 14, 0])];
            tensor<int32, [4]> var_9499_end_0 = const()[name = string("op_9499_end_0"), val = tensor<int32, [4]>([1, 16, 15, 128])];
            tensor<bool, [4]> var_9499_end_mask_0 = const()[name = string("op_9499_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_9499_squeeze_mask_0 = const()[name = string("op_9499_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_9499_cast_fp16 = slice_by_index(begin = var_9499_begin_0, end = var_9499_end_0, end_mask = var_9499_end_mask_0, squeeze_mask = var_9499_squeeze_mask_0, x = _inversed_query_55_cast_fp16)[name = string("op_9499_cast_fp16")];
            tensor<int32, [1]> var_9500_axes_0 = const()[name = string("op_9500_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_9500_cast_fp16 = expand_dims(axes = var_9500_axes_0, x = var_9499_cast_fp16)[name = string("op_9500_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_9501_cast_fp16 = mul(x = state_699_cast_fp16, y = var_9500_cast_fp16)[name = string("op_9501_cast_fp16")];
            tensor<int32, [1]> var_9503_axes_0 = const()[name = string("op_9503_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_9503_keep_dims_0 = const()[name = string("op_9503_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_9503_cast_fp16 = reduce_sum(axes = var_9503_axes_0, keep_dims = var_9503_keep_dims_0, x = var_9501_cast_fp16)[name = string("op_9503_cast_fp16")];
            tensor<int32, [3]> var_9506_begin_0 = const()[name = string("op_9506_begin_0"), val = tensor<int32, [3]>([0, 0, 15])];
            tensor<int32, [3]> var_9506_end_0 = const()[name = string("op_9506_end_0"), val = tensor<int32, [3]>([1, 16, 16])];
            tensor<bool, [3]> var_9506_end_mask_0 = const()[name = string("op_9506_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_9506_squeeze_mask_0 = const()[name = string("op_9506_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_9506_cast_fp16 = slice_by_index(begin = var_9506_begin_0, end = var_9506_end_0, end_mask = var_9506_end_mask_0, squeeze_mask = var_9506_squeeze_mask_0, x = decay_21_cast_fp16)[name = string("op_9506_cast_fp16")];
            tensor<fp16, [1, 16]> var_9507_cast_fp16 = exp(x = var_9506_cast_fp16)[name = string("op_9507_cast_fp16")];
            tensor<int32, [1]> var_9508_axes_0 = const()[name = string("op_9508_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_9508_cast_fp16 = expand_dims(axes = var_9508_axes_0, x = var_9507_cast_fp16)[name = string("op_9508_cast_fp16")];
            tensor<int32, [1]> var_9509_axes_0 = const()[name = string("op_9509_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_9509_cast_fp16 = expand_dims(axes = var_9509_axes_0, x = var_9508_cast_fp16)[name = string("op_9509_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_701_cast_fp16 = mul(x = state_699_cast_fp16, y = var_9509_cast_fp16)[name = string("state_701_cast_fp16")];
            tensor<int32, [4]> key_token_351_begin_0 = const()[name = string("key_token_351_begin_0"), val = tensor<int32, [4]>([0, 0, 15, 0])];
            tensor<int32, [4]> key_token_351_end_0 = const()[name = string("key_token_351_end_0"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<bool, [4]> key_token_351_end_mask_0 = const()[name = string("key_token_351_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_351_squeeze_mask_0 = const()[name = string("key_token_351_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_351_cast_fp16 = slice_by_index(begin = key_token_351_begin_0, end = key_token_351_end_0, end_mask = key_token_351_end_mask_0, squeeze_mask = key_token_351_squeeze_mask_0, x = key_55_cast_fp16)[name = string("key_token_351_cast_fp16")];
            tensor<int32, [4]> value_token_351_begin_0 = const()[name = string("value_token_351_begin_0"), val = tensor<int32, [4]>([0, 0, 15, 0])];
            tensor<int32, [4]> value_token_351_end_0 = const()[name = string("value_token_351_end_0"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<bool, [4]> value_token_351_end_mask_0 = const()[name = string("value_token_351_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_351_squeeze_mask_0 = const()[name = string("value_token_351_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_351_cast_fp16 = slice_by_index(begin = value_token_351_begin_0, end = value_token_351_end_0, end_mask = value_token_351_end_mask_0, squeeze_mask = value_token_351_squeeze_mask_0, x = value_199_cast_fp16)[name = string("value_token_351_cast_fp16")];
            tensor<int32, [1]> var_9517_axes_0 = const()[name = string("op_9517_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_9517_cast_fp16 = expand_dims(axes = var_9517_axes_0, x = key_token_351_cast_fp16)[name = string("op_9517_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_9518_cast_fp16 = mul(x = state_701_cast_fp16, y = var_9517_cast_fp16)[name = string("op_9518_cast_fp16")];
            tensor<int32, [1]> memory_351_axes_0 = const()[name = string("memory_351_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_351_keep_dims_0 = const()[name = string("memory_351_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_351_cast_fp16 = reduce_sum(axes = memory_351_axes_0, keep_dims = memory_351_keep_dims_0, x = var_9518_cast_fp16)[name = string("memory_351_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_9521_cast_fp16 = sub(x = value_token_351_cast_fp16, y = memory_351_cast_fp16)[name = string("op_9521_cast_fp16")];
            tensor<int32, [3]> var_9524_begin_0 = const()[name = string("op_9524_begin_0"), val = tensor<int32, [3]>([0, 0, 15])];
            tensor<int32, [3]> var_9524_end_0 = const()[name = string("op_9524_end_0"), val = tensor<int32, [3]>([1, 16, 16])];
            tensor<bool, [3]> var_9524_end_mask_0 = const()[name = string("op_9524_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_9524_squeeze_mask_0 = const()[name = string("op_9524_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_9524_cast_fp16 = slice_by_index(begin = var_9524_begin_0, end = var_9524_end_0, end_mask = var_9524_end_mask_0, squeeze_mask = var_9524_squeeze_mask_0, x = beta_21_cast_fp16)[name = string("op_9524_cast_fp16")];
            tensor<int32, [1]> var_9525_axes_0 = const()[name = string("op_9525_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_9525_cast_fp16 = expand_dims(axes = var_9525_axes_0, x = var_9524_cast_fp16)[name = string("op_9525_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_351_cast_fp16 = mul(x = var_9521_cast_fp16, y = var_9525_cast_fp16)[name = string("delta_351_cast_fp16")];
            tensor<int32, [1]> var_9528_axes_0 = const()[name = string("op_9528_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_9528_cast_fp16 = expand_dims(axes = var_9528_axes_0, x = delta_351_cast_fp16)[name = string("op_9528_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_9529_cast_fp16 = mul(x = var_9517_cast_fp16, y = var_9528_cast_fp16)[name = string("op_9529_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> rec13_out = add(x = state_701_cast_fp16, y = var_9529_cast_fp16)[name = string("state_703_cast_fp16")];
            tensor<int32, [4]> var_9533_begin_0 = const()[name = string("op_9533_begin_0"), val = tensor<int32, [4]>([0, 0, 15, 0])];
            tensor<int32, [4]> var_9533_end_0 = const()[name = string("op_9533_end_0"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<bool, [4]> var_9533_end_mask_0 = const()[name = string("op_9533_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_9533_squeeze_mask_0 = const()[name = string("op_9533_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_9533_cast_fp16 = slice_by_index(begin = var_9533_begin_0, end = var_9533_end_0, end_mask = var_9533_end_mask_0, squeeze_mask = var_9533_squeeze_mask_0, x = _inversed_query_55_cast_fp16)[name = string("op_9533_cast_fp16")];
            tensor<int32, [1]> var_9534_axes_0 = const()[name = string("op_9534_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_9534_cast_fp16 = expand_dims(axes = var_9534_axes_0, x = var_9533_cast_fp16)[name = string("op_9534_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_9535_cast_fp16 = mul(x = rec13_out, y = var_9534_cast_fp16)[name = string("op_9535_cast_fp16")];
            tensor<int32, [1]> var_9537_axes_0 = const()[name = string("op_9537_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_9537_keep_dims_0 = const()[name = string("op_9537_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_9537_cast_fp16 = reduce_sum(axes = var_9537_axes_0, keep_dims = var_9537_keep_dims_0, x = var_9535_cast_fp16)[name = string("op_9537_cast_fp16")];
            int32 attention_101_axis_0 = const()[name = string("attention_101_axis_0"), val = int32(1)];
            tensor<fp16, [1, 16, 16, 128]> attention_101_cast_fp16 = stack(axis = attention_101_axis_0, values = (var_9027_cast_fp16, var_9061_cast_fp16, var_9095_cast_fp16, var_9129_cast_fp16, var_9163_cast_fp16, var_9197_cast_fp16, var_9231_cast_fp16, var_9265_cast_fp16, var_9299_cast_fp16, var_9333_cast_fp16, var_9367_cast_fp16, var_9401_cast_fp16, var_9435_cast_fp16, var_9469_cast_fp16, var_9503_cast_fp16, var_9537_cast_fp16))[name = string("attention_101_cast_fp16")];
            tensor<int32, [2]> var_9540 = const()[name = string("op_9540"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [256, 128]> attention_103_cast_fp16 = reshape(shape = var_9540, x = attention_101_cast_fp16)[name = string("attention_103_cast_fp16")];
            tensor<int32, [2]> var_9542 = const()[name = string("op_9542"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [256, 128]> input_175_cast_fp16 = reshape(shape = var_9542, x = linear_104_cast_fp16)[name = string("input_175_cast_fp16")];
            tensor<fp16, [256, 128]> var_9544_cast_fp16 = mul(x = attention_103_cast_fp16, y = attention_103_cast_fp16)[name = string("op_9544_cast_fp16")];
            tensor<int32, [1]> variance_87_axes_0 = const()[name = string("variance_87_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_87_keep_dims_0 = const()[name = string("variance_87_keep_dims_0"), val = bool(true)];
            tensor<fp16, [256, 1]> variance_87_cast_fp16 = reduce_mean(axes = variance_87_axes_0, keep_dims = variance_87_keep_dims_0, x = var_9544_cast_fp16)[name = string("variance_87_cast_fp16")];
            fp16 var_9547_to_fp16 = const()[name = string("op_9547_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [256, 1]> var_9548_cast_fp16 = add(x = variance_87_cast_fp16, y = var_9547_to_fp16)[name = string("op_9548_cast_fp16")];
            fp32 var_9549_epsilon_0 = const()[name = string("op_9549_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [256, 1]> var_9549_cast_fp16 = rsqrt(epsilon = var_9549_epsilon_0, x = var_9548_cast_fp16)[name = string("op_9549_cast_fp16")];
            tensor<fp16, [256, 128]> attention_105_cast_fp16 = mul(x = attention_103_cast_fp16, y = var_9549_cast_fp16)[name = string("attention_105_cast_fp16")];
            tensor<fp16, [128]> groups_3_1_gated_norm_promoted_to_fp16 = const()[name = string("groups_3_1_gated_norm_promoted_to_fp16"), val = tensor<fp16, [128]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(211733248)))];
            tensor<fp16, [256, 128]> attention_107_cast_fp16 = mul(x = attention_105_cast_fp16, y = groups_3_1_gated_norm_promoted_to_fp16)[name = string("attention_107_cast_fp16")];
            tensor<fp16, [256, 128]> var_9552_cast_fp16 = silu(x = input_175_cast_fp16)[name = string("op_9552_cast_fp16")];
            tensor<fp16, [256, 128]> attention_109_cast_fp16 = mul(x = attention_107_cast_fp16, y = var_9552_cast_fp16)[name = string("attention_109_cast_fp16")];
            tensor<int32, [2]> var_9554 = const()[name = string("op_9554"), val = tensor<int32, [2]>([16, 2048])];
            tensor<fp16, [16, 2048]> input_177_cast_fp16 = reshape(shape = var_9554, x = attention_109_cast_fp16)[name = string("input_177_cast_fp16")];
            tensor<fp16, [1024, 2048]> groups_3_1_out_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 2048]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(211733568))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(213306496))))[name = string("groups_3_1_out_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 1024]> linear_105_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = groups_3_1_out_proj_weight_promoted_to_fp16_palettized, x = input_177_cast_fp16)[name = string("linear_105_cast_fp16")];
            tensor<fp16, [16, 1024]> value_201_cast_fp16 = add(x = value_189_cast_fp16, y = linear_105_cast_fp16)[name = string("value_201_cast_fp16")];
            tensor<fp16, [16, 1024]> var_9559_cast_fp16 = mul(x = value_201_cast_fp16, y = value_201_cast_fp16)[name = string("op_9559_cast_fp16")];
            tensor<int32, [1]> variance_89_axes_0 = const()[name = string("variance_89_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_89_keep_dims_0 = const()[name = string("variance_89_keep_dims_0"), val = bool(true)];
            tensor<fp16, [16, 1]> variance_89_cast_fp16 = reduce_mean(axes = variance_89_axes_0, keep_dims = variance_89_keep_dims_0, x = var_9559_cast_fp16)[name = string("variance_89_cast_fp16")];
            fp16 var_9562_to_fp16 = const()[name = string("op_9562_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [16, 1]> var_9563_cast_fp16 = add(x = variance_89_cast_fp16, y = var_9562_to_fp16)[name = string("op_9563_cast_fp16")];
            fp32 var_9564_epsilon_0 = const()[name = string("op_9564_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [16, 1]> var_9564_cast_fp16 = rsqrt(epsilon = var_9564_epsilon_0, x = var_9563_cast_fp16)[name = string("op_9564_cast_fp16")];
            tensor<fp16, [16, 1024]> var_9565_cast_fp16 = mul(x = value_201_cast_fp16, y = var_9564_cast_fp16)[name = string("op_9565_cast_fp16")];
            tensor<fp16, [1024]> var_9567_to_fp16 = const()[name = string("op_9567_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(213314752)))];
            tensor<fp16, [16, 1024]> input_179_cast_fp16 = mul(x = var_9565_cast_fp16, y = var_9567_to_fp16)[name = string("input_179_cast_fp16")];
            tensor<fp16, [3584, 1024]> groups_3_1_gate_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(213316864))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(216069440))))[name = string("groups_3_1_gate_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 3584]> linear_106_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = groups_3_1_gate_proj_weight_promoted_to_fp16_palettized, x = input_179_cast_fp16)[name = string("linear_106_cast_fp16")];
            tensor<fp16, [16, 3584]> var_9571_cast_fp16 = silu(x = linear_106_cast_fp16)[name = string("op_9571_cast_fp16")];
            tensor<fp16, [3584, 1024]> groups_3_1_up_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(216098176))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(218850752))))[name = string("groups_3_1_up_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 3584]> linear_107_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = groups_3_1_up_proj_weight_promoted_to_fp16_palettized, x = input_179_cast_fp16)[name = string("linear_107_cast_fp16")];
            tensor<fp16, [16, 3584]> input_183_cast_fp16 = mul(x = var_9571_cast_fp16, y = linear_107_cast_fp16)[name = string("input_183_cast_fp16")];
            tensor<fp16, [1024, 3584]> groups_3_1_down_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 3584]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(218879488))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(221632064))))[name = string("groups_3_1_down_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 1024]> linear_108_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = groups_3_1_down_proj_weight_promoted_to_fp16_palettized, x = input_183_cast_fp16)[name = string("linear_108_cast_fp16")];
            tensor<fp16, [16, 1024]> value_203_cast_fp16 = add(x = value_201_cast_fp16, y = linear_108_cast_fp16)[name = string("value_203_cast_fp16")];
            int32 var_9609 = const()[name = string("op_9609"), val = int32(-1)];
            tensor<fp16, [16, 1024]> var_9624_cast_fp16 = mul(x = value_203_cast_fp16, y = value_203_cast_fp16)[name = string("op_9624_cast_fp16")];
            tensor<int32, [1]> variance_91_axes_0 = const()[name = string("variance_91_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_91_keep_dims_0 = const()[name = string("variance_91_keep_dims_0"), val = bool(true)];
            tensor<fp16, [16, 1]> variance_91_cast_fp16 = reduce_mean(axes = variance_91_axes_0, keep_dims = variance_91_keep_dims_0, x = var_9624_cast_fp16)[name = string("variance_91_cast_fp16")];
            fp16 var_9627_to_fp16 = const()[name = string("op_9627_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [16, 1]> var_9628_cast_fp16 = add(x = variance_91_cast_fp16, y = var_9627_to_fp16)[name = string("op_9628_cast_fp16")];
            fp32 var_9629_epsilon_0 = const()[name = string("op_9629_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [16, 1]> var_9629_cast_fp16 = rsqrt(epsilon = var_9629_epsilon_0, x = var_9628_cast_fp16)[name = string("op_9629_cast_fp16")];
            tensor<fp16, [16, 1024]> var_9630_cast_fp16 = mul(x = value_203_cast_fp16, y = var_9629_cast_fp16)[name = string("op_9630_cast_fp16")];
            tensor<fp16, [1024]> var_9632_to_fp16 = const()[name = string("op_9632_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(221640320)))];
            tensor<fp16, [16, 1024]> input_185_cast_fp16 = mul(x = var_9630_cast_fp16, y = var_9632_to_fp16)[name = string("input_185_cast_fp16")];
            tensor<fp16, [6144, 1024]> groups_3_2_in_proj_qkv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(221642432))), lut = tensor<fp16, [384, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(226361088))))[name = string("groups_3_2_in_proj_qkv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 6144]> linear_109_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = groups_3_2_in_proj_qkv_weight_promoted_to_fp16_palettized, x = input_185_cast_fp16)[name = string("linear_109_cast_fp16")];
            tensor<int32, [2]> var_9636_perm_0 = const()[name = string("op_9636_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> mixed_23_axes_0 = const()[name = string("mixed_23_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [6144, 16]> var_9636_cast_fp16 = transpose(perm = var_9636_perm_0, x = linear_109_cast_fp16)[name = string("transpose_39")];
            tensor<fp16, [1, 6144, 16]> mixed_23_cast_fp16 = expand_dims(axes = mixed_23_axes_0, x = var_9636_cast_fp16)[name = string("mixed_23_cast_fp16")];
            bool convolution_input_23_interleave_0 = const()[name = string("convolution_input_23_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 6144, 19]> convolution_input_23_cast_fp16 = concat(axis = var_9609, interleave = convolution_input_23_interleave_0, values = (conv14, mixed_23_cast_fp16))[name = string("convolution_input_23_cast_fp16")];
            string input_187_pad_type_0 = const()[name = string("input_187_pad_type_0"), val = string("valid")];
            int32 input_187_groups_0 = const()[name = string("input_187_groups_0"), val = int32(6144)];
            tensor<int32, [1]> input_187_strides_0 = const()[name = string("input_187_strides_0"), val = tensor<int32, [1]>([1])];
            tensor<int32, [2]> input_187_pad_0 = const()[name = string("input_187_pad_0"), val = tensor<int32, [2]>([0, 0])];
            tensor<int32, [1]> input_187_dilations_0 = const()[name = string("input_187_dilations_0"), val = tensor<int32, [1]>([1])];
            tensor<fp16, [6144, 1, 4]> groups_3_2_conv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1, 4]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(226410304))), lut = tensor<fp16, [384, 1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(226428800))))[name = string("groups_3_2_conv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 6144, 16]> input_187_cast_fp16 = conv(dilations = input_187_dilations_0, groups = input_187_groups_0, pad = input_187_pad_0, pad_type = input_187_pad_type_0, strides = input_187_strides_0, weight = groups_3_2_conv_weight_promoted_to_fp16_palettized, x = convolution_input_23_cast_fp16)[name = string("input_187_cast_fp16")];
            tensor<fp16, [1, 6144, 16]> var_9645_cast_fp16 = silu(x = input_187_cast_fp16)[name = string("op_9645_cast_fp16")];
            tensor<int32, [3]> var_9646_perm_0 = const()[name = string("op_9646_perm_0"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [3]> var_9649_begin_0 = const()[name = string("op_9649_begin_0"), val = tensor<int32, [3]>([0, 0, 16])];
            tensor<int32, [3]> var_9649_end_0 = const()[name = string("op_9649_end_0"), val = tensor<int32, [3]>([1, 6144, 19])];
            tensor<bool, [3]> var_9649_end_mask_0 = const()[name = string("op_9649_end_mask_0"), val = tensor<bool, [3]>([true, true, true])];
            tensor<fp16, [1, 6144, 3]> conv14_out = slice_by_index(begin = var_9649_begin_0, end = var_9649_end_0, end_mask = var_9649_end_mask_0, x = convolution_input_23_cast_fp16)[name = string("op_9649_cast_fp16")];
            tensor<int32, [3]> var_9650 = const()[name = string("op_9650"), val = tensor<int32, [3]>([2048, 2048, 2048])];
            int32 var_9651_axis_0 = const()[name = string("op_9651_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 16, 6144]> var_9646_cast_fp16 = transpose(perm = var_9646_perm_0, x = var_9645_cast_fp16)[name = string("transpose_38")];
            tensor<fp16, [1, 16, 2048]> var_9651_cast_fp16_0, tensor<fp16, [1, 16, 2048]> var_9651_cast_fp16_1, tensor<fp16, [1, 16, 2048]> var_9651_cast_fp16_2 = split(axis = var_9651_axis_0, split_sizes = var_9650, x = var_9646_cast_fp16)[name = string("op_9651_cast_fp16")];
            tensor<int32, [4]> var_9655 = const()[name = string("op_9655"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<fp16, [1, 16, 16, 128]> value_207_cast_fp16 = reshape(shape = var_9655, x = var_9651_cast_fp16_0)[name = string("value_207_cast_fp16")];
            tensor<int32, [4]> var_9657 = const()[name = string("op_9657"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<fp16, [1, 16, 16, 128]> value_209_cast_fp16 = reshape(shape = var_9657, x = var_9651_cast_fp16_1)[name = string("value_209_cast_fp16")];
            tensor<int32, [4]> var_9659 = const()[name = string("op_9659"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<fp16, [1, 16, 16, 128]> value_211_cast_fp16 = reshape(shape = var_9659, x = var_9651_cast_fp16_2)[name = string("value_211_cast_fp16")];
            tensor<fp16, [1, 16, 16, 128]> var_9661_cast_fp16 = mul(x = value_207_cast_fp16, y = value_207_cast_fp16)[name = string("op_9661_cast_fp16")];
            tensor<int32, [1]> var_9663_axes_0 = const()[name = string("op_9663_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_9663_keep_dims_0 = const()[name = string("op_9663_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 16, 16, 1]> var_9663_cast_fp16 = reduce_sum(axes = var_9663_axes_0, keep_dims = var_9663_keep_dims_0, x = var_9661_cast_fp16)[name = string("op_9663_cast_fp16")];
            fp16 var_9664_to_fp16 = const()[name = string("op_9664_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 16, 16, 1]> var_9665_cast_fp16 = add(x = var_9663_cast_fp16, y = var_9664_to_fp16)[name = string("op_9665_cast_fp16")];
            fp32 var_9666_epsilon_0 = const()[name = string("op_9666_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 16, 16, 1]> var_9666_cast_fp16 = rsqrt(epsilon = var_9666_epsilon_0, x = var_9665_cast_fp16)[name = string("op_9666_cast_fp16")];
            tensor<fp16, [1, 16, 16, 128]> var_9667_cast_fp16 = mul(x = value_207_cast_fp16, y = var_9666_cast_fp16)[name = string("op_9667_cast_fp16")];
            tensor<int32, [4]> var_9668_perm_0 = const()[name = string("op_9668_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            fp16 _inversed_query_59_y_0_to_fp16 = const()[name = string("_inversed_query_59_y_0_to_fp16"), val = fp16(0x1.6ap-4)];
            tensor<fp16, [1, 16, 16, 128]> var_9668_cast_fp16 = transpose(perm = var_9668_perm_0, x = var_9667_cast_fp16)[name = string("transpose_37")];
            tensor<fp16, [1, 16, 16, 128]> _inversed_query_59_cast_fp16 = mul(x = var_9668_cast_fp16, y = _inversed_query_59_y_0_to_fp16)[name = string("_inversed_query_59_cast_fp16")];
            tensor<fp16, [1, 16, 16, 128]> var_9671_cast_fp16 = mul(x = value_209_cast_fp16, y = value_209_cast_fp16)[name = string("op_9671_cast_fp16")];
            tensor<int32, [1]> var_9673_axes_0 = const()[name = string("op_9673_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_9673_keep_dims_0 = const()[name = string("op_9673_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 16, 16, 1]> var_9673_cast_fp16 = reduce_sum(axes = var_9673_axes_0, keep_dims = var_9673_keep_dims_0, x = var_9671_cast_fp16)[name = string("op_9673_cast_fp16")];
            fp16 var_9674_to_fp16 = const()[name = string("op_9674_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 16, 16, 1]> var_9675_cast_fp16 = add(x = var_9673_cast_fp16, y = var_9674_to_fp16)[name = string("op_9675_cast_fp16")];
            fp32 var_9676_epsilon_0 = const()[name = string("op_9676_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 16, 16, 1]> var_9676_cast_fp16 = rsqrt(epsilon = var_9676_epsilon_0, x = var_9675_cast_fp16)[name = string("op_9676_cast_fp16")];
            tensor<fp16, [1, 16, 16, 128]> var_9677_cast_fp16 = mul(x = value_209_cast_fp16, y = var_9676_cast_fp16)[name = string("op_9677_cast_fp16")];
            tensor<int32, [4]> key_59_perm_0 = const()[name = string("key_59_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<int32, [4]> value_213_perm_0 = const()[name = string("value_213_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<fp16, [16, 1024]> groups_3_2_in_proj_b_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(226478016))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(226490368))))[name = string("groups_3_2_in_proj_b_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16]> linear_110_bias_0_to_fp16 = const()[name = string("linear_110_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(4862656)))];
            tensor<fp16, [16, 16]> linear_110_cast_fp16 = linear(bias = linear_110_bias_0_to_fp16, weight = groups_3_2_in_proj_b_weight_promoted_to_fp16_palettized, x = input_185_cast_fp16)[name = string("linear_110_cast_fp16")];
            tensor<fp16, [16, 16]> var_9682_cast_fp16 = sigmoid(x = linear_110_cast_fp16)[name = string("op_9682_cast_fp16")];
            tensor<int32, [2]> var_9683_perm_0 = const()[name = string("op_9683_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> beta_23_axes_0 = const()[name = string("beta_23_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 16]> var_9683_cast_fp16 = transpose(perm = var_9683_perm_0, x = var_9682_cast_fp16)[name = string("transpose_36")];
            tensor<fp16, [1, 16, 16]> beta_23_cast_fp16 = expand_dims(axes = beta_23_axes_0, x = var_9683_cast_fp16)[name = string("beta_23_cast_fp16")];
            tensor<fp16, [16, 1024]> op_9689_weight_0_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(226490560))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(226502912))))[name = string("op_9689_weight_0_to_fp16_palettized")];
            tensor<fp16, [16]> var_9689_bias_0_to_fp16 = const()[name = string("op_9689_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(226503104)))];
            tensor<fp16, [16, 16]> var_9689_cast_fp16 = linear(bias = var_9689_bias_0_to_fp16, weight = op_9689_weight_0_to_fp16_palettized, x = input_185_cast_fp16)[name = string("op_9689_cast_fp16")];
            tensor<fp16, [16, 16]> var_9690_cast_fp16 = softplus(x = var_9689_cast_fp16)[name = string("op_9690_cast_fp16")];
            tensor<fp16, [16]> var_9686_promoted_to_fp16 = const()[name = string("op_9686_promoted_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(226503232)))];
            tensor<fp16, [16, 16]> var_9691_cast_fp16 = mul(x = var_9686_promoted_to_fp16, y = var_9690_cast_fp16)[name = string("op_9691_cast_fp16")];
            tensor<int32, [2]> var_9692_perm_0 = const()[name = string("op_9692_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> decay_23_axes_0 = const()[name = string("decay_23_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 16]> var_9692_cast_fp16 = transpose(perm = var_9692_perm_0, x = var_9691_cast_fp16)[name = string("transpose_35")];
            tensor<fp16, [1, 16, 16]> decay_23_cast_fp16 = expand_dims(axes = decay_23_axes_0, x = var_9692_cast_fp16)[name = string("decay_23_cast_fp16")];
            tensor<fp16, [2048, 1024]> groups_3_2_in_proj_z_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [2048, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(226503360))), lut = tensor<fp16, [128, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(228076288))))[name = string("groups_3_2_in_proj_z_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 2048]> linear_112_cast_fp16 = linear(bias = linear_3_bias_0_to_fp16, weight = groups_3_2_in_proj_z_weight_promoted_to_fp16_palettized, x = input_185_cast_fp16)[name = string("linear_112_cast_fp16")];
            tensor<int32, [3]> var_9700_begin_0 = const()[name = string("op_9700_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_9700_end_0 = const()[name = string("op_9700_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_9700_end_mask_0 = const()[name = string("op_9700_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_9700_squeeze_mask_0 = const()[name = string("op_9700_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_9700_cast_fp16 = slice_by_index(begin = var_9700_begin_0, end = var_9700_end_0, end_mask = var_9700_end_mask_0, squeeze_mask = var_9700_squeeze_mask_0, x = decay_23_cast_fp16)[name = string("op_9700_cast_fp16")];
            tensor<fp16, [1, 16]> var_9701_cast_fp16 = exp(x = var_9700_cast_fp16)[name = string("op_9701_cast_fp16")];
            tensor<int32, [1]> var_9702_axes_0 = const()[name = string("op_9702_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_9702_cast_fp16 = expand_dims(axes = var_9702_axes_0, x = var_9701_cast_fp16)[name = string("op_9702_cast_fp16")];
            tensor<int32, [1]> var_9703_axes_0 = const()[name = string("op_9703_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_9703_cast_fp16 = expand_dims(axes = var_9703_axes_0, x = var_9702_cast_fp16)[name = string("op_9703_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_705_cast_fp16 = mul(x = rec14, y = var_9703_cast_fp16)[name = string("state_705_cast_fp16")];
            tensor<int32, [4]> key_token_353_begin_0 = const()[name = string("key_token_353_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> key_token_353_end_0 = const()[name = string("key_token_353_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> key_token_353_end_mask_0 = const()[name = string("key_token_353_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_353_squeeze_mask_0 = const()[name = string("key_token_353_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 16, 128]> key_59_cast_fp16 = transpose(perm = key_59_perm_0, x = var_9677_cast_fp16)[name = string("transpose_34")];
            tensor<fp16, [1, 16, 128]> key_token_353_cast_fp16 = slice_by_index(begin = key_token_353_begin_0, end = key_token_353_end_0, end_mask = key_token_353_end_mask_0, squeeze_mask = key_token_353_squeeze_mask_0, x = key_59_cast_fp16)[name = string("key_token_353_cast_fp16")];
            tensor<int32, [4]> value_token_353_begin_0 = const()[name = string("value_token_353_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> value_token_353_end_0 = const()[name = string("value_token_353_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> value_token_353_end_mask_0 = const()[name = string("value_token_353_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_353_squeeze_mask_0 = const()[name = string("value_token_353_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 16, 128]> value_213_cast_fp16 = transpose(perm = value_213_perm_0, x = value_211_cast_fp16)[name = string("transpose_33")];
            tensor<fp16, [1, 16, 128]> value_token_353_cast_fp16 = slice_by_index(begin = value_token_353_begin_0, end = value_token_353_end_0, end_mask = value_token_353_end_mask_0, squeeze_mask = value_token_353_squeeze_mask_0, x = value_213_cast_fp16)[name = string("value_token_353_cast_fp16")];
            tensor<int32, [1]> var_9711_axes_0 = const()[name = string("op_9711_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_9711_cast_fp16 = expand_dims(axes = var_9711_axes_0, x = key_token_353_cast_fp16)[name = string("op_9711_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_9712_cast_fp16 = mul(x = state_705_cast_fp16, y = var_9711_cast_fp16)[name = string("op_9712_cast_fp16")];
            tensor<int32, [1]> memory_353_axes_0 = const()[name = string("memory_353_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_353_keep_dims_0 = const()[name = string("memory_353_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_353_cast_fp16 = reduce_sum(axes = memory_353_axes_0, keep_dims = memory_353_keep_dims_0, x = var_9712_cast_fp16)[name = string("memory_353_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_9715_cast_fp16 = sub(x = value_token_353_cast_fp16, y = memory_353_cast_fp16)[name = string("op_9715_cast_fp16")];
            tensor<int32, [3]> var_9718_begin_0 = const()[name = string("op_9718_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_9718_end_0 = const()[name = string("op_9718_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_9718_end_mask_0 = const()[name = string("op_9718_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_9718_squeeze_mask_0 = const()[name = string("op_9718_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_9718_cast_fp16 = slice_by_index(begin = var_9718_begin_0, end = var_9718_end_0, end_mask = var_9718_end_mask_0, squeeze_mask = var_9718_squeeze_mask_0, x = beta_23_cast_fp16)[name = string("op_9718_cast_fp16")];
            tensor<int32, [1]> var_9719_axes_0 = const()[name = string("op_9719_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_9719_cast_fp16 = expand_dims(axes = var_9719_axes_0, x = var_9718_cast_fp16)[name = string("op_9719_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_353_cast_fp16 = mul(x = var_9715_cast_fp16, y = var_9719_cast_fp16)[name = string("delta_353_cast_fp16")];
            tensor<int32, [1]> var_9722_axes_0 = const()[name = string("op_9722_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_9722_cast_fp16 = expand_dims(axes = var_9722_axes_0, x = delta_353_cast_fp16)[name = string("op_9722_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_9723_cast_fp16 = mul(x = var_9711_cast_fp16, y = var_9722_cast_fp16)[name = string("op_9723_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_707_cast_fp16 = add(x = state_705_cast_fp16, y = var_9723_cast_fp16)[name = string("state_707_cast_fp16")];
            tensor<int32, [4]> var_9727_begin_0 = const()[name = string("op_9727_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> var_9727_end_0 = const()[name = string("op_9727_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> var_9727_end_mask_0 = const()[name = string("op_9727_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_9727_squeeze_mask_0 = const()[name = string("op_9727_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_9727_cast_fp16 = slice_by_index(begin = var_9727_begin_0, end = var_9727_end_0, end_mask = var_9727_end_mask_0, squeeze_mask = var_9727_squeeze_mask_0, x = _inversed_query_59_cast_fp16)[name = string("op_9727_cast_fp16")];
            tensor<int32, [1]> var_9728_axes_0 = const()[name = string("op_9728_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_9728_cast_fp16 = expand_dims(axes = var_9728_axes_0, x = var_9727_cast_fp16)[name = string("op_9728_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_9729_cast_fp16 = mul(x = state_707_cast_fp16, y = var_9728_cast_fp16)[name = string("op_9729_cast_fp16")];
            tensor<int32, [1]> var_9731_axes_0 = const()[name = string("op_9731_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_9731_keep_dims_0 = const()[name = string("op_9731_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_9731_cast_fp16 = reduce_sum(axes = var_9731_axes_0, keep_dims = var_9731_keep_dims_0, x = var_9729_cast_fp16)[name = string("op_9731_cast_fp16")];
            tensor<int32, [3]> var_9734_begin_0 = const()[name = string("op_9734_begin_0"), val = tensor<int32, [3]>([0, 0, 1])];
            tensor<int32, [3]> var_9734_end_0 = const()[name = string("op_9734_end_0"), val = tensor<int32, [3]>([1, 16, 2])];
            tensor<bool, [3]> var_9734_end_mask_0 = const()[name = string("op_9734_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_9734_squeeze_mask_0 = const()[name = string("op_9734_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_9734_cast_fp16 = slice_by_index(begin = var_9734_begin_0, end = var_9734_end_0, end_mask = var_9734_end_mask_0, squeeze_mask = var_9734_squeeze_mask_0, x = decay_23_cast_fp16)[name = string("op_9734_cast_fp16")];
            tensor<fp16, [1, 16]> var_9735_cast_fp16 = exp(x = var_9734_cast_fp16)[name = string("op_9735_cast_fp16")];
            tensor<int32, [1]> var_9736_axes_0 = const()[name = string("op_9736_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_9736_cast_fp16 = expand_dims(axes = var_9736_axes_0, x = var_9735_cast_fp16)[name = string("op_9736_cast_fp16")];
            tensor<int32, [1]> var_9737_axes_0 = const()[name = string("op_9737_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_9737_cast_fp16 = expand_dims(axes = var_9737_axes_0, x = var_9736_cast_fp16)[name = string("op_9737_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_709_cast_fp16 = mul(x = state_707_cast_fp16, y = var_9737_cast_fp16)[name = string("state_709_cast_fp16")];
            tensor<int32, [4]> key_token_355_begin_0 = const()[name = string("key_token_355_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> key_token_355_end_0 = const()[name = string("key_token_355_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> key_token_355_end_mask_0 = const()[name = string("key_token_355_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_355_squeeze_mask_0 = const()[name = string("key_token_355_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_355_cast_fp16 = slice_by_index(begin = key_token_355_begin_0, end = key_token_355_end_0, end_mask = key_token_355_end_mask_0, squeeze_mask = key_token_355_squeeze_mask_0, x = key_59_cast_fp16)[name = string("key_token_355_cast_fp16")];
            tensor<int32, [4]> value_token_355_begin_0 = const()[name = string("value_token_355_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> value_token_355_end_0 = const()[name = string("value_token_355_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> value_token_355_end_mask_0 = const()[name = string("value_token_355_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_355_squeeze_mask_0 = const()[name = string("value_token_355_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_355_cast_fp16 = slice_by_index(begin = value_token_355_begin_0, end = value_token_355_end_0, end_mask = value_token_355_end_mask_0, squeeze_mask = value_token_355_squeeze_mask_0, x = value_213_cast_fp16)[name = string("value_token_355_cast_fp16")];
            tensor<int32, [1]> var_9745_axes_0 = const()[name = string("op_9745_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_9745_cast_fp16 = expand_dims(axes = var_9745_axes_0, x = key_token_355_cast_fp16)[name = string("op_9745_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_9746_cast_fp16 = mul(x = state_709_cast_fp16, y = var_9745_cast_fp16)[name = string("op_9746_cast_fp16")];
            tensor<int32, [1]> memory_355_axes_0 = const()[name = string("memory_355_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_355_keep_dims_0 = const()[name = string("memory_355_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_355_cast_fp16 = reduce_sum(axes = memory_355_axes_0, keep_dims = memory_355_keep_dims_0, x = var_9746_cast_fp16)[name = string("memory_355_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_9749_cast_fp16 = sub(x = value_token_355_cast_fp16, y = memory_355_cast_fp16)[name = string("op_9749_cast_fp16")];
            tensor<int32, [3]> var_9752_begin_0 = const()[name = string("op_9752_begin_0"), val = tensor<int32, [3]>([0, 0, 1])];
            tensor<int32, [3]> var_9752_end_0 = const()[name = string("op_9752_end_0"), val = tensor<int32, [3]>([1, 16, 2])];
            tensor<bool, [3]> var_9752_end_mask_0 = const()[name = string("op_9752_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_9752_squeeze_mask_0 = const()[name = string("op_9752_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_9752_cast_fp16 = slice_by_index(begin = var_9752_begin_0, end = var_9752_end_0, end_mask = var_9752_end_mask_0, squeeze_mask = var_9752_squeeze_mask_0, x = beta_23_cast_fp16)[name = string("op_9752_cast_fp16")];
            tensor<int32, [1]> var_9753_axes_0 = const()[name = string("op_9753_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_9753_cast_fp16 = expand_dims(axes = var_9753_axes_0, x = var_9752_cast_fp16)[name = string("op_9753_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_355_cast_fp16 = mul(x = var_9749_cast_fp16, y = var_9753_cast_fp16)[name = string("delta_355_cast_fp16")];
            tensor<int32, [1]> var_9756_axes_0 = const()[name = string("op_9756_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_9756_cast_fp16 = expand_dims(axes = var_9756_axes_0, x = delta_355_cast_fp16)[name = string("op_9756_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_9757_cast_fp16 = mul(x = var_9745_cast_fp16, y = var_9756_cast_fp16)[name = string("op_9757_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_711_cast_fp16 = add(x = state_709_cast_fp16, y = var_9757_cast_fp16)[name = string("state_711_cast_fp16")];
            tensor<int32, [4]> var_9761_begin_0 = const()[name = string("op_9761_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> var_9761_end_0 = const()[name = string("op_9761_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> var_9761_end_mask_0 = const()[name = string("op_9761_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_9761_squeeze_mask_0 = const()[name = string("op_9761_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_9761_cast_fp16 = slice_by_index(begin = var_9761_begin_0, end = var_9761_end_0, end_mask = var_9761_end_mask_0, squeeze_mask = var_9761_squeeze_mask_0, x = _inversed_query_59_cast_fp16)[name = string("op_9761_cast_fp16")];
            tensor<int32, [1]> var_9762_axes_0 = const()[name = string("op_9762_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_9762_cast_fp16 = expand_dims(axes = var_9762_axes_0, x = var_9761_cast_fp16)[name = string("op_9762_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_9763_cast_fp16 = mul(x = state_711_cast_fp16, y = var_9762_cast_fp16)[name = string("op_9763_cast_fp16")];
            tensor<int32, [1]> var_9765_axes_0 = const()[name = string("op_9765_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_9765_keep_dims_0 = const()[name = string("op_9765_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_9765_cast_fp16 = reduce_sum(axes = var_9765_axes_0, keep_dims = var_9765_keep_dims_0, x = var_9763_cast_fp16)[name = string("op_9765_cast_fp16")];
            tensor<int32, [3]> var_9768_begin_0 = const()[name = string("op_9768_begin_0"), val = tensor<int32, [3]>([0, 0, 2])];
            tensor<int32, [3]> var_9768_end_0 = const()[name = string("op_9768_end_0"), val = tensor<int32, [3]>([1, 16, 3])];
            tensor<bool, [3]> var_9768_end_mask_0 = const()[name = string("op_9768_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_9768_squeeze_mask_0 = const()[name = string("op_9768_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_9768_cast_fp16 = slice_by_index(begin = var_9768_begin_0, end = var_9768_end_0, end_mask = var_9768_end_mask_0, squeeze_mask = var_9768_squeeze_mask_0, x = decay_23_cast_fp16)[name = string("op_9768_cast_fp16")];
            tensor<fp16, [1, 16]> var_9769_cast_fp16 = exp(x = var_9768_cast_fp16)[name = string("op_9769_cast_fp16")];
            tensor<int32, [1]> var_9770_axes_0 = const()[name = string("op_9770_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_9770_cast_fp16 = expand_dims(axes = var_9770_axes_0, x = var_9769_cast_fp16)[name = string("op_9770_cast_fp16")];
            tensor<int32, [1]> var_9771_axes_0 = const()[name = string("op_9771_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_9771_cast_fp16 = expand_dims(axes = var_9771_axes_0, x = var_9770_cast_fp16)[name = string("op_9771_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_713_cast_fp16 = mul(x = state_711_cast_fp16, y = var_9771_cast_fp16)[name = string("state_713_cast_fp16")];
            tensor<int32, [4]> key_token_357_begin_0 = const()[name = string("key_token_357_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> key_token_357_end_0 = const()[name = string("key_token_357_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> key_token_357_end_mask_0 = const()[name = string("key_token_357_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_357_squeeze_mask_0 = const()[name = string("key_token_357_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_357_cast_fp16 = slice_by_index(begin = key_token_357_begin_0, end = key_token_357_end_0, end_mask = key_token_357_end_mask_0, squeeze_mask = key_token_357_squeeze_mask_0, x = key_59_cast_fp16)[name = string("key_token_357_cast_fp16")];
            tensor<int32, [4]> value_token_357_begin_0 = const()[name = string("value_token_357_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> value_token_357_end_0 = const()[name = string("value_token_357_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> value_token_357_end_mask_0 = const()[name = string("value_token_357_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_357_squeeze_mask_0 = const()[name = string("value_token_357_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_357_cast_fp16 = slice_by_index(begin = value_token_357_begin_0, end = value_token_357_end_0, end_mask = value_token_357_end_mask_0, squeeze_mask = value_token_357_squeeze_mask_0, x = value_213_cast_fp16)[name = string("value_token_357_cast_fp16")];
            tensor<int32, [1]> var_9779_axes_0 = const()[name = string("op_9779_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_9779_cast_fp16 = expand_dims(axes = var_9779_axes_0, x = key_token_357_cast_fp16)[name = string("op_9779_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_9780_cast_fp16 = mul(x = state_713_cast_fp16, y = var_9779_cast_fp16)[name = string("op_9780_cast_fp16")];
            tensor<int32, [1]> memory_357_axes_0 = const()[name = string("memory_357_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_357_keep_dims_0 = const()[name = string("memory_357_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_357_cast_fp16 = reduce_sum(axes = memory_357_axes_0, keep_dims = memory_357_keep_dims_0, x = var_9780_cast_fp16)[name = string("memory_357_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_9783_cast_fp16 = sub(x = value_token_357_cast_fp16, y = memory_357_cast_fp16)[name = string("op_9783_cast_fp16")];
            tensor<int32, [3]> var_9786_begin_0 = const()[name = string("op_9786_begin_0"), val = tensor<int32, [3]>([0, 0, 2])];
            tensor<int32, [3]> var_9786_end_0 = const()[name = string("op_9786_end_0"), val = tensor<int32, [3]>([1, 16, 3])];
            tensor<bool, [3]> var_9786_end_mask_0 = const()[name = string("op_9786_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_9786_squeeze_mask_0 = const()[name = string("op_9786_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_9786_cast_fp16 = slice_by_index(begin = var_9786_begin_0, end = var_9786_end_0, end_mask = var_9786_end_mask_0, squeeze_mask = var_9786_squeeze_mask_0, x = beta_23_cast_fp16)[name = string("op_9786_cast_fp16")];
            tensor<int32, [1]> var_9787_axes_0 = const()[name = string("op_9787_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_9787_cast_fp16 = expand_dims(axes = var_9787_axes_0, x = var_9786_cast_fp16)[name = string("op_9787_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_357_cast_fp16 = mul(x = var_9783_cast_fp16, y = var_9787_cast_fp16)[name = string("delta_357_cast_fp16")];
            tensor<int32, [1]> var_9790_axes_0 = const()[name = string("op_9790_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_9790_cast_fp16 = expand_dims(axes = var_9790_axes_0, x = delta_357_cast_fp16)[name = string("op_9790_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_9791_cast_fp16 = mul(x = var_9779_cast_fp16, y = var_9790_cast_fp16)[name = string("op_9791_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_715_cast_fp16 = add(x = state_713_cast_fp16, y = var_9791_cast_fp16)[name = string("state_715_cast_fp16")];
            tensor<int32, [4]> var_9795_begin_0 = const()[name = string("op_9795_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> var_9795_end_0 = const()[name = string("op_9795_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> var_9795_end_mask_0 = const()[name = string("op_9795_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_9795_squeeze_mask_0 = const()[name = string("op_9795_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_9795_cast_fp16 = slice_by_index(begin = var_9795_begin_0, end = var_9795_end_0, end_mask = var_9795_end_mask_0, squeeze_mask = var_9795_squeeze_mask_0, x = _inversed_query_59_cast_fp16)[name = string("op_9795_cast_fp16")];
            tensor<int32, [1]> var_9796_axes_0 = const()[name = string("op_9796_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_9796_cast_fp16 = expand_dims(axes = var_9796_axes_0, x = var_9795_cast_fp16)[name = string("op_9796_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_9797_cast_fp16 = mul(x = state_715_cast_fp16, y = var_9796_cast_fp16)[name = string("op_9797_cast_fp16")];
            tensor<int32, [1]> var_9799_axes_0 = const()[name = string("op_9799_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_9799_keep_dims_0 = const()[name = string("op_9799_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_9799_cast_fp16 = reduce_sum(axes = var_9799_axes_0, keep_dims = var_9799_keep_dims_0, x = var_9797_cast_fp16)[name = string("op_9799_cast_fp16")];
            tensor<int32, [3]> var_9802_begin_0 = const()[name = string("op_9802_begin_0"), val = tensor<int32, [3]>([0, 0, 3])];
            tensor<int32, [3]> var_9802_end_0 = const()[name = string("op_9802_end_0"), val = tensor<int32, [3]>([1, 16, 4])];
            tensor<bool, [3]> var_9802_end_mask_0 = const()[name = string("op_9802_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_9802_squeeze_mask_0 = const()[name = string("op_9802_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_9802_cast_fp16 = slice_by_index(begin = var_9802_begin_0, end = var_9802_end_0, end_mask = var_9802_end_mask_0, squeeze_mask = var_9802_squeeze_mask_0, x = decay_23_cast_fp16)[name = string("op_9802_cast_fp16")];
            tensor<fp16, [1, 16]> var_9803_cast_fp16 = exp(x = var_9802_cast_fp16)[name = string("op_9803_cast_fp16")];
            tensor<int32, [1]> var_9804_axes_0 = const()[name = string("op_9804_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_9804_cast_fp16 = expand_dims(axes = var_9804_axes_0, x = var_9803_cast_fp16)[name = string("op_9804_cast_fp16")];
            tensor<int32, [1]> var_9805_axes_0 = const()[name = string("op_9805_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_9805_cast_fp16 = expand_dims(axes = var_9805_axes_0, x = var_9804_cast_fp16)[name = string("op_9805_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_717_cast_fp16 = mul(x = state_715_cast_fp16, y = var_9805_cast_fp16)[name = string("state_717_cast_fp16")];
            tensor<int32, [4]> key_token_359_begin_0 = const()[name = string("key_token_359_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> key_token_359_end_0 = const()[name = string("key_token_359_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> key_token_359_end_mask_0 = const()[name = string("key_token_359_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_359_squeeze_mask_0 = const()[name = string("key_token_359_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_359_cast_fp16 = slice_by_index(begin = key_token_359_begin_0, end = key_token_359_end_0, end_mask = key_token_359_end_mask_0, squeeze_mask = key_token_359_squeeze_mask_0, x = key_59_cast_fp16)[name = string("key_token_359_cast_fp16")];
            tensor<int32, [4]> value_token_359_begin_0 = const()[name = string("value_token_359_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> value_token_359_end_0 = const()[name = string("value_token_359_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> value_token_359_end_mask_0 = const()[name = string("value_token_359_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_359_squeeze_mask_0 = const()[name = string("value_token_359_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_359_cast_fp16 = slice_by_index(begin = value_token_359_begin_0, end = value_token_359_end_0, end_mask = value_token_359_end_mask_0, squeeze_mask = value_token_359_squeeze_mask_0, x = value_213_cast_fp16)[name = string("value_token_359_cast_fp16")];
            tensor<int32, [1]> var_9813_axes_0 = const()[name = string("op_9813_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_9813_cast_fp16 = expand_dims(axes = var_9813_axes_0, x = key_token_359_cast_fp16)[name = string("op_9813_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_9814_cast_fp16 = mul(x = state_717_cast_fp16, y = var_9813_cast_fp16)[name = string("op_9814_cast_fp16")];
            tensor<int32, [1]> memory_359_axes_0 = const()[name = string("memory_359_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_359_keep_dims_0 = const()[name = string("memory_359_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_359_cast_fp16 = reduce_sum(axes = memory_359_axes_0, keep_dims = memory_359_keep_dims_0, x = var_9814_cast_fp16)[name = string("memory_359_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_9817_cast_fp16 = sub(x = value_token_359_cast_fp16, y = memory_359_cast_fp16)[name = string("op_9817_cast_fp16")];
            tensor<int32, [3]> var_9820_begin_0 = const()[name = string("op_9820_begin_0"), val = tensor<int32, [3]>([0, 0, 3])];
            tensor<int32, [3]> var_9820_end_0 = const()[name = string("op_9820_end_0"), val = tensor<int32, [3]>([1, 16, 4])];
            tensor<bool, [3]> var_9820_end_mask_0 = const()[name = string("op_9820_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_9820_squeeze_mask_0 = const()[name = string("op_9820_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_9820_cast_fp16 = slice_by_index(begin = var_9820_begin_0, end = var_9820_end_0, end_mask = var_9820_end_mask_0, squeeze_mask = var_9820_squeeze_mask_0, x = beta_23_cast_fp16)[name = string("op_9820_cast_fp16")];
            tensor<int32, [1]> var_9821_axes_0 = const()[name = string("op_9821_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_9821_cast_fp16 = expand_dims(axes = var_9821_axes_0, x = var_9820_cast_fp16)[name = string("op_9821_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_359_cast_fp16 = mul(x = var_9817_cast_fp16, y = var_9821_cast_fp16)[name = string("delta_359_cast_fp16")];
            tensor<int32, [1]> var_9824_axes_0 = const()[name = string("op_9824_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_9824_cast_fp16 = expand_dims(axes = var_9824_axes_0, x = delta_359_cast_fp16)[name = string("op_9824_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_9825_cast_fp16 = mul(x = var_9813_cast_fp16, y = var_9824_cast_fp16)[name = string("op_9825_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_719_cast_fp16 = add(x = state_717_cast_fp16, y = var_9825_cast_fp16)[name = string("state_719_cast_fp16")];
            tensor<int32, [4]> var_9829_begin_0 = const()[name = string("op_9829_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> var_9829_end_0 = const()[name = string("op_9829_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> var_9829_end_mask_0 = const()[name = string("op_9829_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_9829_squeeze_mask_0 = const()[name = string("op_9829_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_9829_cast_fp16 = slice_by_index(begin = var_9829_begin_0, end = var_9829_end_0, end_mask = var_9829_end_mask_0, squeeze_mask = var_9829_squeeze_mask_0, x = _inversed_query_59_cast_fp16)[name = string("op_9829_cast_fp16")];
            tensor<int32, [1]> var_9830_axes_0 = const()[name = string("op_9830_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_9830_cast_fp16 = expand_dims(axes = var_9830_axes_0, x = var_9829_cast_fp16)[name = string("op_9830_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_9831_cast_fp16 = mul(x = state_719_cast_fp16, y = var_9830_cast_fp16)[name = string("op_9831_cast_fp16")];
            tensor<int32, [1]> var_9833_axes_0 = const()[name = string("op_9833_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_9833_keep_dims_0 = const()[name = string("op_9833_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_9833_cast_fp16 = reduce_sum(axes = var_9833_axes_0, keep_dims = var_9833_keep_dims_0, x = var_9831_cast_fp16)[name = string("op_9833_cast_fp16")];
            tensor<int32, [3]> var_9836_begin_0 = const()[name = string("op_9836_begin_0"), val = tensor<int32, [3]>([0, 0, 4])];
            tensor<int32, [3]> var_9836_end_0 = const()[name = string("op_9836_end_0"), val = tensor<int32, [3]>([1, 16, 5])];
            tensor<bool, [3]> var_9836_end_mask_0 = const()[name = string("op_9836_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_9836_squeeze_mask_0 = const()[name = string("op_9836_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_9836_cast_fp16 = slice_by_index(begin = var_9836_begin_0, end = var_9836_end_0, end_mask = var_9836_end_mask_0, squeeze_mask = var_9836_squeeze_mask_0, x = decay_23_cast_fp16)[name = string("op_9836_cast_fp16")];
            tensor<fp16, [1, 16]> var_9837_cast_fp16 = exp(x = var_9836_cast_fp16)[name = string("op_9837_cast_fp16")];
            tensor<int32, [1]> var_9838_axes_0 = const()[name = string("op_9838_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_9838_cast_fp16 = expand_dims(axes = var_9838_axes_0, x = var_9837_cast_fp16)[name = string("op_9838_cast_fp16")];
            tensor<int32, [1]> var_9839_axes_0 = const()[name = string("op_9839_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_9839_cast_fp16 = expand_dims(axes = var_9839_axes_0, x = var_9838_cast_fp16)[name = string("op_9839_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_721_cast_fp16 = mul(x = state_719_cast_fp16, y = var_9839_cast_fp16)[name = string("state_721_cast_fp16")];
            tensor<int32, [4]> key_token_361_begin_0 = const()[name = string("key_token_361_begin_0"), val = tensor<int32, [4]>([0, 0, 4, 0])];
            tensor<int32, [4]> key_token_361_end_0 = const()[name = string("key_token_361_end_0"), val = tensor<int32, [4]>([1, 16, 5, 128])];
            tensor<bool, [4]> key_token_361_end_mask_0 = const()[name = string("key_token_361_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_361_squeeze_mask_0 = const()[name = string("key_token_361_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_361_cast_fp16 = slice_by_index(begin = key_token_361_begin_0, end = key_token_361_end_0, end_mask = key_token_361_end_mask_0, squeeze_mask = key_token_361_squeeze_mask_0, x = key_59_cast_fp16)[name = string("key_token_361_cast_fp16")];
            tensor<int32, [4]> value_token_361_begin_0 = const()[name = string("value_token_361_begin_0"), val = tensor<int32, [4]>([0, 0, 4, 0])];
            tensor<int32, [4]> value_token_361_end_0 = const()[name = string("value_token_361_end_0"), val = tensor<int32, [4]>([1, 16, 5, 128])];
            tensor<bool, [4]> value_token_361_end_mask_0 = const()[name = string("value_token_361_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_361_squeeze_mask_0 = const()[name = string("value_token_361_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_361_cast_fp16 = slice_by_index(begin = value_token_361_begin_0, end = value_token_361_end_0, end_mask = value_token_361_end_mask_0, squeeze_mask = value_token_361_squeeze_mask_0, x = value_213_cast_fp16)[name = string("value_token_361_cast_fp16")];
            tensor<int32, [1]> var_9847_axes_0 = const()[name = string("op_9847_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_9847_cast_fp16 = expand_dims(axes = var_9847_axes_0, x = key_token_361_cast_fp16)[name = string("op_9847_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_9848_cast_fp16 = mul(x = state_721_cast_fp16, y = var_9847_cast_fp16)[name = string("op_9848_cast_fp16")];
            tensor<int32, [1]> memory_361_axes_0 = const()[name = string("memory_361_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_361_keep_dims_0 = const()[name = string("memory_361_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_361_cast_fp16 = reduce_sum(axes = memory_361_axes_0, keep_dims = memory_361_keep_dims_0, x = var_9848_cast_fp16)[name = string("memory_361_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_9851_cast_fp16 = sub(x = value_token_361_cast_fp16, y = memory_361_cast_fp16)[name = string("op_9851_cast_fp16")];
            tensor<int32, [3]> var_9854_begin_0 = const()[name = string("op_9854_begin_0"), val = tensor<int32, [3]>([0, 0, 4])];
            tensor<int32, [3]> var_9854_end_0 = const()[name = string("op_9854_end_0"), val = tensor<int32, [3]>([1, 16, 5])];
            tensor<bool, [3]> var_9854_end_mask_0 = const()[name = string("op_9854_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_9854_squeeze_mask_0 = const()[name = string("op_9854_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_9854_cast_fp16 = slice_by_index(begin = var_9854_begin_0, end = var_9854_end_0, end_mask = var_9854_end_mask_0, squeeze_mask = var_9854_squeeze_mask_0, x = beta_23_cast_fp16)[name = string("op_9854_cast_fp16")];
            tensor<int32, [1]> var_9855_axes_0 = const()[name = string("op_9855_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_9855_cast_fp16 = expand_dims(axes = var_9855_axes_0, x = var_9854_cast_fp16)[name = string("op_9855_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_361_cast_fp16 = mul(x = var_9851_cast_fp16, y = var_9855_cast_fp16)[name = string("delta_361_cast_fp16")];
            tensor<int32, [1]> var_9858_axes_0 = const()[name = string("op_9858_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_9858_cast_fp16 = expand_dims(axes = var_9858_axes_0, x = delta_361_cast_fp16)[name = string("op_9858_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_9859_cast_fp16 = mul(x = var_9847_cast_fp16, y = var_9858_cast_fp16)[name = string("op_9859_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_723_cast_fp16 = add(x = state_721_cast_fp16, y = var_9859_cast_fp16)[name = string("state_723_cast_fp16")];
            tensor<int32, [4]> var_9863_begin_0 = const()[name = string("op_9863_begin_0"), val = tensor<int32, [4]>([0, 0, 4, 0])];
            tensor<int32, [4]> var_9863_end_0 = const()[name = string("op_9863_end_0"), val = tensor<int32, [4]>([1, 16, 5, 128])];
            tensor<bool, [4]> var_9863_end_mask_0 = const()[name = string("op_9863_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_9863_squeeze_mask_0 = const()[name = string("op_9863_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_9863_cast_fp16 = slice_by_index(begin = var_9863_begin_0, end = var_9863_end_0, end_mask = var_9863_end_mask_0, squeeze_mask = var_9863_squeeze_mask_0, x = _inversed_query_59_cast_fp16)[name = string("op_9863_cast_fp16")];
            tensor<int32, [1]> var_9864_axes_0 = const()[name = string("op_9864_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_9864_cast_fp16 = expand_dims(axes = var_9864_axes_0, x = var_9863_cast_fp16)[name = string("op_9864_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_9865_cast_fp16 = mul(x = state_723_cast_fp16, y = var_9864_cast_fp16)[name = string("op_9865_cast_fp16")];
            tensor<int32, [1]> var_9867_axes_0 = const()[name = string("op_9867_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_9867_keep_dims_0 = const()[name = string("op_9867_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_9867_cast_fp16 = reduce_sum(axes = var_9867_axes_0, keep_dims = var_9867_keep_dims_0, x = var_9865_cast_fp16)[name = string("op_9867_cast_fp16")];
            tensor<int32, [3]> var_9870_begin_0 = const()[name = string("op_9870_begin_0"), val = tensor<int32, [3]>([0, 0, 5])];
            tensor<int32, [3]> var_9870_end_0 = const()[name = string("op_9870_end_0"), val = tensor<int32, [3]>([1, 16, 6])];
            tensor<bool, [3]> var_9870_end_mask_0 = const()[name = string("op_9870_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_9870_squeeze_mask_0 = const()[name = string("op_9870_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_9870_cast_fp16 = slice_by_index(begin = var_9870_begin_0, end = var_9870_end_0, end_mask = var_9870_end_mask_0, squeeze_mask = var_9870_squeeze_mask_0, x = decay_23_cast_fp16)[name = string("op_9870_cast_fp16")];
            tensor<fp16, [1, 16]> var_9871_cast_fp16 = exp(x = var_9870_cast_fp16)[name = string("op_9871_cast_fp16")];
            tensor<int32, [1]> var_9872_axes_0 = const()[name = string("op_9872_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_9872_cast_fp16 = expand_dims(axes = var_9872_axes_0, x = var_9871_cast_fp16)[name = string("op_9872_cast_fp16")];
            tensor<int32, [1]> var_9873_axes_0 = const()[name = string("op_9873_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_9873_cast_fp16 = expand_dims(axes = var_9873_axes_0, x = var_9872_cast_fp16)[name = string("op_9873_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_725_cast_fp16 = mul(x = state_723_cast_fp16, y = var_9873_cast_fp16)[name = string("state_725_cast_fp16")];
            tensor<int32, [4]> key_token_363_begin_0 = const()[name = string("key_token_363_begin_0"), val = tensor<int32, [4]>([0, 0, 5, 0])];
            tensor<int32, [4]> key_token_363_end_0 = const()[name = string("key_token_363_end_0"), val = tensor<int32, [4]>([1, 16, 6, 128])];
            tensor<bool, [4]> key_token_363_end_mask_0 = const()[name = string("key_token_363_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_363_squeeze_mask_0 = const()[name = string("key_token_363_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_363_cast_fp16 = slice_by_index(begin = key_token_363_begin_0, end = key_token_363_end_0, end_mask = key_token_363_end_mask_0, squeeze_mask = key_token_363_squeeze_mask_0, x = key_59_cast_fp16)[name = string("key_token_363_cast_fp16")];
            tensor<int32, [4]> value_token_363_begin_0 = const()[name = string("value_token_363_begin_0"), val = tensor<int32, [4]>([0, 0, 5, 0])];
            tensor<int32, [4]> value_token_363_end_0 = const()[name = string("value_token_363_end_0"), val = tensor<int32, [4]>([1, 16, 6, 128])];
            tensor<bool, [4]> value_token_363_end_mask_0 = const()[name = string("value_token_363_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_363_squeeze_mask_0 = const()[name = string("value_token_363_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_363_cast_fp16 = slice_by_index(begin = value_token_363_begin_0, end = value_token_363_end_0, end_mask = value_token_363_end_mask_0, squeeze_mask = value_token_363_squeeze_mask_0, x = value_213_cast_fp16)[name = string("value_token_363_cast_fp16")];
            tensor<int32, [1]> var_9881_axes_0 = const()[name = string("op_9881_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_9881_cast_fp16 = expand_dims(axes = var_9881_axes_0, x = key_token_363_cast_fp16)[name = string("op_9881_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_9882_cast_fp16 = mul(x = state_725_cast_fp16, y = var_9881_cast_fp16)[name = string("op_9882_cast_fp16")];
            tensor<int32, [1]> memory_363_axes_0 = const()[name = string("memory_363_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_363_keep_dims_0 = const()[name = string("memory_363_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_363_cast_fp16 = reduce_sum(axes = memory_363_axes_0, keep_dims = memory_363_keep_dims_0, x = var_9882_cast_fp16)[name = string("memory_363_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_9885_cast_fp16 = sub(x = value_token_363_cast_fp16, y = memory_363_cast_fp16)[name = string("op_9885_cast_fp16")];
            tensor<int32, [3]> var_9888_begin_0 = const()[name = string("op_9888_begin_0"), val = tensor<int32, [3]>([0, 0, 5])];
            tensor<int32, [3]> var_9888_end_0 = const()[name = string("op_9888_end_0"), val = tensor<int32, [3]>([1, 16, 6])];
            tensor<bool, [3]> var_9888_end_mask_0 = const()[name = string("op_9888_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_9888_squeeze_mask_0 = const()[name = string("op_9888_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_9888_cast_fp16 = slice_by_index(begin = var_9888_begin_0, end = var_9888_end_0, end_mask = var_9888_end_mask_0, squeeze_mask = var_9888_squeeze_mask_0, x = beta_23_cast_fp16)[name = string("op_9888_cast_fp16")];
            tensor<int32, [1]> var_9889_axes_0 = const()[name = string("op_9889_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_9889_cast_fp16 = expand_dims(axes = var_9889_axes_0, x = var_9888_cast_fp16)[name = string("op_9889_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_363_cast_fp16 = mul(x = var_9885_cast_fp16, y = var_9889_cast_fp16)[name = string("delta_363_cast_fp16")];
            tensor<int32, [1]> var_9892_axes_0 = const()[name = string("op_9892_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_9892_cast_fp16 = expand_dims(axes = var_9892_axes_0, x = delta_363_cast_fp16)[name = string("op_9892_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_9893_cast_fp16 = mul(x = var_9881_cast_fp16, y = var_9892_cast_fp16)[name = string("op_9893_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_727_cast_fp16 = add(x = state_725_cast_fp16, y = var_9893_cast_fp16)[name = string("state_727_cast_fp16")];
            tensor<int32, [4]> var_9897_begin_0 = const()[name = string("op_9897_begin_0"), val = tensor<int32, [4]>([0, 0, 5, 0])];
            tensor<int32, [4]> var_9897_end_0 = const()[name = string("op_9897_end_0"), val = tensor<int32, [4]>([1, 16, 6, 128])];
            tensor<bool, [4]> var_9897_end_mask_0 = const()[name = string("op_9897_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_9897_squeeze_mask_0 = const()[name = string("op_9897_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_9897_cast_fp16 = slice_by_index(begin = var_9897_begin_0, end = var_9897_end_0, end_mask = var_9897_end_mask_0, squeeze_mask = var_9897_squeeze_mask_0, x = _inversed_query_59_cast_fp16)[name = string("op_9897_cast_fp16")];
            tensor<int32, [1]> var_9898_axes_0 = const()[name = string("op_9898_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_9898_cast_fp16 = expand_dims(axes = var_9898_axes_0, x = var_9897_cast_fp16)[name = string("op_9898_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_9899_cast_fp16 = mul(x = state_727_cast_fp16, y = var_9898_cast_fp16)[name = string("op_9899_cast_fp16")];
            tensor<int32, [1]> var_9901_axes_0 = const()[name = string("op_9901_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_9901_keep_dims_0 = const()[name = string("op_9901_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_9901_cast_fp16 = reduce_sum(axes = var_9901_axes_0, keep_dims = var_9901_keep_dims_0, x = var_9899_cast_fp16)[name = string("op_9901_cast_fp16")];
            tensor<int32, [3]> var_9904_begin_0 = const()[name = string("op_9904_begin_0"), val = tensor<int32, [3]>([0, 0, 6])];
            tensor<int32, [3]> var_9904_end_0 = const()[name = string("op_9904_end_0"), val = tensor<int32, [3]>([1, 16, 7])];
            tensor<bool, [3]> var_9904_end_mask_0 = const()[name = string("op_9904_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_9904_squeeze_mask_0 = const()[name = string("op_9904_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_9904_cast_fp16 = slice_by_index(begin = var_9904_begin_0, end = var_9904_end_0, end_mask = var_9904_end_mask_0, squeeze_mask = var_9904_squeeze_mask_0, x = decay_23_cast_fp16)[name = string("op_9904_cast_fp16")];
            tensor<fp16, [1, 16]> var_9905_cast_fp16 = exp(x = var_9904_cast_fp16)[name = string("op_9905_cast_fp16")];
            tensor<int32, [1]> var_9906_axes_0 = const()[name = string("op_9906_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_9906_cast_fp16 = expand_dims(axes = var_9906_axes_0, x = var_9905_cast_fp16)[name = string("op_9906_cast_fp16")];
            tensor<int32, [1]> var_9907_axes_0 = const()[name = string("op_9907_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_9907_cast_fp16 = expand_dims(axes = var_9907_axes_0, x = var_9906_cast_fp16)[name = string("op_9907_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_729_cast_fp16 = mul(x = state_727_cast_fp16, y = var_9907_cast_fp16)[name = string("state_729_cast_fp16")];
            tensor<int32, [4]> key_token_365_begin_0 = const()[name = string("key_token_365_begin_0"), val = tensor<int32, [4]>([0, 0, 6, 0])];
            tensor<int32, [4]> key_token_365_end_0 = const()[name = string("key_token_365_end_0"), val = tensor<int32, [4]>([1, 16, 7, 128])];
            tensor<bool, [4]> key_token_365_end_mask_0 = const()[name = string("key_token_365_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_365_squeeze_mask_0 = const()[name = string("key_token_365_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_365_cast_fp16 = slice_by_index(begin = key_token_365_begin_0, end = key_token_365_end_0, end_mask = key_token_365_end_mask_0, squeeze_mask = key_token_365_squeeze_mask_0, x = key_59_cast_fp16)[name = string("key_token_365_cast_fp16")];
            tensor<int32, [4]> value_token_365_begin_0 = const()[name = string("value_token_365_begin_0"), val = tensor<int32, [4]>([0, 0, 6, 0])];
            tensor<int32, [4]> value_token_365_end_0 = const()[name = string("value_token_365_end_0"), val = tensor<int32, [4]>([1, 16, 7, 128])];
            tensor<bool, [4]> value_token_365_end_mask_0 = const()[name = string("value_token_365_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_365_squeeze_mask_0 = const()[name = string("value_token_365_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_365_cast_fp16 = slice_by_index(begin = value_token_365_begin_0, end = value_token_365_end_0, end_mask = value_token_365_end_mask_0, squeeze_mask = value_token_365_squeeze_mask_0, x = value_213_cast_fp16)[name = string("value_token_365_cast_fp16")];
            tensor<int32, [1]> var_9915_axes_0 = const()[name = string("op_9915_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_9915_cast_fp16 = expand_dims(axes = var_9915_axes_0, x = key_token_365_cast_fp16)[name = string("op_9915_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_9916_cast_fp16 = mul(x = state_729_cast_fp16, y = var_9915_cast_fp16)[name = string("op_9916_cast_fp16")];
            tensor<int32, [1]> memory_365_axes_0 = const()[name = string("memory_365_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_365_keep_dims_0 = const()[name = string("memory_365_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_365_cast_fp16 = reduce_sum(axes = memory_365_axes_0, keep_dims = memory_365_keep_dims_0, x = var_9916_cast_fp16)[name = string("memory_365_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_9919_cast_fp16 = sub(x = value_token_365_cast_fp16, y = memory_365_cast_fp16)[name = string("op_9919_cast_fp16")];
            tensor<int32, [3]> var_9922_begin_0 = const()[name = string("op_9922_begin_0"), val = tensor<int32, [3]>([0, 0, 6])];
            tensor<int32, [3]> var_9922_end_0 = const()[name = string("op_9922_end_0"), val = tensor<int32, [3]>([1, 16, 7])];
            tensor<bool, [3]> var_9922_end_mask_0 = const()[name = string("op_9922_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_9922_squeeze_mask_0 = const()[name = string("op_9922_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_9922_cast_fp16 = slice_by_index(begin = var_9922_begin_0, end = var_9922_end_0, end_mask = var_9922_end_mask_0, squeeze_mask = var_9922_squeeze_mask_0, x = beta_23_cast_fp16)[name = string("op_9922_cast_fp16")];
            tensor<int32, [1]> var_9923_axes_0 = const()[name = string("op_9923_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_9923_cast_fp16 = expand_dims(axes = var_9923_axes_0, x = var_9922_cast_fp16)[name = string("op_9923_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_365_cast_fp16 = mul(x = var_9919_cast_fp16, y = var_9923_cast_fp16)[name = string("delta_365_cast_fp16")];
            tensor<int32, [1]> var_9926_axes_0 = const()[name = string("op_9926_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_9926_cast_fp16 = expand_dims(axes = var_9926_axes_0, x = delta_365_cast_fp16)[name = string("op_9926_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_9927_cast_fp16 = mul(x = var_9915_cast_fp16, y = var_9926_cast_fp16)[name = string("op_9927_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_731_cast_fp16 = add(x = state_729_cast_fp16, y = var_9927_cast_fp16)[name = string("state_731_cast_fp16")];
            tensor<int32, [4]> var_9931_begin_0 = const()[name = string("op_9931_begin_0"), val = tensor<int32, [4]>([0, 0, 6, 0])];
            tensor<int32, [4]> var_9931_end_0 = const()[name = string("op_9931_end_0"), val = tensor<int32, [4]>([1, 16, 7, 128])];
            tensor<bool, [4]> var_9931_end_mask_0 = const()[name = string("op_9931_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_9931_squeeze_mask_0 = const()[name = string("op_9931_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_9931_cast_fp16 = slice_by_index(begin = var_9931_begin_0, end = var_9931_end_0, end_mask = var_9931_end_mask_0, squeeze_mask = var_9931_squeeze_mask_0, x = _inversed_query_59_cast_fp16)[name = string("op_9931_cast_fp16")];
            tensor<int32, [1]> var_9932_axes_0 = const()[name = string("op_9932_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_9932_cast_fp16 = expand_dims(axes = var_9932_axes_0, x = var_9931_cast_fp16)[name = string("op_9932_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_9933_cast_fp16 = mul(x = state_731_cast_fp16, y = var_9932_cast_fp16)[name = string("op_9933_cast_fp16")];
            tensor<int32, [1]> var_9935_axes_0 = const()[name = string("op_9935_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_9935_keep_dims_0 = const()[name = string("op_9935_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_9935_cast_fp16 = reduce_sum(axes = var_9935_axes_0, keep_dims = var_9935_keep_dims_0, x = var_9933_cast_fp16)[name = string("op_9935_cast_fp16")];
            tensor<int32, [3]> var_9938_begin_0 = const()[name = string("op_9938_begin_0"), val = tensor<int32, [3]>([0, 0, 7])];
            tensor<int32, [3]> var_9938_end_0 = const()[name = string("op_9938_end_0"), val = tensor<int32, [3]>([1, 16, 8])];
            tensor<bool, [3]> var_9938_end_mask_0 = const()[name = string("op_9938_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_9938_squeeze_mask_0 = const()[name = string("op_9938_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_9938_cast_fp16 = slice_by_index(begin = var_9938_begin_0, end = var_9938_end_0, end_mask = var_9938_end_mask_0, squeeze_mask = var_9938_squeeze_mask_0, x = decay_23_cast_fp16)[name = string("op_9938_cast_fp16")];
            tensor<fp16, [1, 16]> var_9939_cast_fp16 = exp(x = var_9938_cast_fp16)[name = string("op_9939_cast_fp16")];
            tensor<int32, [1]> var_9940_axes_0 = const()[name = string("op_9940_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_9940_cast_fp16 = expand_dims(axes = var_9940_axes_0, x = var_9939_cast_fp16)[name = string("op_9940_cast_fp16")];
            tensor<int32, [1]> var_9941_axes_0 = const()[name = string("op_9941_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_9941_cast_fp16 = expand_dims(axes = var_9941_axes_0, x = var_9940_cast_fp16)[name = string("op_9941_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_733_cast_fp16 = mul(x = state_731_cast_fp16, y = var_9941_cast_fp16)[name = string("state_733_cast_fp16")];
            tensor<int32, [4]> key_token_367_begin_0 = const()[name = string("key_token_367_begin_0"), val = tensor<int32, [4]>([0, 0, 7, 0])];
            tensor<int32, [4]> key_token_367_end_0 = const()[name = string("key_token_367_end_0"), val = tensor<int32, [4]>([1, 16, 8, 128])];
            tensor<bool, [4]> key_token_367_end_mask_0 = const()[name = string("key_token_367_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_367_squeeze_mask_0 = const()[name = string("key_token_367_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_367_cast_fp16 = slice_by_index(begin = key_token_367_begin_0, end = key_token_367_end_0, end_mask = key_token_367_end_mask_0, squeeze_mask = key_token_367_squeeze_mask_0, x = key_59_cast_fp16)[name = string("key_token_367_cast_fp16")];
            tensor<int32, [4]> value_token_367_begin_0 = const()[name = string("value_token_367_begin_0"), val = tensor<int32, [4]>([0, 0, 7, 0])];
            tensor<int32, [4]> value_token_367_end_0 = const()[name = string("value_token_367_end_0"), val = tensor<int32, [4]>([1, 16, 8, 128])];
            tensor<bool, [4]> value_token_367_end_mask_0 = const()[name = string("value_token_367_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_367_squeeze_mask_0 = const()[name = string("value_token_367_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_367_cast_fp16 = slice_by_index(begin = value_token_367_begin_0, end = value_token_367_end_0, end_mask = value_token_367_end_mask_0, squeeze_mask = value_token_367_squeeze_mask_0, x = value_213_cast_fp16)[name = string("value_token_367_cast_fp16")];
            tensor<int32, [1]> var_9949_axes_0 = const()[name = string("op_9949_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_9949_cast_fp16 = expand_dims(axes = var_9949_axes_0, x = key_token_367_cast_fp16)[name = string("op_9949_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_9950_cast_fp16 = mul(x = state_733_cast_fp16, y = var_9949_cast_fp16)[name = string("op_9950_cast_fp16")];
            tensor<int32, [1]> memory_367_axes_0 = const()[name = string("memory_367_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_367_keep_dims_0 = const()[name = string("memory_367_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_367_cast_fp16 = reduce_sum(axes = memory_367_axes_0, keep_dims = memory_367_keep_dims_0, x = var_9950_cast_fp16)[name = string("memory_367_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_9953_cast_fp16 = sub(x = value_token_367_cast_fp16, y = memory_367_cast_fp16)[name = string("op_9953_cast_fp16")];
            tensor<int32, [3]> var_9956_begin_0 = const()[name = string("op_9956_begin_0"), val = tensor<int32, [3]>([0, 0, 7])];
            tensor<int32, [3]> var_9956_end_0 = const()[name = string("op_9956_end_0"), val = tensor<int32, [3]>([1, 16, 8])];
            tensor<bool, [3]> var_9956_end_mask_0 = const()[name = string("op_9956_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_9956_squeeze_mask_0 = const()[name = string("op_9956_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_9956_cast_fp16 = slice_by_index(begin = var_9956_begin_0, end = var_9956_end_0, end_mask = var_9956_end_mask_0, squeeze_mask = var_9956_squeeze_mask_0, x = beta_23_cast_fp16)[name = string("op_9956_cast_fp16")];
            tensor<int32, [1]> var_9957_axes_0 = const()[name = string("op_9957_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_9957_cast_fp16 = expand_dims(axes = var_9957_axes_0, x = var_9956_cast_fp16)[name = string("op_9957_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_367_cast_fp16 = mul(x = var_9953_cast_fp16, y = var_9957_cast_fp16)[name = string("delta_367_cast_fp16")];
            tensor<int32, [1]> var_9960_axes_0 = const()[name = string("op_9960_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_9960_cast_fp16 = expand_dims(axes = var_9960_axes_0, x = delta_367_cast_fp16)[name = string("op_9960_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_9961_cast_fp16 = mul(x = var_9949_cast_fp16, y = var_9960_cast_fp16)[name = string("op_9961_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_735_cast_fp16 = add(x = state_733_cast_fp16, y = var_9961_cast_fp16)[name = string("state_735_cast_fp16")];
            tensor<int32, [4]> var_9965_begin_0 = const()[name = string("op_9965_begin_0"), val = tensor<int32, [4]>([0, 0, 7, 0])];
            tensor<int32, [4]> var_9965_end_0 = const()[name = string("op_9965_end_0"), val = tensor<int32, [4]>([1, 16, 8, 128])];
            tensor<bool, [4]> var_9965_end_mask_0 = const()[name = string("op_9965_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_9965_squeeze_mask_0 = const()[name = string("op_9965_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_9965_cast_fp16 = slice_by_index(begin = var_9965_begin_0, end = var_9965_end_0, end_mask = var_9965_end_mask_0, squeeze_mask = var_9965_squeeze_mask_0, x = _inversed_query_59_cast_fp16)[name = string("op_9965_cast_fp16")];
            tensor<int32, [1]> var_9966_axes_0 = const()[name = string("op_9966_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_9966_cast_fp16 = expand_dims(axes = var_9966_axes_0, x = var_9965_cast_fp16)[name = string("op_9966_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_9967_cast_fp16 = mul(x = state_735_cast_fp16, y = var_9966_cast_fp16)[name = string("op_9967_cast_fp16")];
            tensor<int32, [1]> var_9969_axes_0 = const()[name = string("op_9969_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_9969_keep_dims_0 = const()[name = string("op_9969_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_9969_cast_fp16 = reduce_sum(axes = var_9969_axes_0, keep_dims = var_9969_keep_dims_0, x = var_9967_cast_fp16)[name = string("op_9969_cast_fp16")];
            tensor<int32, [3]> var_9972_begin_0 = const()[name = string("op_9972_begin_0"), val = tensor<int32, [3]>([0, 0, 8])];
            tensor<int32, [3]> var_9972_end_0 = const()[name = string("op_9972_end_0"), val = tensor<int32, [3]>([1, 16, 9])];
            tensor<bool, [3]> var_9972_end_mask_0 = const()[name = string("op_9972_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_9972_squeeze_mask_0 = const()[name = string("op_9972_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_9972_cast_fp16 = slice_by_index(begin = var_9972_begin_0, end = var_9972_end_0, end_mask = var_9972_end_mask_0, squeeze_mask = var_9972_squeeze_mask_0, x = decay_23_cast_fp16)[name = string("op_9972_cast_fp16")];
            tensor<fp16, [1, 16]> var_9973_cast_fp16 = exp(x = var_9972_cast_fp16)[name = string("op_9973_cast_fp16")];
            tensor<int32, [1]> var_9974_axes_0 = const()[name = string("op_9974_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_9974_cast_fp16 = expand_dims(axes = var_9974_axes_0, x = var_9973_cast_fp16)[name = string("op_9974_cast_fp16")];
            tensor<int32, [1]> var_9975_axes_0 = const()[name = string("op_9975_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_9975_cast_fp16 = expand_dims(axes = var_9975_axes_0, x = var_9974_cast_fp16)[name = string("op_9975_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_737_cast_fp16 = mul(x = state_735_cast_fp16, y = var_9975_cast_fp16)[name = string("state_737_cast_fp16")];
            tensor<int32, [4]> key_token_369_begin_0 = const()[name = string("key_token_369_begin_0"), val = tensor<int32, [4]>([0, 0, 8, 0])];
            tensor<int32, [4]> key_token_369_end_0 = const()[name = string("key_token_369_end_0"), val = tensor<int32, [4]>([1, 16, 9, 128])];
            tensor<bool, [4]> key_token_369_end_mask_0 = const()[name = string("key_token_369_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_369_squeeze_mask_0 = const()[name = string("key_token_369_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_369_cast_fp16 = slice_by_index(begin = key_token_369_begin_0, end = key_token_369_end_0, end_mask = key_token_369_end_mask_0, squeeze_mask = key_token_369_squeeze_mask_0, x = key_59_cast_fp16)[name = string("key_token_369_cast_fp16")];
            tensor<int32, [4]> value_token_369_begin_0 = const()[name = string("value_token_369_begin_0"), val = tensor<int32, [4]>([0, 0, 8, 0])];
            tensor<int32, [4]> value_token_369_end_0 = const()[name = string("value_token_369_end_0"), val = tensor<int32, [4]>([1, 16, 9, 128])];
            tensor<bool, [4]> value_token_369_end_mask_0 = const()[name = string("value_token_369_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_369_squeeze_mask_0 = const()[name = string("value_token_369_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_369_cast_fp16 = slice_by_index(begin = value_token_369_begin_0, end = value_token_369_end_0, end_mask = value_token_369_end_mask_0, squeeze_mask = value_token_369_squeeze_mask_0, x = value_213_cast_fp16)[name = string("value_token_369_cast_fp16")];
            tensor<int32, [1]> var_9983_axes_0 = const()[name = string("op_9983_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_9983_cast_fp16 = expand_dims(axes = var_9983_axes_0, x = key_token_369_cast_fp16)[name = string("op_9983_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_9984_cast_fp16 = mul(x = state_737_cast_fp16, y = var_9983_cast_fp16)[name = string("op_9984_cast_fp16")];
            tensor<int32, [1]> memory_369_axes_0 = const()[name = string("memory_369_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_369_keep_dims_0 = const()[name = string("memory_369_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_369_cast_fp16 = reduce_sum(axes = memory_369_axes_0, keep_dims = memory_369_keep_dims_0, x = var_9984_cast_fp16)[name = string("memory_369_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_9987_cast_fp16 = sub(x = value_token_369_cast_fp16, y = memory_369_cast_fp16)[name = string("op_9987_cast_fp16")];
            tensor<int32, [3]> var_9990_begin_0 = const()[name = string("op_9990_begin_0"), val = tensor<int32, [3]>([0, 0, 8])];
            tensor<int32, [3]> var_9990_end_0 = const()[name = string("op_9990_end_0"), val = tensor<int32, [3]>([1, 16, 9])];
            tensor<bool, [3]> var_9990_end_mask_0 = const()[name = string("op_9990_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_9990_squeeze_mask_0 = const()[name = string("op_9990_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_9990_cast_fp16 = slice_by_index(begin = var_9990_begin_0, end = var_9990_end_0, end_mask = var_9990_end_mask_0, squeeze_mask = var_9990_squeeze_mask_0, x = beta_23_cast_fp16)[name = string("op_9990_cast_fp16")];
            tensor<int32, [1]> var_9991_axes_0 = const()[name = string("op_9991_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_9991_cast_fp16 = expand_dims(axes = var_9991_axes_0, x = var_9990_cast_fp16)[name = string("op_9991_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_369_cast_fp16 = mul(x = var_9987_cast_fp16, y = var_9991_cast_fp16)[name = string("delta_369_cast_fp16")];
            tensor<int32, [1]> var_9994_axes_0 = const()[name = string("op_9994_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_9994_cast_fp16 = expand_dims(axes = var_9994_axes_0, x = delta_369_cast_fp16)[name = string("op_9994_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_9995_cast_fp16 = mul(x = var_9983_cast_fp16, y = var_9994_cast_fp16)[name = string("op_9995_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_739_cast_fp16 = add(x = state_737_cast_fp16, y = var_9995_cast_fp16)[name = string("state_739_cast_fp16")];
            tensor<int32, [4]> var_9999_begin_0 = const()[name = string("op_9999_begin_0"), val = tensor<int32, [4]>([0, 0, 8, 0])];
            tensor<int32, [4]> var_9999_end_0 = const()[name = string("op_9999_end_0"), val = tensor<int32, [4]>([1, 16, 9, 128])];
            tensor<bool, [4]> var_9999_end_mask_0 = const()[name = string("op_9999_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_9999_squeeze_mask_0 = const()[name = string("op_9999_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_9999_cast_fp16 = slice_by_index(begin = var_9999_begin_0, end = var_9999_end_0, end_mask = var_9999_end_mask_0, squeeze_mask = var_9999_squeeze_mask_0, x = _inversed_query_59_cast_fp16)[name = string("op_9999_cast_fp16")];
            tensor<int32, [1]> var_10000_axes_0 = const()[name = string("op_10000_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_10000_cast_fp16 = expand_dims(axes = var_10000_axes_0, x = var_9999_cast_fp16)[name = string("op_10000_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_10001_cast_fp16 = mul(x = state_739_cast_fp16, y = var_10000_cast_fp16)[name = string("op_10001_cast_fp16")];
            tensor<int32, [1]> var_10003_axes_0 = const()[name = string("op_10003_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_10003_keep_dims_0 = const()[name = string("op_10003_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_10003_cast_fp16 = reduce_sum(axes = var_10003_axes_0, keep_dims = var_10003_keep_dims_0, x = var_10001_cast_fp16)[name = string("op_10003_cast_fp16")];
            tensor<int32, [3]> var_10006_begin_0 = const()[name = string("op_10006_begin_0"), val = tensor<int32, [3]>([0, 0, 9])];
            tensor<int32, [3]> var_10006_end_0 = const()[name = string("op_10006_end_0"), val = tensor<int32, [3]>([1, 16, 10])];
            tensor<bool, [3]> var_10006_end_mask_0 = const()[name = string("op_10006_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_10006_squeeze_mask_0 = const()[name = string("op_10006_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_10006_cast_fp16 = slice_by_index(begin = var_10006_begin_0, end = var_10006_end_0, end_mask = var_10006_end_mask_0, squeeze_mask = var_10006_squeeze_mask_0, x = decay_23_cast_fp16)[name = string("op_10006_cast_fp16")];
            tensor<fp16, [1, 16]> var_10007_cast_fp16 = exp(x = var_10006_cast_fp16)[name = string("op_10007_cast_fp16")];
            tensor<int32, [1]> var_10008_axes_0 = const()[name = string("op_10008_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_10008_cast_fp16 = expand_dims(axes = var_10008_axes_0, x = var_10007_cast_fp16)[name = string("op_10008_cast_fp16")];
            tensor<int32, [1]> var_10009_axes_0 = const()[name = string("op_10009_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_10009_cast_fp16 = expand_dims(axes = var_10009_axes_0, x = var_10008_cast_fp16)[name = string("op_10009_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_741_cast_fp16 = mul(x = state_739_cast_fp16, y = var_10009_cast_fp16)[name = string("state_741_cast_fp16")];
            tensor<int32, [4]> key_token_371_begin_0 = const()[name = string("key_token_371_begin_0"), val = tensor<int32, [4]>([0, 0, 9, 0])];
            tensor<int32, [4]> key_token_371_end_0 = const()[name = string("key_token_371_end_0"), val = tensor<int32, [4]>([1, 16, 10, 128])];
            tensor<bool, [4]> key_token_371_end_mask_0 = const()[name = string("key_token_371_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_371_squeeze_mask_0 = const()[name = string("key_token_371_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_371_cast_fp16 = slice_by_index(begin = key_token_371_begin_0, end = key_token_371_end_0, end_mask = key_token_371_end_mask_0, squeeze_mask = key_token_371_squeeze_mask_0, x = key_59_cast_fp16)[name = string("key_token_371_cast_fp16")];
            tensor<int32, [4]> value_token_371_begin_0 = const()[name = string("value_token_371_begin_0"), val = tensor<int32, [4]>([0, 0, 9, 0])];
            tensor<int32, [4]> value_token_371_end_0 = const()[name = string("value_token_371_end_0"), val = tensor<int32, [4]>([1, 16, 10, 128])];
            tensor<bool, [4]> value_token_371_end_mask_0 = const()[name = string("value_token_371_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_371_squeeze_mask_0 = const()[name = string("value_token_371_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_371_cast_fp16 = slice_by_index(begin = value_token_371_begin_0, end = value_token_371_end_0, end_mask = value_token_371_end_mask_0, squeeze_mask = value_token_371_squeeze_mask_0, x = value_213_cast_fp16)[name = string("value_token_371_cast_fp16")];
            tensor<int32, [1]> var_10017_axes_0 = const()[name = string("op_10017_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_10017_cast_fp16 = expand_dims(axes = var_10017_axes_0, x = key_token_371_cast_fp16)[name = string("op_10017_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_10018_cast_fp16 = mul(x = state_741_cast_fp16, y = var_10017_cast_fp16)[name = string("op_10018_cast_fp16")];
            tensor<int32, [1]> memory_371_axes_0 = const()[name = string("memory_371_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_371_keep_dims_0 = const()[name = string("memory_371_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_371_cast_fp16 = reduce_sum(axes = memory_371_axes_0, keep_dims = memory_371_keep_dims_0, x = var_10018_cast_fp16)[name = string("memory_371_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_10021_cast_fp16 = sub(x = value_token_371_cast_fp16, y = memory_371_cast_fp16)[name = string("op_10021_cast_fp16")];
            tensor<int32, [3]> var_10024_begin_0 = const()[name = string("op_10024_begin_0"), val = tensor<int32, [3]>([0, 0, 9])];
            tensor<int32, [3]> var_10024_end_0 = const()[name = string("op_10024_end_0"), val = tensor<int32, [3]>([1, 16, 10])];
            tensor<bool, [3]> var_10024_end_mask_0 = const()[name = string("op_10024_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_10024_squeeze_mask_0 = const()[name = string("op_10024_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_10024_cast_fp16 = slice_by_index(begin = var_10024_begin_0, end = var_10024_end_0, end_mask = var_10024_end_mask_0, squeeze_mask = var_10024_squeeze_mask_0, x = beta_23_cast_fp16)[name = string("op_10024_cast_fp16")];
            tensor<int32, [1]> var_10025_axes_0 = const()[name = string("op_10025_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_10025_cast_fp16 = expand_dims(axes = var_10025_axes_0, x = var_10024_cast_fp16)[name = string("op_10025_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_371_cast_fp16 = mul(x = var_10021_cast_fp16, y = var_10025_cast_fp16)[name = string("delta_371_cast_fp16")];
            tensor<int32, [1]> var_10028_axes_0 = const()[name = string("op_10028_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_10028_cast_fp16 = expand_dims(axes = var_10028_axes_0, x = delta_371_cast_fp16)[name = string("op_10028_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_10029_cast_fp16 = mul(x = var_10017_cast_fp16, y = var_10028_cast_fp16)[name = string("op_10029_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_743_cast_fp16 = add(x = state_741_cast_fp16, y = var_10029_cast_fp16)[name = string("state_743_cast_fp16")];
            tensor<int32, [4]> var_10033_begin_0 = const()[name = string("op_10033_begin_0"), val = tensor<int32, [4]>([0, 0, 9, 0])];
            tensor<int32, [4]> var_10033_end_0 = const()[name = string("op_10033_end_0"), val = tensor<int32, [4]>([1, 16, 10, 128])];
            tensor<bool, [4]> var_10033_end_mask_0 = const()[name = string("op_10033_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_10033_squeeze_mask_0 = const()[name = string("op_10033_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_10033_cast_fp16 = slice_by_index(begin = var_10033_begin_0, end = var_10033_end_0, end_mask = var_10033_end_mask_0, squeeze_mask = var_10033_squeeze_mask_0, x = _inversed_query_59_cast_fp16)[name = string("op_10033_cast_fp16")];
            tensor<int32, [1]> var_10034_axes_0 = const()[name = string("op_10034_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_10034_cast_fp16 = expand_dims(axes = var_10034_axes_0, x = var_10033_cast_fp16)[name = string("op_10034_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_10035_cast_fp16 = mul(x = state_743_cast_fp16, y = var_10034_cast_fp16)[name = string("op_10035_cast_fp16")];
            tensor<int32, [1]> var_10037_axes_0 = const()[name = string("op_10037_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_10037_keep_dims_0 = const()[name = string("op_10037_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_10037_cast_fp16 = reduce_sum(axes = var_10037_axes_0, keep_dims = var_10037_keep_dims_0, x = var_10035_cast_fp16)[name = string("op_10037_cast_fp16")];
            tensor<int32, [3]> var_10040_begin_0 = const()[name = string("op_10040_begin_0"), val = tensor<int32, [3]>([0, 0, 10])];
            tensor<int32, [3]> var_10040_end_0 = const()[name = string("op_10040_end_0"), val = tensor<int32, [3]>([1, 16, 11])];
            tensor<bool, [3]> var_10040_end_mask_0 = const()[name = string("op_10040_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_10040_squeeze_mask_0 = const()[name = string("op_10040_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_10040_cast_fp16 = slice_by_index(begin = var_10040_begin_0, end = var_10040_end_0, end_mask = var_10040_end_mask_0, squeeze_mask = var_10040_squeeze_mask_0, x = decay_23_cast_fp16)[name = string("op_10040_cast_fp16")];
            tensor<fp16, [1, 16]> var_10041_cast_fp16 = exp(x = var_10040_cast_fp16)[name = string("op_10041_cast_fp16")];
            tensor<int32, [1]> var_10042_axes_0 = const()[name = string("op_10042_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_10042_cast_fp16 = expand_dims(axes = var_10042_axes_0, x = var_10041_cast_fp16)[name = string("op_10042_cast_fp16")];
            tensor<int32, [1]> var_10043_axes_0 = const()[name = string("op_10043_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_10043_cast_fp16 = expand_dims(axes = var_10043_axes_0, x = var_10042_cast_fp16)[name = string("op_10043_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_745_cast_fp16 = mul(x = state_743_cast_fp16, y = var_10043_cast_fp16)[name = string("state_745_cast_fp16")];
            tensor<int32, [4]> key_token_373_begin_0 = const()[name = string("key_token_373_begin_0"), val = tensor<int32, [4]>([0, 0, 10, 0])];
            tensor<int32, [4]> key_token_373_end_0 = const()[name = string("key_token_373_end_0"), val = tensor<int32, [4]>([1, 16, 11, 128])];
            tensor<bool, [4]> key_token_373_end_mask_0 = const()[name = string("key_token_373_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_373_squeeze_mask_0 = const()[name = string("key_token_373_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_373_cast_fp16 = slice_by_index(begin = key_token_373_begin_0, end = key_token_373_end_0, end_mask = key_token_373_end_mask_0, squeeze_mask = key_token_373_squeeze_mask_0, x = key_59_cast_fp16)[name = string("key_token_373_cast_fp16")];
            tensor<int32, [4]> value_token_373_begin_0 = const()[name = string("value_token_373_begin_0"), val = tensor<int32, [4]>([0, 0, 10, 0])];
            tensor<int32, [4]> value_token_373_end_0 = const()[name = string("value_token_373_end_0"), val = tensor<int32, [4]>([1, 16, 11, 128])];
            tensor<bool, [4]> value_token_373_end_mask_0 = const()[name = string("value_token_373_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_373_squeeze_mask_0 = const()[name = string("value_token_373_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_373_cast_fp16 = slice_by_index(begin = value_token_373_begin_0, end = value_token_373_end_0, end_mask = value_token_373_end_mask_0, squeeze_mask = value_token_373_squeeze_mask_0, x = value_213_cast_fp16)[name = string("value_token_373_cast_fp16")];
            tensor<int32, [1]> var_10051_axes_0 = const()[name = string("op_10051_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_10051_cast_fp16 = expand_dims(axes = var_10051_axes_0, x = key_token_373_cast_fp16)[name = string("op_10051_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_10052_cast_fp16 = mul(x = state_745_cast_fp16, y = var_10051_cast_fp16)[name = string("op_10052_cast_fp16")];
            tensor<int32, [1]> memory_373_axes_0 = const()[name = string("memory_373_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_373_keep_dims_0 = const()[name = string("memory_373_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_373_cast_fp16 = reduce_sum(axes = memory_373_axes_0, keep_dims = memory_373_keep_dims_0, x = var_10052_cast_fp16)[name = string("memory_373_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_10055_cast_fp16 = sub(x = value_token_373_cast_fp16, y = memory_373_cast_fp16)[name = string("op_10055_cast_fp16")];
            tensor<int32, [3]> var_10058_begin_0 = const()[name = string("op_10058_begin_0"), val = tensor<int32, [3]>([0, 0, 10])];
            tensor<int32, [3]> var_10058_end_0 = const()[name = string("op_10058_end_0"), val = tensor<int32, [3]>([1, 16, 11])];
            tensor<bool, [3]> var_10058_end_mask_0 = const()[name = string("op_10058_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_10058_squeeze_mask_0 = const()[name = string("op_10058_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_10058_cast_fp16 = slice_by_index(begin = var_10058_begin_0, end = var_10058_end_0, end_mask = var_10058_end_mask_0, squeeze_mask = var_10058_squeeze_mask_0, x = beta_23_cast_fp16)[name = string("op_10058_cast_fp16")];
            tensor<int32, [1]> var_10059_axes_0 = const()[name = string("op_10059_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_10059_cast_fp16 = expand_dims(axes = var_10059_axes_0, x = var_10058_cast_fp16)[name = string("op_10059_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_373_cast_fp16 = mul(x = var_10055_cast_fp16, y = var_10059_cast_fp16)[name = string("delta_373_cast_fp16")];
            tensor<int32, [1]> var_10062_axes_0 = const()[name = string("op_10062_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_10062_cast_fp16 = expand_dims(axes = var_10062_axes_0, x = delta_373_cast_fp16)[name = string("op_10062_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_10063_cast_fp16 = mul(x = var_10051_cast_fp16, y = var_10062_cast_fp16)[name = string("op_10063_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_747_cast_fp16 = add(x = state_745_cast_fp16, y = var_10063_cast_fp16)[name = string("state_747_cast_fp16")];
            tensor<int32, [4]> var_10067_begin_0 = const()[name = string("op_10067_begin_0"), val = tensor<int32, [4]>([0, 0, 10, 0])];
            tensor<int32, [4]> var_10067_end_0 = const()[name = string("op_10067_end_0"), val = tensor<int32, [4]>([1, 16, 11, 128])];
            tensor<bool, [4]> var_10067_end_mask_0 = const()[name = string("op_10067_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_10067_squeeze_mask_0 = const()[name = string("op_10067_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_10067_cast_fp16 = slice_by_index(begin = var_10067_begin_0, end = var_10067_end_0, end_mask = var_10067_end_mask_0, squeeze_mask = var_10067_squeeze_mask_0, x = _inversed_query_59_cast_fp16)[name = string("op_10067_cast_fp16")];
            tensor<int32, [1]> var_10068_axes_0 = const()[name = string("op_10068_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_10068_cast_fp16 = expand_dims(axes = var_10068_axes_0, x = var_10067_cast_fp16)[name = string("op_10068_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_10069_cast_fp16 = mul(x = state_747_cast_fp16, y = var_10068_cast_fp16)[name = string("op_10069_cast_fp16")];
            tensor<int32, [1]> var_10071_axes_0 = const()[name = string("op_10071_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_10071_keep_dims_0 = const()[name = string("op_10071_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_10071_cast_fp16 = reduce_sum(axes = var_10071_axes_0, keep_dims = var_10071_keep_dims_0, x = var_10069_cast_fp16)[name = string("op_10071_cast_fp16")];
            tensor<int32, [3]> var_10074_begin_0 = const()[name = string("op_10074_begin_0"), val = tensor<int32, [3]>([0, 0, 11])];
            tensor<int32, [3]> var_10074_end_0 = const()[name = string("op_10074_end_0"), val = tensor<int32, [3]>([1, 16, 12])];
            tensor<bool, [3]> var_10074_end_mask_0 = const()[name = string("op_10074_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_10074_squeeze_mask_0 = const()[name = string("op_10074_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_10074_cast_fp16 = slice_by_index(begin = var_10074_begin_0, end = var_10074_end_0, end_mask = var_10074_end_mask_0, squeeze_mask = var_10074_squeeze_mask_0, x = decay_23_cast_fp16)[name = string("op_10074_cast_fp16")];
            tensor<fp16, [1, 16]> var_10075_cast_fp16 = exp(x = var_10074_cast_fp16)[name = string("op_10075_cast_fp16")];
            tensor<int32, [1]> var_10076_axes_0 = const()[name = string("op_10076_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_10076_cast_fp16 = expand_dims(axes = var_10076_axes_0, x = var_10075_cast_fp16)[name = string("op_10076_cast_fp16")];
            tensor<int32, [1]> var_10077_axes_0 = const()[name = string("op_10077_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_10077_cast_fp16 = expand_dims(axes = var_10077_axes_0, x = var_10076_cast_fp16)[name = string("op_10077_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_749_cast_fp16 = mul(x = state_747_cast_fp16, y = var_10077_cast_fp16)[name = string("state_749_cast_fp16")];
            tensor<int32, [4]> key_token_375_begin_0 = const()[name = string("key_token_375_begin_0"), val = tensor<int32, [4]>([0, 0, 11, 0])];
            tensor<int32, [4]> key_token_375_end_0 = const()[name = string("key_token_375_end_0"), val = tensor<int32, [4]>([1, 16, 12, 128])];
            tensor<bool, [4]> key_token_375_end_mask_0 = const()[name = string("key_token_375_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_375_squeeze_mask_0 = const()[name = string("key_token_375_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_375_cast_fp16 = slice_by_index(begin = key_token_375_begin_0, end = key_token_375_end_0, end_mask = key_token_375_end_mask_0, squeeze_mask = key_token_375_squeeze_mask_0, x = key_59_cast_fp16)[name = string("key_token_375_cast_fp16")];
            tensor<int32, [4]> value_token_375_begin_0 = const()[name = string("value_token_375_begin_0"), val = tensor<int32, [4]>([0, 0, 11, 0])];
            tensor<int32, [4]> value_token_375_end_0 = const()[name = string("value_token_375_end_0"), val = tensor<int32, [4]>([1, 16, 12, 128])];
            tensor<bool, [4]> value_token_375_end_mask_0 = const()[name = string("value_token_375_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_375_squeeze_mask_0 = const()[name = string("value_token_375_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_375_cast_fp16 = slice_by_index(begin = value_token_375_begin_0, end = value_token_375_end_0, end_mask = value_token_375_end_mask_0, squeeze_mask = value_token_375_squeeze_mask_0, x = value_213_cast_fp16)[name = string("value_token_375_cast_fp16")];
            tensor<int32, [1]> var_10085_axes_0 = const()[name = string("op_10085_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_10085_cast_fp16 = expand_dims(axes = var_10085_axes_0, x = key_token_375_cast_fp16)[name = string("op_10085_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_10086_cast_fp16 = mul(x = state_749_cast_fp16, y = var_10085_cast_fp16)[name = string("op_10086_cast_fp16")];
            tensor<int32, [1]> memory_375_axes_0 = const()[name = string("memory_375_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_375_keep_dims_0 = const()[name = string("memory_375_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_375_cast_fp16 = reduce_sum(axes = memory_375_axes_0, keep_dims = memory_375_keep_dims_0, x = var_10086_cast_fp16)[name = string("memory_375_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_10089_cast_fp16 = sub(x = value_token_375_cast_fp16, y = memory_375_cast_fp16)[name = string("op_10089_cast_fp16")];
            tensor<int32, [3]> var_10092_begin_0 = const()[name = string("op_10092_begin_0"), val = tensor<int32, [3]>([0, 0, 11])];
            tensor<int32, [3]> var_10092_end_0 = const()[name = string("op_10092_end_0"), val = tensor<int32, [3]>([1, 16, 12])];
            tensor<bool, [3]> var_10092_end_mask_0 = const()[name = string("op_10092_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_10092_squeeze_mask_0 = const()[name = string("op_10092_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_10092_cast_fp16 = slice_by_index(begin = var_10092_begin_0, end = var_10092_end_0, end_mask = var_10092_end_mask_0, squeeze_mask = var_10092_squeeze_mask_0, x = beta_23_cast_fp16)[name = string("op_10092_cast_fp16")];
            tensor<int32, [1]> var_10093_axes_0 = const()[name = string("op_10093_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_10093_cast_fp16 = expand_dims(axes = var_10093_axes_0, x = var_10092_cast_fp16)[name = string("op_10093_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_375_cast_fp16 = mul(x = var_10089_cast_fp16, y = var_10093_cast_fp16)[name = string("delta_375_cast_fp16")];
            tensor<int32, [1]> var_10096_axes_0 = const()[name = string("op_10096_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_10096_cast_fp16 = expand_dims(axes = var_10096_axes_0, x = delta_375_cast_fp16)[name = string("op_10096_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_10097_cast_fp16 = mul(x = var_10085_cast_fp16, y = var_10096_cast_fp16)[name = string("op_10097_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_751_cast_fp16 = add(x = state_749_cast_fp16, y = var_10097_cast_fp16)[name = string("state_751_cast_fp16")];
            tensor<int32, [4]> var_10101_begin_0 = const()[name = string("op_10101_begin_0"), val = tensor<int32, [4]>([0, 0, 11, 0])];
            tensor<int32, [4]> var_10101_end_0 = const()[name = string("op_10101_end_0"), val = tensor<int32, [4]>([1, 16, 12, 128])];
            tensor<bool, [4]> var_10101_end_mask_0 = const()[name = string("op_10101_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_10101_squeeze_mask_0 = const()[name = string("op_10101_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_10101_cast_fp16 = slice_by_index(begin = var_10101_begin_0, end = var_10101_end_0, end_mask = var_10101_end_mask_0, squeeze_mask = var_10101_squeeze_mask_0, x = _inversed_query_59_cast_fp16)[name = string("op_10101_cast_fp16")];
            tensor<int32, [1]> var_10102_axes_0 = const()[name = string("op_10102_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_10102_cast_fp16 = expand_dims(axes = var_10102_axes_0, x = var_10101_cast_fp16)[name = string("op_10102_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_10103_cast_fp16 = mul(x = state_751_cast_fp16, y = var_10102_cast_fp16)[name = string("op_10103_cast_fp16")];
            tensor<int32, [1]> var_10105_axes_0 = const()[name = string("op_10105_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_10105_keep_dims_0 = const()[name = string("op_10105_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_10105_cast_fp16 = reduce_sum(axes = var_10105_axes_0, keep_dims = var_10105_keep_dims_0, x = var_10103_cast_fp16)[name = string("op_10105_cast_fp16")];
            tensor<int32, [3]> var_10108_begin_0 = const()[name = string("op_10108_begin_0"), val = tensor<int32, [3]>([0, 0, 12])];
            tensor<int32, [3]> var_10108_end_0 = const()[name = string("op_10108_end_0"), val = tensor<int32, [3]>([1, 16, 13])];
            tensor<bool, [3]> var_10108_end_mask_0 = const()[name = string("op_10108_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_10108_squeeze_mask_0 = const()[name = string("op_10108_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_10108_cast_fp16 = slice_by_index(begin = var_10108_begin_0, end = var_10108_end_0, end_mask = var_10108_end_mask_0, squeeze_mask = var_10108_squeeze_mask_0, x = decay_23_cast_fp16)[name = string("op_10108_cast_fp16")];
            tensor<fp16, [1, 16]> var_10109_cast_fp16 = exp(x = var_10108_cast_fp16)[name = string("op_10109_cast_fp16")];
            tensor<int32, [1]> var_10110_axes_0 = const()[name = string("op_10110_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_10110_cast_fp16 = expand_dims(axes = var_10110_axes_0, x = var_10109_cast_fp16)[name = string("op_10110_cast_fp16")];
            tensor<int32, [1]> var_10111_axes_0 = const()[name = string("op_10111_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_10111_cast_fp16 = expand_dims(axes = var_10111_axes_0, x = var_10110_cast_fp16)[name = string("op_10111_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_753_cast_fp16 = mul(x = state_751_cast_fp16, y = var_10111_cast_fp16)[name = string("state_753_cast_fp16")];
            tensor<int32, [4]> key_token_377_begin_0 = const()[name = string("key_token_377_begin_0"), val = tensor<int32, [4]>([0, 0, 12, 0])];
            tensor<int32, [4]> key_token_377_end_0 = const()[name = string("key_token_377_end_0"), val = tensor<int32, [4]>([1, 16, 13, 128])];
            tensor<bool, [4]> key_token_377_end_mask_0 = const()[name = string("key_token_377_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_377_squeeze_mask_0 = const()[name = string("key_token_377_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_377_cast_fp16 = slice_by_index(begin = key_token_377_begin_0, end = key_token_377_end_0, end_mask = key_token_377_end_mask_0, squeeze_mask = key_token_377_squeeze_mask_0, x = key_59_cast_fp16)[name = string("key_token_377_cast_fp16")];
            tensor<int32, [4]> value_token_377_begin_0 = const()[name = string("value_token_377_begin_0"), val = tensor<int32, [4]>([0, 0, 12, 0])];
            tensor<int32, [4]> value_token_377_end_0 = const()[name = string("value_token_377_end_0"), val = tensor<int32, [4]>([1, 16, 13, 128])];
            tensor<bool, [4]> value_token_377_end_mask_0 = const()[name = string("value_token_377_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_377_squeeze_mask_0 = const()[name = string("value_token_377_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_377_cast_fp16 = slice_by_index(begin = value_token_377_begin_0, end = value_token_377_end_0, end_mask = value_token_377_end_mask_0, squeeze_mask = value_token_377_squeeze_mask_0, x = value_213_cast_fp16)[name = string("value_token_377_cast_fp16")];
            tensor<int32, [1]> var_10119_axes_0 = const()[name = string("op_10119_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_10119_cast_fp16 = expand_dims(axes = var_10119_axes_0, x = key_token_377_cast_fp16)[name = string("op_10119_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_10120_cast_fp16 = mul(x = state_753_cast_fp16, y = var_10119_cast_fp16)[name = string("op_10120_cast_fp16")];
            tensor<int32, [1]> memory_377_axes_0 = const()[name = string("memory_377_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_377_keep_dims_0 = const()[name = string("memory_377_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_377_cast_fp16 = reduce_sum(axes = memory_377_axes_0, keep_dims = memory_377_keep_dims_0, x = var_10120_cast_fp16)[name = string("memory_377_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_10123_cast_fp16 = sub(x = value_token_377_cast_fp16, y = memory_377_cast_fp16)[name = string("op_10123_cast_fp16")];
            tensor<int32, [3]> var_10126_begin_0 = const()[name = string("op_10126_begin_0"), val = tensor<int32, [3]>([0, 0, 12])];
            tensor<int32, [3]> var_10126_end_0 = const()[name = string("op_10126_end_0"), val = tensor<int32, [3]>([1, 16, 13])];
            tensor<bool, [3]> var_10126_end_mask_0 = const()[name = string("op_10126_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_10126_squeeze_mask_0 = const()[name = string("op_10126_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_10126_cast_fp16 = slice_by_index(begin = var_10126_begin_0, end = var_10126_end_0, end_mask = var_10126_end_mask_0, squeeze_mask = var_10126_squeeze_mask_0, x = beta_23_cast_fp16)[name = string("op_10126_cast_fp16")];
            tensor<int32, [1]> var_10127_axes_0 = const()[name = string("op_10127_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_10127_cast_fp16 = expand_dims(axes = var_10127_axes_0, x = var_10126_cast_fp16)[name = string("op_10127_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_377_cast_fp16 = mul(x = var_10123_cast_fp16, y = var_10127_cast_fp16)[name = string("delta_377_cast_fp16")];
            tensor<int32, [1]> var_10130_axes_0 = const()[name = string("op_10130_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_10130_cast_fp16 = expand_dims(axes = var_10130_axes_0, x = delta_377_cast_fp16)[name = string("op_10130_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_10131_cast_fp16 = mul(x = var_10119_cast_fp16, y = var_10130_cast_fp16)[name = string("op_10131_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_755_cast_fp16 = add(x = state_753_cast_fp16, y = var_10131_cast_fp16)[name = string("state_755_cast_fp16")];
            tensor<int32, [4]> var_10135_begin_0 = const()[name = string("op_10135_begin_0"), val = tensor<int32, [4]>([0, 0, 12, 0])];
            tensor<int32, [4]> var_10135_end_0 = const()[name = string("op_10135_end_0"), val = tensor<int32, [4]>([1, 16, 13, 128])];
            tensor<bool, [4]> var_10135_end_mask_0 = const()[name = string("op_10135_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_10135_squeeze_mask_0 = const()[name = string("op_10135_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_10135_cast_fp16 = slice_by_index(begin = var_10135_begin_0, end = var_10135_end_0, end_mask = var_10135_end_mask_0, squeeze_mask = var_10135_squeeze_mask_0, x = _inversed_query_59_cast_fp16)[name = string("op_10135_cast_fp16")];
            tensor<int32, [1]> var_10136_axes_0 = const()[name = string("op_10136_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_10136_cast_fp16 = expand_dims(axes = var_10136_axes_0, x = var_10135_cast_fp16)[name = string("op_10136_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_10137_cast_fp16 = mul(x = state_755_cast_fp16, y = var_10136_cast_fp16)[name = string("op_10137_cast_fp16")];
            tensor<int32, [1]> var_10139_axes_0 = const()[name = string("op_10139_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_10139_keep_dims_0 = const()[name = string("op_10139_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_10139_cast_fp16 = reduce_sum(axes = var_10139_axes_0, keep_dims = var_10139_keep_dims_0, x = var_10137_cast_fp16)[name = string("op_10139_cast_fp16")];
            tensor<int32, [3]> var_10142_begin_0 = const()[name = string("op_10142_begin_0"), val = tensor<int32, [3]>([0, 0, 13])];
            tensor<int32, [3]> var_10142_end_0 = const()[name = string("op_10142_end_0"), val = tensor<int32, [3]>([1, 16, 14])];
            tensor<bool, [3]> var_10142_end_mask_0 = const()[name = string("op_10142_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_10142_squeeze_mask_0 = const()[name = string("op_10142_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_10142_cast_fp16 = slice_by_index(begin = var_10142_begin_0, end = var_10142_end_0, end_mask = var_10142_end_mask_0, squeeze_mask = var_10142_squeeze_mask_0, x = decay_23_cast_fp16)[name = string("op_10142_cast_fp16")];
            tensor<fp16, [1, 16]> var_10143_cast_fp16 = exp(x = var_10142_cast_fp16)[name = string("op_10143_cast_fp16")];
            tensor<int32, [1]> var_10144_axes_0 = const()[name = string("op_10144_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_10144_cast_fp16 = expand_dims(axes = var_10144_axes_0, x = var_10143_cast_fp16)[name = string("op_10144_cast_fp16")];
            tensor<int32, [1]> var_10145_axes_0 = const()[name = string("op_10145_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_10145_cast_fp16 = expand_dims(axes = var_10145_axes_0, x = var_10144_cast_fp16)[name = string("op_10145_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_757_cast_fp16 = mul(x = state_755_cast_fp16, y = var_10145_cast_fp16)[name = string("state_757_cast_fp16")];
            tensor<int32, [4]> key_token_379_begin_0 = const()[name = string("key_token_379_begin_0"), val = tensor<int32, [4]>([0, 0, 13, 0])];
            tensor<int32, [4]> key_token_379_end_0 = const()[name = string("key_token_379_end_0"), val = tensor<int32, [4]>([1, 16, 14, 128])];
            tensor<bool, [4]> key_token_379_end_mask_0 = const()[name = string("key_token_379_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_379_squeeze_mask_0 = const()[name = string("key_token_379_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_379_cast_fp16 = slice_by_index(begin = key_token_379_begin_0, end = key_token_379_end_0, end_mask = key_token_379_end_mask_0, squeeze_mask = key_token_379_squeeze_mask_0, x = key_59_cast_fp16)[name = string("key_token_379_cast_fp16")];
            tensor<int32, [4]> value_token_379_begin_0 = const()[name = string("value_token_379_begin_0"), val = tensor<int32, [4]>([0, 0, 13, 0])];
            tensor<int32, [4]> value_token_379_end_0 = const()[name = string("value_token_379_end_0"), val = tensor<int32, [4]>([1, 16, 14, 128])];
            tensor<bool, [4]> value_token_379_end_mask_0 = const()[name = string("value_token_379_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_379_squeeze_mask_0 = const()[name = string("value_token_379_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_379_cast_fp16 = slice_by_index(begin = value_token_379_begin_0, end = value_token_379_end_0, end_mask = value_token_379_end_mask_0, squeeze_mask = value_token_379_squeeze_mask_0, x = value_213_cast_fp16)[name = string("value_token_379_cast_fp16")];
            tensor<int32, [1]> var_10153_axes_0 = const()[name = string("op_10153_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_10153_cast_fp16 = expand_dims(axes = var_10153_axes_0, x = key_token_379_cast_fp16)[name = string("op_10153_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_10154_cast_fp16 = mul(x = state_757_cast_fp16, y = var_10153_cast_fp16)[name = string("op_10154_cast_fp16")];
            tensor<int32, [1]> memory_379_axes_0 = const()[name = string("memory_379_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_379_keep_dims_0 = const()[name = string("memory_379_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_379_cast_fp16 = reduce_sum(axes = memory_379_axes_0, keep_dims = memory_379_keep_dims_0, x = var_10154_cast_fp16)[name = string("memory_379_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_10157_cast_fp16 = sub(x = value_token_379_cast_fp16, y = memory_379_cast_fp16)[name = string("op_10157_cast_fp16")];
            tensor<int32, [3]> var_10160_begin_0 = const()[name = string("op_10160_begin_0"), val = tensor<int32, [3]>([0, 0, 13])];
            tensor<int32, [3]> var_10160_end_0 = const()[name = string("op_10160_end_0"), val = tensor<int32, [3]>([1, 16, 14])];
            tensor<bool, [3]> var_10160_end_mask_0 = const()[name = string("op_10160_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_10160_squeeze_mask_0 = const()[name = string("op_10160_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_10160_cast_fp16 = slice_by_index(begin = var_10160_begin_0, end = var_10160_end_0, end_mask = var_10160_end_mask_0, squeeze_mask = var_10160_squeeze_mask_0, x = beta_23_cast_fp16)[name = string("op_10160_cast_fp16")];
            tensor<int32, [1]> var_10161_axes_0 = const()[name = string("op_10161_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_10161_cast_fp16 = expand_dims(axes = var_10161_axes_0, x = var_10160_cast_fp16)[name = string("op_10161_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_379_cast_fp16 = mul(x = var_10157_cast_fp16, y = var_10161_cast_fp16)[name = string("delta_379_cast_fp16")];
            tensor<int32, [1]> var_10164_axes_0 = const()[name = string("op_10164_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_10164_cast_fp16 = expand_dims(axes = var_10164_axes_0, x = delta_379_cast_fp16)[name = string("op_10164_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_10165_cast_fp16 = mul(x = var_10153_cast_fp16, y = var_10164_cast_fp16)[name = string("op_10165_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_759_cast_fp16 = add(x = state_757_cast_fp16, y = var_10165_cast_fp16)[name = string("state_759_cast_fp16")];
            tensor<int32, [4]> var_10169_begin_0 = const()[name = string("op_10169_begin_0"), val = tensor<int32, [4]>([0, 0, 13, 0])];
            tensor<int32, [4]> var_10169_end_0 = const()[name = string("op_10169_end_0"), val = tensor<int32, [4]>([1, 16, 14, 128])];
            tensor<bool, [4]> var_10169_end_mask_0 = const()[name = string("op_10169_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_10169_squeeze_mask_0 = const()[name = string("op_10169_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_10169_cast_fp16 = slice_by_index(begin = var_10169_begin_0, end = var_10169_end_0, end_mask = var_10169_end_mask_0, squeeze_mask = var_10169_squeeze_mask_0, x = _inversed_query_59_cast_fp16)[name = string("op_10169_cast_fp16")];
            tensor<int32, [1]> var_10170_axes_0 = const()[name = string("op_10170_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_10170_cast_fp16 = expand_dims(axes = var_10170_axes_0, x = var_10169_cast_fp16)[name = string("op_10170_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_10171_cast_fp16 = mul(x = state_759_cast_fp16, y = var_10170_cast_fp16)[name = string("op_10171_cast_fp16")];
            tensor<int32, [1]> var_10173_axes_0 = const()[name = string("op_10173_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_10173_keep_dims_0 = const()[name = string("op_10173_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_10173_cast_fp16 = reduce_sum(axes = var_10173_axes_0, keep_dims = var_10173_keep_dims_0, x = var_10171_cast_fp16)[name = string("op_10173_cast_fp16")];
            tensor<int32, [3]> var_10176_begin_0 = const()[name = string("op_10176_begin_0"), val = tensor<int32, [3]>([0, 0, 14])];
            tensor<int32, [3]> var_10176_end_0 = const()[name = string("op_10176_end_0"), val = tensor<int32, [3]>([1, 16, 15])];
            tensor<bool, [3]> var_10176_end_mask_0 = const()[name = string("op_10176_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_10176_squeeze_mask_0 = const()[name = string("op_10176_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_10176_cast_fp16 = slice_by_index(begin = var_10176_begin_0, end = var_10176_end_0, end_mask = var_10176_end_mask_0, squeeze_mask = var_10176_squeeze_mask_0, x = decay_23_cast_fp16)[name = string("op_10176_cast_fp16")];
            tensor<fp16, [1, 16]> var_10177_cast_fp16 = exp(x = var_10176_cast_fp16)[name = string("op_10177_cast_fp16")];
            tensor<int32, [1]> var_10178_axes_0 = const()[name = string("op_10178_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_10178_cast_fp16 = expand_dims(axes = var_10178_axes_0, x = var_10177_cast_fp16)[name = string("op_10178_cast_fp16")];
            tensor<int32, [1]> var_10179_axes_0 = const()[name = string("op_10179_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_10179_cast_fp16 = expand_dims(axes = var_10179_axes_0, x = var_10178_cast_fp16)[name = string("op_10179_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_761_cast_fp16 = mul(x = state_759_cast_fp16, y = var_10179_cast_fp16)[name = string("state_761_cast_fp16")];
            tensor<int32, [4]> key_token_381_begin_0 = const()[name = string("key_token_381_begin_0"), val = tensor<int32, [4]>([0, 0, 14, 0])];
            tensor<int32, [4]> key_token_381_end_0 = const()[name = string("key_token_381_end_0"), val = tensor<int32, [4]>([1, 16, 15, 128])];
            tensor<bool, [4]> key_token_381_end_mask_0 = const()[name = string("key_token_381_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_381_squeeze_mask_0 = const()[name = string("key_token_381_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_381_cast_fp16 = slice_by_index(begin = key_token_381_begin_0, end = key_token_381_end_0, end_mask = key_token_381_end_mask_0, squeeze_mask = key_token_381_squeeze_mask_0, x = key_59_cast_fp16)[name = string("key_token_381_cast_fp16")];
            tensor<int32, [4]> value_token_381_begin_0 = const()[name = string("value_token_381_begin_0"), val = tensor<int32, [4]>([0, 0, 14, 0])];
            tensor<int32, [4]> value_token_381_end_0 = const()[name = string("value_token_381_end_0"), val = tensor<int32, [4]>([1, 16, 15, 128])];
            tensor<bool, [4]> value_token_381_end_mask_0 = const()[name = string("value_token_381_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_381_squeeze_mask_0 = const()[name = string("value_token_381_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_381_cast_fp16 = slice_by_index(begin = value_token_381_begin_0, end = value_token_381_end_0, end_mask = value_token_381_end_mask_0, squeeze_mask = value_token_381_squeeze_mask_0, x = value_213_cast_fp16)[name = string("value_token_381_cast_fp16")];
            tensor<int32, [1]> var_10187_axes_0 = const()[name = string("op_10187_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_10187_cast_fp16 = expand_dims(axes = var_10187_axes_0, x = key_token_381_cast_fp16)[name = string("op_10187_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_10188_cast_fp16 = mul(x = state_761_cast_fp16, y = var_10187_cast_fp16)[name = string("op_10188_cast_fp16")];
            tensor<int32, [1]> memory_381_axes_0 = const()[name = string("memory_381_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_381_keep_dims_0 = const()[name = string("memory_381_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_381_cast_fp16 = reduce_sum(axes = memory_381_axes_0, keep_dims = memory_381_keep_dims_0, x = var_10188_cast_fp16)[name = string("memory_381_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_10191_cast_fp16 = sub(x = value_token_381_cast_fp16, y = memory_381_cast_fp16)[name = string("op_10191_cast_fp16")];
            tensor<int32, [3]> var_10194_begin_0 = const()[name = string("op_10194_begin_0"), val = tensor<int32, [3]>([0, 0, 14])];
            tensor<int32, [3]> var_10194_end_0 = const()[name = string("op_10194_end_0"), val = tensor<int32, [3]>([1, 16, 15])];
            tensor<bool, [3]> var_10194_end_mask_0 = const()[name = string("op_10194_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_10194_squeeze_mask_0 = const()[name = string("op_10194_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_10194_cast_fp16 = slice_by_index(begin = var_10194_begin_0, end = var_10194_end_0, end_mask = var_10194_end_mask_0, squeeze_mask = var_10194_squeeze_mask_0, x = beta_23_cast_fp16)[name = string("op_10194_cast_fp16")];
            tensor<int32, [1]> var_10195_axes_0 = const()[name = string("op_10195_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_10195_cast_fp16 = expand_dims(axes = var_10195_axes_0, x = var_10194_cast_fp16)[name = string("op_10195_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_381_cast_fp16 = mul(x = var_10191_cast_fp16, y = var_10195_cast_fp16)[name = string("delta_381_cast_fp16")];
            tensor<int32, [1]> var_10198_axes_0 = const()[name = string("op_10198_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_10198_cast_fp16 = expand_dims(axes = var_10198_axes_0, x = delta_381_cast_fp16)[name = string("op_10198_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_10199_cast_fp16 = mul(x = var_10187_cast_fp16, y = var_10198_cast_fp16)[name = string("op_10199_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_763_cast_fp16 = add(x = state_761_cast_fp16, y = var_10199_cast_fp16)[name = string("state_763_cast_fp16")];
            tensor<int32, [4]> var_10203_begin_0 = const()[name = string("op_10203_begin_0"), val = tensor<int32, [4]>([0, 0, 14, 0])];
            tensor<int32, [4]> var_10203_end_0 = const()[name = string("op_10203_end_0"), val = tensor<int32, [4]>([1, 16, 15, 128])];
            tensor<bool, [4]> var_10203_end_mask_0 = const()[name = string("op_10203_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_10203_squeeze_mask_0 = const()[name = string("op_10203_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_10203_cast_fp16 = slice_by_index(begin = var_10203_begin_0, end = var_10203_end_0, end_mask = var_10203_end_mask_0, squeeze_mask = var_10203_squeeze_mask_0, x = _inversed_query_59_cast_fp16)[name = string("op_10203_cast_fp16")];
            tensor<int32, [1]> var_10204_axes_0 = const()[name = string("op_10204_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_10204_cast_fp16 = expand_dims(axes = var_10204_axes_0, x = var_10203_cast_fp16)[name = string("op_10204_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_10205_cast_fp16 = mul(x = state_763_cast_fp16, y = var_10204_cast_fp16)[name = string("op_10205_cast_fp16")];
            tensor<int32, [1]> var_10207_axes_0 = const()[name = string("op_10207_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_10207_keep_dims_0 = const()[name = string("op_10207_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_10207_cast_fp16 = reduce_sum(axes = var_10207_axes_0, keep_dims = var_10207_keep_dims_0, x = var_10205_cast_fp16)[name = string("op_10207_cast_fp16")];
            tensor<int32, [3]> var_10210_begin_0 = const()[name = string("op_10210_begin_0"), val = tensor<int32, [3]>([0, 0, 15])];
            tensor<int32, [3]> var_10210_end_0 = const()[name = string("op_10210_end_0"), val = tensor<int32, [3]>([1, 16, 16])];
            tensor<bool, [3]> var_10210_end_mask_0 = const()[name = string("op_10210_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_10210_squeeze_mask_0 = const()[name = string("op_10210_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_10210_cast_fp16 = slice_by_index(begin = var_10210_begin_0, end = var_10210_end_0, end_mask = var_10210_end_mask_0, squeeze_mask = var_10210_squeeze_mask_0, x = decay_23_cast_fp16)[name = string("op_10210_cast_fp16")];
            tensor<fp16, [1, 16]> var_10211_cast_fp16 = exp(x = var_10210_cast_fp16)[name = string("op_10211_cast_fp16")];
            tensor<int32, [1]> var_10212_axes_0 = const()[name = string("op_10212_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_10212_cast_fp16 = expand_dims(axes = var_10212_axes_0, x = var_10211_cast_fp16)[name = string("op_10212_cast_fp16")];
            tensor<int32, [1]> var_10213_axes_0 = const()[name = string("op_10213_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_10213_cast_fp16 = expand_dims(axes = var_10213_axes_0, x = var_10212_cast_fp16)[name = string("op_10213_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_765_cast_fp16 = mul(x = state_763_cast_fp16, y = var_10213_cast_fp16)[name = string("state_765_cast_fp16")];
            tensor<int32, [4]> key_token_383_begin_0 = const()[name = string("key_token_383_begin_0"), val = tensor<int32, [4]>([0, 0, 15, 0])];
            tensor<int32, [4]> key_token_383_end_0 = const()[name = string("key_token_383_end_0"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<bool, [4]> key_token_383_end_mask_0 = const()[name = string("key_token_383_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_383_squeeze_mask_0 = const()[name = string("key_token_383_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_383_cast_fp16 = slice_by_index(begin = key_token_383_begin_0, end = key_token_383_end_0, end_mask = key_token_383_end_mask_0, squeeze_mask = key_token_383_squeeze_mask_0, x = key_59_cast_fp16)[name = string("key_token_383_cast_fp16")];
            tensor<int32, [4]> value_token_383_begin_0 = const()[name = string("value_token_383_begin_0"), val = tensor<int32, [4]>([0, 0, 15, 0])];
            tensor<int32, [4]> value_token_383_end_0 = const()[name = string("value_token_383_end_0"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<bool, [4]> value_token_383_end_mask_0 = const()[name = string("value_token_383_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_383_squeeze_mask_0 = const()[name = string("value_token_383_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_383_cast_fp16 = slice_by_index(begin = value_token_383_begin_0, end = value_token_383_end_0, end_mask = value_token_383_end_mask_0, squeeze_mask = value_token_383_squeeze_mask_0, x = value_213_cast_fp16)[name = string("value_token_383_cast_fp16")];
            tensor<int32, [1]> var_10221_axes_0 = const()[name = string("op_10221_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_10221_cast_fp16 = expand_dims(axes = var_10221_axes_0, x = key_token_383_cast_fp16)[name = string("op_10221_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_10222_cast_fp16 = mul(x = state_765_cast_fp16, y = var_10221_cast_fp16)[name = string("op_10222_cast_fp16")];
            tensor<int32, [1]> memory_383_axes_0 = const()[name = string("memory_383_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_383_keep_dims_0 = const()[name = string("memory_383_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_383_cast_fp16 = reduce_sum(axes = memory_383_axes_0, keep_dims = memory_383_keep_dims_0, x = var_10222_cast_fp16)[name = string("memory_383_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_10225_cast_fp16 = sub(x = value_token_383_cast_fp16, y = memory_383_cast_fp16)[name = string("op_10225_cast_fp16")];
            tensor<int32, [3]> var_10228_begin_0 = const()[name = string("op_10228_begin_0"), val = tensor<int32, [3]>([0, 0, 15])];
            tensor<int32, [3]> var_10228_end_0 = const()[name = string("op_10228_end_0"), val = tensor<int32, [3]>([1, 16, 16])];
            tensor<bool, [3]> var_10228_end_mask_0 = const()[name = string("op_10228_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_10228_squeeze_mask_0 = const()[name = string("op_10228_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_10228_cast_fp16 = slice_by_index(begin = var_10228_begin_0, end = var_10228_end_0, end_mask = var_10228_end_mask_0, squeeze_mask = var_10228_squeeze_mask_0, x = beta_23_cast_fp16)[name = string("op_10228_cast_fp16")];
            tensor<int32, [1]> var_10229_axes_0 = const()[name = string("op_10229_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_10229_cast_fp16 = expand_dims(axes = var_10229_axes_0, x = var_10228_cast_fp16)[name = string("op_10229_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_383_cast_fp16 = mul(x = var_10225_cast_fp16, y = var_10229_cast_fp16)[name = string("delta_383_cast_fp16")];
            tensor<int32, [1]> var_10232_axes_0 = const()[name = string("op_10232_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_10232_cast_fp16 = expand_dims(axes = var_10232_axes_0, x = delta_383_cast_fp16)[name = string("op_10232_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_10233_cast_fp16 = mul(x = var_10221_cast_fp16, y = var_10232_cast_fp16)[name = string("op_10233_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> rec14_out = add(x = state_765_cast_fp16, y = var_10233_cast_fp16)[name = string("state_767_cast_fp16")];
            tensor<int32, [4]> var_10237_begin_0 = const()[name = string("op_10237_begin_0"), val = tensor<int32, [4]>([0, 0, 15, 0])];
            tensor<int32, [4]> var_10237_end_0 = const()[name = string("op_10237_end_0"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<bool, [4]> var_10237_end_mask_0 = const()[name = string("op_10237_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_10237_squeeze_mask_0 = const()[name = string("op_10237_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_10237_cast_fp16 = slice_by_index(begin = var_10237_begin_0, end = var_10237_end_0, end_mask = var_10237_end_mask_0, squeeze_mask = var_10237_squeeze_mask_0, x = _inversed_query_59_cast_fp16)[name = string("op_10237_cast_fp16")];
            tensor<int32, [1]> var_10238_axes_0 = const()[name = string("op_10238_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_10238_cast_fp16 = expand_dims(axes = var_10238_axes_0, x = var_10237_cast_fp16)[name = string("op_10238_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_10239_cast_fp16 = mul(x = rec14_out, y = var_10238_cast_fp16)[name = string("op_10239_cast_fp16")];
            tensor<int32, [1]> var_10241_axes_0 = const()[name = string("op_10241_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_10241_keep_dims_0 = const()[name = string("op_10241_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_10241_cast_fp16 = reduce_sum(axes = var_10241_axes_0, keep_dims = var_10241_keep_dims_0, x = var_10239_cast_fp16)[name = string("op_10241_cast_fp16")];
            int32 attention_111_axis_0 = const()[name = string("attention_111_axis_0"), val = int32(1)];
            tensor<fp16, [1, 16, 16, 128]> attention_111_cast_fp16 = stack(axis = attention_111_axis_0, values = (var_9731_cast_fp16, var_9765_cast_fp16, var_9799_cast_fp16, var_9833_cast_fp16, var_9867_cast_fp16, var_9901_cast_fp16, var_9935_cast_fp16, var_9969_cast_fp16, var_10003_cast_fp16, var_10037_cast_fp16, var_10071_cast_fp16, var_10105_cast_fp16, var_10139_cast_fp16, var_10173_cast_fp16, var_10207_cast_fp16, var_10241_cast_fp16))[name = string("attention_111_cast_fp16")];
            tensor<int32, [2]> var_10244 = const()[name = string("op_10244"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [256, 128]> attention_113_cast_fp16 = reshape(shape = var_10244, x = attention_111_cast_fp16)[name = string("attention_113_cast_fp16")];
            tensor<int32, [2]> var_10246 = const()[name = string("op_10246"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [256, 128]> input_189_cast_fp16 = reshape(shape = var_10246, x = linear_112_cast_fp16)[name = string("input_189_cast_fp16")];
            tensor<fp16, [256, 128]> var_10248_cast_fp16 = mul(x = attention_113_cast_fp16, y = attention_113_cast_fp16)[name = string("op_10248_cast_fp16")];
            tensor<int32, [1]> variance_93_axes_0 = const()[name = string("variance_93_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_93_keep_dims_0 = const()[name = string("variance_93_keep_dims_0"), val = bool(true)];
            tensor<fp16, [256, 1]> variance_93_cast_fp16 = reduce_mean(axes = variance_93_axes_0, keep_dims = variance_93_keep_dims_0, x = var_10248_cast_fp16)[name = string("variance_93_cast_fp16")];
            fp16 var_10251_to_fp16 = const()[name = string("op_10251_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [256, 1]> var_10252_cast_fp16 = add(x = variance_93_cast_fp16, y = var_10251_to_fp16)[name = string("op_10252_cast_fp16")];
            fp32 var_10253_epsilon_0 = const()[name = string("op_10253_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [256, 1]> var_10253_cast_fp16 = rsqrt(epsilon = var_10253_epsilon_0, x = var_10252_cast_fp16)[name = string("op_10253_cast_fp16")];
            tensor<fp16, [256, 128]> attention_115_cast_fp16 = mul(x = attention_113_cast_fp16, y = var_10253_cast_fp16)[name = string("attention_115_cast_fp16")];
            tensor<fp16, [128]> groups_3_2_gated_norm_promoted_to_fp16 = const()[name = string("groups_3_2_gated_norm_promoted_to_fp16"), val = tensor<fp16, [128]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(228092736)))];
            tensor<fp16, [256, 128]> attention_117_cast_fp16 = mul(x = attention_115_cast_fp16, y = groups_3_2_gated_norm_promoted_to_fp16)[name = string("attention_117_cast_fp16")];
            tensor<fp16, [256, 128]> var_10256_cast_fp16 = silu(x = input_189_cast_fp16)[name = string("op_10256_cast_fp16")];
            tensor<fp16, [256, 128]> attention_119_cast_fp16 = mul(x = attention_117_cast_fp16, y = var_10256_cast_fp16)[name = string("attention_119_cast_fp16")];
            tensor<int32, [2]> var_10258 = const()[name = string("op_10258"), val = tensor<int32, [2]>([16, 2048])];
            tensor<fp16, [16, 2048]> input_191_cast_fp16 = reshape(shape = var_10258, x = attention_119_cast_fp16)[name = string("input_191_cast_fp16")];
            tensor<fp16, [1024, 2048]> groups_3_2_out_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 2048]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(228093056))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(229665984))))[name = string("groups_3_2_out_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 1024]> linear_113_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = groups_3_2_out_proj_weight_promoted_to_fp16_palettized, x = input_191_cast_fp16)[name = string("linear_113_cast_fp16")];
            tensor<fp16, [16, 1024]> value_215_cast_fp16 = add(x = value_203_cast_fp16, y = linear_113_cast_fp16)[name = string("value_215_cast_fp16")];
            tensor<fp16, [16, 1024]> var_10263_cast_fp16 = mul(x = value_215_cast_fp16, y = value_215_cast_fp16)[name = string("op_10263_cast_fp16")];
            tensor<int32, [1]> variance_95_axes_0 = const()[name = string("variance_95_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_95_keep_dims_0 = const()[name = string("variance_95_keep_dims_0"), val = bool(true)];
            tensor<fp16, [16, 1]> variance_95_cast_fp16 = reduce_mean(axes = variance_95_axes_0, keep_dims = variance_95_keep_dims_0, x = var_10263_cast_fp16)[name = string("variance_95_cast_fp16")];
            fp16 var_10266_to_fp16 = const()[name = string("op_10266_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [16, 1]> var_10267_cast_fp16 = add(x = variance_95_cast_fp16, y = var_10266_to_fp16)[name = string("op_10267_cast_fp16")];
            fp32 var_10268_epsilon_0 = const()[name = string("op_10268_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [16, 1]> var_10268_cast_fp16 = rsqrt(epsilon = var_10268_epsilon_0, x = var_10267_cast_fp16)[name = string("op_10268_cast_fp16")];
            tensor<fp16, [16, 1024]> var_10269_cast_fp16 = mul(x = value_215_cast_fp16, y = var_10268_cast_fp16)[name = string("op_10269_cast_fp16")];
            tensor<fp16, [1024]> var_10271_to_fp16 = const()[name = string("op_10271_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(229674240)))];
            tensor<fp16, [16, 1024]> input_193_cast_fp16 = mul(x = var_10269_cast_fp16, y = var_10271_to_fp16)[name = string("input_193_cast_fp16")];
            tensor<fp16, [3584, 1024]> groups_3_2_gate_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(229676352))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(232428928))))[name = string("groups_3_2_gate_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 3584]> linear_114_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = groups_3_2_gate_proj_weight_promoted_to_fp16_palettized, x = input_193_cast_fp16)[name = string("linear_114_cast_fp16")];
            tensor<fp16, [16, 3584]> var_10275_cast_fp16 = silu(x = linear_114_cast_fp16)[name = string("op_10275_cast_fp16")];
            tensor<fp16, [3584, 1024]> groups_3_2_up_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(232457664))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(235210240))))[name = string("groups_3_2_up_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 3584]> linear_115_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = groups_3_2_up_proj_weight_promoted_to_fp16_palettized, x = input_193_cast_fp16)[name = string("linear_115_cast_fp16")];
            tensor<fp16, [16, 3584]> input_197_cast_fp16 = mul(x = var_10275_cast_fp16, y = linear_115_cast_fp16)[name = string("input_197_cast_fp16")];
            tensor<fp16, [1024, 3584]> groups_3_2_down_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 3584]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(235238976))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(237991552))))[name = string("groups_3_2_down_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 1024]> linear_116_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = groups_3_2_down_proj_weight_promoted_to_fp16_palettized, x = input_197_cast_fp16)[name = string("linear_116_cast_fp16")];
            tensor<fp16, [16, 1024]> value_217_cast_fp16 = add(x = value_215_cast_fp16, y = linear_116_cast_fp16)[name = string("value_217_cast_fp16")];
            int32 var_10300 = const()[name = string("op_10300"), val = int32(-1)];
            tensor<fp16, [16, 1024]> var_10307_cast_fp16 = mul(x = value_217_cast_fp16, y = value_217_cast_fp16)[name = string("op_10307_cast_fp16")];
            tensor<int32, [1]> variance_97_axes_0 = const()[name = string("variance_97_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_97_keep_dims_0 = const()[name = string("variance_97_keep_dims_0"), val = bool(true)];
            tensor<fp16, [16, 1]> variance_97_cast_fp16 = reduce_mean(axes = variance_97_axes_0, keep_dims = variance_97_keep_dims_0, x = var_10307_cast_fp16)[name = string("variance_97_cast_fp16")];
            fp16 var_10310_to_fp16 = const()[name = string("op_10310_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [16, 1]> var_10311_cast_fp16 = add(x = variance_97_cast_fp16, y = var_10310_to_fp16)[name = string("op_10311_cast_fp16")];
            fp32 var_10312_epsilon_0 = const()[name = string("op_10312_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [16, 1]> var_10312_cast_fp16 = rsqrt(epsilon = var_10312_epsilon_0, x = var_10311_cast_fp16)[name = string("op_10312_cast_fp16")];
            tensor<fp16, [16, 1024]> var_10313_cast_fp16 = mul(x = value_217_cast_fp16, y = var_10312_cast_fp16)[name = string("op_10313_cast_fp16")];
            tensor<fp16, [1024]> var_10315_to_fp16 = const()[name = string("op_10315_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(237999808)))];
            tensor<fp16, [16, 1024]> input_199_cast_fp16 = mul(x = var_10313_cast_fp16, y = var_10315_to_fp16)[name = string("input_199_cast_fp16")];
            tensor<fp16, [4096, 1024]> projections_3_q_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [4096, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(238001920))), lut = tensor<fp16, [256, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(241147712))))[name = string("projections_3_q_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 4096]> linear_117_cast_fp16 = linear(bias = linear_24_bias_0_to_fp16, weight = projections_3_q_proj_weight_promoted_to_fp16_palettized, x = input_199_cast_fp16)[name = string("linear_117_cast_fp16")];
            tensor<int32, [3]> var_10319 = const()[name = string("op_10319"), val = tensor<int32, [3]>([16, 8, 512])];
            tensor<fp16, [16, 8, 512]> query_and_gate_7_cast_fp16 = reshape(shape = var_10319, x = linear_117_cast_fp16)[name = string("query_and_gate_7_cast_fp16")];
            tensor<int32, [2]> var_10321_split_sizes_0 = const()[name = string("op_10321_split_sizes_0"), val = tensor<int32, [2]>([256, 256])];
            int32 var_10321_axis_0 = const()[name = string("op_10321_axis_0"), val = int32(-1)];
            tensor<fp16, [16, 8, 256]> var_10321_cast_fp16_0, tensor<fp16, [16, 8, 256]> var_10321_cast_fp16_1 = split(axis = var_10321_axis_0, split_sizes = var_10321_split_sizes_0, x = query_and_gate_7_cast_fp16)[name = string("op_10321_cast_fp16")];
            tensor<fp16, [512, 1024]> projections_3_k_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [512, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(241180544))), lut = tensor<fp16, [32, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(241573824))))[name = string("projections_3_k_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 512]> linear_118_cast_fp16 = linear(bias = linear_25_bias_0_to_fp16, weight = projections_3_k_proj_weight_promoted_to_fp16_palettized, x = input_199_cast_fp16)[name = string("linear_118_cast_fp16")];
            tensor<int32, [3]> var_10325 = const()[name = string("op_10325"), val = tensor<int32, [3]>([16, 2, 256])];
            tensor<fp16, [16, 2, 256]> value_221_cast_fp16 = reshape(shape = var_10325, x = linear_118_cast_fp16)[name = string("value_221_cast_fp16")];
            tensor<fp16, [512, 1024]> projections_3_v_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [512, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(241577984))), lut = tensor<fp16, [32, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(241971264))))[name = string("projections_3_v_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 512]> linear_119_cast_fp16 = linear(bias = linear_25_bias_0_to_fp16, weight = projections_3_v_proj_weight_promoted_to_fp16_palettized, x = input_199_cast_fp16)[name = string("linear_119_cast_fp16")];
            tensor<int32, [3]> var_10329 = const()[name = string("op_10329"), val = tensor<int32, [3]>([16, 2, 256])];
            tensor<fp16, [16, 2, 256]> value_227_cast_fp16 = reshape(shape = var_10329, x = linear_119_cast_fp16)[name = string("value_227_cast_fp16")];
            tensor<fp16, [16, 8, 256]> var_10331_cast_fp16 = mul(x = var_10321_cast_fp16_0, y = var_10321_cast_fp16_0)[name = string("op_10331_cast_fp16")];
            tensor<int32, [1]> variance_99_axes_0 = const()[name = string("variance_99_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_99_keep_dims_0 = const()[name = string("variance_99_keep_dims_0"), val = bool(true)];
            tensor<fp16, [16, 8, 1]> variance_99_cast_fp16 = reduce_mean(axes = variance_99_axes_0, keep_dims = variance_99_keep_dims_0, x = var_10331_cast_fp16)[name = string("variance_99_cast_fp16")];
            fp16 var_10334_to_fp16 = const()[name = string("op_10334_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [16, 8, 1]> var_10335_cast_fp16 = add(x = variance_99_cast_fp16, y = var_10334_to_fp16)[name = string("op_10335_cast_fp16")];
            fp32 var_10336_epsilon_0 = const()[name = string("op_10336_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [16, 8, 1]> var_10336_cast_fp16 = rsqrt(epsilon = var_10336_epsilon_0, x = var_10335_cast_fp16)[name = string("op_10336_cast_fp16")];
            tensor<fp16, [16, 8, 256]> var_10337_cast_fp16 = mul(x = var_10321_cast_fp16_0, y = var_10336_cast_fp16)[name = string("op_10337_cast_fp16")];
            tensor<fp16, [256]> var_10339_to_fp16 = const()[name = string("op_10339_to_fp16"), val = tensor<fp16, [256]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(241975424)))];
            tensor<fp16, [16, 8, 256]> var_10340_cast_fp16 = mul(x = var_10337_cast_fp16, y = var_10339_to_fp16)[name = string("op_10340_cast_fp16")];
            tensor<int32, [3]> var_10341 = const()[name = string("op_10341"), val = tensor<int32, [3]>([1, 0, 2])];
            tensor<int32, [1]> value_223_axes_0 = const()[name = string("value_223_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [8, 16, 256]> var_10342_cast_fp16 = transpose(perm = var_10341, x = var_10340_cast_fp16)[name = string("transpose_32")];
            tensor<fp16, [1, 8, 16, 256]> value_223_cast_fp16 = expand_dims(axes = value_223_axes_0, x = var_10342_cast_fp16)[name = string("value_223_cast_fp16")];
            tensor<fp16, [16, 2, 256]> var_10344_cast_fp16 = mul(x = value_221_cast_fp16, y = value_221_cast_fp16)[name = string("op_10344_cast_fp16")];
            tensor<int32, [1]> variance_101_axes_0 = const()[name = string("variance_101_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_101_keep_dims_0 = const()[name = string("variance_101_keep_dims_0"), val = bool(true)];
            tensor<fp16, [16, 2, 1]> variance_101_cast_fp16 = reduce_mean(axes = variance_101_axes_0, keep_dims = variance_101_keep_dims_0, x = var_10344_cast_fp16)[name = string("variance_101_cast_fp16")];
            fp16 var_10347_to_fp16 = const()[name = string("op_10347_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [16, 2, 1]> var_10348_cast_fp16 = add(x = variance_101_cast_fp16, y = var_10347_to_fp16)[name = string("op_10348_cast_fp16")];
            fp32 var_10349_epsilon_0 = const()[name = string("op_10349_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [16, 2, 1]> var_10349_cast_fp16 = rsqrt(epsilon = var_10349_epsilon_0, x = var_10348_cast_fp16)[name = string("op_10349_cast_fp16")];
            tensor<fp16, [16, 2, 256]> var_10350_cast_fp16 = mul(x = value_221_cast_fp16, y = var_10349_cast_fp16)[name = string("op_10350_cast_fp16")];
            tensor<fp16, [256]> var_10352_to_fp16 = const()[name = string("op_10352_to_fp16"), val = tensor<fp16, [256]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(241976000)))];
            tensor<fp16, [16, 2, 256]> var_10353_cast_fp16 = mul(x = var_10350_cast_fp16, y = var_10352_to_fp16)[name = string("op_10353_cast_fp16")];
            tensor<int32, [3]> var_10354 = const()[name = string("op_10354"), val = tensor<int32, [3]>([1, 0, 2])];
            tensor<int32, [1]> value_225_axes_0 = const()[name = string("value_225_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [2, 16, 256]> var_10355_cast_fp16 = transpose(perm = var_10354, x = var_10353_cast_fp16)[name = string("transpose_31")];
            tensor<fp16, [1, 2, 16, 256]> value_225_cast_fp16 = expand_dims(axes = value_225_axes_0, x = var_10355_cast_fp16)[name = string("value_225_cast_fp16")];
            tensor<int32, [4]> rotated_13_begin_0 = const()[name = string("rotated_13_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> rotated_13_end_0 = const()[name = string("rotated_13_end_0"), val = tensor<int32, [4]>([1, 8, 16, 64])];
            tensor<bool, [4]> rotated_13_end_mask_0 = const()[name = string("rotated_13_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
            tensor<fp16, [1, 8, 16, 64]> rotated_13_cast_fp16 = slice_by_index(begin = rotated_13_begin_0, end = rotated_13_end_0, end_mask = rotated_13_end_mask_0, x = value_223_cast_fp16)[name = string("rotated_13_cast_fp16")];
            tensor<int32, [4]> passthrough_13_begin_0 = const()[name = string("passthrough_13_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 64])];
            tensor<int32, [4]> passthrough_13_end_0 = const()[name = string("passthrough_13_end_0"), val = tensor<int32, [4]>([1, 8, 16, 256])];
            tensor<bool, [4]> passthrough_13_end_mask_0 = const()[name = string("passthrough_13_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
            tensor<fp16, [1, 8, 16, 192]> passthrough_13_cast_fp16 = slice_by_index(begin = passthrough_13_begin_0, end = passthrough_13_end_0, end_mask = passthrough_13_end_mask_0, x = value_223_cast_fp16)[name = string("passthrough_13_cast_fp16")];
            tensor<int32, [2]> var_10359_split_sizes_0 = const()[name = string("op_10359_split_sizes_0"), val = tensor<int32, [2]>([32, 32])];
            int32 var_10359_axis_0 = const()[name = string("op_10359_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 8, 16, 32]> var_10359_cast_fp16_0, tensor<fp16, [1, 8, 16, 32]> var_10359_cast_fp16_1 = split(axis = var_10359_axis_0, split_sizes = var_10359_split_sizes_0, x = rotated_13_cast_fp16)[name = string("op_10359_cast_fp16")];
            fp16 const_132_promoted_to_fp16 = const()[name = string("const_132_promoted_to_fp16"), val = fp16(-0x1p+0)];
            tensor<fp16, [1, 8, 16, 32]> var_10361_cast_fp16 = mul(x = var_10359_cast_fp16_1, y = const_132_promoted_to_fp16)[name = string("op_10361_cast_fp16")];
            bool rotated_half_13_interleave_0 = const()[name = string("rotated_half_13_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 8, 16, 64]> rotated_half_13_cast_fp16 = concat(axis = var_10300, interleave = rotated_half_13_interleave_0, values = (var_10361_cast_fp16, var_10359_cast_fp16_0))[name = string("rotated_half_13_cast_fp16")];
            tensor<fp16, [1, 8, 16, 64]> var_10364_cast_fp16 = mul(x = rotated_13_cast_fp16, y = cos)[name = string("op_10364_cast_fp16")];
            tensor<fp16, [1, 8, 16, 64]> var_10365_cast_fp16 = mul(x = rotated_half_13_cast_fp16, y = sin)[name = string("op_10365_cast_fp16")];
            tensor<fp16, [1, 8, 16, 64]> var_10366_cast_fp16 = add(x = var_10364_cast_fp16, y = var_10365_cast_fp16)[name = string("op_10366_cast_fp16")];
            bool query_61_interleave_0 = const()[name = string("query_61_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 8, 16, 256]> query_61_cast_fp16 = concat(axis = var_10300, interleave = query_61_interleave_0, values = (var_10366_cast_fp16, passthrough_13_cast_fp16))[name = string("query_61_cast_fp16")];
            tensor<int32, [4]> rotated_15_begin_0 = const()[name = string("rotated_15_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> rotated_15_end_0 = const()[name = string("rotated_15_end_0"), val = tensor<int32, [4]>([1, 2, 16, 64])];
            tensor<bool, [4]> rotated_15_end_mask_0 = const()[name = string("rotated_15_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
            tensor<fp16, [1, 2, 16, 64]> rotated_15_cast_fp16 = slice_by_index(begin = rotated_15_begin_0, end = rotated_15_end_0, end_mask = rotated_15_end_mask_0, x = value_225_cast_fp16)[name = string("rotated_15_cast_fp16")];
            tensor<int32, [4]> passthrough_15_begin_0 = const()[name = string("passthrough_15_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 64])];
            tensor<int32, [4]> passthrough_15_end_0 = const()[name = string("passthrough_15_end_0"), val = tensor<int32, [4]>([1, 2, 16, 256])];
            tensor<bool, [4]> passthrough_15_end_mask_0 = const()[name = string("passthrough_15_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
            tensor<fp16, [1, 2, 16, 192]> passthrough_15_cast_fp16 = slice_by_index(begin = passthrough_15_begin_0, end = passthrough_15_end_0, end_mask = passthrough_15_end_mask_0, x = value_225_cast_fp16)[name = string("passthrough_15_cast_fp16")];
            tensor<int32, [2]> var_10371_split_sizes_0 = const()[name = string("op_10371_split_sizes_0"), val = tensor<int32, [2]>([32, 32])];
            int32 var_10371_axis_0 = const()[name = string("op_10371_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 2, 16, 32]> var_10371_cast_fp16_0, tensor<fp16, [1, 2, 16, 32]> var_10371_cast_fp16_1 = split(axis = var_10371_axis_0, split_sizes = var_10371_split_sizes_0, x = rotated_15_cast_fp16)[name = string("op_10371_cast_fp16")];
            fp16 const_133_promoted_to_fp16 = const()[name = string("const_133_promoted_to_fp16"), val = fp16(-0x1p+0)];
            tensor<fp16, [1, 2, 16, 32]> var_10373_cast_fp16 = mul(x = var_10371_cast_fp16_1, y = const_133_promoted_to_fp16)[name = string("op_10373_cast_fp16")];
            bool rotated_half_15_interleave_0 = const()[name = string("rotated_half_15_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 2, 16, 64]> rotated_half_15_cast_fp16 = concat(axis = var_10300, interleave = rotated_half_15_interleave_0, values = (var_10373_cast_fp16, var_10371_cast_fp16_0))[name = string("rotated_half_15_cast_fp16")];
            tensor<fp16, [1, 2, 16, 64]> var_10376_cast_fp16 = mul(x = rotated_15_cast_fp16, y = cos)[name = string("op_10376_cast_fp16")];
            tensor<fp16, [1, 2, 16, 64]> var_10377_cast_fp16 = mul(x = rotated_half_15_cast_fp16, y = sin)[name = string("op_10377_cast_fp16")];
            tensor<fp16, [1, 2, 16, 64]> var_10378_cast_fp16 = add(x = var_10376_cast_fp16, y = var_10377_cast_fp16)[name = string("op_10378_cast_fp16")];
            bool key_61_interleave_0 = const()[name = string("key_61_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 2, 16, 256]> key_61_cast_fp16 = concat(axis = var_10300, interleave = key_61_interleave_0, values = (var_10378_cast_fp16, passthrough_15_cast_fp16))[name = string("key_61_cast_fp16")];
            tensor<int32, [4]> var_10381 = const()[name = string("op_10381"), val = tensor<int32, [4]>([2, 4, 16, 256])];
            tensor<fp16, [2, 4, 16, 256]> var_10382_cast_fp16 = reshape(shape = var_10381, x = query_61_cast_fp16)[name = string("op_10382_cast_fp16")];
            tensor<int32, [4]> transpose_6_perm_0 = const()[name = string("transpose_6_perm_0"), val = tensor<int32, [4]>([0, 1, 3, 2])];
            tensor<int32, [3]> var_10385 = const()[name = string("op_10385"), val = tensor<int32, [3]>([2, 16, 256])];
            tensor<fp16, [2, 16, 256]> key_63_cast_fp16 = reshape(shape = var_10385, x = key_61_cast_fp16)[name = string("key_63_cast_fp16")];
            tensor<int32, [3]> var_10387 = const()[name = string("op_10387"), val = tensor<int32, [3]>([1, 0, 2])];
            tensor<int32, [2]> var_10389 = const()[name = string("op_10389"), val = tensor<int32, [2]>([16, 2048])];
            tensor<fp16, [16, 2048]> gate_15_cast_fp16 = reshape(shape = var_10389, x = var_10321_cast_fp16_1)[name = string("gate_15_cast_fp16")];
            tensor<int32, [1]> var_10397_axes_0 = const()[name = string("op_10397_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [1, 2, 16, 256]> var_10397_cast_fp16 = expand_dims(axes = var_10397_axes_0, x = key_63_cast_fp16)[name = string("op_10397_cast_fp16")];
            tensor<int32, [1]> var_10399_axes_0 = const()[name = string("op_10399_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [1, 1, 2, 16, 256]> var_10399_cast_fp16 = expand_dims(axes = var_10399_axes_0, x = var_10397_cast_fp16)[name = string("op_10399_cast_fp16")];
            tensor<int32, [1]> expand_dims_48 = const()[name = string("expand_dims_48"), val = tensor<int32, [1]>([3])];
            tensor<int32, [1]> expand_dims_49 = const()[name = string("expand_dims_49"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_50 = const()[name = string("expand_dims_50"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_52 = const()[name = string("expand_dims_52"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_53 = const()[name = string("expand_dims_53"), val = tensor<int32, [1]>([4])];
            tensor<int32, [1]> expand_dims_54 = const()[name = string("expand_dims_54"), val = tensor<int32, [1]>([1])];
            int32 concat_26_axis_0 = const()[name = string("concat_26_axis_0"), val = int32(0)];
            bool concat_26_interleave_0 = const()[name = string("concat_26_interleave_0"), val = bool(false)];
            tensor<int32, [5]> concat_26 = concat(axis = concat_26_axis_0, interleave = concat_26_interleave_0, values = (expand_dims_48, expand_dims_49, expand_dims_50, current_pos, expand_dims_52))[name = string("concat_26")];
            tensor<int32, [1]> concat_27_values2_0 = const()[name = string("concat_27_values2_0"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> concat_27_values4_0 = const()[name = string("concat_27_values4_0"), val = tensor<int32, [1]>([0])];
            int32 concat_27_axis_0 = const()[name = string("concat_27_axis_0"), val = int32(0)];
            bool concat_27_interleave_0 = const()[name = string("concat_27_interleave_0"), val = bool(false)];
            tensor<int32, [5]> concat_27 = concat(axis = concat_27_axis_0, interleave = concat_27_interleave_0, values = (expand_dims_53, expand_dims_54, concat_27_values2_0, var_2332, concat_27_values4_0))[name = string("concat_27")];
            tensor<int32, [5]> kv_cache_internal_tensor_assign_7_stride_0 = const()[name = string("kv_cache_internal_tensor_assign_7_stride_0"), val = tensor<int32, [5]>([1, 1, 1, 1, 1])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_7_begin_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_7_begin_mask_0"), val = tensor<bool, [5]>([false, false, false, false, false])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_7_end_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_7_end_mask_0"), val = tensor<bool, [5]>([false, false, true, false, true])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_7_squeeze_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_7_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, false, false, false])];
            tensor<fp16, [4, 2, 2, 2048, 256]> kv_cache_internal_tensor_assign_7_cast_fp16 = slice_update(begin = concat_26, begin_mask = kv_cache_internal_tensor_assign_7_begin_mask_0, end = concat_27, end_mask = kv_cache_internal_tensor_assign_7_end_mask_0, squeeze_mask = kv_cache_internal_tensor_assign_7_squeeze_mask_0, stride = kv_cache_internal_tensor_assign_7_stride_0, update = var_10399_cast_fp16, x = coreml_update_state_13)[name = string("kv_cache_internal_tensor_assign_7_cast_fp16")];
            write_state(data = kv_cache_internal_tensor_assign_7_cast_fp16, input = kv_cache)[name = string("coreml_update_state_14_write_state")];
            tensor<fp16, [4, 2, 2, 2048, 256]> coreml_update_state_14 = read_state(input = kv_cache)[name = string("coreml_update_state_14")];
            tensor<int32, [1]> var_10430_axes_0 = const()[name = string("op_10430_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [2, 16, 256]> value_229_cast_fp16 = transpose(perm = var_10387, x = value_227_cast_fp16)[name = string("transpose_30")];
            tensor<fp16, [1, 2, 16, 256]> var_10430_cast_fp16 = expand_dims(axes = var_10430_axes_0, x = value_229_cast_fp16)[name = string("op_10430_cast_fp16")];
            tensor<int32, [1]> var_10432_axes_0 = const()[name = string("op_10432_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [1, 1, 2, 16, 256]> var_10432_cast_fp16 = expand_dims(axes = var_10432_axes_0, x = var_10430_cast_fp16)[name = string("op_10432_cast_fp16")];
            tensor<int32, [1]> expand_dims_56 = const()[name = string("expand_dims_56"), val = tensor<int32, [1]>([3])];
            tensor<int32, [1]> expand_dims_57 = const()[name = string("expand_dims_57"), val = tensor<int32, [1]>([1])];
            tensor<int32, [1]> expand_dims_58 = const()[name = string("expand_dims_58"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_60 = const()[name = string("expand_dims_60"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_61 = const()[name = string("expand_dims_61"), val = tensor<int32, [1]>([4])];
            tensor<int32, [1]> expand_dims_62 = const()[name = string("expand_dims_62"), val = tensor<int32, [1]>([2])];
            int32 concat_30_axis_0 = const()[name = string("concat_30_axis_0"), val = int32(0)];
            bool concat_30_interleave_0 = const()[name = string("concat_30_interleave_0"), val = bool(false)];
            tensor<int32, [5]> concat_30 = concat(axis = concat_30_axis_0, interleave = concat_30_interleave_0, values = (expand_dims_56, expand_dims_57, expand_dims_58, current_pos, expand_dims_60))[name = string("concat_30")];
            tensor<int32, [1]> concat_31_values2_0 = const()[name = string("concat_31_values2_0"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> concat_31_values4_0 = const()[name = string("concat_31_values4_0"), val = tensor<int32, [1]>([0])];
            int32 concat_31_axis_0 = const()[name = string("concat_31_axis_0"), val = int32(0)];
            bool concat_31_interleave_0 = const()[name = string("concat_31_interleave_0"), val = bool(false)];
            tensor<int32, [5]> concat_31 = concat(axis = concat_31_axis_0, interleave = concat_31_interleave_0, values = (expand_dims_61, expand_dims_62, concat_31_values2_0, var_2332, concat_31_values4_0))[name = string("concat_31")];
            tensor<int32, [5]> kv_cache_internal_tensor_assign_8_stride_0 = const()[name = string("kv_cache_internal_tensor_assign_8_stride_0"), val = tensor<int32, [5]>([1, 1, 1, 1, 1])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_8_begin_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_8_begin_mask_0"), val = tensor<bool, [5]>([false, false, false, false, false])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_8_end_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_8_end_mask_0"), val = tensor<bool, [5]>([false, false, true, false, true])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_8_squeeze_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_8_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, false, false, false])];
            tensor<fp16, [4, 2, 2, 2048, 256]> kv_cache_internal_tensor_assign_8_cast_fp16 = slice_update(begin = concat_30, begin_mask = kv_cache_internal_tensor_assign_8_begin_mask_0, end = concat_31, end_mask = kv_cache_internal_tensor_assign_8_end_mask_0, squeeze_mask = kv_cache_internal_tensor_assign_8_squeeze_mask_0, stride = kv_cache_internal_tensor_assign_8_stride_0, update = var_10432_cast_fp16, x = coreml_update_state_14)[name = string("kv_cache_internal_tensor_assign_8_cast_fp16")];
            write_state(data = kv_cache_internal_tensor_assign_8_cast_fp16, input = kv_cache)[name = string("coreml_update_state_15_write_state")];
            tensor<fp16, [4, 2, 2, 2048, 256]> coreml_update_state_15 = read_state(input = kv_cache)[name = string("coreml_update_state_15")];
            tensor<int32, [5]> var_10464_begin_0 = const()[name = string("op_10464_begin_0"), val = tensor<int32, [5]>([3, 0, 0, 0, 0])];
            tensor<int32, [5]> var_10464_end_0 = const()[name = string("op_10464_end_0"), val = tensor<int32, [5]>([4, 2, 2, 2048, 256])];
            tensor<bool, [5]> var_10464_end_mask_0 = const()[name = string("op_10464_end_mask_0"), val = tensor<bool, [5]>([false, true, true, true, true])];
            tensor<bool, [5]> var_10464_squeeze_mask_0 = const()[name = string("op_10464_squeeze_mask_0"), val = tensor<bool, [5]>([true, false, false, false, false])];
            tensor<fp16, [2, 2, 2048, 256]> var_10464_cast_fp16 = slice_by_index(begin = var_10464_begin_0, end = var_10464_end_0, end_mask = var_10464_end_mask_0, squeeze_mask = var_10464_squeeze_mask_0, x = coreml_update_state_15)[name = string("op_10464_cast_fp16")];
            tensor<int32, [4]> keys_begin_0 = const()[name = string("keys_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> keys_end_0 = const()[name = string("keys_end_0"), val = tensor<int32, [4]>([1, 2, 2048, 256])];
            tensor<bool, [4]> keys_end_mask_0 = const()[name = string("keys_end_mask_0"), val = tensor<bool, [4]>([false, true, true, true])];
            tensor<bool, [4]> keys_squeeze_mask_0 = const()[name = string("keys_squeeze_mask_0"), val = tensor<bool, [4]>([true, false, false, false])];
            tensor<fp16, [2, 2048, 256]> keys_cast_fp16 = slice_by_index(begin = keys_begin_0, end = keys_end_0, end_mask = keys_end_mask_0, squeeze_mask = keys_squeeze_mask_0, x = var_10464_cast_fp16)[name = string("keys_cast_fp16")];
            tensor<int32, [4]> values_begin_0 = const()[name = string("values_begin_0"), val = tensor<int32, [4]>([1, 0, 0, 0])];
            tensor<int32, [4]> values_end_0 = const()[name = string("values_end_0"), val = tensor<int32, [4]>([2, 2, 2048, 256])];
            tensor<bool, [4]> values_end_mask_0 = const()[name = string("values_end_mask_0"), val = tensor<bool, [4]>([false, true, true, true])];
            tensor<bool, [4]> values_squeeze_mask_0 = const()[name = string("values_squeeze_mask_0"), val = tensor<bool, [4]>([true, false, false, false])];
            tensor<fp16, [2, 2048, 256]> values_cast_fp16 = slice_by_index(begin = values_begin_0, end = values_end_0, end_mask = values_end_mask_0, squeeze_mask = values_squeeze_mask_0, x = var_10464_cast_fp16)[name = string("values_cast_fp16")];
            int32 var_10482 = const()[name = string("op_10482"), val = int32(1)];
            tensor<int32, [4]> var_10489 = const()[name = string("op_10489"), val = tensor<int32, [4]>([1, 2048, 1, 16])];
            tensor<fp16, [2, 4, 256, 16]> transpose_6_cast_fp16 = transpose(perm = transpose_6_perm_0, x = var_10382_cast_fp16)[name = string("transpose_29")];
            tensor<fp16, [1, 2048, 1, 16]> var_10490_cast_fp16 = reshape(shape = var_10489, x = transpose_6_cast_fp16)[name = string("op_10490_cast_fp16")];
            tensor<int32, [3]> var_10491 = const()[name = string("op_10491"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [4]> var_10494 = const()[name = string("op_10494"), val = tensor<int32, [4]>([1, 512, 1, 2048])];
            tensor<fp16, [2, 256, 2048]> var_10492_cast_fp16 = transpose(perm = var_10491, x = keys_cast_fp16)[name = string("transpose_28")];
            tensor<fp16, [1, 512, 1, 2048]> key_native_cast_fp16 = reshape(shape = var_10494, x = var_10492_cast_fp16)[name = string("key_native_cast_fp16")];
            tensor<int32, [3]> var_10496 = const()[name = string("op_10496"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [4]> var_10499 = const()[name = string("op_10499"), val = tensor<int32, [4]>([1, 512, 1, 2048])];
            tensor<fp16, [2, 256, 2048]> var_10497_cast_fp16 = transpose(perm = var_10496, x = values_cast_fp16)[name = string("transpose_27")];
            tensor<fp16, [1, 512, 1, 2048]> var_10500_cast_fp16 = reshape(shape = var_10499, x = var_10497_cast_fp16)[name = string("op_10500_cast_fp16")];
            tensor<int32, [8]> tile_57 = const()[name = string("tile_57"), val = tensor<int32, [8]>([256, 256, 256, 256, 256, 256, 256, 256])];
            int32 var_10504_axis_0 = const()[name = string("op_10504_axis_0"), val = int32(1)];
            tensor<fp16, [1, 256, 1, 16]> var_10504_cast_fp16_0, tensor<fp16, [1, 256, 1, 16]> var_10504_cast_fp16_1, tensor<fp16, [1, 256, 1, 16]> var_10504_cast_fp16_2, tensor<fp16, [1, 256, 1, 16]> var_10504_cast_fp16_3, tensor<fp16, [1, 256, 1, 16]> var_10504_cast_fp16_4, tensor<fp16, [1, 256, 1, 16]> var_10504_cast_fp16_5, tensor<fp16, [1, 256, 1, 16]> var_10504_cast_fp16_6, tensor<fp16, [1, 256, 1, 16]> var_10504_cast_fp16_7 = split(axis = var_10504_axis_0, split_sizes = tile_57, x = var_10490_cast_fp16)[name = string("op_10504_cast_fp16")];
            tensor<int32, [4]> var_10513_perm_0 = const()[name = string("op_10513_perm_0"), val = tensor<int32, [4]>([0, 3, 2, 1])];
            tensor<int32, [2]> tile_58 = const()[name = string("tile_58"), val = tensor<int32, [2]>([256, 256])];
            int32 var_10514_axis_0 = const()[name = string("op_10514_axis_0"), val = int32(3)];
            tensor<fp16, [1, 2048, 1, 512]> var_10513_cast_fp16 = transpose(perm = var_10513_perm_0, x = key_native_cast_fp16)[name = string("transpose_26")];
            tensor<fp16, [1, 2048, 1, 256]> var_10514_cast_fp16_0, tensor<fp16, [1, 2048, 1, 256]> var_10514_cast_fp16_1 = split(axis = var_10514_axis_0, split_sizes = tile_58, x = var_10513_cast_fp16)[name = string("op_10514_cast_fp16")];
            tensor<int32, [2]> tile_59 = const()[name = string("tile_59"), val = tensor<int32, [2]>([256, 256])];
            int32 var_10517_axis_0 = const()[name = string("op_10517_axis_0"), val = int32(1)];
            tensor<fp16, [1, 256, 1, 2048]> var_10517_cast_fp16_0, tensor<fp16, [1, 256, 1, 2048]> var_10517_cast_fp16_1 = split(axis = var_10517_axis_0, split_sizes = tile_59, x = var_10500_cast_fp16)[name = string("op_10517_cast_fp16")];
            string scores_49_equation_0 = const()[name = string("scores_49_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 16]> scores_49_cast_fp16 = einsum(equation = scores_49_equation_0, values = (var_10514_cast_fp16_0, var_10504_cast_fp16_0))[name = string("scores_49_cast_fp16")];
            fp16 var_10522_to_fp16 = const()[name = string("op_10522_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 16]> var_10523_cast_fp16 = mul(x = scores_49_cast_fp16, y = var_10522_to_fp16)[name = string("op_10523_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_10524_cast_fp16 = add(x = var_10523_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_10524_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_10525_cast_fp16 = softmax(axis = var_10482, x = var_10524_cast_fp16)[name = string("op_10525_cast_fp16")];
            tensor<int32, [4]> tile_60 = const()[name = string("tile_60"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_10526_axis_0 = const()[name = string("op_10526_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 16]> var_10526_cast_fp16_0, tensor<fp16, [1, 512, 1, 16]> var_10526_cast_fp16_1, tensor<fp16, [1, 512, 1, 16]> var_10526_cast_fp16_2, tensor<fp16, [1, 512, 1, 16]> var_10526_cast_fp16_3 = split(axis = var_10526_axis_0, split_sizes = tile_60, x = var_10525_cast_fp16)[name = string("op_10526_cast_fp16")];
            tensor<int32, [4]> tile_61 = const()[name = string("tile_61"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_10531_axis_0 = const()[name = string("op_10531_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_10531_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_10531_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_10531_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_10531_cast_fp16_3 = split(axis = var_10531_axis_0, split_sizes = tile_61, x = var_10517_cast_fp16_0)[name = string("op_10531_cast_fp16")];
            fp16 var_10536_to_fp16 = const()[name = string("op_10536_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_10537_cast_fp16 = mul(x = var_10526_cast_fp16_0, y = var_10536_to_fp16)[name = string("op_10537_cast_fp16")];
            string var_10539_equation_0 = const()[name = string("op_10539_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_10539_cast_fp16 = einsum(equation = var_10539_equation_0, values = (var_10531_cast_fp16_0, var_10537_cast_fp16))[name = string("op_10539_cast_fp16")];
            fp16 var_10540_to_fp16 = const()[name = string("op_10540_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_10541_cast_fp16 = mul(x = var_10526_cast_fp16_1, y = var_10540_to_fp16)[name = string("op_10541_cast_fp16")];
            string var_10543_equation_0 = const()[name = string("op_10543_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_10543_cast_fp16 = einsum(equation = var_10543_equation_0, values = (var_10531_cast_fp16_1, var_10541_cast_fp16))[name = string("op_10543_cast_fp16")];
            fp16 var_10544_to_fp16 = const()[name = string("op_10544_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_10545_cast_fp16 = mul(x = var_10526_cast_fp16_2, y = var_10544_to_fp16)[name = string("op_10545_cast_fp16")];
            string var_10547_equation_0 = const()[name = string("op_10547_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_10547_cast_fp16 = einsum(equation = var_10547_equation_0, values = (var_10531_cast_fp16_2, var_10545_cast_fp16))[name = string("op_10547_cast_fp16")];
            fp16 var_10548_to_fp16 = const()[name = string("op_10548_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_10549_cast_fp16 = mul(x = var_10526_cast_fp16_3, y = var_10548_to_fp16)[name = string("op_10549_cast_fp16")];
            string var_10551_equation_0 = const()[name = string("op_10551_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_10551_cast_fp16 = einsum(equation = var_10551_equation_0, values = (var_10531_cast_fp16_3, var_10549_cast_fp16))[name = string("op_10551_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_10552_cast_fp16 = add(x = var_10539_cast_fp16, y = var_10543_cast_fp16)[name = string("op_10552_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_10553_cast_fp16 = add(x = var_10547_cast_fp16, y = var_10551_cast_fp16)[name = string("op_10553_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_10554_cast_fp16 = add(x = var_10552_cast_fp16, y = var_10553_cast_fp16)[name = string("op_10554_cast_fp16")];
            fp16 _inversed_10556_y_0_to_fp16 = const()[name = string("_inversed_10556_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 16]> _inversed_10556_cast_fp16 = mul(x = var_10554_cast_fp16, y = _inversed_10556_y_0_to_fp16)[name = string("_inversed_10556_cast_fp16")];
            string scores_51_equation_0 = const()[name = string("scores_51_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 16]> scores_51_cast_fp16 = einsum(equation = scores_51_equation_0, values = (var_10514_cast_fp16_0, var_10504_cast_fp16_1))[name = string("scores_51_cast_fp16")];
            fp16 var_10559_to_fp16 = const()[name = string("op_10559_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 16]> var_10560_cast_fp16 = mul(x = scores_51_cast_fp16, y = var_10559_to_fp16)[name = string("op_10560_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_10561_cast_fp16 = add(x = var_10560_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_10561_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_10562_cast_fp16 = softmax(axis = var_10482, x = var_10561_cast_fp16)[name = string("op_10562_cast_fp16")];
            tensor<int32, [4]> tile_62 = const()[name = string("tile_62"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_10563_axis_0 = const()[name = string("op_10563_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 16]> var_10563_cast_fp16_0, tensor<fp16, [1, 512, 1, 16]> var_10563_cast_fp16_1, tensor<fp16, [1, 512, 1, 16]> var_10563_cast_fp16_2, tensor<fp16, [1, 512, 1, 16]> var_10563_cast_fp16_3 = split(axis = var_10563_axis_0, split_sizes = tile_62, x = var_10562_cast_fp16)[name = string("op_10563_cast_fp16")];
            tensor<int32, [4]> tile_63 = const()[name = string("tile_63"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_10568_axis_0 = const()[name = string("op_10568_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_10568_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_10568_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_10568_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_10568_cast_fp16_3 = split(axis = var_10568_axis_0, split_sizes = tile_63, x = var_10517_cast_fp16_0)[name = string("op_10568_cast_fp16")];
            fp16 var_10573_to_fp16 = const()[name = string("op_10573_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_10574_cast_fp16 = mul(x = var_10563_cast_fp16_0, y = var_10573_to_fp16)[name = string("op_10574_cast_fp16")];
            string var_10576_equation_0 = const()[name = string("op_10576_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_10576_cast_fp16 = einsum(equation = var_10576_equation_0, values = (var_10568_cast_fp16_0, var_10574_cast_fp16))[name = string("op_10576_cast_fp16")];
            fp16 var_10577_to_fp16 = const()[name = string("op_10577_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_10578_cast_fp16 = mul(x = var_10563_cast_fp16_1, y = var_10577_to_fp16)[name = string("op_10578_cast_fp16")];
            string var_10580_equation_0 = const()[name = string("op_10580_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_10580_cast_fp16 = einsum(equation = var_10580_equation_0, values = (var_10568_cast_fp16_1, var_10578_cast_fp16))[name = string("op_10580_cast_fp16")];
            fp16 var_10581_to_fp16 = const()[name = string("op_10581_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_10582_cast_fp16 = mul(x = var_10563_cast_fp16_2, y = var_10581_to_fp16)[name = string("op_10582_cast_fp16")];
            string var_10584_equation_0 = const()[name = string("op_10584_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_10584_cast_fp16 = einsum(equation = var_10584_equation_0, values = (var_10568_cast_fp16_2, var_10582_cast_fp16))[name = string("op_10584_cast_fp16")];
            fp16 var_10585_to_fp16 = const()[name = string("op_10585_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_10586_cast_fp16 = mul(x = var_10563_cast_fp16_3, y = var_10585_to_fp16)[name = string("op_10586_cast_fp16")];
            string var_10588_equation_0 = const()[name = string("op_10588_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_10588_cast_fp16 = einsum(equation = var_10588_equation_0, values = (var_10568_cast_fp16_3, var_10586_cast_fp16))[name = string("op_10588_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_10589_cast_fp16 = add(x = var_10576_cast_fp16, y = var_10580_cast_fp16)[name = string("op_10589_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_10590_cast_fp16 = add(x = var_10584_cast_fp16, y = var_10588_cast_fp16)[name = string("op_10590_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_10591_cast_fp16 = add(x = var_10589_cast_fp16, y = var_10590_cast_fp16)[name = string("op_10591_cast_fp16")];
            fp16 _inversed_10593_y_0_to_fp16 = const()[name = string("_inversed_10593_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 16]> _inversed_10593_cast_fp16 = mul(x = var_10591_cast_fp16, y = _inversed_10593_y_0_to_fp16)[name = string("_inversed_10593_cast_fp16")];
            string scores_53_equation_0 = const()[name = string("scores_53_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 16]> scores_53_cast_fp16 = einsum(equation = scores_53_equation_0, values = (var_10514_cast_fp16_0, var_10504_cast_fp16_2))[name = string("scores_53_cast_fp16")];
            fp16 var_10596_to_fp16 = const()[name = string("op_10596_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 16]> var_10597_cast_fp16 = mul(x = scores_53_cast_fp16, y = var_10596_to_fp16)[name = string("op_10597_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_10598_cast_fp16 = add(x = var_10597_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_10598_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_10599_cast_fp16 = softmax(axis = var_10482, x = var_10598_cast_fp16)[name = string("op_10599_cast_fp16")];
            tensor<int32, [4]> tile_64 = const()[name = string("tile_64"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_10600_axis_0 = const()[name = string("op_10600_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 16]> var_10600_cast_fp16_0, tensor<fp16, [1, 512, 1, 16]> var_10600_cast_fp16_1, tensor<fp16, [1, 512, 1, 16]> var_10600_cast_fp16_2, tensor<fp16, [1, 512, 1, 16]> var_10600_cast_fp16_3 = split(axis = var_10600_axis_0, split_sizes = tile_64, x = var_10599_cast_fp16)[name = string("op_10600_cast_fp16")];
            tensor<int32, [4]> tile_65 = const()[name = string("tile_65"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_10605_axis_0 = const()[name = string("op_10605_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_10605_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_10605_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_10605_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_10605_cast_fp16_3 = split(axis = var_10605_axis_0, split_sizes = tile_65, x = var_10517_cast_fp16_0)[name = string("op_10605_cast_fp16")];
            fp16 var_10610_to_fp16 = const()[name = string("op_10610_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_10611_cast_fp16 = mul(x = var_10600_cast_fp16_0, y = var_10610_to_fp16)[name = string("op_10611_cast_fp16")];
            string var_10613_equation_0 = const()[name = string("op_10613_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_10613_cast_fp16 = einsum(equation = var_10613_equation_0, values = (var_10605_cast_fp16_0, var_10611_cast_fp16))[name = string("op_10613_cast_fp16")];
            fp16 var_10614_to_fp16 = const()[name = string("op_10614_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_10615_cast_fp16 = mul(x = var_10600_cast_fp16_1, y = var_10614_to_fp16)[name = string("op_10615_cast_fp16")];
            string var_10617_equation_0 = const()[name = string("op_10617_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_10617_cast_fp16 = einsum(equation = var_10617_equation_0, values = (var_10605_cast_fp16_1, var_10615_cast_fp16))[name = string("op_10617_cast_fp16")];
            fp16 var_10618_to_fp16 = const()[name = string("op_10618_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_10619_cast_fp16 = mul(x = var_10600_cast_fp16_2, y = var_10618_to_fp16)[name = string("op_10619_cast_fp16")];
            string var_10621_equation_0 = const()[name = string("op_10621_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_10621_cast_fp16 = einsum(equation = var_10621_equation_0, values = (var_10605_cast_fp16_2, var_10619_cast_fp16))[name = string("op_10621_cast_fp16")];
            fp16 var_10622_to_fp16 = const()[name = string("op_10622_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_10623_cast_fp16 = mul(x = var_10600_cast_fp16_3, y = var_10622_to_fp16)[name = string("op_10623_cast_fp16")];
            string var_10625_equation_0 = const()[name = string("op_10625_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_10625_cast_fp16 = einsum(equation = var_10625_equation_0, values = (var_10605_cast_fp16_3, var_10623_cast_fp16))[name = string("op_10625_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_10626_cast_fp16 = add(x = var_10613_cast_fp16, y = var_10617_cast_fp16)[name = string("op_10626_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_10627_cast_fp16 = add(x = var_10621_cast_fp16, y = var_10625_cast_fp16)[name = string("op_10627_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_10628_cast_fp16 = add(x = var_10626_cast_fp16, y = var_10627_cast_fp16)[name = string("op_10628_cast_fp16")];
            fp16 _inversed_10630_y_0_to_fp16 = const()[name = string("_inversed_10630_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 16]> _inversed_10630_cast_fp16 = mul(x = var_10628_cast_fp16, y = _inversed_10630_y_0_to_fp16)[name = string("_inversed_10630_cast_fp16")];
            string scores_55_equation_0 = const()[name = string("scores_55_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 16]> scores_55_cast_fp16 = einsum(equation = scores_55_equation_0, values = (var_10514_cast_fp16_0, var_10504_cast_fp16_3))[name = string("scores_55_cast_fp16")];
            fp16 var_10633_to_fp16 = const()[name = string("op_10633_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 16]> var_10634_cast_fp16 = mul(x = scores_55_cast_fp16, y = var_10633_to_fp16)[name = string("op_10634_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_10635_cast_fp16 = add(x = var_10634_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_10635_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_10636_cast_fp16 = softmax(axis = var_10482, x = var_10635_cast_fp16)[name = string("op_10636_cast_fp16")];
            tensor<int32, [4]> tile_66 = const()[name = string("tile_66"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_10637_axis_0 = const()[name = string("op_10637_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 16]> var_10637_cast_fp16_0, tensor<fp16, [1, 512, 1, 16]> var_10637_cast_fp16_1, tensor<fp16, [1, 512, 1, 16]> var_10637_cast_fp16_2, tensor<fp16, [1, 512, 1, 16]> var_10637_cast_fp16_3 = split(axis = var_10637_axis_0, split_sizes = tile_66, x = var_10636_cast_fp16)[name = string("op_10637_cast_fp16")];
            tensor<int32, [4]> tile_67 = const()[name = string("tile_67"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_10642_axis_0 = const()[name = string("op_10642_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_10642_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_10642_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_10642_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_10642_cast_fp16_3 = split(axis = var_10642_axis_0, split_sizes = tile_67, x = var_10517_cast_fp16_0)[name = string("op_10642_cast_fp16")];
            fp16 var_10647_to_fp16 = const()[name = string("op_10647_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_10648_cast_fp16 = mul(x = var_10637_cast_fp16_0, y = var_10647_to_fp16)[name = string("op_10648_cast_fp16")];
            string var_10650_equation_0 = const()[name = string("op_10650_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_10650_cast_fp16 = einsum(equation = var_10650_equation_0, values = (var_10642_cast_fp16_0, var_10648_cast_fp16))[name = string("op_10650_cast_fp16")];
            fp16 var_10651_to_fp16 = const()[name = string("op_10651_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_10652_cast_fp16 = mul(x = var_10637_cast_fp16_1, y = var_10651_to_fp16)[name = string("op_10652_cast_fp16")];
            string var_10654_equation_0 = const()[name = string("op_10654_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_10654_cast_fp16 = einsum(equation = var_10654_equation_0, values = (var_10642_cast_fp16_1, var_10652_cast_fp16))[name = string("op_10654_cast_fp16")];
            fp16 var_10655_to_fp16 = const()[name = string("op_10655_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_10656_cast_fp16 = mul(x = var_10637_cast_fp16_2, y = var_10655_to_fp16)[name = string("op_10656_cast_fp16")];
            string var_10658_equation_0 = const()[name = string("op_10658_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_10658_cast_fp16 = einsum(equation = var_10658_equation_0, values = (var_10642_cast_fp16_2, var_10656_cast_fp16))[name = string("op_10658_cast_fp16")];
            fp16 var_10659_to_fp16 = const()[name = string("op_10659_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_10660_cast_fp16 = mul(x = var_10637_cast_fp16_3, y = var_10659_to_fp16)[name = string("op_10660_cast_fp16")];
            string var_10662_equation_0 = const()[name = string("op_10662_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_10662_cast_fp16 = einsum(equation = var_10662_equation_0, values = (var_10642_cast_fp16_3, var_10660_cast_fp16))[name = string("op_10662_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_10663_cast_fp16 = add(x = var_10650_cast_fp16, y = var_10654_cast_fp16)[name = string("op_10663_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_10664_cast_fp16 = add(x = var_10658_cast_fp16, y = var_10662_cast_fp16)[name = string("op_10664_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_10665_cast_fp16 = add(x = var_10663_cast_fp16, y = var_10664_cast_fp16)[name = string("op_10665_cast_fp16")];
            fp16 _inversed_10667_y_0_to_fp16 = const()[name = string("_inversed_10667_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 16]> _inversed_10667_cast_fp16 = mul(x = var_10665_cast_fp16, y = _inversed_10667_y_0_to_fp16)[name = string("_inversed_10667_cast_fp16")];
            string scores_57_equation_0 = const()[name = string("scores_57_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 16]> scores_57_cast_fp16 = einsum(equation = scores_57_equation_0, values = (var_10514_cast_fp16_1, var_10504_cast_fp16_4))[name = string("scores_57_cast_fp16")];
            fp16 var_10670_to_fp16 = const()[name = string("op_10670_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 16]> var_10671_cast_fp16 = mul(x = scores_57_cast_fp16, y = var_10670_to_fp16)[name = string("op_10671_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_10672_cast_fp16 = add(x = var_10671_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_10672_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_10673_cast_fp16 = softmax(axis = var_10482, x = var_10672_cast_fp16)[name = string("op_10673_cast_fp16")];
            tensor<int32, [4]> tile_68 = const()[name = string("tile_68"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_10674_axis_0 = const()[name = string("op_10674_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 16]> var_10674_cast_fp16_0, tensor<fp16, [1, 512, 1, 16]> var_10674_cast_fp16_1, tensor<fp16, [1, 512, 1, 16]> var_10674_cast_fp16_2, tensor<fp16, [1, 512, 1, 16]> var_10674_cast_fp16_3 = split(axis = var_10674_axis_0, split_sizes = tile_68, x = var_10673_cast_fp16)[name = string("op_10674_cast_fp16")];
            tensor<int32, [4]> tile_69 = const()[name = string("tile_69"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_10679_axis_0 = const()[name = string("op_10679_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_10679_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_10679_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_10679_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_10679_cast_fp16_3 = split(axis = var_10679_axis_0, split_sizes = tile_69, x = var_10517_cast_fp16_1)[name = string("op_10679_cast_fp16")];
            fp16 var_10684_to_fp16 = const()[name = string("op_10684_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_10685_cast_fp16 = mul(x = var_10674_cast_fp16_0, y = var_10684_to_fp16)[name = string("op_10685_cast_fp16")];
            string var_10687_equation_0 = const()[name = string("op_10687_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_10687_cast_fp16 = einsum(equation = var_10687_equation_0, values = (var_10679_cast_fp16_0, var_10685_cast_fp16))[name = string("op_10687_cast_fp16")];
            fp16 var_10688_to_fp16 = const()[name = string("op_10688_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_10689_cast_fp16 = mul(x = var_10674_cast_fp16_1, y = var_10688_to_fp16)[name = string("op_10689_cast_fp16")];
            string var_10691_equation_0 = const()[name = string("op_10691_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_10691_cast_fp16 = einsum(equation = var_10691_equation_0, values = (var_10679_cast_fp16_1, var_10689_cast_fp16))[name = string("op_10691_cast_fp16")];
            fp16 var_10692_to_fp16 = const()[name = string("op_10692_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_10693_cast_fp16 = mul(x = var_10674_cast_fp16_2, y = var_10692_to_fp16)[name = string("op_10693_cast_fp16")];
            string var_10695_equation_0 = const()[name = string("op_10695_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_10695_cast_fp16 = einsum(equation = var_10695_equation_0, values = (var_10679_cast_fp16_2, var_10693_cast_fp16))[name = string("op_10695_cast_fp16")];
            fp16 var_10696_to_fp16 = const()[name = string("op_10696_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_10697_cast_fp16 = mul(x = var_10674_cast_fp16_3, y = var_10696_to_fp16)[name = string("op_10697_cast_fp16")];
            string var_10699_equation_0 = const()[name = string("op_10699_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_10699_cast_fp16 = einsum(equation = var_10699_equation_0, values = (var_10679_cast_fp16_3, var_10697_cast_fp16))[name = string("op_10699_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_10700_cast_fp16 = add(x = var_10687_cast_fp16, y = var_10691_cast_fp16)[name = string("op_10700_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_10701_cast_fp16 = add(x = var_10695_cast_fp16, y = var_10699_cast_fp16)[name = string("op_10701_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_10702_cast_fp16 = add(x = var_10700_cast_fp16, y = var_10701_cast_fp16)[name = string("op_10702_cast_fp16")];
            fp16 _inversed_10704_y_0_to_fp16 = const()[name = string("_inversed_10704_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 16]> _inversed_10704_cast_fp16 = mul(x = var_10702_cast_fp16, y = _inversed_10704_y_0_to_fp16)[name = string("_inversed_10704_cast_fp16")];
            string scores_59_equation_0 = const()[name = string("scores_59_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 16]> scores_59_cast_fp16 = einsum(equation = scores_59_equation_0, values = (var_10514_cast_fp16_1, var_10504_cast_fp16_5))[name = string("scores_59_cast_fp16")];
            fp16 var_10707_to_fp16 = const()[name = string("op_10707_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 16]> var_10708_cast_fp16 = mul(x = scores_59_cast_fp16, y = var_10707_to_fp16)[name = string("op_10708_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_10709_cast_fp16 = add(x = var_10708_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_10709_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_10710_cast_fp16 = softmax(axis = var_10482, x = var_10709_cast_fp16)[name = string("op_10710_cast_fp16")];
            tensor<int32, [4]> tile_70 = const()[name = string("tile_70"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_10711_axis_0 = const()[name = string("op_10711_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 16]> var_10711_cast_fp16_0, tensor<fp16, [1, 512, 1, 16]> var_10711_cast_fp16_1, tensor<fp16, [1, 512, 1, 16]> var_10711_cast_fp16_2, tensor<fp16, [1, 512, 1, 16]> var_10711_cast_fp16_3 = split(axis = var_10711_axis_0, split_sizes = tile_70, x = var_10710_cast_fp16)[name = string("op_10711_cast_fp16")];
            tensor<int32, [4]> tile_71 = const()[name = string("tile_71"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_10716_axis_0 = const()[name = string("op_10716_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_10716_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_10716_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_10716_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_10716_cast_fp16_3 = split(axis = var_10716_axis_0, split_sizes = tile_71, x = var_10517_cast_fp16_1)[name = string("op_10716_cast_fp16")];
            fp16 var_10721_to_fp16 = const()[name = string("op_10721_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_10722_cast_fp16 = mul(x = var_10711_cast_fp16_0, y = var_10721_to_fp16)[name = string("op_10722_cast_fp16")];
            string var_10724_equation_0 = const()[name = string("op_10724_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_10724_cast_fp16 = einsum(equation = var_10724_equation_0, values = (var_10716_cast_fp16_0, var_10722_cast_fp16))[name = string("op_10724_cast_fp16")];
            fp16 var_10725_to_fp16 = const()[name = string("op_10725_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_10726_cast_fp16 = mul(x = var_10711_cast_fp16_1, y = var_10725_to_fp16)[name = string("op_10726_cast_fp16")];
            string var_10728_equation_0 = const()[name = string("op_10728_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_10728_cast_fp16 = einsum(equation = var_10728_equation_0, values = (var_10716_cast_fp16_1, var_10726_cast_fp16))[name = string("op_10728_cast_fp16")];
            fp16 var_10729_to_fp16 = const()[name = string("op_10729_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_10730_cast_fp16 = mul(x = var_10711_cast_fp16_2, y = var_10729_to_fp16)[name = string("op_10730_cast_fp16")];
            string var_10732_equation_0 = const()[name = string("op_10732_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_10732_cast_fp16 = einsum(equation = var_10732_equation_0, values = (var_10716_cast_fp16_2, var_10730_cast_fp16))[name = string("op_10732_cast_fp16")];
            fp16 var_10733_to_fp16 = const()[name = string("op_10733_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_10734_cast_fp16 = mul(x = var_10711_cast_fp16_3, y = var_10733_to_fp16)[name = string("op_10734_cast_fp16")];
            string var_10736_equation_0 = const()[name = string("op_10736_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_10736_cast_fp16 = einsum(equation = var_10736_equation_0, values = (var_10716_cast_fp16_3, var_10734_cast_fp16))[name = string("op_10736_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_10737_cast_fp16 = add(x = var_10724_cast_fp16, y = var_10728_cast_fp16)[name = string("op_10737_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_10738_cast_fp16 = add(x = var_10732_cast_fp16, y = var_10736_cast_fp16)[name = string("op_10738_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_10739_cast_fp16 = add(x = var_10737_cast_fp16, y = var_10738_cast_fp16)[name = string("op_10739_cast_fp16")];
            fp16 _inversed_10741_y_0_to_fp16 = const()[name = string("_inversed_10741_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 16]> _inversed_10741_cast_fp16 = mul(x = var_10739_cast_fp16, y = _inversed_10741_y_0_to_fp16)[name = string("_inversed_10741_cast_fp16")];
            string scores_61_equation_0 = const()[name = string("scores_61_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 16]> scores_61_cast_fp16 = einsum(equation = scores_61_equation_0, values = (var_10514_cast_fp16_1, var_10504_cast_fp16_6))[name = string("scores_61_cast_fp16")];
            fp16 var_10744_to_fp16 = const()[name = string("op_10744_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 16]> var_10745_cast_fp16 = mul(x = scores_61_cast_fp16, y = var_10744_to_fp16)[name = string("op_10745_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_10746_cast_fp16 = add(x = var_10745_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_10746_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_10747_cast_fp16 = softmax(axis = var_10482, x = var_10746_cast_fp16)[name = string("op_10747_cast_fp16")];
            tensor<int32, [4]> tile_72 = const()[name = string("tile_72"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_10748_axis_0 = const()[name = string("op_10748_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 16]> var_10748_cast_fp16_0, tensor<fp16, [1, 512, 1, 16]> var_10748_cast_fp16_1, tensor<fp16, [1, 512, 1, 16]> var_10748_cast_fp16_2, tensor<fp16, [1, 512, 1, 16]> var_10748_cast_fp16_3 = split(axis = var_10748_axis_0, split_sizes = tile_72, x = var_10747_cast_fp16)[name = string("op_10748_cast_fp16")];
            tensor<int32, [4]> tile_73 = const()[name = string("tile_73"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_10753_axis_0 = const()[name = string("op_10753_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_10753_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_10753_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_10753_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_10753_cast_fp16_3 = split(axis = var_10753_axis_0, split_sizes = tile_73, x = var_10517_cast_fp16_1)[name = string("op_10753_cast_fp16")];
            fp16 var_10758_to_fp16 = const()[name = string("op_10758_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_10759_cast_fp16 = mul(x = var_10748_cast_fp16_0, y = var_10758_to_fp16)[name = string("op_10759_cast_fp16")];
            string var_10761_equation_0 = const()[name = string("op_10761_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_10761_cast_fp16 = einsum(equation = var_10761_equation_0, values = (var_10753_cast_fp16_0, var_10759_cast_fp16))[name = string("op_10761_cast_fp16")];
            fp16 var_10762_to_fp16 = const()[name = string("op_10762_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_10763_cast_fp16 = mul(x = var_10748_cast_fp16_1, y = var_10762_to_fp16)[name = string("op_10763_cast_fp16")];
            string var_10765_equation_0 = const()[name = string("op_10765_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_10765_cast_fp16 = einsum(equation = var_10765_equation_0, values = (var_10753_cast_fp16_1, var_10763_cast_fp16))[name = string("op_10765_cast_fp16")];
            fp16 var_10766_to_fp16 = const()[name = string("op_10766_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_10767_cast_fp16 = mul(x = var_10748_cast_fp16_2, y = var_10766_to_fp16)[name = string("op_10767_cast_fp16")];
            string var_10769_equation_0 = const()[name = string("op_10769_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_10769_cast_fp16 = einsum(equation = var_10769_equation_0, values = (var_10753_cast_fp16_2, var_10767_cast_fp16))[name = string("op_10769_cast_fp16")];
            fp16 var_10770_to_fp16 = const()[name = string("op_10770_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_10771_cast_fp16 = mul(x = var_10748_cast_fp16_3, y = var_10770_to_fp16)[name = string("op_10771_cast_fp16")];
            string var_10773_equation_0 = const()[name = string("op_10773_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_10773_cast_fp16 = einsum(equation = var_10773_equation_0, values = (var_10753_cast_fp16_3, var_10771_cast_fp16))[name = string("op_10773_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_10774_cast_fp16 = add(x = var_10761_cast_fp16, y = var_10765_cast_fp16)[name = string("op_10774_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_10775_cast_fp16 = add(x = var_10769_cast_fp16, y = var_10773_cast_fp16)[name = string("op_10775_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_10776_cast_fp16 = add(x = var_10774_cast_fp16, y = var_10775_cast_fp16)[name = string("op_10776_cast_fp16")];
            fp16 _inversed_10778_y_0_to_fp16 = const()[name = string("_inversed_10778_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 16]> _inversed_10778_cast_fp16 = mul(x = var_10776_cast_fp16, y = _inversed_10778_y_0_to_fp16)[name = string("_inversed_10778_cast_fp16")];
            string scores_equation_0 = const()[name = string("scores_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 16]> scores_cast_fp16 = einsum(equation = scores_equation_0, values = (var_10514_cast_fp16_1, var_10504_cast_fp16_7))[name = string("scores_cast_fp16")];
            fp16 var_10781_to_fp16 = const()[name = string("op_10781_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 16]> var_10782_cast_fp16 = mul(x = scores_cast_fp16, y = var_10781_to_fp16)[name = string("op_10782_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_10783_cast_fp16 = add(x = var_10782_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_10783_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_10784_cast_fp16 = softmax(axis = var_10482, x = var_10783_cast_fp16)[name = string("op_10784_cast_fp16")];
            tensor<int32, [4]> tile_74 = const()[name = string("tile_74"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_10785_axis_0 = const()[name = string("op_10785_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 16]> var_10785_cast_fp16_0, tensor<fp16, [1, 512, 1, 16]> var_10785_cast_fp16_1, tensor<fp16, [1, 512, 1, 16]> var_10785_cast_fp16_2, tensor<fp16, [1, 512, 1, 16]> var_10785_cast_fp16_3 = split(axis = var_10785_axis_0, split_sizes = tile_74, x = var_10784_cast_fp16)[name = string("op_10785_cast_fp16")];
            tensor<int32, [4]> tile_75 = const()[name = string("tile_75"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_10790_axis_0 = const()[name = string("op_10790_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_10790_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_10790_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_10790_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_10790_cast_fp16_3 = split(axis = var_10790_axis_0, split_sizes = tile_75, x = var_10517_cast_fp16_1)[name = string("op_10790_cast_fp16")];
            fp16 var_10795_to_fp16 = const()[name = string("op_10795_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_10796_cast_fp16 = mul(x = var_10785_cast_fp16_0, y = var_10795_to_fp16)[name = string("op_10796_cast_fp16")];
            string var_10798_equation_0 = const()[name = string("op_10798_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_10798_cast_fp16 = einsum(equation = var_10798_equation_0, values = (var_10790_cast_fp16_0, var_10796_cast_fp16))[name = string("op_10798_cast_fp16")];
            fp16 var_10799_to_fp16 = const()[name = string("op_10799_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_10800_cast_fp16 = mul(x = var_10785_cast_fp16_1, y = var_10799_to_fp16)[name = string("op_10800_cast_fp16")];
            string var_10802_equation_0 = const()[name = string("op_10802_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_10802_cast_fp16 = einsum(equation = var_10802_equation_0, values = (var_10790_cast_fp16_1, var_10800_cast_fp16))[name = string("op_10802_cast_fp16")];
            fp16 var_10803_to_fp16 = const()[name = string("op_10803_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_10804_cast_fp16 = mul(x = var_10785_cast_fp16_2, y = var_10803_to_fp16)[name = string("op_10804_cast_fp16")];
            string var_10806_equation_0 = const()[name = string("op_10806_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_10806_cast_fp16 = einsum(equation = var_10806_equation_0, values = (var_10790_cast_fp16_2, var_10804_cast_fp16))[name = string("op_10806_cast_fp16")];
            fp16 var_10807_to_fp16 = const()[name = string("op_10807_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_10808_cast_fp16 = mul(x = var_10785_cast_fp16_3, y = var_10807_to_fp16)[name = string("op_10808_cast_fp16")];
            string var_10810_equation_0 = const()[name = string("op_10810_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_10810_cast_fp16 = einsum(equation = var_10810_equation_0, values = (var_10790_cast_fp16_3, var_10808_cast_fp16))[name = string("op_10810_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_10811_cast_fp16 = add(x = var_10798_cast_fp16, y = var_10802_cast_fp16)[name = string("op_10811_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_10812_cast_fp16 = add(x = var_10806_cast_fp16, y = var_10810_cast_fp16)[name = string("op_10812_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_10813_cast_fp16 = add(x = var_10811_cast_fp16, y = var_10812_cast_fp16)[name = string("op_10813_cast_fp16")];
            fp16 _inversed_10815_y_0_to_fp16 = const()[name = string("_inversed_10815_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 16]> _inversed_10815_cast_fp16 = mul(x = var_10813_cast_fp16, y = _inversed_10815_y_0_to_fp16)[name = string("_inversed_10815_cast_fp16")];
            bool var_10817_interleave_0 = const()[name = string("op_10817_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 2048, 1, 16]> var_10817_cast_fp16 = concat(axis = var_10482, interleave = var_10817_interleave_0, values = (_inversed_10556_cast_fp16, _inversed_10593_cast_fp16, _inversed_10630_cast_fp16, _inversed_10667_cast_fp16, _inversed_10704_cast_fp16, _inversed_10741_cast_fp16, _inversed_10778_cast_fp16, _inversed_10815_cast_fp16))[name = string("op_10817_cast_fp16")];
            tensor<int32, [4]> var_10818 = const()[name = string("op_10818"), val = tensor<int32, [4]>([2, 4, 256, 16])];
            tensor<fp16, [2, 4, 256, 16]> var_10819_cast_fp16 = reshape(shape = var_10818, x = var_10817_cast_fp16)[name = string("op_10819_cast_fp16")];
            tensor<int32, [4]> transpose_7_perm_0 = const()[name = string("transpose_7_perm_0"), val = tensor<int32, [4]>([3, 0, 1, 2])];
            tensor<int32, [2]> var_10838 = const()[name = string("op_10838"), val = tensor<int32, [2]>([16, 2048])];
            tensor<fp16, [16, 2, 4, 256]> transpose_7_cast_fp16 = transpose(perm = transpose_7_perm_0, x = var_10819_cast_fp16)[name = string("transpose_25")];
            tensor<fp16, [16, 2048]> attention_output_cast_fp16 = reshape(shape = var_10838, x = transpose_7_cast_fp16)[name = string("attention_output_cast_fp16")];
            tensor<fp16, [16, 2048]> var_10840_cast_fp16 = sigmoid(x = gate_15_cast_fp16)[name = string("op_10840_cast_fp16")];
            tensor<fp16, [16, 2048]> input_201_cast_fp16 = mul(x = attention_output_cast_fp16, y = var_10840_cast_fp16)[name = string("input_201_cast_fp16")];
            tensor<fp16, [1024, 2048]> completions_3_o_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 2048]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(241976576))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(243549504))))[name = string("completions_3_o_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 1024]> linear_120_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = completions_3_o_proj_weight_promoted_to_fp16_palettized, x = input_201_cast_fp16)[name = string("linear_120_cast_fp16")];
            tensor<fp16, [16, 1024]> value_231_cast_fp16 = add(x = value_217_cast_fp16, y = linear_120_cast_fp16)[name = string("value_231_cast_fp16")];
            tensor<fp16, [16, 1024]> var_10845_cast_fp16 = mul(x = value_231_cast_fp16, y = value_231_cast_fp16)[name = string("op_10845_cast_fp16")];
            tensor<int32, [1]> variance_103_axes_0 = const()[name = string("variance_103_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_103_keep_dims_0 = const()[name = string("variance_103_keep_dims_0"), val = bool(true)];
            tensor<fp16, [16, 1]> variance_103_cast_fp16 = reduce_mean(axes = variance_103_axes_0, keep_dims = variance_103_keep_dims_0, x = var_10845_cast_fp16)[name = string("variance_103_cast_fp16")];
            fp16 var_10848_to_fp16 = const()[name = string("op_10848_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [16, 1]> var_10849_cast_fp16 = add(x = variance_103_cast_fp16, y = var_10848_to_fp16)[name = string("op_10849_cast_fp16")];
            fp32 var_10850_epsilon_0 = const()[name = string("op_10850_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [16, 1]> var_10850_cast_fp16 = rsqrt(epsilon = var_10850_epsilon_0, x = var_10849_cast_fp16)[name = string("op_10850_cast_fp16")];
            tensor<fp16, [16, 1024]> var_10851_cast_fp16 = mul(x = value_231_cast_fp16, y = var_10850_cast_fp16)[name = string("op_10851_cast_fp16")];
            tensor<fp16, [1024]> var_10853_to_fp16 = const()[name = string("op_10853_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(243557760)))];
            tensor<fp16, [16, 1024]> input_203_cast_fp16 = mul(x = var_10851_cast_fp16, y = var_10853_to_fp16)[name = string("input_203_cast_fp16")];
            tensor<fp16, [3584, 1024]> completions_3_gate_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(243559872))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(246312448))))[name = string("completions_3_gate_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 3584]> linear_121_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = completions_3_gate_proj_weight_promoted_to_fp16_palettized, x = input_203_cast_fp16)[name = string("linear_121_cast_fp16")];
            tensor<fp16, [16, 3584]> var_10857_cast_fp16 = silu(x = linear_121_cast_fp16)[name = string("op_10857_cast_fp16")];
            tensor<fp16, [3584, 1024]> completions_3_up_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(246341184))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(249093760))))[name = string("completions_3_up_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 3584]> linear_122_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = completions_3_up_proj_weight_promoted_to_fp16_palettized, x = input_203_cast_fp16)[name = string("linear_122_cast_fp16")];
            tensor<fp16, [16, 3584]> input_207_cast_fp16 = mul(x = var_10857_cast_fp16, y = linear_122_cast_fp16)[name = string("input_207_cast_fp16")];
            tensor<fp16, [1024, 3584]> completions_3_down_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 3584]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(249122496))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(251875072))))[name = string("completions_3_down_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 1024]> linear_123_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = completions_3_down_proj_weight_promoted_to_fp16_palettized, x = input_207_cast_fp16)[name = string("linear_123_cast_fp16")];
            tensor<fp16, [16, 1024]> value_233_cast_fp16 = add(x = value_231_cast_fp16, y = linear_123_cast_fp16)[name = string("value_233_cast_fp16")];
            int32 var_10891 = const()[name = string("op_10891"), val = int32(-1)];
            tensor<fp16, [16, 1024]> var_10906_cast_fp16 = mul(x = value_233_cast_fp16, y = value_233_cast_fp16)[name = string("op_10906_cast_fp16")];
            tensor<int32, [1]> variance_105_axes_0 = const()[name = string("variance_105_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_105_keep_dims_0 = const()[name = string("variance_105_keep_dims_0"), val = bool(true)];
            tensor<fp16, [16, 1]> variance_105_cast_fp16 = reduce_mean(axes = variance_105_axes_0, keep_dims = variance_105_keep_dims_0, x = var_10906_cast_fp16)[name = string("variance_105_cast_fp16")];
            fp16 var_10909_to_fp16 = const()[name = string("op_10909_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [16, 1]> var_10910_cast_fp16 = add(x = variance_105_cast_fp16, y = var_10909_to_fp16)[name = string("op_10910_cast_fp16")];
            fp32 var_10911_epsilon_0 = const()[name = string("op_10911_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [16, 1]> var_10911_cast_fp16 = rsqrt(epsilon = var_10911_epsilon_0, x = var_10910_cast_fp16)[name = string("op_10911_cast_fp16")];
            tensor<fp16, [16, 1024]> var_10912_cast_fp16 = mul(x = value_233_cast_fp16, y = var_10911_cast_fp16)[name = string("op_10912_cast_fp16")];
            tensor<fp16, [1024]> var_10914_to_fp16 = const()[name = string("op_10914_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(251883328)))];
            tensor<fp16, [16, 1024]> input_209_cast_fp16 = mul(x = var_10912_cast_fp16, y = var_10914_to_fp16)[name = string("input_209_cast_fp16")];
            tensor<fp16, [6144, 1024]> final_group_0_in_proj_qkv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(251885440))), lut = tensor<fp16, [384, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(256604096))))[name = string("final_group_0_in_proj_qkv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 6144]> linear_124_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = final_group_0_in_proj_qkv_weight_promoted_to_fp16_palettized, x = input_209_cast_fp16)[name = string("linear_124_cast_fp16")];
            tensor<int32, [2]> var_10918_perm_0 = const()[name = string("op_10918_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> mixed_25_axes_0 = const()[name = string("mixed_25_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [6144, 16]> var_10918_cast_fp16 = transpose(perm = var_10918_perm_0, x = linear_124_cast_fp16)[name = string("transpose_24")];
            tensor<fp16, [1, 6144, 16]> mixed_25_cast_fp16 = expand_dims(axes = mixed_25_axes_0, x = var_10918_cast_fp16)[name = string("mixed_25_cast_fp16")];
            bool convolution_input_25_interleave_0 = const()[name = string("convolution_input_25_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 6144, 19]> convolution_input_25_cast_fp16 = concat(axis = var_10891, interleave = convolution_input_25_interleave_0, values = (conv16, mixed_25_cast_fp16))[name = string("convolution_input_25_cast_fp16")];
            string input_211_pad_type_0 = const()[name = string("input_211_pad_type_0"), val = string("valid")];
            int32 input_211_groups_0 = const()[name = string("input_211_groups_0"), val = int32(6144)];
            tensor<int32, [1]> input_211_strides_0 = const()[name = string("input_211_strides_0"), val = tensor<int32, [1]>([1])];
            tensor<int32, [2]> input_211_pad_0 = const()[name = string("input_211_pad_0"), val = tensor<int32, [2]>([0, 0])];
            tensor<int32, [1]> input_211_dilations_0 = const()[name = string("input_211_dilations_0"), val = tensor<int32, [1]>([1])];
            tensor<fp16, [6144, 1, 4]> final_group_0_conv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1, 4]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(256653312))), lut = tensor<fp16, [384, 1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(256671808))))[name = string("final_group_0_conv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 6144, 16]> input_211_cast_fp16 = conv(dilations = input_211_dilations_0, groups = input_211_groups_0, pad = input_211_pad_0, pad_type = input_211_pad_type_0, strides = input_211_strides_0, weight = final_group_0_conv_weight_promoted_to_fp16_palettized, x = convolution_input_25_cast_fp16)[name = string("input_211_cast_fp16")];
            tensor<fp16, [1, 6144, 16]> var_10927_cast_fp16 = silu(x = input_211_cast_fp16)[name = string("op_10927_cast_fp16")];
            tensor<int32, [3]> var_10928_perm_0 = const()[name = string("op_10928_perm_0"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [3]> var_10931_begin_0 = const()[name = string("op_10931_begin_0"), val = tensor<int32, [3]>([0, 0, 16])];
            tensor<int32, [3]> var_10931_end_0 = const()[name = string("op_10931_end_0"), val = tensor<int32, [3]>([1, 6144, 19])];
            tensor<bool, [3]> var_10931_end_mask_0 = const()[name = string("op_10931_end_mask_0"), val = tensor<bool, [3]>([true, true, true])];
            tensor<fp16, [1, 6144, 3]> conv16_out = slice_by_index(begin = var_10931_begin_0, end = var_10931_end_0, end_mask = var_10931_end_mask_0, x = convolution_input_25_cast_fp16)[name = string("op_10931_cast_fp16")];
            tensor<int32, [3]> var_10932 = const()[name = string("op_10932"), val = tensor<int32, [3]>([2048, 2048, 2048])];
            int32 var_10933_axis_0 = const()[name = string("op_10933_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 16, 6144]> var_10928_cast_fp16 = transpose(perm = var_10928_perm_0, x = var_10927_cast_fp16)[name = string("transpose_23")];
            tensor<fp16, [1, 16, 2048]> var_10933_cast_fp16_0, tensor<fp16, [1, 16, 2048]> var_10933_cast_fp16_1, tensor<fp16, [1, 16, 2048]> var_10933_cast_fp16_2 = split(axis = var_10933_axis_0, split_sizes = var_10932, x = var_10928_cast_fp16)[name = string("op_10933_cast_fp16")];
            tensor<int32, [4]> var_10937 = const()[name = string("op_10937"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<fp16, [1, 16, 16, 128]> value_237_cast_fp16 = reshape(shape = var_10937, x = var_10933_cast_fp16_0)[name = string("value_237_cast_fp16")];
            tensor<int32, [4]> var_10939 = const()[name = string("op_10939"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<fp16, [1, 16, 16, 128]> value_239_cast_fp16 = reshape(shape = var_10939, x = var_10933_cast_fp16_1)[name = string("value_239_cast_fp16")];
            tensor<int32, [4]> var_10941 = const()[name = string("op_10941"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<fp16, [1, 16, 16, 128]> value_241_cast_fp16 = reshape(shape = var_10941, x = var_10933_cast_fp16_2)[name = string("value_241_cast_fp16")];
            tensor<fp16, [1, 16, 16, 128]> var_10943_cast_fp16 = mul(x = value_237_cast_fp16, y = value_237_cast_fp16)[name = string("op_10943_cast_fp16")];
            tensor<int32, [1]> var_10945_axes_0 = const()[name = string("op_10945_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_10945_keep_dims_0 = const()[name = string("op_10945_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 16, 16, 1]> var_10945_cast_fp16 = reduce_sum(axes = var_10945_axes_0, keep_dims = var_10945_keep_dims_0, x = var_10943_cast_fp16)[name = string("op_10945_cast_fp16")];
            fp16 var_10946_to_fp16 = const()[name = string("op_10946_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 16, 16, 1]> var_10947_cast_fp16 = add(x = var_10945_cast_fp16, y = var_10946_to_fp16)[name = string("op_10947_cast_fp16")];
            fp32 var_10948_epsilon_0 = const()[name = string("op_10948_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 16, 16, 1]> var_10948_cast_fp16 = rsqrt(epsilon = var_10948_epsilon_0, x = var_10947_cast_fp16)[name = string("op_10948_cast_fp16")];
            tensor<fp16, [1, 16, 16, 128]> var_10949_cast_fp16 = mul(x = value_237_cast_fp16, y = var_10948_cast_fp16)[name = string("op_10949_cast_fp16")];
            tensor<int32, [4]> var_10950_perm_0 = const()[name = string("op_10950_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            fp16 _inversed_query_67_y_0_to_fp16 = const()[name = string("_inversed_query_67_y_0_to_fp16"), val = fp16(0x1.6ap-4)];
            tensor<fp16, [1, 16, 16, 128]> var_10950_cast_fp16 = transpose(perm = var_10950_perm_0, x = var_10949_cast_fp16)[name = string("transpose_22")];
            tensor<fp16, [1, 16, 16, 128]> _inversed_query_67_cast_fp16 = mul(x = var_10950_cast_fp16, y = _inversed_query_67_y_0_to_fp16)[name = string("_inversed_query_67_cast_fp16")];
            tensor<fp16, [1, 16, 16, 128]> var_10953_cast_fp16 = mul(x = value_239_cast_fp16, y = value_239_cast_fp16)[name = string("op_10953_cast_fp16")];
            tensor<int32, [1]> var_10955_axes_0 = const()[name = string("op_10955_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_10955_keep_dims_0 = const()[name = string("op_10955_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 16, 16, 1]> var_10955_cast_fp16 = reduce_sum(axes = var_10955_axes_0, keep_dims = var_10955_keep_dims_0, x = var_10953_cast_fp16)[name = string("op_10955_cast_fp16")];
            fp16 var_10956_to_fp16 = const()[name = string("op_10956_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 16, 16, 1]> var_10957_cast_fp16 = add(x = var_10955_cast_fp16, y = var_10956_to_fp16)[name = string("op_10957_cast_fp16")];
            fp32 var_10958_epsilon_0 = const()[name = string("op_10958_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 16, 16, 1]> var_10958_cast_fp16 = rsqrt(epsilon = var_10958_epsilon_0, x = var_10957_cast_fp16)[name = string("op_10958_cast_fp16")];
            tensor<fp16, [1, 16, 16, 128]> var_10959_cast_fp16 = mul(x = value_239_cast_fp16, y = var_10958_cast_fp16)[name = string("op_10959_cast_fp16")];
            tensor<int32, [4]> key_67_perm_0 = const()[name = string("key_67_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<int32, [4]> value_243_perm_0 = const()[name = string("value_243_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<fp16, [16, 1024]> final_group_0_in_proj_b_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(256721024))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(256733376))))[name = string("final_group_0_in_proj_b_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16]> linear_125_bias_0_to_fp16 = const()[name = string("linear_125_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(4862656)))];
            tensor<fp16, [16, 16]> linear_125_cast_fp16 = linear(bias = linear_125_bias_0_to_fp16, weight = final_group_0_in_proj_b_weight_promoted_to_fp16_palettized, x = input_209_cast_fp16)[name = string("linear_125_cast_fp16")];
            tensor<fp16, [16, 16]> var_10964_cast_fp16 = sigmoid(x = linear_125_cast_fp16)[name = string("op_10964_cast_fp16")];
            tensor<int32, [2]> var_10965_perm_0 = const()[name = string("op_10965_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> beta_25_axes_0 = const()[name = string("beta_25_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 16]> var_10965_cast_fp16 = transpose(perm = var_10965_perm_0, x = var_10964_cast_fp16)[name = string("transpose_21")];
            tensor<fp16, [1, 16, 16]> beta_25_cast_fp16 = expand_dims(axes = beta_25_axes_0, x = var_10965_cast_fp16)[name = string("beta_25_cast_fp16")];
            tensor<fp16, [16, 1024]> op_10971_weight_0_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(256733568))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(256745920))))[name = string("op_10971_weight_0_to_fp16_palettized")];
            tensor<fp16, [16]> var_10971_bias_0_to_fp16 = const()[name = string("op_10971_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(256746112)))];
            tensor<fp16, [16, 16]> var_10971_cast_fp16 = linear(bias = var_10971_bias_0_to_fp16, weight = op_10971_weight_0_to_fp16_palettized, x = input_209_cast_fp16)[name = string("op_10971_cast_fp16")];
            tensor<fp16, [16, 16]> var_10972_cast_fp16 = softplus(x = var_10971_cast_fp16)[name = string("op_10972_cast_fp16")];
            tensor<fp16, [16]> var_10968_promoted_to_fp16 = const()[name = string("op_10968_promoted_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(256746240)))];
            tensor<fp16, [16, 16]> var_10973_cast_fp16 = mul(x = var_10968_promoted_to_fp16, y = var_10972_cast_fp16)[name = string("op_10973_cast_fp16")];
            tensor<int32, [2]> var_10974_perm_0 = const()[name = string("op_10974_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> decay_25_axes_0 = const()[name = string("decay_25_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 16]> var_10974_cast_fp16 = transpose(perm = var_10974_perm_0, x = var_10973_cast_fp16)[name = string("transpose_20")];
            tensor<fp16, [1, 16, 16]> decay_25_cast_fp16 = expand_dims(axes = decay_25_axes_0, x = var_10974_cast_fp16)[name = string("decay_25_cast_fp16")];
            tensor<fp16, [2048, 1024]> final_group_0_in_proj_z_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [2048, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(256746368))), lut = tensor<fp16, [128, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(258319296))))[name = string("final_group_0_in_proj_z_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 2048]> linear_127_cast_fp16 = linear(bias = linear_3_bias_0_to_fp16, weight = final_group_0_in_proj_z_weight_promoted_to_fp16_palettized, x = input_209_cast_fp16)[name = string("linear_127_cast_fp16")];
            tensor<int32, [3]> var_10982_begin_0 = const()[name = string("op_10982_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_10982_end_0 = const()[name = string("op_10982_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_10982_end_mask_0 = const()[name = string("op_10982_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_10982_squeeze_mask_0 = const()[name = string("op_10982_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_10982_cast_fp16 = slice_by_index(begin = var_10982_begin_0, end = var_10982_end_0, end_mask = var_10982_end_mask_0, squeeze_mask = var_10982_squeeze_mask_0, x = decay_25_cast_fp16)[name = string("op_10982_cast_fp16")];
            tensor<fp16, [1, 16]> var_10983_cast_fp16 = exp(x = var_10982_cast_fp16)[name = string("op_10983_cast_fp16")];
            tensor<int32, [1]> var_10984_axes_0 = const()[name = string("op_10984_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_10984_cast_fp16 = expand_dims(axes = var_10984_axes_0, x = var_10983_cast_fp16)[name = string("op_10984_cast_fp16")];
            tensor<int32, [1]> var_10985_axes_0 = const()[name = string("op_10985_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_10985_cast_fp16 = expand_dims(axes = var_10985_axes_0, x = var_10984_cast_fp16)[name = string("op_10985_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_769_cast_fp16 = mul(x = rec16, y = var_10985_cast_fp16)[name = string("state_769_cast_fp16")];
            tensor<int32, [4]> key_token_385_begin_0 = const()[name = string("key_token_385_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> key_token_385_end_0 = const()[name = string("key_token_385_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> key_token_385_end_mask_0 = const()[name = string("key_token_385_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_385_squeeze_mask_0 = const()[name = string("key_token_385_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 16, 128]> key_67_cast_fp16 = transpose(perm = key_67_perm_0, x = var_10959_cast_fp16)[name = string("transpose_19")];
            tensor<fp16, [1, 16, 128]> key_token_385_cast_fp16 = slice_by_index(begin = key_token_385_begin_0, end = key_token_385_end_0, end_mask = key_token_385_end_mask_0, squeeze_mask = key_token_385_squeeze_mask_0, x = key_67_cast_fp16)[name = string("key_token_385_cast_fp16")];
            tensor<int32, [4]> value_token_385_begin_0 = const()[name = string("value_token_385_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> value_token_385_end_0 = const()[name = string("value_token_385_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> value_token_385_end_mask_0 = const()[name = string("value_token_385_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_385_squeeze_mask_0 = const()[name = string("value_token_385_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 16, 128]> value_243_cast_fp16 = transpose(perm = value_243_perm_0, x = value_241_cast_fp16)[name = string("transpose_18")];
            tensor<fp16, [1, 16, 128]> value_token_385_cast_fp16 = slice_by_index(begin = value_token_385_begin_0, end = value_token_385_end_0, end_mask = value_token_385_end_mask_0, squeeze_mask = value_token_385_squeeze_mask_0, x = value_243_cast_fp16)[name = string("value_token_385_cast_fp16")];
            tensor<int32, [1]> var_10993_axes_0 = const()[name = string("op_10993_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_10993_cast_fp16 = expand_dims(axes = var_10993_axes_0, x = key_token_385_cast_fp16)[name = string("op_10993_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_10994_cast_fp16 = mul(x = state_769_cast_fp16, y = var_10993_cast_fp16)[name = string("op_10994_cast_fp16")];
            tensor<int32, [1]> memory_385_axes_0 = const()[name = string("memory_385_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_385_keep_dims_0 = const()[name = string("memory_385_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_385_cast_fp16 = reduce_sum(axes = memory_385_axes_0, keep_dims = memory_385_keep_dims_0, x = var_10994_cast_fp16)[name = string("memory_385_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_10997_cast_fp16 = sub(x = value_token_385_cast_fp16, y = memory_385_cast_fp16)[name = string("op_10997_cast_fp16")];
            tensor<int32, [3]> var_11000_begin_0 = const()[name = string("op_11000_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_11000_end_0 = const()[name = string("op_11000_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_11000_end_mask_0 = const()[name = string("op_11000_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_11000_squeeze_mask_0 = const()[name = string("op_11000_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_11000_cast_fp16 = slice_by_index(begin = var_11000_begin_0, end = var_11000_end_0, end_mask = var_11000_end_mask_0, squeeze_mask = var_11000_squeeze_mask_0, x = beta_25_cast_fp16)[name = string("op_11000_cast_fp16")];
            tensor<int32, [1]> var_11001_axes_0 = const()[name = string("op_11001_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_11001_cast_fp16 = expand_dims(axes = var_11001_axes_0, x = var_11000_cast_fp16)[name = string("op_11001_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_385_cast_fp16 = mul(x = var_10997_cast_fp16, y = var_11001_cast_fp16)[name = string("delta_385_cast_fp16")];
            tensor<int32, [1]> var_11004_axes_0 = const()[name = string("op_11004_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_11004_cast_fp16 = expand_dims(axes = var_11004_axes_0, x = delta_385_cast_fp16)[name = string("op_11004_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_11005_cast_fp16 = mul(x = var_10993_cast_fp16, y = var_11004_cast_fp16)[name = string("op_11005_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_771_cast_fp16 = add(x = state_769_cast_fp16, y = var_11005_cast_fp16)[name = string("state_771_cast_fp16")];
            tensor<int32, [4]> var_11009_begin_0 = const()[name = string("op_11009_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> var_11009_end_0 = const()[name = string("op_11009_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> var_11009_end_mask_0 = const()[name = string("op_11009_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_11009_squeeze_mask_0 = const()[name = string("op_11009_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_11009_cast_fp16 = slice_by_index(begin = var_11009_begin_0, end = var_11009_end_0, end_mask = var_11009_end_mask_0, squeeze_mask = var_11009_squeeze_mask_0, x = _inversed_query_67_cast_fp16)[name = string("op_11009_cast_fp16")];
            tensor<int32, [1]> var_11010_axes_0 = const()[name = string("op_11010_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_11010_cast_fp16 = expand_dims(axes = var_11010_axes_0, x = var_11009_cast_fp16)[name = string("op_11010_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_11011_cast_fp16 = mul(x = state_771_cast_fp16, y = var_11010_cast_fp16)[name = string("op_11011_cast_fp16")];
            tensor<int32, [1]> var_11013_axes_0 = const()[name = string("op_11013_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_11013_keep_dims_0 = const()[name = string("op_11013_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_11013_cast_fp16 = reduce_sum(axes = var_11013_axes_0, keep_dims = var_11013_keep_dims_0, x = var_11011_cast_fp16)[name = string("op_11013_cast_fp16")];
            tensor<int32, [3]> var_11016_begin_0 = const()[name = string("op_11016_begin_0"), val = tensor<int32, [3]>([0, 0, 1])];
            tensor<int32, [3]> var_11016_end_0 = const()[name = string("op_11016_end_0"), val = tensor<int32, [3]>([1, 16, 2])];
            tensor<bool, [3]> var_11016_end_mask_0 = const()[name = string("op_11016_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_11016_squeeze_mask_0 = const()[name = string("op_11016_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_11016_cast_fp16 = slice_by_index(begin = var_11016_begin_0, end = var_11016_end_0, end_mask = var_11016_end_mask_0, squeeze_mask = var_11016_squeeze_mask_0, x = decay_25_cast_fp16)[name = string("op_11016_cast_fp16")];
            tensor<fp16, [1, 16]> var_11017_cast_fp16 = exp(x = var_11016_cast_fp16)[name = string("op_11017_cast_fp16")];
            tensor<int32, [1]> var_11018_axes_0 = const()[name = string("op_11018_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_11018_cast_fp16 = expand_dims(axes = var_11018_axes_0, x = var_11017_cast_fp16)[name = string("op_11018_cast_fp16")];
            tensor<int32, [1]> var_11019_axes_0 = const()[name = string("op_11019_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_11019_cast_fp16 = expand_dims(axes = var_11019_axes_0, x = var_11018_cast_fp16)[name = string("op_11019_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_773_cast_fp16 = mul(x = state_771_cast_fp16, y = var_11019_cast_fp16)[name = string("state_773_cast_fp16")];
            tensor<int32, [4]> key_token_387_begin_0 = const()[name = string("key_token_387_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> key_token_387_end_0 = const()[name = string("key_token_387_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> key_token_387_end_mask_0 = const()[name = string("key_token_387_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_387_squeeze_mask_0 = const()[name = string("key_token_387_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_387_cast_fp16 = slice_by_index(begin = key_token_387_begin_0, end = key_token_387_end_0, end_mask = key_token_387_end_mask_0, squeeze_mask = key_token_387_squeeze_mask_0, x = key_67_cast_fp16)[name = string("key_token_387_cast_fp16")];
            tensor<int32, [4]> value_token_387_begin_0 = const()[name = string("value_token_387_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> value_token_387_end_0 = const()[name = string("value_token_387_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> value_token_387_end_mask_0 = const()[name = string("value_token_387_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_387_squeeze_mask_0 = const()[name = string("value_token_387_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_387_cast_fp16 = slice_by_index(begin = value_token_387_begin_0, end = value_token_387_end_0, end_mask = value_token_387_end_mask_0, squeeze_mask = value_token_387_squeeze_mask_0, x = value_243_cast_fp16)[name = string("value_token_387_cast_fp16")];
            tensor<int32, [1]> var_11027_axes_0 = const()[name = string("op_11027_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_11027_cast_fp16 = expand_dims(axes = var_11027_axes_0, x = key_token_387_cast_fp16)[name = string("op_11027_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_11028_cast_fp16 = mul(x = state_773_cast_fp16, y = var_11027_cast_fp16)[name = string("op_11028_cast_fp16")];
            tensor<int32, [1]> memory_387_axes_0 = const()[name = string("memory_387_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_387_keep_dims_0 = const()[name = string("memory_387_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_387_cast_fp16 = reduce_sum(axes = memory_387_axes_0, keep_dims = memory_387_keep_dims_0, x = var_11028_cast_fp16)[name = string("memory_387_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_11031_cast_fp16 = sub(x = value_token_387_cast_fp16, y = memory_387_cast_fp16)[name = string("op_11031_cast_fp16")];
            tensor<int32, [3]> var_11034_begin_0 = const()[name = string("op_11034_begin_0"), val = tensor<int32, [3]>([0, 0, 1])];
            tensor<int32, [3]> var_11034_end_0 = const()[name = string("op_11034_end_0"), val = tensor<int32, [3]>([1, 16, 2])];
            tensor<bool, [3]> var_11034_end_mask_0 = const()[name = string("op_11034_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_11034_squeeze_mask_0 = const()[name = string("op_11034_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_11034_cast_fp16 = slice_by_index(begin = var_11034_begin_0, end = var_11034_end_0, end_mask = var_11034_end_mask_0, squeeze_mask = var_11034_squeeze_mask_0, x = beta_25_cast_fp16)[name = string("op_11034_cast_fp16")];
            tensor<int32, [1]> var_11035_axes_0 = const()[name = string("op_11035_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_11035_cast_fp16 = expand_dims(axes = var_11035_axes_0, x = var_11034_cast_fp16)[name = string("op_11035_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_387_cast_fp16 = mul(x = var_11031_cast_fp16, y = var_11035_cast_fp16)[name = string("delta_387_cast_fp16")];
            tensor<int32, [1]> var_11038_axes_0 = const()[name = string("op_11038_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_11038_cast_fp16 = expand_dims(axes = var_11038_axes_0, x = delta_387_cast_fp16)[name = string("op_11038_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_11039_cast_fp16 = mul(x = var_11027_cast_fp16, y = var_11038_cast_fp16)[name = string("op_11039_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_775_cast_fp16 = add(x = state_773_cast_fp16, y = var_11039_cast_fp16)[name = string("state_775_cast_fp16")];
            tensor<int32, [4]> var_11043_begin_0 = const()[name = string("op_11043_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> var_11043_end_0 = const()[name = string("op_11043_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> var_11043_end_mask_0 = const()[name = string("op_11043_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_11043_squeeze_mask_0 = const()[name = string("op_11043_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_11043_cast_fp16 = slice_by_index(begin = var_11043_begin_0, end = var_11043_end_0, end_mask = var_11043_end_mask_0, squeeze_mask = var_11043_squeeze_mask_0, x = _inversed_query_67_cast_fp16)[name = string("op_11043_cast_fp16")];
            tensor<int32, [1]> var_11044_axes_0 = const()[name = string("op_11044_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_11044_cast_fp16 = expand_dims(axes = var_11044_axes_0, x = var_11043_cast_fp16)[name = string("op_11044_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_11045_cast_fp16 = mul(x = state_775_cast_fp16, y = var_11044_cast_fp16)[name = string("op_11045_cast_fp16")];
            tensor<int32, [1]> var_11047_axes_0 = const()[name = string("op_11047_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_11047_keep_dims_0 = const()[name = string("op_11047_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_11047_cast_fp16 = reduce_sum(axes = var_11047_axes_0, keep_dims = var_11047_keep_dims_0, x = var_11045_cast_fp16)[name = string("op_11047_cast_fp16")];
            tensor<int32, [3]> var_11050_begin_0 = const()[name = string("op_11050_begin_0"), val = tensor<int32, [3]>([0, 0, 2])];
            tensor<int32, [3]> var_11050_end_0 = const()[name = string("op_11050_end_0"), val = tensor<int32, [3]>([1, 16, 3])];
            tensor<bool, [3]> var_11050_end_mask_0 = const()[name = string("op_11050_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_11050_squeeze_mask_0 = const()[name = string("op_11050_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_11050_cast_fp16 = slice_by_index(begin = var_11050_begin_0, end = var_11050_end_0, end_mask = var_11050_end_mask_0, squeeze_mask = var_11050_squeeze_mask_0, x = decay_25_cast_fp16)[name = string("op_11050_cast_fp16")];
            tensor<fp16, [1, 16]> var_11051_cast_fp16 = exp(x = var_11050_cast_fp16)[name = string("op_11051_cast_fp16")];
            tensor<int32, [1]> var_11052_axes_0 = const()[name = string("op_11052_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_11052_cast_fp16 = expand_dims(axes = var_11052_axes_0, x = var_11051_cast_fp16)[name = string("op_11052_cast_fp16")];
            tensor<int32, [1]> var_11053_axes_0 = const()[name = string("op_11053_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_11053_cast_fp16 = expand_dims(axes = var_11053_axes_0, x = var_11052_cast_fp16)[name = string("op_11053_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_777_cast_fp16 = mul(x = state_775_cast_fp16, y = var_11053_cast_fp16)[name = string("state_777_cast_fp16")];
            tensor<int32, [4]> key_token_389_begin_0 = const()[name = string("key_token_389_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> key_token_389_end_0 = const()[name = string("key_token_389_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> key_token_389_end_mask_0 = const()[name = string("key_token_389_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_389_squeeze_mask_0 = const()[name = string("key_token_389_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_389_cast_fp16 = slice_by_index(begin = key_token_389_begin_0, end = key_token_389_end_0, end_mask = key_token_389_end_mask_0, squeeze_mask = key_token_389_squeeze_mask_0, x = key_67_cast_fp16)[name = string("key_token_389_cast_fp16")];
            tensor<int32, [4]> value_token_389_begin_0 = const()[name = string("value_token_389_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> value_token_389_end_0 = const()[name = string("value_token_389_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> value_token_389_end_mask_0 = const()[name = string("value_token_389_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_389_squeeze_mask_0 = const()[name = string("value_token_389_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_389_cast_fp16 = slice_by_index(begin = value_token_389_begin_0, end = value_token_389_end_0, end_mask = value_token_389_end_mask_0, squeeze_mask = value_token_389_squeeze_mask_0, x = value_243_cast_fp16)[name = string("value_token_389_cast_fp16")];
            tensor<int32, [1]> var_11061_axes_0 = const()[name = string("op_11061_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_11061_cast_fp16 = expand_dims(axes = var_11061_axes_0, x = key_token_389_cast_fp16)[name = string("op_11061_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_11062_cast_fp16 = mul(x = state_777_cast_fp16, y = var_11061_cast_fp16)[name = string("op_11062_cast_fp16")];
            tensor<int32, [1]> memory_389_axes_0 = const()[name = string("memory_389_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_389_keep_dims_0 = const()[name = string("memory_389_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_389_cast_fp16 = reduce_sum(axes = memory_389_axes_0, keep_dims = memory_389_keep_dims_0, x = var_11062_cast_fp16)[name = string("memory_389_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_11065_cast_fp16 = sub(x = value_token_389_cast_fp16, y = memory_389_cast_fp16)[name = string("op_11065_cast_fp16")];
            tensor<int32, [3]> var_11068_begin_0 = const()[name = string("op_11068_begin_0"), val = tensor<int32, [3]>([0, 0, 2])];
            tensor<int32, [3]> var_11068_end_0 = const()[name = string("op_11068_end_0"), val = tensor<int32, [3]>([1, 16, 3])];
            tensor<bool, [3]> var_11068_end_mask_0 = const()[name = string("op_11068_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_11068_squeeze_mask_0 = const()[name = string("op_11068_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_11068_cast_fp16 = slice_by_index(begin = var_11068_begin_0, end = var_11068_end_0, end_mask = var_11068_end_mask_0, squeeze_mask = var_11068_squeeze_mask_0, x = beta_25_cast_fp16)[name = string("op_11068_cast_fp16")];
            tensor<int32, [1]> var_11069_axes_0 = const()[name = string("op_11069_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_11069_cast_fp16 = expand_dims(axes = var_11069_axes_0, x = var_11068_cast_fp16)[name = string("op_11069_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_389_cast_fp16 = mul(x = var_11065_cast_fp16, y = var_11069_cast_fp16)[name = string("delta_389_cast_fp16")];
            tensor<int32, [1]> var_11072_axes_0 = const()[name = string("op_11072_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_11072_cast_fp16 = expand_dims(axes = var_11072_axes_0, x = delta_389_cast_fp16)[name = string("op_11072_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_11073_cast_fp16 = mul(x = var_11061_cast_fp16, y = var_11072_cast_fp16)[name = string("op_11073_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_779_cast_fp16 = add(x = state_777_cast_fp16, y = var_11073_cast_fp16)[name = string("state_779_cast_fp16")];
            tensor<int32, [4]> var_11077_begin_0 = const()[name = string("op_11077_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> var_11077_end_0 = const()[name = string("op_11077_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> var_11077_end_mask_0 = const()[name = string("op_11077_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_11077_squeeze_mask_0 = const()[name = string("op_11077_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_11077_cast_fp16 = slice_by_index(begin = var_11077_begin_0, end = var_11077_end_0, end_mask = var_11077_end_mask_0, squeeze_mask = var_11077_squeeze_mask_0, x = _inversed_query_67_cast_fp16)[name = string("op_11077_cast_fp16")];
            tensor<int32, [1]> var_11078_axes_0 = const()[name = string("op_11078_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_11078_cast_fp16 = expand_dims(axes = var_11078_axes_0, x = var_11077_cast_fp16)[name = string("op_11078_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_11079_cast_fp16 = mul(x = state_779_cast_fp16, y = var_11078_cast_fp16)[name = string("op_11079_cast_fp16")];
            tensor<int32, [1]> var_11081_axes_0 = const()[name = string("op_11081_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_11081_keep_dims_0 = const()[name = string("op_11081_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_11081_cast_fp16 = reduce_sum(axes = var_11081_axes_0, keep_dims = var_11081_keep_dims_0, x = var_11079_cast_fp16)[name = string("op_11081_cast_fp16")];
            tensor<int32, [3]> var_11084_begin_0 = const()[name = string("op_11084_begin_0"), val = tensor<int32, [3]>([0, 0, 3])];
            tensor<int32, [3]> var_11084_end_0 = const()[name = string("op_11084_end_0"), val = tensor<int32, [3]>([1, 16, 4])];
            tensor<bool, [3]> var_11084_end_mask_0 = const()[name = string("op_11084_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_11084_squeeze_mask_0 = const()[name = string("op_11084_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_11084_cast_fp16 = slice_by_index(begin = var_11084_begin_0, end = var_11084_end_0, end_mask = var_11084_end_mask_0, squeeze_mask = var_11084_squeeze_mask_0, x = decay_25_cast_fp16)[name = string("op_11084_cast_fp16")];
            tensor<fp16, [1, 16]> var_11085_cast_fp16 = exp(x = var_11084_cast_fp16)[name = string("op_11085_cast_fp16")];
            tensor<int32, [1]> var_11086_axes_0 = const()[name = string("op_11086_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_11086_cast_fp16 = expand_dims(axes = var_11086_axes_0, x = var_11085_cast_fp16)[name = string("op_11086_cast_fp16")];
            tensor<int32, [1]> var_11087_axes_0 = const()[name = string("op_11087_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_11087_cast_fp16 = expand_dims(axes = var_11087_axes_0, x = var_11086_cast_fp16)[name = string("op_11087_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_781_cast_fp16 = mul(x = state_779_cast_fp16, y = var_11087_cast_fp16)[name = string("state_781_cast_fp16")];
            tensor<int32, [4]> key_token_391_begin_0 = const()[name = string("key_token_391_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> key_token_391_end_0 = const()[name = string("key_token_391_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> key_token_391_end_mask_0 = const()[name = string("key_token_391_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_391_squeeze_mask_0 = const()[name = string("key_token_391_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_391_cast_fp16 = slice_by_index(begin = key_token_391_begin_0, end = key_token_391_end_0, end_mask = key_token_391_end_mask_0, squeeze_mask = key_token_391_squeeze_mask_0, x = key_67_cast_fp16)[name = string("key_token_391_cast_fp16")];
            tensor<int32, [4]> value_token_391_begin_0 = const()[name = string("value_token_391_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> value_token_391_end_0 = const()[name = string("value_token_391_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> value_token_391_end_mask_0 = const()[name = string("value_token_391_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_391_squeeze_mask_0 = const()[name = string("value_token_391_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_391_cast_fp16 = slice_by_index(begin = value_token_391_begin_0, end = value_token_391_end_0, end_mask = value_token_391_end_mask_0, squeeze_mask = value_token_391_squeeze_mask_0, x = value_243_cast_fp16)[name = string("value_token_391_cast_fp16")];
            tensor<int32, [1]> var_11095_axes_0 = const()[name = string("op_11095_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_11095_cast_fp16 = expand_dims(axes = var_11095_axes_0, x = key_token_391_cast_fp16)[name = string("op_11095_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_11096_cast_fp16 = mul(x = state_781_cast_fp16, y = var_11095_cast_fp16)[name = string("op_11096_cast_fp16")];
            tensor<int32, [1]> memory_391_axes_0 = const()[name = string("memory_391_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_391_keep_dims_0 = const()[name = string("memory_391_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_391_cast_fp16 = reduce_sum(axes = memory_391_axes_0, keep_dims = memory_391_keep_dims_0, x = var_11096_cast_fp16)[name = string("memory_391_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_11099_cast_fp16 = sub(x = value_token_391_cast_fp16, y = memory_391_cast_fp16)[name = string("op_11099_cast_fp16")];
            tensor<int32, [3]> var_11102_begin_0 = const()[name = string("op_11102_begin_0"), val = tensor<int32, [3]>([0, 0, 3])];
            tensor<int32, [3]> var_11102_end_0 = const()[name = string("op_11102_end_0"), val = tensor<int32, [3]>([1, 16, 4])];
            tensor<bool, [3]> var_11102_end_mask_0 = const()[name = string("op_11102_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_11102_squeeze_mask_0 = const()[name = string("op_11102_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_11102_cast_fp16 = slice_by_index(begin = var_11102_begin_0, end = var_11102_end_0, end_mask = var_11102_end_mask_0, squeeze_mask = var_11102_squeeze_mask_0, x = beta_25_cast_fp16)[name = string("op_11102_cast_fp16")];
            tensor<int32, [1]> var_11103_axes_0 = const()[name = string("op_11103_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_11103_cast_fp16 = expand_dims(axes = var_11103_axes_0, x = var_11102_cast_fp16)[name = string("op_11103_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_391_cast_fp16 = mul(x = var_11099_cast_fp16, y = var_11103_cast_fp16)[name = string("delta_391_cast_fp16")];
            tensor<int32, [1]> var_11106_axes_0 = const()[name = string("op_11106_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_11106_cast_fp16 = expand_dims(axes = var_11106_axes_0, x = delta_391_cast_fp16)[name = string("op_11106_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_11107_cast_fp16 = mul(x = var_11095_cast_fp16, y = var_11106_cast_fp16)[name = string("op_11107_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_783_cast_fp16 = add(x = state_781_cast_fp16, y = var_11107_cast_fp16)[name = string("state_783_cast_fp16")];
            tensor<int32, [4]> var_11111_begin_0 = const()[name = string("op_11111_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> var_11111_end_0 = const()[name = string("op_11111_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> var_11111_end_mask_0 = const()[name = string("op_11111_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_11111_squeeze_mask_0 = const()[name = string("op_11111_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_11111_cast_fp16 = slice_by_index(begin = var_11111_begin_0, end = var_11111_end_0, end_mask = var_11111_end_mask_0, squeeze_mask = var_11111_squeeze_mask_0, x = _inversed_query_67_cast_fp16)[name = string("op_11111_cast_fp16")];
            tensor<int32, [1]> var_11112_axes_0 = const()[name = string("op_11112_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_11112_cast_fp16 = expand_dims(axes = var_11112_axes_0, x = var_11111_cast_fp16)[name = string("op_11112_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_11113_cast_fp16 = mul(x = state_783_cast_fp16, y = var_11112_cast_fp16)[name = string("op_11113_cast_fp16")];
            tensor<int32, [1]> var_11115_axes_0 = const()[name = string("op_11115_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_11115_keep_dims_0 = const()[name = string("op_11115_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_11115_cast_fp16 = reduce_sum(axes = var_11115_axes_0, keep_dims = var_11115_keep_dims_0, x = var_11113_cast_fp16)[name = string("op_11115_cast_fp16")];
            tensor<int32, [3]> var_11118_begin_0 = const()[name = string("op_11118_begin_0"), val = tensor<int32, [3]>([0, 0, 4])];
            tensor<int32, [3]> var_11118_end_0 = const()[name = string("op_11118_end_0"), val = tensor<int32, [3]>([1, 16, 5])];
            tensor<bool, [3]> var_11118_end_mask_0 = const()[name = string("op_11118_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_11118_squeeze_mask_0 = const()[name = string("op_11118_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_11118_cast_fp16 = slice_by_index(begin = var_11118_begin_0, end = var_11118_end_0, end_mask = var_11118_end_mask_0, squeeze_mask = var_11118_squeeze_mask_0, x = decay_25_cast_fp16)[name = string("op_11118_cast_fp16")];
            tensor<fp16, [1, 16]> var_11119_cast_fp16 = exp(x = var_11118_cast_fp16)[name = string("op_11119_cast_fp16")];
            tensor<int32, [1]> var_11120_axes_0 = const()[name = string("op_11120_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_11120_cast_fp16 = expand_dims(axes = var_11120_axes_0, x = var_11119_cast_fp16)[name = string("op_11120_cast_fp16")];
            tensor<int32, [1]> var_11121_axes_0 = const()[name = string("op_11121_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_11121_cast_fp16 = expand_dims(axes = var_11121_axes_0, x = var_11120_cast_fp16)[name = string("op_11121_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_785_cast_fp16 = mul(x = state_783_cast_fp16, y = var_11121_cast_fp16)[name = string("state_785_cast_fp16")];
            tensor<int32, [4]> key_token_393_begin_0 = const()[name = string("key_token_393_begin_0"), val = tensor<int32, [4]>([0, 0, 4, 0])];
            tensor<int32, [4]> key_token_393_end_0 = const()[name = string("key_token_393_end_0"), val = tensor<int32, [4]>([1, 16, 5, 128])];
            tensor<bool, [4]> key_token_393_end_mask_0 = const()[name = string("key_token_393_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_393_squeeze_mask_0 = const()[name = string("key_token_393_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_393_cast_fp16 = slice_by_index(begin = key_token_393_begin_0, end = key_token_393_end_0, end_mask = key_token_393_end_mask_0, squeeze_mask = key_token_393_squeeze_mask_0, x = key_67_cast_fp16)[name = string("key_token_393_cast_fp16")];
            tensor<int32, [4]> value_token_393_begin_0 = const()[name = string("value_token_393_begin_0"), val = tensor<int32, [4]>([0, 0, 4, 0])];
            tensor<int32, [4]> value_token_393_end_0 = const()[name = string("value_token_393_end_0"), val = tensor<int32, [4]>([1, 16, 5, 128])];
            tensor<bool, [4]> value_token_393_end_mask_0 = const()[name = string("value_token_393_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_393_squeeze_mask_0 = const()[name = string("value_token_393_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_393_cast_fp16 = slice_by_index(begin = value_token_393_begin_0, end = value_token_393_end_0, end_mask = value_token_393_end_mask_0, squeeze_mask = value_token_393_squeeze_mask_0, x = value_243_cast_fp16)[name = string("value_token_393_cast_fp16")];
            tensor<int32, [1]> var_11129_axes_0 = const()[name = string("op_11129_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_11129_cast_fp16 = expand_dims(axes = var_11129_axes_0, x = key_token_393_cast_fp16)[name = string("op_11129_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_11130_cast_fp16 = mul(x = state_785_cast_fp16, y = var_11129_cast_fp16)[name = string("op_11130_cast_fp16")];
            tensor<int32, [1]> memory_393_axes_0 = const()[name = string("memory_393_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_393_keep_dims_0 = const()[name = string("memory_393_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_393_cast_fp16 = reduce_sum(axes = memory_393_axes_0, keep_dims = memory_393_keep_dims_0, x = var_11130_cast_fp16)[name = string("memory_393_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_11133_cast_fp16 = sub(x = value_token_393_cast_fp16, y = memory_393_cast_fp16)[name = string("op_11133_cast_fp16")];
            tensor<int32, [3]> var_11136_begin_0 = const()[name = string("op_11136_begin_0"), val = tensor<int32, [3]>([0, 0, 4])];
            tensor<int32, [3]> var_11136_end_0 = const()[name = string("op_11136_end_0"), val = tensor<int32, [3]>([1, 16, 5])];
            tensor<bool, [3]> var_11136_end_mask_0 = const()[name = string("op_11136_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_11136_squeeze_mask_0 = const()[name = string("op_11136_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_11136_cast_fp16 = slice_by_index(begin = var_11136_begin_0, end = var_11136_end_0, end_mask = var_11136_end_mask_0, squeeze_mask = var_11136_squeeze_mask_0, x = beta_25_cast_fp16)[name = string("op_11136_cast_fp16")];
            tensor<int32, [1]> var_11137_axes_0 = const()[name = string("op_11137_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_11137_cast_fp16 = expand_dims(axes = var_11137_axes_0, x = var_11136_cast_fp16)[name = string("op_11137_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_393_cast_fp16 = mul(x = var_11133_cast_fp16, y = var_11137_cast_fp16)[name = string("delta_393_cast_fp16")];
            tensor<int32, [1]> var_11140_axes_0 = const()[name = string("op_11140_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_11140_cast_fp16 = expand_dims(axes = var_11140_axes_0, x = delta_393_cast_fp16)[name = string("op_11140_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_11141_cast_fp16 = mul(x = var_11129_cast_fp16, y = var_11140_cast_fp16)[name = string("op_11141_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_787_cast_fp16 = add(x = state_785_cast_fp16, y = var_11141_cast_fp16)[name = string("state_787_cast_fp16")];
            tensor<int32, [4]> var_11145_begin_0 = const()[name = string("op_11145_begin_0"), val = tensor<int32, [4]>([0, 0, 4, 0])];
            tensor<int32, [4]> var_11145_end_0 = const()[name = string("op_11145_end_0"), val = tensor<int32, [4]>([1, 16, 5, 128])];
            tensor<bool, [4]> var_11145_end_mask_0 = const()[name = string("op_11145_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_11145_squeeze_mask_0 = const()[name = string("op_11145_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_11145_cast_fp16 = slice_by_index(begin = var_11145_begin_0, end = var_11145_end_0, end_mask = var_11145_end_mask_0, squeeze_mask = var_11145_squeeze_mask_0, x = _inversed_query_67_cast_fp16)[name = string("op_11145_cast_fp16")];
            tensor<int32, [1]> var_11146_axes_0 = const()[name = string("op_11146_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_11146_cast_fp16 = expand_dims(axes = var_11146_axes_0, x = var_11145_cast_fp16)[name = string("op_11146_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_11147_cast_fp16 = mul(x = state_787_cast_fp16, y = var_11146_cast_fp16)[name = string("op_11147_cast_fp16")];
            tensor<int32, [1]> var_11149_axes_0 = const()[name = string("op_11149_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_11149_keep_dims_0 = const()[name = string("op_11149_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_11149_cast_fp16 = reduce_sum(axes = var_11149_axes_0, keep_dims = var_11149_keep_dims_0, x = var_11147_cast_fp16)[name = string("op_11149_cast_fp16")];
            tensor<int32, [3]> var_11152_begin_0 = const()[name = string("op_11152_begin_0"), val = tensor<int32, [3]>([0, 0, 5])];
            tensor<int32, [3]> var_11152_end_0 = const()[name = string("op_11152_end_0"), val = tensor<int32, [3]>([1, 16, 6])];
            tensor<bool, [3]> var_11152_end_mask_0 = const()[name = string("op_11152_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_11152_squeeze_mask_0 = const()[name = string("op_11152_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_11152_cast_fp16 = slice_by_index(begin = var_11152_begin_0, end = var_11152_end_0, end_mask = var_11152_end_mask_0, squeeze_mask = var_11152_squeeze_mask_0, x = decay_25_cast_fp16)[name = string("op_11152_cast_fp16")];
            tensor<fp16, [1, 16]> var_11153_cast_fp16 = exp(x = var_11152_cast_fp16)[name = string("op_11153_cast_fp16")];
            tensor<int32, [1]> var_11154_axes_0 = const()[name = string("op_11154_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_11154_cast_fp16 = expand_dims(axes = var_11154_axes_0, x = var_11153_cast_fp16)[name = string("op_11154_cast_fp16")];
            tensor<int32, [1]> var_11155_axes_0 = const()[name = string("op_11155_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_11155_cast_fp16 = expand_dims(axes = var_11155_axes_0, x = var_11154_cast_fp16)[name = string("op_11155_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_789_cast_fp16 = mul(x = state_787_cast_fp16, y = var_11155_cast_fp16)[name = string("state_789_cast_fp16")];
            tensor<int32, [4]> key_token_395_begin_0 = const()[name = string("key_token_395_begin_0"), val = tensor<int32, [4]>([0, 0, 5, 0])];
            tensor<int32, [4]> key_token_395_end_0 = const()[name = string("key_token_395_end_0"), val = tensor<int32, [4]>([1, 16, 6, 128])];
            tensor<bool, [4]> key_token_395_end_mask_0 = const()[name = string("key_token_395_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_395_squeeze_mask_0 = const()[name = string("key_token_395_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_395_cast_fp16 = slice_by_index(begin = key_token_395_begin_0, end = key_token_395_end_0, end_mask = key_token_395_end_mask_0, squeeze_mask = key_token_395_squeeze_mask_0, x = key_67_cast_fp16)[name = string("key_token_395_cast_fp16")];
            tensor<int32, [4]> value_token_395_begin_0 = const()[name = string("value_token_395_begin_0"), val = tensor<int32, [4]>([0, 0, 5, 0])];
            tensor<int32, [4]> value_token_395_end_0 = const()[name = string("value_token_395_end_0"), val = tensor<int32, [4]>([1, 16, 6, 128])];
            tensor<bool, [4]> value_token_395_end_mask_0 = const()[name = string("value_token_395_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_395_squeeze_mask_0 = const()[name = string("value_token_395_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_395_cast_fp16 = slice_by_index(begin = value_token_395_begin_0, end = value_token_395_end_0, end_mask = value_token_395_end_mask_0, squeeze_mask = value_token_395_squeeze_mask_0, x = value_243_cast_fp16)[name = string("value_token_395_cast_fp16")];
            tensor<int32, [1]> var_11163_axes_0 = const()[name = string("op_11163_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_11163_cast_fp16 = expand_dims(axes = var_11163_axes_0, x = key_token_395_cast_fp16)[name = string("op_11163_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_11164_cast_fp16 = mul(x = state_789_cast_fp16, y = var_11163_cast_fp16)[name = string("op_11164_cast_fp16")];
            tensor<int32, [1]> memory_395_axes_0 = const()[name = string("memory_395_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_395_keep_dims_0 = const()[name = string("memory_395_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_395_cast_fp16 = reduce_sum(axes = memory_395_axes_0, keep_dims = memory_395_keep_dims_0, x = var_11164_cast_fp16)[name = string("memory_395_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_11167_cast_fp16 = sub(x = value_token_395_cast_fp16, y = memory_395_cast_fp16)[name = string("op_11167_cast_fp16")];
            tensor<int32, [3]> var_11170_begin_0 = const()[name = string("op_11170_begin_0"), val = tensor<int32, [3]>([0, 0, 5])];
            tensor<int32, [3]> var_11170_end_0 = const()[name = string("op_11170_end_0"), val = tensor<int32, [3]>([1, 16, 6])];
            tensor<bool, [3]> var_11170_end_mask_0 = const()[name = string("op_11170_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_11170_squeeze_mask_0 = const()[name = string("op_11170_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_11170_cast_fp16 = slice_by_index(begin = var_11170_begin_0, end = var_11170_end_0, end_mask = var_11170_end_mask_0, squeeze_mask = var_11170_squeeze_mask_0, x = beta_25_cast_fp16)[name = string("op_11170_cast_fp16")];
            tensor<int32, [1]> var_11171_axes_0 = const()[name = string("op_11171_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_11171_cast_fp16 = expand_dims(axes = var_11171_axes_0, x = var_11170_cast_fp16)[name = string("op_11171_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_395_cast_fp16 = mul(x = var_11167_cast_fp16, y = var_11171_cast_fp16)[name = string("delta_395_cast_fp16")];
            tensor<int32, [1]> var_11174_axes_0 = const()[name = string("op_11174_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_11174_cast_fp16 = expand_dims(axes = var_11174_axes_0, x = delta_395_cast_fp16)[name = string("op_11174_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_11175_cast_fp16 = mul(x = var_11163_cast_fp16, y = var_11174_cast_fp16)[name = string("op_11175_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_791_cast_fp16 = add(x = state_789_cast_fp16, y = var_11175_cast_fp16)[name = string("state_791_cast_fp16")];
            tensor<int32, [4]> var_11179_begin_0 = const()[name = string("op_11179_begin_0"), val = tensor<int32, [4]>([0, 0, 5, 0])];
            tensor<int32, [4]> var_11179_end_0 = const()[name = string("op_11179_end_0"), val = tensor<int32, [4]>([1, 16, 6, 128])];
            tensor<bool, [4]> var_11179_end_mask_0 = const()[name = string("op_11179_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_11179_squeeze_mask_0 = const()[name = string("op_11179_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_11179_cast_fp16 = slice_by_index(begin = var_11179_begin_0, end = var_11179_end_0, end_mask = var_11179_end_mask_0, squeeze_mask = var_11179_squeeze_mask_0, x = _inversed_query_67_cast_fp16)[name = string("op_11179_cast_fp16")];
            tensor<int32, [1]> var_11180_axes_0 = const()[name = string("op_11180_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_11180_cast_fp16 = expand_dims(axes = var_11180_axes_0, x = var_11179_cast_fp16)[name = string("op_11180_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_11181_cast_fp16 = mul(x = state_791_cast_fp16, y = var_11180_cast_fp16)[name = string("op_11181_cast_fp16")];
            tensor<int32, [1]> var_11183_axes_0 = const()[name = string("op_11183_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_11183_keep_dims_0 = const()[name = string("op_11183_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_11183_cast_fp16 = reduce_sum(axes = var_11183_axes_0, keep_dims = var_11183_keep_dims_0, x = var_11181_cast_fp16)[name = string("op_11183_cast_fp16")];
            tensor<int32, [3]> var_11186_begin_0 = const()[name = string("op_11186_begin_0"), val = tensor<int32, [3]>([0, 0, 6])];
            tensor<int32, [3]> var_11186_end_0 = const()[name = string("op_11186_end_0"), val = tensor<int32, [3]>([1, 16, 7])];
            tensor<bool, [3]> var_11186_end_mask_0 = const()[name = string("op_11186_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_11186_squeeze_mask_0 = const()[name = string("op_11186_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_11186_cast_fp16 = slice_by_index(begin = var_11186_begin_0, end = var_11186_end_0, end_mask = var_11186_end_mask_0, squeeze_mask = var_11186_squeeze_mask_0, x = decay_25_cast_fp16)[name = string("op_11186_cast_fp16")];
            tensor<fp16, [1, 16]> var_11187_cast_fp16 = exp(x = var_11186_cast_fp16)[name = string("op_11187_cast_fp16")];
            tensor<int32, [1]> var_11188_axes_0 = const()[name = string("op_11188_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_11188_cast_fp16 = expand_dims(axes = var_11188_axes_0, x = var_11187_cast_fp16)[name = string("op_11188_cast_fp16")];
            tensor<int32, [1]> var_11189_axes_0 = const()[name = string("op_11189_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_11189_cast_fp16 = expand_dims(axes = var_11189_axes_0, x = var_11188_cast_fp16)[name = string("op_11189_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_793_cast_fp16 = mul(x = state_791_cast_fp16, y = var_11189_cast_fp16)[name = string("state_793_cast_fp16")];
            tensor<int32, [4]> key_token_397_begin_0 = const()[name = string("key_token_397_begin_0"), val = tensor<int32, [4]>([0, 0, 6, 0])];
            tensor<int32, [4]> key_token_397_end_0 = const()[name = string("key_token_397_end_0"), val = tensor<int32, [4]>([1, 16, 7, 128])];
            tensor<bool, [4]> key_token_397_end_mask_0 = const()[name = string("key_token_397_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_397_squeeze_mask_0 = const()[name = string("key_token_397_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_397_cast_fp16 = slice_by_index(begin = key_token_397_begin_0, end = key_token_397_end_0, end_mask = key_token_397_end_mask_0, squeeze_mask = key_token_397_squeeze_mask_0, x = key_67_cast_fp16)[name = string("key_token_397_cast_fp16")];
            tensor<int32, [4]> value_token_397_begin_0 = const()[name = string("value_token_397_begin_0"), val = tensor<int32, [4]>([0, 0, 6, 0])];
            tensor<int32, [4]> value_token_397_end_0 = const()[name = string("value_token_397_end_0"), val = tensor<int32, [4]>([1, 16, 7, 128])];
            tensor<bool, [4]> value_token_397_end_mask_0 = const()[name = string("value_token_397_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_397_squeeze_mask_0 = const()[name = string("value_token_397_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_397_cast_fp16 = slice_by_index(begin = value_token_397_begin_0, end = value_token_397_end_0, end_mask = value_token_397_end_mask_0, squeeze_mask = value_token_397_squeeze_mask_0, x = value_243_cast_fp16)[name = string("value_token_397_cast_fp16")];
            tensor<int32, [1]> var_11197_axes_0 = const()[name = string("op_11197_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_11197_cast_fp16 = expand_dims(axes = var_11197_axes_0, x = key_token_397_cast_fp16)[name = string("op_11197_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_11198_cast_fp16 = mul(x = state_793_cast_fp16, y = var_11197_cast_fp16)[name = string("op_11198_cast_fp16")];
            tensor<int32, [1]> memory_397_axes_0 = const()[name = string("memory_397_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_397_keep_dims_0 = const()[name = string("memory_397_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_397_cast_fp16 = reduce_sum(axes = memory_397_axes_0, keep_dims = memory_397_keep_dims_0, x = var_11198_cast_fp16)[name = string("memory_397_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_11201_cast_fp16 = sub(x = value_token_397_cast_fp16, y = memory_397_cast_fp16)[name = string("op_11201_cast_fp16")];
            tensor<int32, [3]> var_11204_begin_0 = const()[name = string("op_11204_begin_0"), val = tensor<int32, [3]>([0, 0, 6])];
            tensor<int32, [3]> var_11204_end_0 = const()[name = string("op_11204_end_0"), val = tensor<int32, [3]>([1, 16, 7])];
            tensor<bool, [3]> var_11204_end_mask_0 = const()[name = string("op_11204_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_11204_squeeze_mask_0 = const()[name = string("op_11204_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_11204_cast_fp16 = slice_by_index(begin = var_11204_begin_0, end = var_11204_end_0, end_mask = var_11204_end_mask_0, squeeze_mask = var_11204_squeeze_mask_0, x = beta_25_cast_fp16)[name = string("op_11204_cast_fp16")];
            tensor<int32, [1]> var_11205_axes_0 = const()[name = string("op_11205_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_11205_cast_fp16 = expand_dims(axes = var_11205_axes_0, x = var_11204_cast_fp16)[name = string("op_11205_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_397_cast_fp16 = mul(x = var_11201_cast_fp16, y = var_11205_cast_fp16)[name = string("delta_397_cast_fp16")];
            tensor<int32, [1]> var_11208_axes_0 = const()[name = string("op_11208_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_11208_cast_fp16 = expand_dims(axes = var_11208_axes_0, x = delta_397_cast_fp16)[name = string("op_11208_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_11209_cast_fp16 = mul(x = var_11197_cast_fp16, y = var_11208_cast_fp16)[name = string("op_11209_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_795_cast_fp16 = add(x = state_793_cast_fp16, y = var_11209_cast_fp16)[name = string("state_795_cast_fp16")];
            tensor<int32, [4]> var_11213_begin_0 = const()[name = string("op_11213_begin_0"), val = tensor<int32, [4]>([0, 0, 6, 0])];
            tensor<int32, [4]> var_11213_end_0 = const()[name = string("op_11213_end_0"), val = tensor<int32, [4]>([1, 16, 7, 128])];
            tensor<bool, [4]> var_11213_end_mask_0 = const()[name = string("op_11213_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_11213_squeeze_mask_0 = const()[name = string("op_11213_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_11213_cast_fp16 = slice_by_index(begin = var_11213_begin_0, end = var_11213_end_0, end_mask = var_11213_end_mask_0, squeeze_mask = var_11213_squeeze_mask_0, x = _inversed_query_67_cast_fp16)[name = string("op_11213_cast_fp16")];
            tensor<int32, [1]> var_11214_axes_0 = const()[name = string("op_11214_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_11214_cast_fp16 = expand_dims(axes = var_11214_axes_0, x = var_11213_cast_fp16)[name = string("op_11214_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_11215_cast_fp16 = mul(x = state_795_cast_fp16, y = var_11214_cast_fp16)[name = string("op_11215_cast_fp16")];
            tensor<int32, [1]> var_11217_axes_0 = const()[name = string("op_11217_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_11217_keep_dims_0 = const()[name = string("op_11217_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_11217_cast_fp16 = reduce_sum(axes = var_11217_axes_0, keep_dims = var_11217_keep_dims_0, x = var_11215_cast_fp16)[name = string("op_11217_cast_fp16")];
            tensor<int32, [3]> var_11220_begin_0 = const()[name = string("op_11220_begin_0"), val = tensor<int32, [3]>([0, 0, 7])];
            tensor<int32, [3]> var_11220_end_0 = const()[name = string("op_11220_end_0"), val = tensor<int32, [3]>([1, 16, 8])];
            tensor<bool, [3]> var_11220_end_mask_0 = const()[name = string("op_11220_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_11220_squeeze_mask_0 = const()[name = string("op_11220_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_11220_cast_fp16 = slice_by_index(begin = var_11220_begin_0, end = var_11220_end_0, end_mask = var_11220_end_mask_0, squeeze_mask = var_11220_squeeze_mask_0, x = decay_25_cast_fp16)[name = string("op_11220_cast_fp16")];
            tensor<fp16, [1, 16]> var_11221_cast_fp16 = exp(x = var_11220_cast_fp16)[name = string("op_11221_cast_fp16")];
            tensor<int32, [1]> var_11222_axes_0 = const()[name = string("op_11222_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_11222_cast_fp16 = expand_dims(axes = var_11222_axes_0, x = var_11221_cast_fp16)[name = string("op_11222_cast_fp16")];
            tensor<int32, [1]> var_11223_axes_0 = const()[name = string("op_11223_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_11223_cast_fp16 = expand_dims(axes = var_11223_axes_0, x = var_11222_cast_fp16)[name = string("op_11223_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_797_cast_fp16 = mul(x = state_795_cast_fp16, y = var_11223_cast_fp16)[name = string("state_797_cast_fp16")];
            tensor<int32, [4]> key_token_399_begin_0 = const()[name = string("key_token_399_begin_0"), val = tensor<int32, [4]>([0, 0, 7, 0])];
            tensor<int32, [4]> key_token_399_end_0 = const()[name = string("key_token_399_end_0"), val = tensor<int32, [4]>([1, 16, 8, 128])];
            tensor<bool, [4]> key_token_399_end_mask_0 = const()[name = string("key_token_399_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_399_squeeze_mask_0 = const()[name = string("key_token_399_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_399_cast_fp16 = slice_by_index(begin = key_token_399_begin_0, end = key_token_399_end_0, end_mask = key_token_399_end_mask_0, squeeze_mask = key_token_399_squeeze_mask_0, x = key_67_cast_fp16)[name = string("key_token_399_cast_fp16")];
            tensor<int32, [4]> value_token_399_begin_0 = const()[name = string("value_token_399_begin_0"), val = tensor<int32, [4]>([0, 0, 7, 0])];
            tensor<int32, [4]> value_token_399_end_0 = const()[name = string("value_token_399_end_0"), val = tensor<int32, [4]>([1, 16, 8, 128])];
            tensor<bool, [4]> value_token_399_end_mask_0 = const()[name = string("value_token_399_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_399_squeeze_mask_0 = const()[name = string("value_token_399_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_399_cast_fp16 = slice_by_index(begin = value_token_399_begin_0, end = value_token_399_end_0, end_mask = value_token_399_end_mask_0, squeeze_mask = value_token_399_squeeze_mask_0, x = value_243_cast_fp16)[name = string("value_token_399_cast_fp16")];
            tensor<int32, [1]> var_11231_axes_0 = const()[name = string("op_11231_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_11231_cast_fp16 = expand_dims(axes = var_11231_axes_0, x = key_token_399_cast_fp16)[name = string("op_11231_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_11232_cast_fp16 = mul(x = state_797_cast_fp16, y = var_11231_cast_fp16)[name = string("op_11232_cast_fp16")];
            tensor<int32, [1]> memory_399_axes_0 = const()[name = string("memory_399_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_399_keep_dims_0 = const()[name = string("memory_399_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_399_cast_fp16 = reduce_sum(axes = memory_399_axes_0, keep_dims = memory_399_keep_dims_0, x = var_11232_cast_fp16)[name = string("memory_399_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_11235_cast_fp16 = sub(x = value_token_399_cast_fp16, y = memory_399_cast_fp16)[name = string("op_11235_cast_fp16")];
            tensor<int32, [3]> var_11238_begin_0 = const()[name = string("op_11238_begin_0"), val = tensor<int32, [3]>([0, 0, 7])];
            tensor<int32, [3]> var_11238_end_0 = const()[name = string("op_11238_end_0"), val = tensor<int32, [3]>([1, 16, 8])];
            tensor<bool, [3]> var_11238_end_mask_0 = const()[name = string("op_11238_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_11238_squeeze_mask_0 = const()[name = string("op_11238_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_11238_cast_fp16 = slice_by_index(begin = var_11238_begin_0, end = var_11238_end_0, end_mask = var_11238_end_mask_0, squeeze_mask = var_11238_squeeze_mask_0, x = beta_25_cast_fp16)[name = string("op_11238_cast_fp16")];
            tensor<int32, [1]> var_11239_axes_0 = const()[name = string("op_11239_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_11239_cast_fp16 = expand_dims(axes = var_11239_axes_0, x = var_11238_cast_fp16)[name = string("op_11239_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_399_cast_fp16 = mul(x = var_11235_cast_fp16, y = var_11239_cast_fp16)[name = string("delta_399_cast_fp16")];
            tensor<int32, [1]> var_11242_axes_0 = const()[name = string("op_11242_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_11242_cast_fp16 = expand_dims(axes = var_11242_axes_0, x = delta_399_cast_fp16)[name = string("op_11242_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_11243_cast_fp16 = mul(x = var_11231_cast_fp16, y = var_11242_cast_fp16)[name = string("op_11243_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_799_cast_fp16 = add(x = state_797_cast_fp16, y = var_11243_cast_fp16)[name = string("state_799_cast_fp16")];
            tensor<int32, [4]> var_11247_begin_0 = const()[name = string("op_11247_begin_0"), val = tensor<int32, [4]>([0, 0, 7, 0])];
            tensor<int32, [4]> var_11247_end_0 = const()[name = string("op_11247_end_0"), val = tensor<int32, [4]>([1, 16, 8, 128])];
            tensor<bool, [4]> var_11247_end_mask_0 = const()[name = string("op_11247_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_11247_squeeze_mask_0 = const()[name = string("op_11247_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_11247_cast_fp16 = slice_by_index(begin = var_11247_begin_0, end = var_11247_end_0, end_mask = var_11247_end_mask_0, squeeze_mask = var_11247_squeeze_mask_0, x = _inversed_query_67_cast_fp16)[name = string("op_11247_cast_fp16")];
            tensor<int32, [1]> var_11248_axes_0 = const()[name = string("op_11248_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_11248_cast_fp16 = expand_dims(axes = var_11248_axes_0, x = var_11247_cast_fp16)[name = string("op_11248_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_11249_cast_fp16 = mul(x = state_799_cast_fp16, y = var_11248_cast_fp16)[name = string("op_11249_cast_fp16")];
            tensor<int32, [1]> var_11251_axes_0 = const()[name = string("op_11251_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_11251_keep_dims_0 = const()[name = string("op_11251_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_11251_cast_fp16 = reduce_sum(axes = var_11251_axes_0, keep_dims = var_11251_keep_dims_0, x = var_11249_cast_fp16)[name = string("op_11251_cast_fp16")];
            tensor<int32, [3]> var_11254_begin_0 = const()[name = string("op_11254_begin_0"), val = tensor<int32, [3]>([0, 0, 8])];
            tensor<int32, [3]> var_11254_end_0 = const()[name = string("op_11254_end_0"), val = tensor<int32, [3]>([1, 16, 9])];
            tensor<bool, [3]> var_11254_end_mask_0 = const()[name = string("op_11254_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_11254_squeeze_mask_0 = const()[name = string("op_11254_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_11254_cast_fp16 = slice_by_index(begin = var_11254_begin_0, end = var_11254_end_0, end_mask = var_11254_end_mask_0, squeeze_mask = var_11254_squeeze_mask_0, x = decay_25_cast_fp16)[name = string("op_11254_cast_fp16")];
            tensor<fp16, [1, 16]> var_11255_cast_fp16 = exp(x = var_11254_cast_fp16)[name = string("op_11255_cast_fp16")];
            tensor<int32, [1]> var_11256_axes_0 = const()[name = string("op_11256_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_11256_cast_fp16 = expand_dims(axes = var_11256_axes_0, x = var_11255_cast_fp16)[name = string("op_11256_cast_fp16")];
            tensor<int32, [1]> var_11257_axes_0 = const()[name = string("op_11257_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_11257_cast_fp16 = expand_dims(axes = var_11257_axes_0, x = var_11256_cast_fp16)[name = string("op_11257_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_801_cast_fp16 = mul(x = state_799_cast_fp16, y = var_11257_cast_fp16)[name = string("state_801_cast_fp16")];
            tensor<int32, [4]> key_token_401_begin_0 = const()[name = string("key_token_401_begin_0"), val = tensor<int32, [4]>([0, 0, 8, 0])];
            tensor<int32, [4]> key_token_401_end_0 = const()[name = string("key_token_401_end_0"), val = tensor<int32, [4]>([1, 16, 9, 128])];
            tensor<bool, [4]> key_token_401_end_mask_0 = const()[name = string("key_token_401_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_401_squeeze_mask_0 = const()[name = string("key_token_401_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_401_cast_fp16 = slice_by_index(begin = key_token_401_begin_0, end = key_token_401_end_0, end_mask = key_token_401_end_mask_0, squeeze_mask = key_token_401_squeeze_mask_0, x = key_67_cast_fp16)[name = string("key_token_401_cast_fp16")];
            tensor<int32, [4]> value_token_401_begin_0 = const()[name = string("value_token_401_begin_0"), val = tensor<int32, [4]>([0, 0, 8, 0])];
            tensor<int32, [4]> value_token_401_end_0 = const()[name = string("value_token_401_end_0"), val = tensor<int32, [4]>([1, 16, 9, 128])];
            tensor<bool, [4]> value_token_401_end_mask_0 = const()[name = string("value_token_401_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_401_squeeze_mask_0 = const()[name = string("value_token_401_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_401_cast_fp16 = slice_by_index(begin = value_token_401_begin_0, end = value_token_401_end_0, end_mask = value_token_401_end_mask_0, squeeze_mask = value_token_401_squeeze_mask_0, x = value_243_cast_fp16)[name = string("value_token_401_cast_fp16")];
            tensor<int32, [1]> var_11265_axes_0 = const()[name = string("op_11265_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_11265_cast_fp16 = expand_dims(axes = var_11265_axes_0, x = key_token_401_cast_fp16)[name = string("op_11265_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_11266_cast_fp16 = mul(x = state_801_cast_fp16, y = var_11265_cast_fp16)[name = string("op_11266_cast_fp16")];
            tensor<int32, [1]> memory_401_axes_0 = const()[name = string("memory_401_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_401_keep_dims_0 = const()[name = string("memory_401_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_401_cast_fp16 = reduce_sum(axes = memory_401_axes_0, keep_dims = memory_401_keep_dims_0, x = var_11266_cast_fp16)[name = string("memory_401_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_11269_cast_fp16 = sub(x = value_token_401_cast_fp16, y = memory_401_cast_fp16)[name = string("op_11269_cast_fp16")];
            tensor<int32, [3]> var_11272_begin_0 = const()[name = string("op_11272_begin_0"), val = tensor<int32, [3]>([0, 0, 8])];
            tensor<int32, [3]> var_11272_end_0 = const()[name = string("op_11272_end_0"), val = tensor<int32, [3]>([1, 16, 9])];
            tensor<bool, [3]> var_11272_end_mask_0 = const()[name = string("op_11272_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_11272_squeeze_mask_0 = const()[name = string("op_11272_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_11272_cast_fp16 = slice_by_index(begin = var_11272_begin_0, end = var_11272_end_0, end_mask = var_11272_end_mask_0, squeeze_mask = var_11272_squeeze_mask_0, x = beta_25_cast_fp16)[name = string("op_11272_cast_fp16")];
            tensor<int32, [1]> var_11273_axes_0 = const()[name = string("op_11273_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_11273_cast_fp16 = expand_dims(axes = var_11273_axes_0, x = var_11272_cast_fp16)[name = string("op_11273_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_401_cast_fp16 = mul(x = var_11269_cast_fp16, y = var_11273_cast_fp16)[name = string("delta_401_cast_fp16")];
            tensor<int32, [1]> var_11276_axes_0 = const()[name = string("op_11276_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_11276_cast_fp16 = expand_dims(axes = var_11276_axes_0, x = delta_401_cast_fp16)[name = string("op_11276_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_11277_cast_fp16 = mul(x = var_11265_cast_fp16, y = var_11276_cast_fp16)[name = string("op_11277_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_803_cast_fp16 = add(x = state_801_cast_fp16, y = var_11277_cast_fp16)[name = string("state_803_cast_fp16")];
            tensor<int32, [4]> var_11281_begin_0 = const()[name = string("op_11281_begin_0"), val = tensor<int32, [4]>([0, 0, 8, 0])];
            tensor<int32, [4]> var_11281_end_0 = const()[name = string("op_11281_end_0"), val = tensor<int32, [4]>([1, 16, 9, 128])];
            tensor<bool, [4]> var_11281_end_mask_0 = const()[name = string("op_11281_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_11281_squeeze_mask_0 = const()[name = string("op_11281_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_11281_cast_fp16 = slice_by_index(begin = var_11281_begin_0, end = var_11281_end_0, end_mask = var_11281_end_mask_0, squeeze_mask = var_11281_squeeze_mask_0, x = _inversed_query_67_cast_fp16)[name = string("op_11281_cast_fp16")];
            tensor<int32, [1]> var_11282_axes_0 = const()[name = string("op_11282_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_11282_cast_fp16 = expand_dims(axes = var_11282_axes_0, x = var_11281_cast_fp16)[name = string("op_11282_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_11283_cast_fp16 = mul(x = state_803_cast_fp16, y = var_11282_cast_fp16)[name = string("op_11283_cast_fp16")];
            tensor<int32, [1]> var_11285_axes_0 = const()[name = string("op_11285_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_11285_keep_dims_0 = const()[name = string("op_11285_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_11285_cast_fp16 = reduce_sum(axes = var_11285_axes_0, keep_dims = var_11285_keep_dims_0, x = var_11283_cast_fp16)[name = string("op_11285_cast_fp16")];
            tensor<int32, [3]> var_11288_begin_0 = const()[name = string("op_11288_begin_0"), val = tensor<int32, [3]>([0, 0, 9])];
            tensor<int32, [3]> var_11288_end_0 = const()[name = string("op_11288_end_0"), val = tensor<int32, [3]>([1, 16, 10])];
            tensor<bool, [3]> var_11288_end_mask_0 = const()[name = string("op_11288_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_11288_squeeze_mask_0 = const()[name = string("op_11288_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_11288_cast_fp16 = slice_by_index(begin = var_11288_begin_0, end = var_11288_end_0, end_mask = var_11288_end_mask_0, squeeze_mask = var_11288_squeeze_mask_0, x = decay_25_cast_fp16)[name = string("op_11288_cast_fp16")];
            tensor<fp16, [1, 16]> var_11289_cast_fp16 = exp(x = var_11288_cast_fp16)[name = string("op_11289_cast_fp16")];
            tensor<int32, [1]> var_11290_axes_0 = const()[name = string("op_11290_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_11290_cast_fp16 = expand_dims(axes = var_11290_axes_0, x = var_11289_cast_fp16)[name = string("op_11290_cast_fp16")];
            tensor<int32, [1]> var_11291_axes_0 = const()[name = string("op_11291_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_11291_cast_fp16 = expand_dims(axes = var_11291_axes_0, x = var_11290_cast_fp16)[name = string("op_11291_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_805_cast_fp16 = mul(x = state_803_cast_fp16, y = var_11291_cast_fp16)[name = string("state_805_cast_fp16")];
            tensor<int32, [4]> key_token_403_begin_0 = const()[name = string("key_token_403_begin_0"), val = tensor<int32, [4]>([0, 0, 9, 0])];
            tensor<int32, [4]> key_token_403_end_0 = const()[name = string("key_token_403_end_0"), val = tensor<int32, [4]>([1, 16, 10, 128])];
            tensor<bool, [4]> key_token_403_end_mask_0 = const()[name = string("key_token_403_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_403_squeeze_mask_0 = const()[name = string("key_token_403_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_403_cast_fp16 = slice_by_index(begin = key_token_403_begin_0, end = key_token_403_end_0, end_mask = key_token_403_end_mask_0, squeeze_mask = key_token_403_squeeze_mask_0, x = key_67_cast_fp16)[name = string("key_token_403_cast_fp16")];
            tensor<int32, [4]> value_token_403_begin_0 = const()[name = string("value_token_403_begin_0"), val = tensor<int32, [4]>([0, 0, 9, 0])];
            tensor<int32, [4]> value_token_403_end_0 = const()[name = string("value_token_403_end_0"), val = tensor<int32, [4]>([1, 16, 10, 128])];
            tensor<bool, [4]> value_token_403_end_mask_0 = const()[name = string("value_token_403_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_403_squeeze_mask_0 = const()[name = string("value_token_403_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_403_cast_fp16 = slice_by_index(begin = value_token_403_begin_0, end = value_token_403_end_0, end_mask = value_token_403_end_mask_0, squeeze_mask = value_token_403_squeeze_mask_0, x = value_243_cast_fp16)[name = string("value_token_403_cast_fp16")];
            tensor<int32, [1]> var_11299_axes_0 = const()[name = string("op_11299_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_11299_cast_fp16 = expand_dims(axes = var_11299_axes_0, x = key_token_403_cast_fp16)[name = string("op_11299_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_11300_cast_fp16 = mul(x = state_805_cast_fp16, y = var_11299_cast_fp16)[name = string("op_11300_cast_fp16")];
            tensor<int32, [1]> memory_403_axes_0 = const()[name = string("memory_403_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_403_keep_dims_0 = const()[name = string("memory_403_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_403_cast_fp16 = reduce_sum(axes = memory_403_axes_0, keep_dims = memory_403_keep_dims_0, x = var_11300_cast_fp16)[name = string("memory_403_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_11303_cast_fp16 = sub(x = value_token_403_cast_fp16, y = memory_403_cast_fp16)[name = string("op_11303_cast_fp16")];
            tensor<int32, [3]> var_11306_begin_0 = const()[name = string("op_11306_begin_0"), val = tensor<int32, [3]>([0, 0, 9])];
            tensor<int32, [3]> var_11306_end_0 = const()[name = string("op_11306_end_0"), val = tensor<int32, [3]>([1, 16, 10])];
            tensor<bool, [3]> var_11306_end_mask_0 = const()[name = string("op_11306_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_11306_squeeze_mask_0 = const()[name = string("op_11306_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_11306_cast_fp16 = slice_by_index(begin = var_11306_begin_0, end = var_11306_end_0, end_mask = var_11306_end_mask_0, squeeze_mask = var_11306_squeeze_mask_0, x = beta_25_cast_fp16)[name = string("op_11306_cast_fp16")];
            tensor<int32, [1]> var_11307_axes_0 = const()[name = string("op_11307_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_11307_cast_fp16 = expand_dims(axes = var_11307_axes_0, x = var_11306_cast_fp16)[name = string("op_11307_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_403_cast_fp16 = mul(x = var_11303_cast_fp16, y = var_11307_cast_fp16)[name = string("delta_403_cast_fp16")];
            tensor<int32, [1]> var_11310_axes_0 = const()[name = string("op_11310_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_11310_cast_fp16 = expand_dims(axes = var_11310_axes_0, x = delta_403_cast_fp16)[name = string("op_11310_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_11311_cast_fp16 = mul(x = var_11299_cast_fp16, y = var_11310_cast_fp16)[name = string("op_11311_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_807_cast_fp16 = add(x = state_805_cast_fp16, y = var_11311_cast_fp16)[name = string("state_807_cast_fp16")];
            tensor<int32, [4]> var_11315_begin_0 = const()[name = string("op_11315_begin_0"), val = tensor<int32, [4]>([0, 0, 9, 0])];
            tensor<int32, [4]> var_11315_end_0 = const()[name = string("op_11315_end_0"), val = tensor<int32, [4]>([1, 16, 10, 128])];
            tensor<bool, [4]> var_11315_end_mask_0 = const()[name = string("op_11315_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_11315_squeeze_mask_0 = const()[name = string("op_11315_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_11315_cast_fp16 = slice_by_index(begin = var_11315_begin_0, end = var_11315_end_0, end_mask = var_11315_end_mask_0, squeeze_mask = var_11315_squeeze_mask_0, x = _inversed_query_67_cast_fp16)[name = string("op_11315_cast_fp16")];
            tensor<int32, [1]> var_11316_axes_0 = const()[name = string("op_11316_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_11316_cast_fp16 = expand_dims(axes = var_11316_axes_0, x = var_11315_cast_fp16)[name = string("op_11316_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_11317_cast_fp16 = mul(x = state_807_cast_fp16, y = var_11316_cast_fp16)[name = string("op_11317_cast_fp16")];
            tensor<int32, [1]> var_11319_axes_0 = const()[name = string("op_11319_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_11319_keep_dims_0 = const()[name = string("op_11319_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_11319_cast_fp16 = reduce_sum(axes = var_11319_axes_0, keep_dims = var_11319_keep_dims_0, x = var_11317_cast_fp16)[name = string("op_11319_cast_fp16")];
            tensor<int32, [3]> var_11322_begin_0 = const()[name = string("op_11322_begin_0"), val = tensor<int32, [3]>([0, 0, 10])];
            tensor<int32, [3]> var_11322_end_0 = const()[name = string("op_11322_end_0"), val = tensor<int32, [3]>([1, 16, 11])];
            tensor<bool, [3]> var_11322_end_mask_0 = const()[name = string("op_11322_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_11322_squeeze_mask_0 = const()[name = string("op_11322_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_11322_cast_fp16 = slice_by_index(begin = var_11322_begin_0, end = var_11322_end_0, end_mask = var_11322_end_mask_0, squeeze_mask = var_11322_squeeze_mask_0, x = decay_25_cast_fp16)[name = string("op_11322_cast_fp16")];
            tensor<fp16, [1, 16]> var_11323_cast_fp16 = exp(x = var_11322_cast_fp16)[name = string("op_11323_cast_fp16")];
            tensor<int32, [1]> var_11324_axes_0 = const()[name = string("op_11324_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_11324_cast_fp16 = expand_dims(axes = var_11324_axes_0, x = var_11323_cast_fp16)[name = string("op_11324_cast_fp16")];
            tensor<int32, [1]> var_11325_axes_0 = const()[name = string("op_11325_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_11325_cast_fp16 = expand_dims(axes = var_11325_axes_0, x = var_11324_cast_fp16)[name = string("op_11325_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_809_cast_fp16 = mul(x = state_807_cast_fp16, y = var_11325_cast_fp16)[name = string("state_809_cast_fp16")];
            tensor<int32, [4]> key_token_405_begin_0 = const()[name = string("key_token_405_begin_0"), val = tensor<int32, [4]>([0, 0, 10, 0])];
            tensor<int32, [4]> key_token_405_end_0 = const()[name = string("key_token_405_end_0"), val = tensor<int32, [4]>([1, 16, 11, 128])];
            tensor<bool, [4]> key_token_405_end_mask_0 = const()[name = string("key_token_405_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_405_squeeze_mask_0 = const()[name = string("key_token_405_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_405_cast_fp16 = slice_by_index(begin = key_token_405_begin_0, end = key_token_405_end_0, end_mask = key_token_405_end_mask_0, squeeze_mask = key_token_405_squeeze_mask_0, x = key_67_cast_fp16)[name = string("key_token_405_cast_fp16")];
            tensor<int32, [4]> value_token_405_begin_0 = const()[name = string("value_token_405_begin_0"), val = tensor<int32, [4]>([0, 0, 10, 0])];
            tensor<int32, [4]> value_token_405_end_0 = const()[name = string("value_token_405_end_0"), val = tensor<int32, [4]>([1, 16, 11, 128])];
            tensor<bool, [4]> value_token_405_end_mask_0 = const()[name = string("value_token_405_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_405_squeeze_mask_0 = const()[name = string("value_token_405_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_405_cast_fp16 = slice_by_index(begin = value_token_405_begin_0, end = value_token_405_end_0, end_mask = value_token_405_end_mask_0, squeeze_mask = value_token_405_squeeze_mask_0, x = value_243_cast_fp16)[name = string("value_token_405_cast_fp16")];
            tensor<int32, [1]> var_11333_axes_0 = const()[name = string("op_11333_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_11333_cast_fp16 = expand_dims(axes = var_11333_axes_0, x = key_token_405_cast_fp16)[name = string("op_11333_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_11334_cast_fp16 = mul(x = state_809_cast_fp16, y = var_11333_cast_fp16)[name = string("op_11334_cast_fp16")];
            tensor<int32, [1]> memory_405_axes_0 = const()[name = string("memory_405_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_405_keep_dims_0 = const()[name = string("memory_405_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_405_cast_fp16 = reduce_sum(axes = memory_405_axes_0, keep_dims = memory_405_keep_dims_0, x = var_11334_cast_fp16)[name = string("memory_405_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_11337_cast_fp16 = sub(x = value_token_405_cast_fp16, y = memory_405_cast_fp16)[name = string("op_11337_cast_fp16")];
            tensor<int32, [3]> var_11340_begin_0 = const()[name = string("op_11340_begin_0"), val = tensor<int32, [3]>([0, 0, 10])];
            tensor<int32, [3]> var_11340_end_0 = const()[name = string("op_11340_end_0"), val = tensor<int32, [3]>([1, 16, 11])];
            tensor<bool, [3]> var_11340_end_mask_0 = const()[name = string("op_11340_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_11340_squeeze_mask_0 = const()[name = string("op_11340_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_11340_cast_fp16 = slice_by_index(begin = var_11340_begin_0, end = var_11340_end_0, end_mask = var_11340_end_mask_0, squeeze_mask = var_11340_squeeze_mask_0, x = beta_25_cast_fp16)[name = string("op_11340_cast_fp16")];
            tensor<int32, [1]> var_11341_axes_0 = const()[name = string("op_11341_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_11341_cast_fp16 = expand_dims(axes = var_11341_axes_0, x = var_11340_cast_fp16)[name = string("op_11341_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_405_cast_fp16 = mul(x = var_11337_cast_fp16, y = var_11341_cast_fp16)[name = string("delta_405_cast_fp16")];
            tensor<int32, [1]> var_11344_axes_0 = const()[name = string("op_11344_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_11344_cast_fp16 = expand_dims(axes = var_11344_axes_0, x = delta_405_cast_fp16)[name = string("op_11344_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_11345_cast_fp16 = mul(x = var_11333_cast_fp16, y = var_11344_cast_fp16)[name = string("op_11345_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_811_cast_fp16 = add(x = state_809_cast_fp16, y = var_11345_cast_fp16)[name = string("state_811_cast_fp16")];
            tensor<int32, [4]> var_11349_begin_0 = const()[name = string("op_11349_begin_0"), val = tensor<int32, [4]>([0, 0, 10, 0])];
            tensor<int32, [4]> var_11349_end_0 = const()[name = string("op_11349_end_0"), val = tensor<int32, [4]>([1, 16, 11, 128])];
            tensor<bool, [4]> var_11349_end_mask_0 = const()[name = string("op_11349_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_11349_squeeze_mask_0 = const()[name = string("op_11349_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_11349_cast_fp16 = slice_by_index(begin = var_11349_begin_0, end = var_11349_end_0, end_mask = var_11349_end_mask_0, squeeze_mask = var_11349_squeeze_mask_0, x = _inversed_query_67_cast_fp16)[name = string("op_11349_cast_fp16")];
            tensor<int32, [1]> var_11350_axes_0 = const()[name = string("op_11350_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_11350_cast_fp16 = expand_dims(axes = var_11350_axes_0, x = var_11349_cast_fp16)[name = string("op_11350_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_11351_cast_fp16 = mul(x = state_811_cast_fp16, y = var_11350_cast_fp16)[name = string("op_11351_cast_fp16")];
            tensor<int32, [1]> var_11353_axes_0 = const()[name = string("op_11353_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_11353_keep_dims_0 = const()[name = string("op_11353_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_11353_cast_fp16 = reduce_sum(axes = var_11353_axes_0, keep_dims = var_11353_keep_dims_0, x = var_11351_cast_fp16)[name = string("op_11353_cast_fp16")];
            tensor<int32, [3]> var_11356_begin_0 = const()[name = string("op_11356_begin_0"), val = tensor<int32, [3]>([0, 0, 11])];
            tensor<int32, [3]> var_11356_end_0 = const()[name = string("op_11356_end_0"), val = tensor<int32, [3]>([1, 16, 12])];
            tensor<bool, [3]> var_11356_end_mask_0 = const()[name = string("op_11356_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_11356_squeeze_mask_0 = const()[name = string("op_11356_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_11356_cast_fp16 = slice_by_index(begin = var_11356_begin_0, end = var_11356_end_0, end_mask = var_11356_end_mask_0, squeeze_mask = var_11356_squeeze_mask_0, x = decay_25_cast_fp16)[name = string("op_11356_cast_fp16")];
            tensor<fp16, [1, 16]> var_11357_cast_fp16 = exp(x = var_11356_cast_fp16)[name = string("op_11357_cast_fp16")];
            tensor<int32, [1]> var_11358_axes_0 = const()[name = string("op_11358_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_11358_cast_fp16 = expand_dims(axes = var_11358_axes_0, x = var_11357_cast_fp16)[name = string("op_11358_cast_fp16")];
            tensor<int32, [1]> var_11359_axes_0 = const()[name = string("op_11359_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_11359_cast_fp16 = expand_dims(axes = var_11359_axes_0, x = var_11358_cast_fp16)[name = string("op_11359_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_813_cast_fp16 = mul(x = state_811_cast_fp16, y = var_11359_cast_fp16)[name = string("state_813_cast_fp16")];
            tensor<int32, [4]> key_token_407_begin_0 = const()[name = string("key_token_407_begin_0"), val = tensor<int32, [4]>([0, 0, 11, 0])];
            tensor<int32, [4]> key_token_407_end_0 = const()[name = string("key_token_407_end_0"), val = tensor<int32, [4]>([1, 16, 12, 128])];
            tensor<bool, [4]> key_token_407_end_mask_0 = const()[name = string("key_token_407_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_407_squeeze_mask_0 = const()[name = string("key_token_407_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_407_cast_fp16 = slice_by_index(begin = key_token_407_begin_0, end = key_token_407_end_0, end_mask = key_token_407_end_mask_0, squeeze_mask = key_token_407_squeeze_mask_0, x = key_67_cast_fp16)[name = string("key_token_407_cast_fp16")];
            tensor<int32, [4]> value_token_407_begin_0 = const()[name = string("value_token_407_begin_0"), val = tensor<int32, [4]>([0, 0, 11, 0])];
            tensor<int32, [4]> value_token_407_end_0 = const()[name = string("value_token_407_end_0"), val = tensor<int32, [4]>([1, 16, 12, 128])];
            tensor<bool, [4]> value_token_407_end_mask_0 = const()[name = string("value_token_407_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_407_squeeze_mask_0 = const()[name = string("value_token_407_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_407_cast_fp16 = slice_by_index(begin = value_token_407_begin_0, end = value_token_407_end_0, end_mask = value_token_407_end_mask_0, squeeze_mask = value_token_407_squeeze_mask_0, x = value_243_cast_fp16)[name = string("value_token_407_cast_fp16")];
            tensor<int32, [1]> var_11367_axes_0 = const()[name = string("op_11367_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_11367_cast_fp16 = expand_dims(axes = var_11367_axes_0, x = key_token_407_cast_fp16)[name = string("op_11367_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_11368_cast_fp16 = mul(x = state_813_cast_fp16, y = var_11367_cast_fp16)[name = string("op_11368_cast_fp16")];
            tensor<int32, [1]> memory_407_axes_0 = const()[name = string("memory_407_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_407_keep_dims_0 = const()[name = string("memory_407_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_407_cast_fp16 = reduce_sum(axes = memory_407_axes_0, keep_dims = memory_407_keep_dims_0, x = var_11368_cast_fp16)[name = string("memory_407_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_11371_cast_fp16 = sub(x = value_token_407_cast_fp16, y = memory_407_cast_fp16)[name = string("op_11371_cast_fp16")];
            tensor<int32, [3]> var_11374_begin_0 = const()[name = string("op_11374_begin_0"), val = tensor<int32, [3]>([0, 0, 11])];
            tensor<int32, [3]> var_11374_end_0 = const()[name = string("op_11374_end_0"), val = tensor<int32, [3]>([1, 16, 12])];
            tensor<bool, [3]> var_11374_end_mask_0 = const()[name = string("op_11374_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_11374_squeeze_mask_0 = const()[name = string("op_11374_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_11374_cast_fp16 = slice_by_index(begin = var_11374_begin_0, end = var_11374_end_0, end_mask = var_11374_end_mask_0, squeeze_mask = var_11374_squeeze_mask_0, x = beta_25_cast_fp16)[name = string("op_11374_cast_fp16")];
            tensor<int32, [1]> var_11375_axes_0 = const()[name = string("op_11375_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_11375_cast_fp16 = expand_dims(axes = var_11375_axes_0, x = var_11374_cast_fp16)[name = string("op_11375_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_407_cast_fp16 = mul(x = var_11371_cast_fp16, y = var_11375_cast_fp16)[name = string("delta_407_cast_fp16")];
            tensor<int32, [1]> var_11378_axes_0 = const()[name = string("op_11378_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_11378_cast_fp16 = expand_dims(axes = var_11378_axes_0, x = delta_407_cast_fp16)[name = string("op_11378_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_11379_cast_fp16 = mul(x = var_11367_cast_fp16, y = var_11378_cast_fp16)[name = string("op_11379_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_815_cast_fp16 = add(x = state_813_cast_fp16, y = var_11379_cast_fp16)[name = string("state_815_cast_fp16")];
            tensor<int32, [4]> var_11383_begin_0 = const()[name = string("op_11383_begin_0"), val = tensor<int32, [4]>([0, 0, 11, 0])];
            tensor<int32, [4]> var_11383_end_0 = const()[name = string("op_11383_end_0"), val = tensor<int32, [4]>([1, 16, 12, 128])];
            tensor<bool, [4]> var_11383_end_mask_0 = const()[name = string("op_11383_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_11383_squeeze_mask_0 = const()[name = string("op_11383_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_11383_cast_fp16 = slice_by_index(begin = var_11383_begin_0, end = var_11383_end_0, end_mask = var_11383_end_mask_0, squeeze_mask = var_11383_squeeze_mask_0, x = _inversed_query_67_cast_fp16)[name = string("op_11383_cast_fp16")];
            tensor<int32, [1]> var_11384_axes_0 = const()[name = string("op_11384_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_11384_cast_fp16 = expand_dims(axes = var_11384_axes_0, x = var_11383_cast_fp16)[name = string("op_11384_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_11385_cast_fp16 = mul(x = state_815_cast_fp16, y = var_11384_cast_fp16)[name = string("op_11385_cast_fp16")];
            tensor<int32, [1]> var_11387_axes_0 = const()[name = string("op_11387_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_11387_keep_dims_0 = const()[name = string("op_11387_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_11387_cast_fp16 = reduce_sum(axes = var_11387_axes_0, keep_dims = var_11387_keep_dims_0, x = var_11385_cast_fp16)[name = string("op_11387_cast_fp16")];
            tensor<int32, [3]> var_11390_begin_0 = const()[name = string("op_11390_begin_0"), val = tensor<int32, [3]>([0, 0, 12])];
            tensor<int32, [3]> var_11390_end_0 = const()[name = string("op_11390_end_0"), val = tensor<int32, [3]>([1, 16, 13])];
            tensor<bool, [3]> var_11390_end_mask_0 = const()[name = string("op_11390_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_11390_squeeze_mask_0 = const()[name = string("op_11390_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_11390_cast_fp16 = slice_by_index(begin = var_11390_begin_0, end = var_11390_end_0, end_mask = var_11390_end_mask_0, squeeze_mask = var_11390_squeeze_mask_0, x = decay_25_cast_fp16)[name = string("op_11390_cast_fp16")];
            tensor<fp16, [1, 16]> var_11391_cast_fp16 = exp(x = var_11390_cast_fp16)[name = string("op_11391_cast_fp16")];
            tensor<int32, [1]> var_11392_axes_0 = const()[name = string("op_11392_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_11392_cast_fp16 = expand_dims(axes = var_11392_axes_0, x = var_11391_cast_fp16)[name = string("op_11392_cast_fp16")];
            tensor<int32, [1]> var_11393_axes_0 = const()[name = string("op_11393_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_11393_cast_fp16 = expand_dims(axes = var_11393_axes_0, x = var_11392_cast_fp16)[name = string("op_11393_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_817_cast_fp16 = mul(x = state_815_cast_fp16, y = var_11393_cast_fp16)[name = string("state_817_cast_fp16")];
            tensor<int32, [4]> key_token_409_begin_0 = const()[name = string("key_token_409_begin_0"), val = tensor<int32, [4]>([0, 0, 12, 0])];
            tensor<int32, [4]> key_token_409_end_0 = const()[name = string("key_token_409_end_0"), val = tensor<int32, [4]>([1, 16, 13, 128])];
            tensor<bool, [4]> key_token_409_end_mask_0 = const()[name = string("key_token_409_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_409_squeeze_mask_0 = const()[name = string("key_token_409_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_409_cast_fp16 = slice_by_index(begin = key_token_409_begin_0, end = key_token_409_end_0, end_mask = key_token_409_end_mask_0, squeeze_mask = key_token_409_squeeze_mask_0, x = key_67_cast_fp16)[name = string("key_token_409_cast_fp16")];
            tensor<int32, [4]> value_token_409_begin_0 = const()[name = string("value_token_409_begin_0"), val = tensor<int32, [4]>([0, 0, 12, 0])];
            tensor<int32, [4]> value_token_409_end_0 = const()[name = string("value_token_409_end_0"), val = tensor<int32, [4]>([1, 16, 13, 128])];
            tensor<bool, [4]> value_token_409_end_mask_0 = const()[name = string("value_token_409_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_409_squeeze_mask_0 = const()[name = string("value_token_409_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_409_cast_fp16 = slice_by_index(begin = value_token_409_begin_0, end = value_token_409_end_0, end_mask = value_token_409_end_mask_0, squeeze_mask = value_token_409_squeeze_mask_0, x = value_243_cast_fp16)[name = string("value_token_409_cast_fp16")];
            tensor<int32, [1]> var_11401_axes_0 = const()[name = string("op_11401_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_11401_cast_fp16 = expand_dims(axes = var_11401_axes_0, x = key_token_409_cast_fp16)[name = string("op_11401_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_11402_cast_fp16 = mul(x = state_817_cast_fp16, y = var_11401_cast_fp16)[name = string("op_11402_cast_fp16")];
            tensor<int32, [1]> memory_409_axes_0 = const()[name = string("memory_409_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_409_keep_dims_0 = const()[name = string("memory_409_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_409_cast_fp16 = reduce_sum(axes = memory_409_axes_0, keep_dims = memory_409_keep_dims_0, x = var_11402_cast_fp16)[name = string("memory_409_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_11405_cast_fp16 = sub(x = value_token_409_cast_fp16, y = memory_409_cast_fp16)[name = string("op_11405_cast_fp16")];
            tensor<int32, [3]> var_11408_begin_0 = const()[name = string("op_11408_begin_0"), val = tensor<int32, [3]>([0, 0, 12])];
            tensor<int32, [3]> var_11408_end_0 = const()[name = string("op_11408_end_0"), val = tensor<int32, [3]>([1, 16, 13])];
            tensor<bool, [3]> var_11408_end_mask_0 = const()[name = string("op_11408_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_11408_squeeze_mask_0 = const()[name = string("op_11408_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_11408_cast_fp16 = slice_by_index(begin = var_11408_begin_0, end = var_11408_end_0, end_mask = var_11408_end_mask_0, squeeze_mask = var_11408_squeeze_mask_0, x = beta_25_cast_fp16)[name = string("op_11408_cast_fp16")];
            tensor<int32, [1]> var_11409_axes_0 = const()[name = string("op_11409_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_11409_cast_fp16 = expand_dims(axes = var_11409_axes_0, x = var_11408_cast_fp16)[name = string("op_11409_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_409_cast_fp16 = mul(x = var_11405_cast_fp16, y = var_11409_cast_fp16)[name = string("delta_409_cast_fp16")];
            tensor<int32, [1]> var_11412_axes_0 = const()[name = string("op_11412_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_11412_cast_fp16 = expand_dims(axes = var_11412_axes_0, x = delta_409_cast_fp16)[name = string("op_11412_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_11413_cast_fp16 = mul(x = var_11401_cast_fp16, y = var_11412_cast_fp16)[name = string("op_11413_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_819_cast_fp16 = add(x = state_817_cast_fp16, y = var_11413_cast_fp16)[name = string("state_819_cast_fp16")];
            tensor<int32, [4]> var_11417_begin_0 = const()[name = string("op_11417_begin_0"), val = tensor<int32, [4]>([0, 0, 12, 0])];
            tensor<int32, [4]> var_11417_end_0 = const()[name = string("op_11417_end_0"), val = tensor<int32, [4]>([1, 16, 13, 128])];
            tensor<bool, [4]> var_11417_end_mask_0 = const()[name = string("op_11417_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_11417_squeeze_mask_0 = const()[name = string("op_11417_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_11417_cast_fp16 = slice_by_index(begin = var_11417_begin_0, end = var_11417_end_0, end_mask = var_11417_end_mask_0, squeeze_mask = var_11417_squeeze_mask_0, x = _inversed_query_67_cast_fp16)[name = string("op_11417_cast_fp16")];
            tensor<int32, [1]> var_11418_axes_0 = const()[name = string("op_11418_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_11418_cast_fp16 = expand_dims(axes = var_11418_axes_0, x = var_11417_cast_fp16)[name = string("op_11418_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_11419_cast_fp16 = mul(x = state_819_cast_fp16, y = var_11418_cast_fp16)[name = string("op_11419_cast_fp16")];
            tensor<int32, [1]> var_11421_axes_0 = const()[name = string("op_11421_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_11421_keep_dims_0 = const()[name = string("op_11421_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_11421_cast_fp16 = reduce_sum(axes = var_11421_axes_0, keep_dims = var_11421_keep_dims_0, x = var_11419_cast_fp16)[name = string("op_11421_cast_fp16")];
            tensor<int32, [3]> var_11424_begin_0 = const()[name = string("op_11424_begin_0"), val = tensor<int32, [3]>([0, 0, 13])];
            tensor<int32, [3]> var_11424_end_0 = const()[name = string("op_11424_end_0"), val = tensor<int32, [3]>([1, 16, 14])];
            tensor<bool, [3]> var_11424_end_mask_0 = const()[name = string("op_11424_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_11424_squeeze_mask_0 = const()[name = string("op_11424_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_11424_cast_fp16 = slice_by_index(begin = var_11424_begin_0, end = var_11424_end_0, end_mask = var_11424_end_mask_0, squeeze_mask = var_11424_squeeze_mask_0, x = decay_25_cast_fp16)[name = string("op_11424_cast_fp16")];
            tensor<fp16, [1, 16]> var_11425_cast_fp16 = exp(x = var_11424_cast_fp16)[name = string("op_11425_cast_fp16")];
            tensor<int32, [1]> var_11426_axes_0 = const()[name = string("op_11426_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_11426_cast_fp16 = expand_dims(axes = var_11426_axes_0, x = var_11425_cast_fp16)[name = string("op_11426_cast_fp16")];
            tensor<int32, [1]> var_11427_axes_0 = const()[name = string("op_11427_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_11427_cast_fp16 = expand_dims(axes = var_11427_axes_0, x = var_11426_cast_fp16)[name = string("op_11427_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_821_cast_fp16 = mul(x = state_819_cast_fp16, y = var_11427_cast_fp16)[name = string("state_821_cast_fp16")];
            tensor<int32, [4]> key_token_411_begin_0 = const()[name = string("key_token_411_begin_0"), val = tensor<int32, [4]>([0, 0, 13, 0])];
            tensor<int32, [4]> key_token_411_end_0 = const()[name = string("key_token_411_end_0"), val = tensor<int32, [4]>([1, 16, 14, 128])];
            tensor<bool, [4]> key_token_411_end_mask_0 = const()[name = string("key_token_411_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_411_squeeze_mask_0 = const()[name = string("key_token_411_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_411_cast_fp16 = slice_by_index(begin = key_token_411_begin_0, end = key_token_411_end_0, end_mask = key_token_411_end_mask_0, squeeze_mask = key_token_411_squeeze_mask_0, x = key_67_cast_fp16)[name = string("key_token_411_cast_fp16")];
            tensor<int32, [4]> value_token_411_begin_0 = const()[name = string("value_token_411_begin_0"), val = tensor<int32, [4]>([0, 0, 13, 0])];
            tensor<int32, [4]> value_token_411_end_0 = const()[name = string("value_token_411_end_0"), val = tensor<int32, [4]>([1, 16, 14, 128])];
            tensor<bool, [4]> value_token_411_end_mask_0 = const()[name = string("value_token_411_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_411_squeeze_mask_0 = const()[name = string("value_token_411_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_411_cast_fp16 = slice_by_index(begin = value_token_411_begin_0, end = value_token_411_end_0, end_mask = value_token_411_end_mask_0, squeeze_mask = value_token_411_squeeze_mask_0, x = value_243_cast_fp16)[name = string("value_token_411_cast_fp16")];
            tensor<int32, [1]> var_11435_axes_0 = const()[name = string("op_11435_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_11435_cast_fp16 = expand_dims(axes = var_11435_axes_0, x = key_token_411_cast_fp16)[name = string("op_11435_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_11436_cast_fp16 = mul(x = state_821_cast_fp16, y = var_11435_cast_fp16)[name = string("op_11436_cast_fp16")];
            tensor<int32, [1]> memory_411_axes_0 = const()[name = string("memory_411_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_411_keep_dims_0 = const()[name = string("memory_411_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_411_cast_fp16 = reduce_sum(axes = memory_411_axes_0, keep_dims = memory_411_keep_dims_0, x = var_11436_cast_fp16)[name = string("memory_411_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_11439_cast_fp16 = sub(x = value_token_411_cast_fp16, y = memory_411_cast_fp16)[name = string("op_11439_cast_fp16")];
            tensor<int32, [3]> var_11442_begin_0 = const()[name = string("op_11442_begin_0"), val = tensor<int32, [3]>([0, 0, 13])];
            tensor<int32, [3]> var_11442_end_0 = const()[name = string("op_11442_end_0"), val = tensor<int32, [3]>([1, 16, 14])];
            tensor<bool, [3]> var_11442_end_mask_0 = const()[name = string("op_11442_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_11442_squeeze_mask_0 = const()[name = string("op_11442_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_11442_cast_fp16 = slice_by_index(begin = var_11442_begin_0, end = var_11442_end_0, end_mask = var_11442_end_mask_0, squeeze_mask = var_11442_squeeze_mask_0, x = beta_25_cast_fp16)[name = string("op_11442_cast_fp16")];
            tensor<int32, [1]> var_11443_axes_0 = const()[name = string("op_11443_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_11443_cast_fp16 = expand_dims(axes = var_11443_axes_0, x = var_11442_cast_fp16)[name = string("op_11443_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_411_cast_fp16 = mul(x = var_11439_cast_fp16, y = var_11443_cast_fp16)[name = string("delta_411_cast_fp16")];
            tensor<int32, [1]> var_11446_axes_0 = const()[name = string("op_11446_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_11446_cast_fp16 = expand_dims(axes = var_11446_axes_0, x = delta_411_cast_fp16)[name = string("op_11446_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_11447_cast_fp16 = mul(x = var_11435_cast_fp16, y = var_11446_cast_fp16)[name = string("op_11447_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_823_cast_fp16 = add(x = state_821_cast_fp16, y = var_11447_cast_fp16)[name = string("state_823_cast_fp16")];
            tensor<int32, [4]> var_11451_begin_0 = const()[name = string("op_11451_begin_0"), val = tensor<int32, [4]>([0, 0, 13, 0])];
            tensor<int32, [4]> var_11451_end_0 = const()[name = string("op_11451_end_0"), val = tensor<int32, [4]>([1, 16, 14, 128])];
            tensor<bool, [4]> var_11451_end_mask_0 = const()[name = string("op_11451_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_11451_squeeze_mask_0 = const()[name = string("op_11451_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_11451_cast_fp16 = slice_by_index(begin = var_11451_begin_0, end = var_11451_end_0, end_mask = var_11451_end_mask_0, squeeze_mask = var_11451_squeeze_mask_0, x = _inversed_query_67_cast_fp16)[name = string("op_11451_cast_fp16")];
            tensor<int32, [1]> var_11452_axes_0 = const()[name = string("op_11452_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_11452_cast_fp16 = expand_dims(axes = var_11452_axes_0, x = var_11451_cast_fp16)[name = string("op_11452_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_11453_cast_fp16 = mul(x = state_823_cast_fp16, y = var_11452_cast_fp16)[name = string("op_11453_cast_fp16")];
            tensor<int32, [1]> var_11455_axes_0 = const()[name = string("op_11455_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_11455_keep_dims_0 = const()[name = string("op_11455_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_11455_cast_fp16 = reduce_sum(axes = var_11455_axes_0, keep_dims = var_11455_keep_dims_0, x = var_11453_cast_fp16)[name = string("op_11455_cast_fp16")];
            tensor<int32, [3]> var_11458_begin_0 = const()[name = string("op_11458_begin_0"), val = tensor<int32, [3]>([0, 0, 14])];
            tensor<int32, [3]> var_11458_end_0 = const()[name = string("op_11458_end_0"), val = tensor<int32, [3]>([1, 16, 15])];
            tensor<bool, [3]> var_11458_end_mask_0 = const()[name = string("op_11458_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_11458_squeeze_mask_0 = const()[name = string("op_11458_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_11458_cast_fp16 = slice_by_index(begin = var_11458_begin_0, end = var_11458_end_0, end_mask = var_11458_end_mask_0, squeeze_mask = var_11458_squeeze_mask_0, x = decay_25_cast_fp16)[name = string("op_11458_cast_fp16")];
            tensor<fp16, [1, 16]> var_11459_cast_fp16 = exp(x = var_11458_cast_fp16)[name = string("op_11459_cast_fp16")];
            tensor<int32, [1]> var_11460_axes_0 = const()[name = string("op_11460_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_11460_cast_fp16 = expand_dims(axes = var_11460_axes_0, x = var_11459_cast_fp16)[name = string("op_11460_cast_fp16")];
            tensor<int32, [1]> var_11461_axes_0 = const()[name = string("op_11461_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_11461_cast_fp16 = expand_dims(axes = var_11461_axes_0, x = var_11460_cast_fp16)[name = string("op_11461_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_825_cast_fp16 = mul(x = state_823_cast_fp16, y = var_11461_cast_fp16)[name = string("state_825_cast_fp16")];
            tensor<int32, [4]> key_token_413_begin_0 = const()[name = string("key_token_413_begin_0"), val = tensor<int32, [4]>([0, 0, 14, 0])];
            tensor<int32, [4]> key_token_413_end_0 = const()[name = string("key_token_413_end_0"), val = tensor<int32, [4]>([1, 16, 15, 128])];
            tensor<bool, [4]> key_token_413_end_mask_0 = const()[name = string("key_token_413_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_413_squeeze_mask_0 = const()[name = string("key_token_413_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_413_cast_fp16 = slice_by_index(begin = key_token_413_begin_0, end = key_token_413_end_0, end_mask = key_token_413_end_mask_0, squeeze_mask = key_token_413_squeeze_mask_0, x = key_67_cast_fp16)[name = string("key_token_413_cast_fp16")];
            tensor<int32, [4]> value_token_413_begin_0 = const()[name = string("value_token_413_begin_0"), val = tensor<int32, [4]>([0, 0, 14, 0])];
            tensor<int32, [4]> value_token_413_end_0 = const()[name = string("value_token_413_end_0"), val = tensor<int32, [4]>([1, 16, 15, 128])];
            tensor<bool, [4]> value_token_413_end_mask_0 = const()[name = string("value_token_413_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_413_squeeze_mask_0 = const()[name = string("value_token_413_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_413_cast_fp16 = slice_by_index(begin = value_token_413_begin_0, end = value_token_413_end_0, end_mask = value_token_413_end_mask_0, squeeze_mask = value_token_413_squeeze_mask_0, x = value_243_cast_fp16)[name = string("value_token_413_cast_fp16")];
            tensor<int32, [1]> var_11469_axes_0 = const()[name = string("op_11469_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_11469_cast_fp16 = expand_dims(axes = var_11469_axes_0, x = key_token_413_cast_fp16)[name = string("op_11469_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_11470_cast_fp16 = mul(x = state_825_cast_fp16, y = var_11469_cast_fp16)[name = string("op_11470_cast_fp16")];
            tensor<int32, [1]> memory_413_axes_0 = const()[name = string("memory_413_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_413_keep_dims_0 = const()[name = string("memory_413_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_413_cast_fp16 = reduce_sum(axes = memory_413_axes_0, keep_dims = memory_413_keep_dims_0, x = var_11470_cast_fp16)[name = string("memory_413_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_11473_cast_fp16 = sub(x = value_token_413_cast_fp16, y = memory_413_cast_fp16)[name = string("op_11473_cast_fp16")];
            tensor<int32, [3]> var_11476_begin_0 = const()[name = string("op_11476_begin_0"), val = tensor<int32, [3]>([0, 0, 14])];
            tensor<int32, [3]> var_11476_end_0 = const()[name = string("op_11476_end_0"), val = tensor<int32, [3]>([1, 16, 15])];
            tensor<bool, [3]> var_11476_end_mask_0 = const()[name = string("op_11476_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_11476_squeeze_mask_0 = const()[name = string("op_11476_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_11476_cast_fp16 = slice_by_index(begin = var_11476_begin_0, end = var_11476_end_0, end_mask = var_11476_end_mask_0, squeeze_mask = var_11476_squeeze_mask_0, x = beta_25_cast_fp16)[name = string("op_11476_cast_fp16")];
            tensor<int32, [1]> var_11477_axes_0 = const()[name = string("op_11477_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_11477_cast_fp16 = expand_dims(axes = var_11477_axes_0, x = var_11476_cast_fp16)[name = string("op_11477_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_413_cast_fp16 = mul(x = var_11473_cast_fp16, y = var_11477_cast_fp16)[name = string("delta_413_cast_fp16")];
            tensor<int32, [1]> var_11480_axes_0 = const()[name = string("op_11480_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_11480_cast_fp16 = expand_dims(axes = var_11480_axes_0, x = delta_413_cast_fp16)[name = string("op_11480_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_11481_cast_fp16 = mul(x = var_11469_cast_fp16, y = var_11480_cast_fp16)[name = string("op_11481_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_827_cast_fp16 = add(x = state_825_cast_fp16, y = var_11481_cast_fp16)[name = string("state_827_cast_fp16")];
            tensor<int32, [4]> var_11485_begin_0 = const()[name = string("op_11485_begin_0"), val = tensor<int32, [4]>([0, 0, 14, 0])];
            tensor<int32, [4]> var_11485_end_0 = const()[name = string("op_11485_end_0"), val = tensor<int32, [4]>([1, 16, 15, 128])];
            tensor<bool, [4]> var_11485_end_mask_0 = const()[name = string("op_11485_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_11485_squeeze_mask_0 = const()[name = string("op_11485_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_11485_cast_fp16 = slice_by_index(begin = var_11485_begin_0, end = var_11485_end_0, end_mask = var_11485_end_mask_0, squeeze_mask = var_11485_squeeze_mask_0, x = _inversed_query_67_cast_fp16)[name = string("op_11485_cast_fp16")];
            tensor<int32, [1]> var_11486_axes_0 = const()[name = string("op_11486_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_11486_cast_fp16 = expand_dims(axes = var_11486_axes_0, x = var_11485_cast_fp16)[name = string("op_11486_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_11487_cast_fp16 = mul(x = state_827_cast_fp16, y = var_11486_cast_fp16)[name = string("op_11487_cast_fp16")];
            tensor<int32, [1]> var_11489_axes_0 = const()[name = string("op_11489_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_11489_keep_dims_0 = const()[name = string("op_11489_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_11489_cast_fp16 = reduce_sum(axes = var_11489_axes_0, keep_dims = var_11489_keep_dims_0, x = var_11487_cast_fp16)[name = string("op_11489_cast_fp16")];
            tensor<int32, [3]> var_11492_begin_0 = const()[name = string("op_11492_begin_0"), val = tensor<int32, [3]>([0, 0, 15])];
            tensor<int32, [3]> var_11492_end_0 = const()[name = string("op_11492_end_0"), val = tensor<int32, [3]>([1, 16, 16])];
            tensor<bool, [3]> var_11492_end_mask_0 = const()[name = string("op_11492_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_11492_squeeze_mask_0 = const()[name = string("op_11492_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_11492_cast_fp16 = slice_by_index(begin = var_11492_begin_0, end = var_11492_end_0, end_mask = var_11492_end_mask_0, squeeze_mask = var_11492_squeeze_mask_0, x = decay_25_cast_fp16)[name = string("op_11492_cast_fp16")];
            tensor<fp16, [1, 16]> var_11493_cast_fp16 = exp(x = var_11492_cast_fp16)[name = string("op_11493_cast_fp16")];
            tensor<int32, [1]> var_11494_axes_0 = const()[name = string("op_11494_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_11494_cast_fp16 = expand_dims(axes = var_11494_axes_0, x = var_11493_cast_fp16)[name = string("op_11494_cast_fp16")];
            tensor<int32, [1]> var_11495_axes_0 = const()[name = string("op_11495_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_11495_cast_fp16 = expand_dims(axes = var_11495_axes_0, x = var_11494_cast_fp16)[name = string("op_11495_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_829_cast_fp16 = mul(x = state_827_cast_fp16, y = var_11495_cast_fp16)[name = string("state_829_cast_fp16")];
            tensor<int32, [4]> key_token_415_begin_0 = const()[name = string("key_token_415_begin_0"), val = tensor<int32, [4]>([0, 0, 15, 0])];
            tensor<int32, [4]> key_token_415_end_0 = const()[name = string("key_token_415_end_0"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<bool, [4]> key_token_415_end_mask_0 = const()[name = string("key_token_415_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_415_squeeze_mask_0 = const()[name = string("key_token_415_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_415_cast_fp16 = slice_by_index(begin = key_token_415_begin_0, end = key_token_415_end_0, end_mask = key_token_415_end_mask_0, squeeze_mask = key_token_415_squeeze_mask_0, x = key_67_cast_fp16)[name = string("key_token_415_cast_fp16")];
            tensor<int32, [4]> value_token_415_begin_0 = const()[name = string("value_token_415_begin_0"), val = tensor<int32, [4]>([0, 0, 15, 0])];
            tensor<int32, [4]> value_token_415_end_0 = const()[name = string("value_token_415_end_0"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<bool, [4]> value_token_415_end_mask_0 = const()[name = string("value_token_415_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_415_squeeze_mask_0 = const()[name = string("value_token_415_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_415_cast_fp16 = slice_by_index(begin = value_token_415_begin_0, end = value_token_415_end_0, end_mask = value_token_415_end_mask_0, squeeze_mask = value_token_415_squeeze_mask_0, x = value_243_cast_fp16)[name = string("value_token_415_cast_fp16")];
            tensor<int32, [1]> var_11503_axes_0 = const()[name = string("op_11503_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_11503_cast_fp16 = expand_dims(axes = var_11503_axes_0, x = key_token_415_cast_fp16)[name = string("op_11503_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_11504_cast_fp16 = mul(x = state_829_cast_fp16, y = var_11503_cast_fp16)[name = string("op_11504_cast_fp16")];
            tensor<int32, [1]> memory_415_axes_0 = const()[name = string("memory_415_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_415_keep_dims_0 = const()[name = string("memory_415_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_415_cast_fp16 = reduce_sum(axes = memory_415_axes_0, keep_dims = memory_415_keep_dims_0, x = var_11504_cast_fp16)[name = string("memory_415_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_11507_cast_fp16 = sub(x = value_token_415_cast_fp16, y = memory_415_cast_fp16)[name = string("op_11507_cast_fp16")];
            tensor<int32, [3]> var_11510_begin_0 = const()[name = string("op_11510_begin_0"), val = tensor<int32, [3]>([0, 0, 15])];
            tensor<int32, [3]> var_11510_end_0 = const()[name = string("op_11510_end_0"), val = tensor<int32, [3]>([1, 16, 16])];
            tensor<bool, [3]> var_11510_end_mask_0 = const()[name = string("op_11510_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_11510_squeeze_mask_0 = const()[name = string("op_11510_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_11510_cast_fp16 = slice_by_index(begin = var_11510_begin_0, end = var_11510_end_0, end_mask = var_11510_end_mask_0, squeeze_mask = var_11510_squeeze_mask_0, x = beta_25_cast_fp16)[name = string("op_11510_cast_fp16")];
            tensor<int32, [1]> var_11511_axes_0 = const()[name = string("op_11511_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_11511_cast_fp16 = expand_dims(axes = var_11511_axes_0, x = var_11510_cast_fp16)[name = string("op_11511_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_415_cast_fp16 = mul(x = var_11507_cast_fp16, y = var_11511_cast_fp16)[name = string("delta_415_cast_fp16")];
            tensor<int32, [1]> var_11514_axes_0 = const()[name = string("op_11514_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_11514_cast_fp16 = expand_dims(axes = var_11514_axes_0, x = delta_415_cast_fp16)[name = string("op_11514_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_11515_cast_fp16 = mul(x = var_11503_cast_fp16, y = var_11514_cast_fp16)[name = string("op_11515_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> rec16_out = add(x = state_829_cast_fp16, y = var_11515_cast_fp16)[name = string("state_831_cast_fp16")];
            tensor<int32, [4]> var_11519_begin_0 = const()[name = string("op_11519_begin_0"), val = tensor<int32, [4]>([0, 0, 15, 0])];
            tensor<int32, [4]> var_11519_end_0 = const()[name = string("op_11519_end_0"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<bool, [4]> var_11519_end_mask_0 = const()[name = string("op_11519_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_11519_squeeze_mask_0 = const()[name = string("op_11519_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_11519_cast_fp16 = slice_by_index(begin = var_11519_begin_0, end = var_11519_end_0, end_mask = var_11519_end_mask_0, squeeze_mask = var_11519_squeeze_mask_0, x = _inversed_query_67_cast_fp16)[name = string("op_11519_cast_fp16")];
            tensor<int32, [1]> var_11520_axes_0 = const()[name = string("op_11520_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_11520_cast_fp16 = expand_dims(axes = var_11520_axes_0, x = var_11519_cast_fp16)[name = string("op_11520_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_11521_cast_fp16 = mul(x = rec16_out, y = var_11520_cast_fp16)[name = string("op_11521_cast_fp16")];
            tensor<int32, [1]> var_11523_axes_0 = const()[name = string("op_11523_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_11523_keep_dims_0 = const()[name = string("op_11523_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_11523_cast_fp16 = reduce_sum(axes = var_11523_axes_0, keep_dims = var_11523_keep_dims_0, x = var_11521_cast_fp16)[name = string("op_11523_cast_fp16")];
            int32 attention_121_axis_0 = const()[name = string("attention_121_axis_0"), val = int32(1)];
            tensor<fp16, [1, 16, 16, 128]> attention_121_cast_fp16 = stack(axis = attention_121_axis_0, values = (var_11013_cast_fp16, var_11047_cast_fp16, var_11081_cast_fp16, var_11115_cast_fp16, var_11149_cast_fp16, var_11183_cast_fp16, var_11217_cast_fp16, var_11251_cast_fp16, var_11285_cast_fp16, var_11319_cast_fp16, var_11353_cast_fp16, var_11387_cast_fp16, var_11421_cast_fp16, var_11455_cast_fp16, var_11489_cast_fp16, var_11523_cast_fp16))[name = string("attention_121_cast_fp16")];
            tensor<int32, [2]> var_11526 = const()[name = string("op_11526"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [256, 128]> attention_123_cast_fp16 = reshape(shape = var_11526, x = attention_121_cast_fp16)[name = string("attention_123_cast_fp16")];
            tensor<int32, [2]> var_11528 = const()[name = string("op_11528"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [256, 128]> input_213_cast_fp16 = reshape(shape = var_11528, x = linear_127_cast_fp16)[name = string("input_213_cast_fp16")];
            tensor<fp16, [256, 128]> var_11530_cast_fp16 = mul(x = attention_123_cast_fp16, y = attention_123_cast_fp16)[name = string("op_11530_cast_fp16")];
            tensor<int32, [1]> variance_107_axes_0 = const()[name = string("variance_107_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_107_keep_dims_0 = const()[name = string("variance_107_keep_dims_0"), val = bool(true)];
            tensor<fp16, [256, 1]> variance_107_cast_fp16 = reduce_mean(axes = variance_107_axes_0, keep_dims = variance_107_keep_dims_0, x = var_11530_cast_fp16)[name = string("variance_107_cast_fp16")];
            fp16 var_11533_to_fp16 = const()[name = string("op_11533_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [256, 1]> var_11534_cast_fp16 = add(x = variance_107_cast_fp16, y = var_11533_to_fp16)[name = string("op_11534_cast_fp16")];
            fp32 var_11535_epsilon_0 = const()[name = string("op_11535_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [256, 1]> var_11535_cast_fp16 = rsqrt(epsilon = var_11535_epsilon_0, x = var_11534_cast_fp16)[name = string("op_11535_cast_fp16")];
            tensor<fp16, [256, 128]> attention_125_cast_fp16 = mul(x = attention_123_cast_fp16, y = var_11535_cast_fp16)[name = string("attention_125_cast_fp16")];
            tensor<fp16, [128]> final_group_0_gated_norm_promoted_to_fp16 = const()[name = string("final_group_0_gated_norm_promoted_to_fp16"), val = tensor<fp16, [128]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(258335744)))];
            tensor<fp16, [256, 128]> attention_127_cast_fp16 = mul(x = attention_125_cast_fp16, y = final_group_0_gated_norm_promoted_to_fp16)[name = string("attention_127_cast_fp16")];
            tensor<fp16, [256, 128]> var_11538_cast_fp16 = silu(x = input_213_cast_fp16)[name = string("op_11538_cast_fp16")];
            tensor<fp16, [256, 128]> attention_129_cast_fp16 = mul(x = attention_127_cast_fp16, y = var_11538_cast_fp16)[name = string("attention_129_cast_fp16")];
            tensor<int32, [2]> var_11540 = const()[name = string("op_11540"), val = tensor<int32, [2]>([16, 2048])];
            tensor<fp16, [16, 2048]> input_215_cast_fp16 = reshape(shape = var_11540, x = attention_129_cast_fp16)[name = string("input_215_cast_fp16")];
            tensor<fp16, [1024, 2048]> final_group_0_out_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 2048]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(258336064))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(259908992))))[name = string("final_group_0_out_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 1024]> linear_128_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = final_group_0_out_proj_weight_promoted_to_fp16_palettized, x = input_215_cast_fp16)[name = string("linear_128_cast_fp16")];
            tensor<fp16, [16, 1024]> value_245_cast_fp16 = add(x = value_233_cast_fp16, y = linear_128_cast_fp16)[name = string("value_245_cast_fp16")];
            tensor<fp16, [16, 1024]> var_11545_cast_fp16 = mul(x = value_245_cast_fp16, y = value_245_cast_fp16)[name = string("op_11545_cast_fp16")];
            tensor<int32, [1]> variance_109_axes_0 = const()[name = string("variance_109_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_109_keep_dims_0 = const()[name = string("variance_109_keep_dims_0"), val = bool(true)];
            tensor<fp16, [16, 1]> variance_109_cast_fp16 = reduce_mean(axes = variance_109_axes_0, keep_dims = variance_109_keep_dims_0, x = var_11545_cast_fp16)[name = string("variance_109_cast_fp16")];
            fp16 var_11548_to_fp16 = const()[name = string("op_11548_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [16, 1]> var_11549_cast_fp16 = add(x = variance_109_cast_fp16, y = var_11548_to_fp16)[name = string("op_11549_cast_fp16")];
            fp32 var_11550_epsilon_0 = const()[name = string("op_11550_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [16, 1]> var_11550_cast_fp16 = rsqrt(epsilon = var_11550_epsilon_0, x = var_11549_cast_fp16)[name = string("op_11550_cast_fp16")];
            tensor<fp16, [16, 1024]> var_11551_cast_fp16 = mul(x = value_245_cast_fp16, y = var_11550_cast_fp16)[name = string("op_11551_cast_fp16")];
            tensor<fp16, [1024]> var_11553_to_fp16 = const()[name = string("op_11553_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(259917248)))];
            tensor<fp16, [16, 1024]> input_217_cast_fp16 = mul(x = var_11551_cast_fp16, y = var_11553_to_fp16)[name = string("input_217_cast_fp16")];
            tensor<fp16, [3584, 1024]> final_group_0_gate_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(259919360))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(262671936))))[name = string("final_group_0_gate_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 3584]> linear_129_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = final_group_0_gate_proj_weight_promoted_to_fp16_palettized, x = input_217_cast_fp16)[name = string("linear_129_cast_fp16")];
            tensor<fp16, [16, 3584]> var_11557_cast_fp16 = silu(x = linear_129_cast_fp16)[name = string("op_11557_cast_fp16")];
            tensor<fp16, [3584, 1024]> final_group_0_up_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(262700672))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(265453248))))[name = string("final_group_0_up_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 3584]> linear_130_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = final_group_0_up_proj_weight_promoted_to_fp16_palettized, x = input_217_cast_fp16)[name = string("linear_130_cast_fp16")];
            tensor<fp16, [16, 3584]> input_221_cast_fp16 = mul(x = var_11557_cast_fp16, y = linear_130_cast_fp16)[name = string("input_221_cast_fp16")];
            tensor<fp16, [1024, 3584]> final_group_0_down_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 3584]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(265481984))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(268234560))))[name = string("final_group_0_down_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 1024]> linear_131_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = final_group_0_down_proj_weight_promoted_to_fp16_palettized, x = input_221_cast_fp16)[name = string("linear_131_cast_fp16")];
            tensor<fp16, [16, 1024]> value_247_cast_fp16 = add(x = value_245_cast_fp16, y = linear_131_cast_fp16)[name = string("value_247_cast_fp16")];
            int32 var_11595 = const()[name = string("op_11595"), val = int32(-1)];
            tensor<fp16, [16, 1024]> var_11610_cast_fp16 = mul(x = value_247_cast_fp16, y = value_247_cast_fp16)[name = string("op_11610_cast_fp16")];
            tensor<int32, [1]> variance_111_axes_0 = const()[name = string("variance_111_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_111_keep_dims_0 = const()[name = string("variance_111_keep_dims_0"), val = bool(true)];
            tensor<fp16, [16, 1]> variance_111_cast_fp16 = reduce_mean(axes = variance_111_axes_0, keep_dims = variance_111_keep_dims_0, x = var_11610_cast_fp16)[name = string("variance_111_cast_fp16")];
            fp16 var_11613_to_fp16 = const()[name = string("op_11613_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [16, 1]> var_11614_cast_fp16 = add(x = variance_111_cast_fp16, y = var_11613_to_fp16)[name = string("op_11614_cast_fp16")];
            fp32 var_11615_epsilon_0 = const()[name = string("op_11615_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [16, 1]> var_11615_cast_fp16 = rsqrt(epsilon = var_11615_epsilon_0, x = var_11614_cast_fp16)[name = string("op_11615_cast_fp16")];
            tensor<fp16, [16, 1024]> var_11616_cast_fp16 = mul(x = value_247_cast_fp16, y = var_11615_cast_fp16)[name = string("op_11616_cast_fp16")];
            tensor<fp16, [1024]> var_11618_to_fp16 = const()[name = string("op_11618_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(268242816)))];
            tensor<fp16, [16, 1024]> input_223_cast_fp16 = mul(x = var_11616_cast_fp16, y = var_11618_to_fp16)[name = string("input_223_cast_fp16")];
            tensor<fp16, [6144, 1024]> final_group_1_in_proj_qkv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(268244928))), lut = tensor<fp16, [384, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(272963584))))[name = string("final_group_1_in_proj_qkv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 6144]> linear_132_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = final_group_1_in_proj_qkv_weight_promoted_to_fp16_palettized, x = input_223_cast_fp16)[name = string("linear_132_cast_fp16")];
            tensor<int32, [2]> var_11622_perm_0 = const()[name = string("op_11622_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> mixed_27_axes_0 = const()[name = string("mixed_27_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [6144, 16]> var_11622_cast_fp16 = transpose(perm = var_11622_perm_0, x = linear_132_cast_fp16)[name = string("transpose_17")];
            tensor<fp16, [1, 6144, 16]> mixed_27_cast_fp16 = expand_dims(axes = mixed_27_axes_0, x = var_11622_cast_fp16)[name = string("mixed_27_cast_fp16")];
            bool convolution_input_27_interleave_0 = const()[name = string("convolution_input_27_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 6144, 19]> convolution_input_27_cast_fp16 = concat(axis = var_11595, interleave = convolution_input_27_interleave_0, values = (conv17, mixed_27_cast_fp16))[name = string("convolution_input_27_cast_fp16")];
            string input_225_pad_type_0 = const()[name = string("input_225_pad_type_0"), val = string("valid")];
            int32 input_225_groups_0 = const()[name = string("input_225_groups_0"), val = int32(6144)];
            tensor<int32, [1]> input_225_strides_0 = const()[name = string("input_225_strides_0"), val = tensor<int32, [1]>([1])];
            tensor<int32, [2]> input_225_pad_0 = const()[name = string("input_225_pad_0"), val = tensor<int32, [2]>([0, 0])];
            tensor<int32, [1]> input_225_dilations_0 = const()[name = string("input_225_dilations_0"), val = tensor<int32, [1]>([1])];
            tensor<fp16, [6144, 1, 4]> final_group_1_conv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1, 4]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(273012800))), lut = tensor<fp16, [384, 1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(273031296))))[name = string("final_group_1_conv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 6144, 16]> input_225_cast_fp16 = conv(dilations = input_225_dilations_0, groups = input_225_groups_0, pad = input_225_pad_0, pad_type = input_225_pad_type_0, strides = input_225_strides_0, weight = final_group_1_conv_weight_promoted_to_fp16_palettized, x = convolution_input_27_cast_fp16)[name = string("input_225_cast_fp16")];
            tensor<fp16, [1, 6144, 16]> var_11631_cast_fp16 = silu(x = input_225_cast_fp16)[name = string("op_11631_cast_fp16")];
            tensor<int32, [3]> var_11632_perm_0 = const()[name = string("op_11632_perm_0"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [3]> var_11635_begin_0 = const()[name = string("op_11635_begin_0"), val = tensor<int32, [3]>([0, 0, 16])];
            tensor<int32, [3]> var_11635_end_0 = const()[name = string("op_11635_end_0"), val = tensor<int32, [3]>([1, 6144, 19])];
            tensor<bool, [3]> var_11635_end_mask_0 = const()[name = string("op_11635_end_mask_0"), val = tensor<bool, [3]>([true, true, true])];
            tensor<fp16, [1, 6144, 3]> conv17_out = slice_by_index(begin = var_11635_begin_0, end = var_11635_end_0, end_mask = var_11635_end_mask_0, x = convolution_input_27_cast_fp16)[name = string("op_11635_cast_fp16")];
            tensor<int32, [3]> var_11636 = const()[name = string("op_11636"), val = tensor<int32, [3]>([2048, 2048, 2048])];
            int32 var_11637_axis_0 = const()[name = string("op_11637_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 16, 6144]> var_11632_cast_fp16 = transpose(perm = var_11632_perm_0, x = var_11631_cast_fp16)[name = string("transpose_16")];
            tensor<fp16, [1, 16, 2048]> var_11637_cast_fp16_0, tensor<fp16, [1, 16, 2048]> var_11637_cast_fp16_1, tensor<fp16, [1, 16, 2048]> var_11637_cast_fp16_2 = split(axis = var_11637_axis_0, split_sizes = var_11636, x = var_11632_cast_fp16)[name = string("op_11637_cast_fp16")];
            tensor<int32, [4]> var_11641 = const()[name = string("op_11641"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<fp16, [1, 16, 16, 128]> value_251_cast_fp16 = reshape(shape = var_11641, x = var_11637_cast_fp16_0)[name = string("value_251_cast_fp16")];
            tensor<int32, [4]> var_11643 = const()[name = string("op_11643"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<fp16, [1, 16, 16, 128]> value_253_cast_fp16 = reshape(shape = var_11643, x = var_11637_cast_fp16_1)[name = string("value_253_cast_fp16")];
            tensor<int32, [4]> var_11645 = const()[name = string("op_11645"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<fp16, [1, 16, 16, 128]> value_255_cast_fp16 = reshape(shape = var_11645, x = var_11637_cast_fp16_2)[name = string("value_255_cast_fp16")];
            tensor<fp16, [1, 16, 16, 128]> var_11647_cast_fp16 = mul(x = value_251_cast_fp16, y = value_251_cast_fp16)[name = string("op_11647_cast_fp16")];
            tensor<int32, [1]> var_11649_axes_0 = const()[name = string("op_11649_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_11649_keep_dims_0 = const()[name = string("op_11649_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 16, 16, 1]> var_11649_cast_fp16 = reduce_sum(axes = var_11649_axes_0, keep_dims = var_11649_keep_dims_0, x = var_11647_cast_fp16)[name = string("op_11649_cast_fp16")];
            fp16 var_11650_to_fp16 = const()[name = string("op_11650_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 16, 16, 1]> var_11651_cast_fp16 = add(x = var_11649_cast_fp16, y = var_11650_to_fp16)[name = string("op_11651_cast_fp16")];
            fp32 var_11652_epsilon_0 = const()[name = string("op_11652_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 16, 16, 1]> var_11652_cast_fp16 = rsqrt(epsilon = var_11652_epsilon_0, x = var_11651_cast_fp16)[name = string("op_11652_cast_fp16")];
            tensor<fp16, [1, 16, 16, 128]> var_11653_cast_fp16 = mul(x = value_251_cast_fp16, y = var_11652_cast_fp16)[name = string("op_11653_cast_fp16")];
            tensor<int32, [4]> var_11654_perm_0 = const()[name = string("op_11654_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            fp16 _inversed_query_71_y_0_to_fp16 = const()[name = string("_inversed_query_71_y_0_to_fp16"), val = fp16(0x1.6ap-4)];
            tensor<fp16, [1, 16, 16, 128]> var_11654_cast_fp16 = transpose(perm = var_11654_perm_0, x = var_11653_cast_fp16)[name = string("transpose_15")];
            tensor<fp16, [1, 16, 16, 128]> _inversed_query_71_cast_fp16 = mul(x = var_11654_cast_fp16, y = _inversed_query_71_y_0_to_fp16)[name = string("_inversed_query_71_cast_fp16")];
            tensor<fp16, [1, 16, 16, 128]> var_11657_cast_fp16 = mul(x = value_253_cast_fp16, y = value_253_cast_fp16)[name = string("op_11657_cast_fp16")];
            tensor<int32, [1]> var_11659_axes_0 = const()[name = string("op_11659_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_11659_keep_dims_0 = const()[name = string("op_11659_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 16, 16, 1]> var_11659_cast_fp16 = reduce_sum(axes = var_11659_axes_0, keep_dims = var_11659_keep_dims_0, x = var_11657_cast_fp16)[name = string("op_11659_cast_fp16")];
            fp16 var_11660_to_fp16 = const()[name = string("op_11660_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 16, 16, 1]> var_11661_cast_fp16 = add(x = var_11659_cast_fp16, y = var_11660_to_fp16)[name = string("op_11661_cast_fp16")];
            fp32 var_11662_epsilon_0 = const()[name = string("op_11662_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 16, 16, 1]> var_11662_cast_fp16 = rsqrt(epsilon = var_11662_epsilon_0, x = var_11661_cast_fp16)[name = string("op_11662_cast_fp16")];
            tensor<fp16, [1, 16, 16, 128]> var_11663_cast_fp16 = mul(x = value_253_cast_fp16, y = var_11662_cast_fp16)[name = string("op_11663_cast_fp16")];
            tensor<int32, [4]> key_71_perm_0 = const()[name = string("key_71_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<int32, [4]> value_257_perm_0 = const()[name = string("value_257_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<fp16, [16, 1024]> final_group_1_in_proj_b_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(273080512))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(273092864))))[name = string("final_group_1_in_proj_b_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16]> linear_133_bias_0_to_fp16 = const()[name = string("linear_133_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(4862656)))];
            tensor<fp16, [16, 16]> linear_133_cast_fp16 = linear(bias = linear_133_bias_0_to_fp16, weight = final_group_1_in_proj_b_weight_promoted_to_fp16_palettized, x = input_223_cast_fp16)[name = string("linear_133_cast_fp16")];
            tensor<fp16, [16, 16]> var_11668_cast_fp16 = sigmoid(x = linear_133_cast_fp16)[name = string("op_11668_cast_fp16")];
            tensor<int32, [2]> var_11669_perm_0 = const()[name = string("op_11669_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> beta_27_axes_0 = const()[name = string("beta_27_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 16]> var_11669_cast_fp16 = transpose(perm = var_11669_perm_0, x = var_11668_cast_fp16)[name = string("transpose_14")];
            tensor<fp16, [1, 16, 16]> beta_27_cast_fp16 = expand_dims(axes = beta_27_axes_0, x = var_11669_cast_fp16)[name = string("beta_27_cast_fp16")];
            tensor<fp16, [16, 1024]> op_11675_weight_0_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(273093056))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(273105408))))[name = string("op_11675_weight_0_to_fp16_palettized")];
            tensor<fp16, [16]> var_11675_bias_0_to_fp16 = const()[name = string("op_11675_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(273105600)))];
            tensor<fp16, [16, 16]> var_11675_cast_fp16 = linear(bias = var_11675_bias_0_to_fp16, weight = op_11675_weight_0_to_fp16_palettized, x = input_223_cast_fp16)[name = string("op_11675_cast_fp16")];
            tensor<fp16, [16, 16]> var_11676_cast_fp16 = softplus(x = var_11675_cast_fp16)[name = string("op_11676_cast_fp16")];
            tensor<fp16, [16]> var_11672_promoted_to_fp16 = const()[name = string("op_11672_promoted_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(273105728)))];
            tensor<fp16, [16, 16]> var_11677_cast_fp16 = mul(x = var_11672_promoted_to_fp16, y = var_11676_cast_fp16)[name = string("op_11677_cast_fp16")];
            tensor<int32, [2]> var_11678_perm_0 = const()[name = string("op_11678_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> decay_27_axes_0 = const()[name = string("decay_27_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 16]> var_11678_cast_fp16 = transpose(perm = var_11678_perm_0, x = var_11677_cast_fp16)[name = string("transpose_13")];
            tensor<fp16, [1, 16, 16]> decay_27_cast_fp16 = expand_dims(axes = decay_27_axes_0, x = var_11678_cast_fp16)[name = string("decay_27_cast_fp16")];
            tensor<fp16, [2048, 1024]> final_group_1_in_proj_z_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [2048, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(273105856))), lut = tensor<fp16, [128, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(274678784))))[name = string("final_group_1_in_proj_z_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 2048]> linear_135_cast_fp16 = linear(bias = linear_3_bias_0_to_fp16, weight = final_group_1_in_proj_z_weight_promoted_to_fp16_palettized, x = input_223_cast_fp16)[name = string("linear_135_cast_fp16")];
            tensor<int32, [3]> var_11686_begin_0 = const()[name = string("op_11686_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_11686_end_0 = const()[name = string("op_11686_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_11686_end_mask_0 = const()[name = string("op_11686_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_11686_squeeze_mask_0 = const()[name = string("op_11686_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_11686_cast_fp16 = slice_by_index(begin = var_11686_begin_0, end = var_11686_end_0, end_mask = var_11686_end_mask_0, squeeze_mask = var_11686_squeeze_mask_0, x = decay_27_cast_fp16)[name = string("op_11686_cast_fp16")];
            tensor<fp16, [1, 16]> var_11687_cast_fp16 = exp(x = var_11686_cast_fp16)[name = string("op_11687_cast_fp16")];
            tensor<int32, [1]> var_11688_axes_0 = const()[name = string("op_11688_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_11688_cast_fp16 = expand_dims(axes = var_11688_axes_0, x = var_11687_cast_fp16)[name = string("op_11688_cast_fp16")];
            tensor<int32, [1]> var_11689_axes_0 = const()[name = string("op_11689_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_11689_cast_fp16 = expand_dims(axes = var_11689_axes_0, x = var_11688_cast_fp16)[name = string("op_11689_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_833_cast_fp16 = mul(x = rec17, y = var_11689_cast_fp16)[name = string("state_833_cast_fp16")];
            tensor<int32, [4]> key_token_417_begin_0 = const()[name = string("key_token_417_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> key_token_417_end_0 = const()[name = string("key_token_417_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> key_token_417_end_mask_0 = const()[name = string("key_token_417_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_417_squeeze_mask_0 = const()[name = string("key_token_417_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 16, 128]> key_71_cast_fp16 = transpose(perm = key_71_perm_0, x = var_11663_cast_fp16)[name = string("transpose_12")];
            tensor<fp16, [1, 16, 128]> key_token_417_cast_fp16 = slice_by_index(begin = key_token_417_begin_0, end = key_token_417_end_0, end_mask = key_token_417_end_mask_0, squeeze_mask = key_token_417_squeeze_mask_0, x = key_71_cast_fp16)[name = string("key_token_417_cast_fp16")];
            tensor<int32, [4]> value_token_417_begin_0 = const()[name = string("value_token_417_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> value_token_417_end_0 = const()[name = string("value_token_417_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> value_token_417_end_mask_0 = const()[name = string("value_token_417_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_417_squeeze_mask_0 = const()[name = string("value_token_417_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 16, 128]> value_257_cast_fp16 = transpose(perm = value_257_perm_0, x = value_255_cast_fp16)[name = string("transpose_11")];
            tensor<fp16, [1, 16, 128]> value_token_417_cast_fp16 = slice_by_index(begin = value_token_417_begin_0, end = value_token_417_end_0, end_mask = value_token_417_end_mask_0, squeeze_mask = value_token_417_squeeze_mask_0, x = value_257_cast_fp16)[name = string("value_token_417_cast_fp16")];
            tensor<int32, [1]> var_11697_axes_0 = const()[name = string("op_11697_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_11697_cast_fp16 = expand_dims(axes = var_11697_axes_0, x = key_token_417_cast_fp16)[name = string("op_11697_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_11698_cast_fp16 = mul(x = state_833_cast_fp16, y = var_11697_cast_fp16)[name = string("op_11698_cast_fp16")];
            tensor<int32, [1]> memory_417_axes_0 = const()[name = string("memory_417_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_417_keep_dims_0 = const()[name = string("memory_417_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_417_cast_fp16 = reduce_sum(axes = memory_417_axes_0, keep_dims = memory_417_keep_dims_0, x = var_11698_cast_fp16)[name = string("memory_417_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_11701_cast_fp16 = sub(x = value_token_417_cast_fp16, y = memory_417_cast_fp16)[name = string("op_11701_cast_fp16")];
            tensor<int32, [3]> var_11704_begin_0 = const()[name = string("op_11704_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_11704_end_0 = const()[name = string("op_11704_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_11704_end_mask_0 = const()[name = string("op_11704_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_11704_squeeze_mask_0 = const()[name = string("op_11704_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_11704_cast_fp16 = slice_by_index(begin = var_11704_begin_0, end = var_11704_end_0, end_mask = var_11704_end_mask_0, squeeze_mask = var_11704_squeeze_mask_0, x = beta_27_cast_fp16)[name = string("op_11704_cast_fp16")];
            tensor<int32, [1]> var_11705_axes_0 = const()[name = string("op_11705_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_11705_cast_fp16 = expand_dims(axes = var_11705_axes_0, x = var_11704_cast_fp16)[name = string("op_11705_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_417_cast_fp16 = mul(x = var_11701_cast_fp16, y = var_11705_cast_fp16)[name = string("delta_417_cast_fp16")];
            tensor<int32, [1]> var_11708_axes_0 = const()[name = string("op_11708_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_11708_cast_fp16 = expand_dims(axes = var_11708_axes_0, x = delta_417_cast_fp16)[name = string("op_11708_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_11709_cast_fp16 = mul(x = var_11697_cast_fp16, y = var_11708_cast_fp16)[name = string("op_11709_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_835_cast_fp16 = add(x = state_833_cast_fp16, y = var_11709_cast_fp16)[name = string("state_835_cast_fp16")];
            tensor<int32, [4]> var_11713_begin_0 = const()[name = string("op_11713_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> var_11713_end_0 = const()[name = string("op_11713_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> var_11713_end_mask_0 = const()[name = string("op_11713_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_11713_squeeze_mask_0 = const()[name = string("op_11713_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_11713_cast_fp16 = slice_by_index(begin = var_11713_begin_0, end = var_11713_end_0, end_mask = var_11713_end_mask_0, squeeze_mask = var_11713_squeeze_mask_0, x = _inversed_query_71_cast_fp16)[name = string("op_11713_cast_fp16")];
            tensor<int32, [1]> var_11714_axes_0 = const()[name = string("op_11714_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_11714_cast_fp16 = expand_dims(axes = var_11714_axes_0, x = var_11713_cast_fp16)[name = string("op_11714_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_11715_cast_fp16 = mul(x = state_835_cast_fp16, y = var_11714_cast_fp16)[name = string("op_11715_cast_fp16")];
            tensor<int32, [1]> var_11717_axes_0 = const()[name = string("op_11717_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_11717_keep_dims_0 = const()[name = string("op_11717_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_11717_cast_fp16 = reduce_sum(axes = var_11717_axes_0, keep_dims = var_11717_keep_dims_0, x = var_11715_cast_fp16)[name = string("op_11717_cast_fp16")];
            tensor<int32, [3]> var_11720_begin_0 = const()[name = string("op_11720_begin_0"), val = tensor<int32, [3]>([0, 0, 1])];
            tensor<int32, [3]> var_11720_end_0 = const()[name = string("op_11720_end_0"), val = tensor<int32, [3]>([1, 16, 2])];
            tensor<bool, [3]> var_11720_end_mask_0 = const()[name = string("op_11720_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_11720_squeeze_mask_0 = const()[name = string("op_11720_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_11720_cast_fp16 = slice_by_index(begin = var_11720_begin_0, end = var_11720_end_0, end_mask = var_11720_end_mask_0, squeeze_mask = var_11720_squeeze_mask_0, x = decay_27_cast_fp16)[name = string("op_11720_cast_fp16")];
            tensor<fp16, [1, 16]> var_11721_cast_fp16 = exp(x = var_11720_cast_fp16)[name = string("op_11721_cast_fp16")];
            tensor<int32, [1]> var_11722_axes_0 = const()[name = string("op_11722_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_11722_cast_fp16 = expand_dims(axes = var_11722_axes_0, x = var_11721_cast_fp16)[name = string("op_11722_cast_fp16")];
            tensor<int32, [1]> var_11723_axes_0 = const()[name = string("op_11723_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_11723_cast_fp16 = expand_dims(axes = var_11723_axes_0, x = var_11722_cast_fp16)[name = string("op_11723_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_837_cast_fp16 = mul(x = state_835_cast_fp16, y = var_11723_cast_fp16)[name = string("state_837_cast_fp16")];
            tensor<int32, [4]> key_token_419_begin_0 = const()[name = string("key_token_419_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> key_token_419_end_0 = const()[name = string("key_token_419_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> key_token_419_end_mask_0 = const()[name = string("key_token_419_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_419_squeeze_mask_0 = const()[name = string("key_token_419_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_419_cast_fp16 = slice_by_index(begin = key_token_419_begin_0, end = key_token_419_end_0, end_mask = key_token_419_end_mask_0, squeeze_mask = key_token_419_squeeze_mask_0, x = key_71_cast_fp16)[name = string("key_token_419_cast_fp16")];
            tensor<int32, [4]> value_token_419_begin_0 = const()[name = string("value_token_419_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> value_token_419_end_0 = const()[name = string("value_token_419_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> value_token_419_end_mask_0 = const()[name = string("value_token_419_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_419_squeeze_mask_0 = const()[name = string("value_token_419_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_419_cast_fp16 = slice_by_index(begin = value_token_419_begin_0, end = value_token_419_end_0, end_mask = value_token_419_end_mask_0, squeeze_mask = value_token_419_squeeze_mask_0, x = value_257_cast_fp16)[name = string("value_token_419_cast_fp16")];
            tensor<int32, [1]> var_11731_axes_0 = const()[name = string("op_11731_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_11731_cast_fp16 = expand_dims(axes = var_11731_axes_0, x = key_token_419_cast_fp16)[name = string("op_11731_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_11732_cast_fp16 = mul(x = state_837_cast_fp16, y = var_11731_cast_fp16)[name = string("op_11732_cast_fp16")];
            tensor<int32, [1]> memory_419_axes_0 = const()[name = string("memory_419_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_419_keep_dims_0 = const()[name = string("memory_419_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_419_cast_fp16 = reduce_sum(axes = memory_419_axes_0, keep_dims = memory_419_keep_dims_0, x = var_11732_cast_fp16)[name = string("memory_419_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_11735_cast_fp16 = sub(x = value_token_419_cast_fp16, y = memory_419_cast_fp16)[name = string("op_11735_cast_fp16")];
            tensor<int32, [3]> var_11738_begin_0 = const()[name = string("op_11738_begin_0"), val = tensor<int32, [3]>([0, 0, 1])];
            tensor<int32, [3]> var_11738_end_0 = const()[name = string("op_11738_end_0"), val = tensor<int32, [3]>([1, 16, 2])];
            tensor<bool, [3]> var_11738_end_mask_0 = const()[name = string("op_11738_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_11738_squeeze_mask_0 = const()[name = string("op_11738_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_11738_cast_fp16 = slice_by_index(begin = var_11738_begin_0, end = var_11738_end_0, end_mask = var_11738_end_mask_0, squeeze_mask = var_11738_squeeze_mask_0, x = beta_27_cast_fp16)[name = string("op_11738_cast_fp16")];
            tensor<int32, [1]> var_11739_axes_0 = const()[name = string("op_11739_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_11739_cast_fp16 = expand_dims(axes = var_11739_axes_0, x = var_11738_cast_fp16)[name = string("op_11739_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_419_cast_fp16 = mul(x = var_11735_cast_fp16, y = var_11739_cast_fp16)[name = string("delta_419_cast_fp16")];
            tensor<int32, [1]> var_11742_axes_0 = const()[name = string("op_11742_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_11742_cast_fp16 = expand_dims(axes = var_11742_axes_0, x = delta_419_cast_fp16)[name = string("op_11742_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_11743_cast_fp16 = mul(x = var_11731_cast_fp16, y = var_11742_cast_fp16)[name = string("op_11743_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_839_cast_fp16 = add(x = state_837_cast_fp16, y = var_11743_cast_fp16)[name = string("state_839_cast_fp16")];
            tensor<int32, [4]> var_11747_begin_0 = const()[name = string("op_11747_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> var_11747_end_0 = const()[name = string("op_11747_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> var_11747_end_mask_0 = const()[name = string("op_11747_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_11747_squeeze_mask_0 = const()[name = string("op_11747_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_11747_cast_fp16 = slice_by_index(begin = var_11747_begin_0, end = var_11747_end_0, end_mask = var_11747_end_mask_0, squeeze_mask = var_11747_squeeze_mask_0, x = _inversed_query_71_cast_fp16)[name = string("op_11747_cast_fp16")];
            tensor<int32, [1]> var_11748_axes_0 = const()[name = string("op_11748_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_11748_cast_fp16 = expand_dims(axes = var_11748_axes_0, x = var_11747_cast_fp16)[name = string("op_11748_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_11749_cast_fp16 = mul(x = state_839_cast_fp16, y = var_11748_cast_fp16)[name = string("op_11749_cast_fp16")];
            tensor<int32, [1]> var_11751_axes_0 = const()[name = string("op_11751_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_11751_keep_dims_0 = const()[name = string("op_11751_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_11751_cast_fp16 = reduce_sum(axes = var_11751_axes_0, keep_dims = var_11751_keep_dims_0, x = var_11749_cast_fp16)[name = string("op_11751_cast_fp16")];
            tensor<int32, [3]> var_11754_begin_0 = const()[name = string("op_11754_begin_0"), val = tensor<int32, [3]>([0, 0, 2])];
            tensor<int32, [3]> var_11754_end_0 = const()[name = string("op_11754_end_0"), val = tensor<int32, [3]>([1, 16, 3])];
            tensor<bool, [3]> var_11754_end_mask_0 = const()[name = string("op_11754_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_11754_squeeze_mask_0 = const()[name = string("op_11754_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_11754_cast_fp16 = slice_by_index(begin = var_11754_begin_0, end = var_11754_end_0, end_mask = var_11754_end_mask_0, squeeze_mask = var_11754_squeeze_mask_0, x = decay_27_cast_fp16)[name = string("op_11754_cast_fp16")];
            tensor<fp16, [1, 16]> var_11755_cast_fp16 = exp(x = var_11754_cast_fp16)[name = string("op_11755_cast_fp16")];
            tensor<int32, [1]> var_11756_axes_0 = const()[name = string("op_11756_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_11756_cast_fp16 = expand_dims(axes = var_11756_axes_0, x = var_11755_cast_fp16)[name = string("op_11756_cast_fp16")];
            tensor<int32, [1]> var_11757_axes_0 = const()[name = string("op_11757_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_11757_cast_fp16 = expand_dims(axes = var_11757_axes_0, x = var_11756_cast_fp16)[name = string("op_11757_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_841_cast_fp16 = mul(x = state_839_cast_fp16, y = var_11757_cast_fp16)[name = string("state_841_cast_fp16")];
            tensor<int32, [4]> key_token_421_begin_0 = const()[name = string("key_token_421_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> key_token_421_end_0 = const()[name = string("key_token_421_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> key_token_421_end_mask_0 = const()[name = string("key_token_421_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_421_squeeze_mask_0 = const()[name = string("key_token_421_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_421_cast_fp16 = slice_by_index(begin = key_token_421_begin_0, end = key_token_421_end_0, end_mask = key_token_421_end_mask_0, squeeze_mask = key_token_421_squeeze_mask_0, x = key_71_cast_fp16)[name = string("key_token_421_cast_fp16")];
            tensor<int32, [4]> value_token_421_begin_0 = const()[name = string("value_token_421_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> value_token_421_end_0 = const()[name = string("value_token_421_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> value_token_421_end_mask_0 = const()[name = string("value_token_421_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_421_squeeze_mask_0 = const()[name = string("value_token_421_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_421_cast_fp16 = slice_by_index(begin = value_token_421_begin_0, end = value_token_421_end_0, end_mask = value_token_421_end_mask_0, squeeze_mask = value_token_421_squeeze_mask_0, x = value_257_cast_fp16)[name = string("value_token_421_cast_fp16")];
            tensor<int32, [1]> var_11765_axes_0 = const()[name = string("op_11765_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_11765_cast_fp16 = expand_dims(axes = var_11765_axes_0, x = key_token_421_cast_fp16)[name = string("op_11765_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_11766_cast_fp16 = mul(x = state_841_cast_fp16, y = var_11765_cast_fp16)[name = string("op_11766_cast_fp16")];
            tensor<int32, [1]> memory_421_axes_0 = const()[name = string("memory_421_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_421_keep_dims_0 = const()[name = string("memory_421_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_421_cast_fp16 = reduce_sum(axes = memory_421_axes_0, keep_dims = memory_421_keep_dims_0, x = var_11766_cast_fp16)[name = string("memory_421_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_11769_cast_fp16 = sub(x = value_token_421_cast_fp16, y = memory_421_cast_fp16)[name = string("op_11769_cast_fp16")];
            tensor<int32, [3]> var_11772_begin_0 = const()[name = string("op_11772_begin_0"), val = tensor<int32, [3]>([0, 0, 2])];
            tensor<int32, [3]> var_11772_end_0 = const()[name = string("op_11772_end_0"), val = tensor<int32, [3]>([1, 16, 3])];
            tensor<bool, [3]> var_11772_end_mask_0 = const()[name = string("op_11772_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_11772_squeeze_mask_0 = const()[name = string("op_11772_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_11772_cast_fp16 = slice_by_index(begin = var_11772_begin_0, end = var_11772_end_0, end_mask = var_11772_end_mask_0, squeeze_mask = var_11772_squeeze_mask_0, x = beta_27_cast_fp16)[name = string("op_11772_cast_fp16")];
            tensor<int32, [1]> var_11773_axes_0 = const()[name = string("op_11773_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_11773_cast_fp16 = expand_dims(axes = var_11773_axes_0, x = var_11772_cast_fp16)[name = string("op_11773_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_421_cast_fp16 = mul(x = var_11769_cast_fp16, y = var_11773_cast_fp16)[name = string("delta_421_cast_fp16")];
            tensor<int32, [1]> var_11776_axes_0 = const()[name = string("op_11776_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_11776_cast_fp16 = expand_dims(axes = var_11776_axes_0, x = delta_421_cast_fp16)[name = string("op_11776_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_11777_cast_fp16 = mul(x = var_11765_cast_fp16, y = var_11776_cast_fp16)[name = string("op_11777_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_843_cast_fp16 = add(x = state_841_cast_fp16, y = var_11777_cast_fp16)[name = string("state_843_cast_fp16")];
            tensor<int32, [4]> var_11781_begin_0 = const()[name = string("op_11781_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> var_11781_end_0 = const()[name = string("op_11781_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> var_11781_end_mask_0 = const()[name = string("op_11781_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_11781_squeeze_mask_0 = const()[name = string("op_11781_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_11781_cast_fp16 = slice_by_index(begin = var_11781_begin_0, end = var_11781_end_0, end_mask = var_11781_end_mask_0, squeeze_mask = var_11781_squeeze_mask_0, x = _inversed_query_71_cast_fp16)[name = string("op_11781_cast_fp16")];
            tensor<int32, [1]> var_11782_axes_0 = const()[name = string("op_11782_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_11782_cast_fp16 = expand_dims(axes = var_11782_axes_0, x = var_11781_cast_fp16)[name = string("op_11782_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_11783_cast_fp16 = mul(x = state_843_cast_fp16, y = var_11782_cast_fp16)[name = string("op_11783_cast_fp16")];
            tensor<int32, [1]> var_11785_axes_0 = const()[name = string("op_11785_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_11785_keep_dims_0 = const()[name = string("op_11785_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_11785_cast_fp16 = reduce_sum(axes = var_11785_axes_0, keep_dims = var_11785_keep_dims_0, x = var_11783_cast_fp16)[name = string("op_11785_cast_fp16")];
            tensor<int32, [3]> var_11788_begin_0 = const()[name = string("op_11788_begin_0"), val = tensor<int32, [3]>([0, 0, 3])];
            tensor<int32, [3]> var_11788_end_0 = const()[name = string("op_11788_end_0"), val = tensor<int32, [3]>([1, 16, 4])];
            tensor<bool, [3]> var_11788_end_mask_0 = const()[name = string("op_11788_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_11788_squeeze_mask_0 = const()[name = string("op_11788_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_11788_cast_fp16 = slice_by_index(begin = var_11788_begin_0, end = var_11788_end_0, end_mask = var_11788_end_mask_0, squeeze_mask = var_11788_squeeze_mask_0, x = decay_27_cast_fp16)[name = string("op_11788_cast_fp16")];
            tensor<fp16, [1, 16]> var_11789_cast_fp16 = exp(x = var_11788_cast_fp16)[name = string("op_11789_cast_fp16")];
            tensor<int32, [1]> var_11790_axes_0 = const()[name = string("op_11790_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_11790_cast_fp16 = expand_dims(axes = var_11790_axes_0, x = var_11789_cast_fp16)[name = string("op_11790_cast_fp16")];
            tensor<int32, [1]> var_11791_axes_0 = const()[name = string("op_11791_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_11791_cast_fp16 = expand_dims(axes = var_11791_axes_0, x = var_11790_cast_fp16)[name = string("op_11791_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_845_cast_fp16 = mul(x = state_843_cast_fp16, y = var_11791_cast_fp16)[name = string("state_845_cast_fp16")];
            tensor<int32, [4]> key_token_423_begin_0 = const()[name = string("key_token_423_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> key_token_423_end_0 = const()[name = string("key_token_423_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> key_token_423_end_mask_0 = const()[name = string("key_token_423_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_423_squeeze_mask_0 = const()[name = string("key_token_423_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_423_cast_fp16 = slice_by_index(begin = key_token_423_begin_0, end = key_token_423_end_0, end_mask = key_token_423_end_mask_0, squeeze_mask = key_token_423_squeeze_mask_0, x = key_71_cast_fp16)[name = string("key_token_423_cast_fp16")];
            tensor<int32, [4]> value_token_423_begin_0 = const()[name = string("value_token_423_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> value_token_423_end_0 = const()[name = string("value_token_423_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> value_token_423_end_mask_0 = const()[name = string("value_token_423_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_423_squeeze_mask_0 = const()[name = string("value_token_423_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_423_cast_fp16 = slice_by_index(begin = value_token_423_begin_0, end = value_token_423_end_0, end_mask = value_token_423_end_mask_0, squeeze_mask = value_token_423_squeeze_mask_0, x = value_257_cast_fp16)[name = string("value_token_423_cast_fp16")];
            tensor<int32, [1]> var_11799_axes_0 = const()[name = string("op_11799_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_11799_cast_fp16 = expand_dims(axes = var_11799_axes_0, x = key_token_423_cast_fp16)[name = string("op_11799_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_11800_cast_fp16 = mul(x = state_845_cast_fp16, y = var_11799_cast_fp16)[name = string("op_11800_cast_fp16")];
            tensor<int32, [1]> memory_423_axes_0 = const()[name = string("memory_423_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_423_keep_dims_0 = const()[name = string("memory_423_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_423_cast_fp16 = reduce_sum(axes = memory_423_axes_0, keep_dims = memory_423_keep_dims_0, x = var_11800_cast_fp16)[name = string("memory_423_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_11803_cast_fp16 = sub(x = value_token_423_cast_fp16, y = memory_423_cast_fp16)[name = string("op_11803_cast_fp16")];
            tensor<int32, [3]> var_11806_begin_0 = const()[name = string("op_11806_begin_0"), val = tensor<int32, [3]>([0, 0, 3])];
            tensor<int32, [3]> var_11806_end_0 = const()[name = string("op_11806_end_0"), val = tensor<int32, [3]>([1, 16, 4])];
            tensor<bool, [3]> var_11806_end_mask_0 = const()[name = string("op_11806_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_11806_squeeze_mask_0 = const()[name = string("op_11806_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_11806_cast_fp16 = slice_by_index(begin = var_11806_begin_0, end = var_11806_end_0, end_mask = var_11806_end_mask_0, squeeze_mask = var_11806_squeeze_mask_0, x = beta_27_cast_fp16)[name = string("op_11806_cast_fp16")];
            tensor<int32, [1]> var_11807_axes_0 = const()[name = string("op_11807_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_11807_cast_fp16 = expand_dims(axes = var_11807_axes_0, x = var_11806_cast_fp16)[name = string("op_11807_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_423_cast_fp16 = mul(x = var_11803_cast_fp16, y = var_11807_cast_fp16)[name = string("delta_423_cast_fp16")];
            tensor<int32, [1]> var_11810_axes_0 = const()[name = string("op_11810_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_11810_cast_fp16 = expand_dims(axes = var_11810_axes_0, x = delta_423_cast_fp16)[name = string("op_11810_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_11811_cast_fp16 = mul(x = var_11799_cast_fp16, y = var_11810_cast_fp16)[name = string("op_11811_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_847_cast_fp16 = add(x = state_845_cast_fp16, y = var_11811_cast_fp16)[name = string("state_847_cast_fp16")];
            tensor<int32, [4]> var_11815_begin_0 = const()[name = string("op_11815_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> var_11815_end_0 = const()[name = string("op_11815_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> var_11815_end_mask_0 = const()[name = string("op_11815_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_11815_squeeze_mask_0 = const()[name = string("op_11815_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_11815_cast_fp16 = slice_by_index(begin = var_11815_begin_0, end = var_11815_end_0, end_mask = var_11815_end_mask_0, squeeze_mask = var_11815_squeeze_mask_0, x = _inversed_query_71_cast_fp16)[name = string("op_11815_cast_fp16")];
            tensor<int32, [1]> var_11816_axes_0 = const()[name = string("op_11816_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_11816_cast_fp16 = expand_dims(axes = var_11816_axes_0, x = var_11815_cast_fp16)[name = string("op_11816_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_11817_cast_fp16 = mul(x = state_847_cast_fp16, y = var_11816_cast_fp16)[name = string("op_11817_cast_fp16")];
            tensor<int32, [1]> var_11819_axes_0 = const()[name = string("op_11819_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_11819_keep_dims_0 = const()[name = string("op_11819_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_11819_cast_fp16 = reduce_sum(axes = var_11819_axes_0, keep_dims = var_11819_keep_dims_0, x = var_11817_cast_fp16)[name = string("op_11819_cast_fp16")];
            tensor<int32, [3]> var_11822_begin_0 = const()[name = string("op_11822_begin_0"), val = tensor<int32, [3]>([0, 0, 4])];
            tensor<int32, [3]> var_11822_end_0 = const()[name = string("op_11822_end_0"), val = tensor<int32, [3]>([1, 16, 5])];
            tensor<bool, [3]> var_11822_end_mask_0 = const()[name = string("op_11822_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_11822_squeeze_mask_0 = const()[name = string("op_11822_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_11822_cast_fp16 = slice_by_index(begin = var_11822_begin_0, end = var_11822_end_0, end_mask = var_11822_end_mask_0, squeeze_mask = var_11822_squeeze_mask_0, x = decay_27_cast_fp16)[name = string("op_11822_cast_fp16")];
            tensor<fp16, [1, 16]> var_11823_cast_fp16 = exp(x = var_11822_cast_fp16)[name = string("op_11823_cast_fp16")];
            tensor<int32, [1]> var_11824_axes_0 = const()[name = string("op_11824_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_11824_cast_fp16 = expand_dims(axes = var_11824_axes_0, x = var_11823_cast_fp16)[name = string("op_11824_cast_fp16")];
            tensor<int32, [1]> var_11825_axes_0 = const()[name = string("op_11825_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_11825_cast_fp16 = expand_dims(axes = var_11825_axes_0, x = var_11824_cast_fp16)[name = string("op_11825_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_849_cast_fp16 = mul(x = state_847_cast_fp16, y = var_11825_cast_fp16)[name = string("state_849_cast_fp16")];
            tensor<int32, [4]> key_token_425_begin_0 = const()[name = string("key_token_425_begin_0"), val = tensor<int32, [4]>([0, 0, 4, 0])];
            tensor<int32, [4]> key_token_425_end_0 = const()[name = string("key_token_425_end_0"), val = tensor<int32, [4]>([1, 16, 5, 128])];
            tensor<bool, [4]> key_token_425_end_mask_0 = const()[name = string("key_token_425_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_425_squeeze_mask_0 = const()[name = string("key_token_425_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_425_cast_fp16 = slice_by_index(begin = key_token_425_begin_0, end = key_token_425_end_0, end_mask = key_token_425_end_mask_0, squeeze_mask = key_token_425_squeeze_mask_0, x = key_71_cast_fp16)[name = string("key_token_425_cast_fp16")];
            tensor<int32, [4]> value_token_425_begin_0 = const()[name = string("value_token_425_begin_0"), val = tensor<int32, [4]>([0, 0, 4, 0])];
            tensor<int32, [4]> value_token_425_end_0 = const()[name = string("value_token_425_end_0"), val = tensor<int32, [4]>([1, 16, 5, 128])];
            tensor<bool, [4]> value_token_425_end_mask_0 = const()[name = string("value_token_425_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_425_squeeze_mask_0 = const()[name = string("value_token_425_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_425_cast_fp16 = slice_by_index(begin = value_token_425_begin_0, end = value_token_425_end_0, end_mask = value_token_425_end_mask_0, squeeze_mask = value_token_425_squeeze_mask_0, x = value_257_cast_fp16)[name = string("value_token_425_cast_fp16")];
            tensor<int32, [1]> var_11833_axes_0 = const()[name = string("op_11833_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_11833_cast_fp16 = expand_dims(axes = var_11833_axes_0, x = key_token_425_cast_fp16)[name = string("op_11833_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_11834_cast_fp16 = mul(x = state_849_cast_fp16, y = var_11833_cast_fp16)[name = string("op_11834_cast_fp16")];
            tensor<int32, [1]> memory_425_axes_0 = const()[name = string("memory_425_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_425_keep_dims_0 = const()[name = string("memory_425_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_425_cast_fp16 = reduce_sum(axes = memory_425_axes_0, keep_dims = memory_425_keep_dims_0, x = var_11834_cast_fp16)[name = string("memory_425_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_11837_cast_fp16 = sub(x = value_token_425_cast_fp16, y = memory_425_cast_fp16)[name = string("op_11837_cast_fp16")];
            tensor<int32, [3]> var_11840_begin_0 = const()[name = string("op_11840_begin_0"), val = tensor<int32, [3]>([0, 0, 4])];
            tensor<int32, [3]> var_11840_end_0 = const()[name = string("op_11840_end_0"), val = tensor<int32, [3]>([1, 16, 5])];
            tensor<bool, [3]> var_11840_end_mask_0 = const()[name = string("op_11840_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_11840_squeeze_mask_0 = const()[name = string("op_11840_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_11840_cast_fp16 = slice_by_index(begin = var_11840_begin_0, end = var_11840_end_0, end_mask = var_11840_end_mask_0, squeeze_mask = var_11840_squeeze_mask_0, x = beta_27_cast_fp16)[name = string("op_11840_cast_fp16")];
            tensor<int32, [1]> var_11841_axes_0 = const()[name = string("op_11841_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_11841_cast_fp16 = expand_dims(axes = var_11841_axes_0, x = var_11840_cast_fp16)[name = string("op_11841_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_425_cast_fp16 = mul(x = var_11837_cast_fp16, y = var_11841_cast_fp16)[name = string("delta_425_cast_fp16")];
            tensor<int32, [1]> var_11844_axes_0 = const()[name = string("op_11844_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_11844_cast_fp16 = expand_dims(axes = var_11844_axes_0, x = delta_425_cast_fp16)[name = string("op_11844_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_11845_cast_fp16 = mul(x = var_11833_cast_fp16, y = var_11844_cast_fp16)[name = string("op_11845_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_851_cast_fp16 = add(x = state_849_cast_fp16, y = var_11845_cast_fp16)[name = string("state_851_cast_fp16")];
            tensor<int32, [4]> var_11849_begin_0 = const()[name = string("op_11849_begin_0"), val = tensor<int32, [4]>([0, 0, 4, 0])];
            tensor<int32, [4]> var_11849_end_0 = const()[name = string("op_11849_end_0"), val = tensor<int32, [4]>([1, 16, 5, 128])];
            tensor<bool, [4]> var_11849_end_mask_0 = const()[name = string("op_11849_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_11849_squeeze_mask_0 = const()[name = string("op_11849_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_11849_cast_fp16 = slice_by_index(begin = var_11849_begin_0, end = var_11849_end_0, end_mask = var_11849_end_mask_0, squeeze_mask = var_11849_squeeze_mask_0, x = _inversed_query_71_cast_fp16)[name = string("op_11849_cast_fp16")];
            tensor<int32, [1]> var_11850_axes_0 = const()[name = string("op_11850_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_11850_cast_fp16 = expand_dims(axes = var_11850_axes_0, x = var_11849_cast_fp16)[name = string("op_11850_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_11851_cast_fp16 = mul(x = state_851_cast_fp16, y = var_11850_cast_fp16)[name = string("op_11851_cast_fp16")];
            tensor<int32, [1]> var_11853_axes_0 = const()[name = string("op_11853_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_11853_keep_dims_0 = const()[name = string("op_11853_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_11853_cast_fp16 = reduce_sum(axes = var_11853_axes_0, keep_dims = var_11853_keep_dims_0, x = var_11851_cast_fp16)[name = string("op_11853_cast_fp16")];
            tensor<int32, [3]> var_11856_begin_0 = const()[name = string("op_11856_begin_0"), val = tensor<int32, [3]>([0, 0, 5])];
            tensor<int32, [3]> var_11856_end_0 = const()[name = string("op_11856_end_0"), val = tensor<int32, [3]>([1, 16, 6])];
            tensor<bool, [3]> var_11856_end_mask_0 = const()[name = string("op_11856_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_11856_squeeze_mask_0 = const()[name = string("op_11856_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_11856_cast_fp16 = slice_by_index(begin = var_11856_begin_0, end = var_11856_end_0, end_mask = var_11856_end_mask_0, squeeze_mask = var_11856_squeeze_mask_0, x = decay_27_cast_fp16)[name = string("op_11856_cast_fp16")];
            tensor<fp16, [1, 16]> var_11857_cast_fp16 = exp(x = var_11856_cast_fp16)[name = string("op_11857_cast_fp16")];
            tensor<int32, [1]> var_11858_axes_0 = const()[name = string("op_11858_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_11858_cast_fp16 = expand_dims(axes = var_11858_axes_0, x = var_11857_cast_fp16)[name = string("op_11858_cast_fp16")];
            tensor<int32, [1]> var_11859_axes_0 = const()[name = string("op_11859_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_11859_cast_fp16 = expand_dims(axes = var_11859_axes_0, x = var_11858_cast_fp16)[name = string("op_11859_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_853_cast_fp16 = mul(x = state_851_cast_fp16, y = var_11859_cast_fp16)[name = string("state_853_cast_fp16")];
            tensor<int32, [4]> key_token_427_begin_0 = const()[name = string("key_token_427_begin_0"), val = tensor<int32, [4]>([0, 0, 5, 0])];
            tensor<int32, [4]> key_token_427_end_0 = const()[name = string("key_token_427_end_0"), val = tensor<int32, [4]>([1, 16, 6, 128])];
            tensor<bool, [4]> key_token_427_end_mask_0 = const()[name = string("key_token_427_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_427_squeeze_mask_0 = const()[name = string("key_token_427_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_427_cast_fp16 = slice_by_index(begin = key_token_427_begin_0, end = key_token_427_end_0, end_mask = key_token_427_end_mask_0, squeeze_mask = key_token_427_squeeze_mask_0, x = key_71_cast_fp16)[name = string("key_token_427_cast_fp16")];
            tensor<int32, [4]> value_token_427_begin_0 = const()[name = string("value_token_427_begin_0"), val = tensor<int32, [4]>([0, 0, 5, 0])];
            tensor<int32, [4]> value_token_427_end_0 = const()[name = string("value_token_427_end_0"), val = tensor<int32, [4]>([1, 16, 6, 128])];
            tensor<bool, [4]> value_token_427_end_mask_0 = const()[name = string("value_token_427_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_427_squeeze_mask_0 = const()[name = string("value_token_427_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_427_cast_fp16 = slice_by_index(begin = value_token_427_begin_0, end = value_token_427_end_0, end_mask = value_token_427_end_mask_0, squeeze_mask = value_token_427_squeeze_mask_0, x = value_257_cast_fp16)[name = string("value_token_427_cast_fp16")];
            tensor<int32, [1]> var_11867_axes_0 = const()[name = string("op_11867_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_11867_cast_fp16 = expand_dims(axes = var_11867_axes_0, x = key_token_427_cast_fp16)[name = string("op_11867_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_11868_cast_fp16 = mul(x = state_853_cast_fp16, y = var_11867_cast_fp16)[name = string("op_11868_cast_fp16")];
            tensor<int32, [1]> memory_427_axes_0 = const()[name = string("memory_427_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_427_keep_dims_0 = const()[name = string("memory_427_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_427_cast_fp16 = reduce_sum(axes = memory_427_axes_0, keep_dims = memory_427_keep_dims_0, x = var_11868_cast_fp16)[name = string("memory_427_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_11871_cast_fp16 = sub(x = value_token_427_cast_fp16, y = memory_427_cast_fp16)[name = string("op_11871_cast_fp16")];
            tensor<int32, [3]> var_11874_begin_0 = const()[name = string("op_11874_begin_0"), val = tensor<int32, [3]>([0, 0, 5])];
            tensor<int32, [3]> var_11874_end_0 = const()[name = string("op_11874_end_0"), val = tensor<int32, [3]>([1, 16, 6])];
            tensor<bool, [3]> var_11874_end_mask_0 = const()[name = string("op_11874_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_11874_squeeze_mask_0 = const()[name = string("op_11874_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_11874_cast_fp16 = slice_by_index(begin = var_11874_begin_0, end = var_11874_end_0, end_mask = var_11874_end_mask_0, squeeze_mask = var_11874_squeeze_mask_0, x = beta_27_cast_fp16)[name = string("op_11874_cast_fp16")];
            tensor<int32, [1]> var_11875_axes_0 = const()[name = string("op_11875_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_11875_cast_fp16 = expand_dims(axes = var_11875_axes_0, x = var_11874_cast_fp16)[name = string("op_11875_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_427_cast_fp16 = mul(x = var_11871_cast_fp16, y = var_11875_cast_fp16)[name = string("delta_427_cast_fp16")];
            tensor<int32, [1]> var_11878_axes_0 = const()[name = string("op_11878_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_11878_cast_fp16 = expand_dims(axes = var_11878_axes_0, x = delta_427_cast_fp16)[name = string("op_11878_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_11879_cast_fp16 = mul(x = var_11867_cast_fp16, y = var_11878_cast_fp16)[name = string("op_11879_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_855_cast_fp16 = add(x = state_853_cast_fp16, y = var_11879_cast_fp16)[name = string("state_855_cast_fp16")];
            tensor<int32, [4]> var_11883_begin_0 = const()[name = string("op_11883_begin_0"), val = tensor<int32, [4]>([0, 0, 5, 0])];
            tensor<int32, [4]> var_11883_end_0 = const()[name = string("op_11883_end_0"), val = tensor<int32, [4]>([1, 16, 6, 128])];
            tensor<bool, [4]> var_11883_end_mask_0 = const()[name = string("op_11883_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_11883_squeeze_mask_0 = const()[name = string("op_11883_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_11883_cast_fp16 = slice_by_index(begin = var_11883_begin_0, end = var_11883_end_0, end_mask = var_11883_end_mask_0, squeeze_mask = var_11883_squeeze_mask_0, x = _inversed_query_71_cast_fp16)[name = string("op_11883_cast_fp16")];
            tensor<int32, [1]> var_11884_axes_0 = const()[name = string("op_11884_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_11884_cast_fp16 = expand_dims(axes = var_11884_axes_0, x = var_11883_cast_fp16)[name = string("op_11884_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_11885_cast_fp16 = mul(x = state_855_cast_fp16, y = var_11884_cast_fp16)[name = string("op_11885_cast_fp16")];
            tensor<int32, [1]> var_11887_axes_0 = const()[name = string("op_11887_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_11887_keep_dims_0 = const()[name = string("op_11887_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_11887_cast_fp16 = reduce_sum(axes = var_11887_axes_0, keep_dims = var_11887_keep_dims_0, x = var_11885_cast_fp16)[name = string("op_11887_cast_fp16")];
            tensor<int32, [3]> var_11890_begin_0 = const()[name = string("op_11890_begin_0"), val = tensor<int32, [3]>([0, 0, 6])];
            tensor<int32, [3]> var_11890_end_0 = const()[name = string("op_11890_end_0"), val = tensor<int32, [3]>([1, 16, 7])];
            tensor<bool, [3]> var_11890_end_mask_0 = const()[name = string("op_11890_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_11890_squeeze_mask_0 = const()[name = string("op_11890_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_11890_cast_fp16 = slice_by_index(begin = var_11890_begin_0, end = var_11890_end_0, end_mask = var_11890_end_mask_0, squeeze_mask = var_11890_squeeze_mask_0, x = decay_27_cast_fp16)[name = string("op_11890_cast_fp16")];
            tensor<fp16, [1, 16]> var_11891_cast_fp16 = exp(x = var_11890_cast_fp16)[name = string("op_11891_cast_fp16")];
            tensor<int32, [1]> var_11892_axes_0 = const()[name = string("op_11892_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_11892_cast_fp16 = expand_dims(axes = var_11892_axes_0, x = var_11891_cast_fp16)[name = string("op_11892_cast_fp16")];
            tensor<int32, [1]> var_11893_axes_0 = const()[name = string("op_11893_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_11893_cast_fp16 = expand_dims(axes = var_11893_axes_0, x = var_11892_cast_fp16)[name = string("op_11893_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_857_cast_fp16 = mul(x = state_855_cast_fp16, y = var_11893_cast_fp16)[name = string("state_857_cast_fp16")];
            tensor<int32, [4]> key_token_429_begin_0 = const()[name = string("key_token_429_begin_0"), val = tensor<int32, [4]>([0, 0, 6, 0])];
            tensor<int32, [4]> key_token_429_end_0 = const()[name = string("key_token_429_end_0"), val = tensor<int32, [4]>([1, 16, 7, 128])];
            tensor<bool, [4]> key_token_429_end_mask_0 = const()[name = string("key_token_429_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_429_squeeze_mask_0 = const()[name = string("key_token_429_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_429_cast_fp16 = slice_by_index(begin = key_token_429_begin_0, end = key_token_429_end_0, end_mask = key_token_429_end_mask_0, squeeze_mask = key_token_429_squeeze_mask_0, x = key_71_cast_fp16)[name = string("key_token_429_cast_fp16")];
            tensor<int32, [4]> value_token_429_begin_0 = const()[name = string("value_token_429_begin_0"), val = tensor<int32, [4]>([0, 0, 6, 0])];
            tensor<int32, [4]> value_token_429_end_0 = const()[name = string("value_token_429_end_0"), val = tensor<int32, [4]>([1, 16, 7, 128])];
            tensor<bool, [4]> value_token_429_end_mask_0 = const()[name = string("value_token_429_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_429_squeeze_mask_0 = const()[name = string("value_token_429_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_429_cast_fp16 = slice_by_index(begin = value_token_429_begin_0, end = value_token_429_end_0, end_mask = value_token_429_end_mask_0, squeeze_mask = value_token_429_squeeze_mask_0, x = value_257_cast_fp16)[name = string("value_token_429_cast_fp16")];
            tensor<int32, [1]> var_11901_axes_0 = const()[name = string("op_11901_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_11901_cast_fp16 = expand_dims(axes = var_11901_axes_0, x = key_token_429_cast_fp16)[name = string("op_11901_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_11902_cast_fp16 = mul(x = state_857_cast_fp16, y = var_11901_cast_fp16)[name = string("op_11902_cast_fp16")];
            tensor<int32, [1]> memory_429_axes_0 = const()[name = string("memory_429_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_429_keep_dims_0 = const()[name = string("memory_429_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_429_cast_fp16 = reduce_sum(axes = memory_429_axes_0, keep_dims = memory_429_keep_dims_0, x = var_11902_cast_fp16)[name = string("memory_429_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_11905_cast_fp16 = sub(x = value_token_429_cast_fp16, y = memory_429_cast_fp16)[name = string("op_11905_cast_fp16")];
            tensor<int32, [3]> var_11908_begin_0 = const()[name = string("op_11908_begin_0"), val = tensor<int32, [3]>([0, 0, 6])];
            tensor<int32, [3]> var_11908_end_0 = const()[name = string("op_11908_end_0"), val = tensor<int32, [3]>([1, 16, 7])];
            tensor<bool, [3]> var_11908_end_mask_0 = const()[name = string("op_11908_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_11908_squeeze_mask_0 = const()[name = string("op_11908_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_11908_cast_fp16 = slice_by_index(begin = var_11908_begin_0, end = var_11908_end_0, end_mask = var_11908_end_mask_0, squeeze_mask = var_11908_squeeze_mask_0, x = beta_27_cast_fp16)[name = string("op_11908_cast_fp16")];
            tensor<int32, [1]> var_11909_axes_0 = const()[name = string("op_11909_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_11909_cast_fp16 = expand_dims(axes = var_11909_axes_0, x = var_11908_cast_fp16)[name = string("op_11909_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_429_cast_fp16 = mul(x = var_11905_cast_fp16, y = var_11909_cast_fp16)[name = string("delta_429_cast_fp16")];
            tensor<int32, [1]> var_11912_axes_0 = const()[name = string("op_11912_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_11912_cast_fp16 = expand_dims(axes = var_11912_axes_0, x = delta_429_cast_fp16)[name = string("op_11912_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_11913_cast_fp16 = mul(x = var_11901_cast_fp16, y = var_11912_cast_fp16)[name = string("op_11913_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_859_cast_fp16 = add(x = state_857_cast_fp16, y = var_11913_cast_fp16)[name = string("state_859_cast_fp16")];
            tensor<int32, [4]> var_11917_begin_0 = const()[name = string("op_11917_begin_0"), val = tensor<int32, [4]>([0, 0, 6, 0])];
            tensor<int32, [4]> var_11917_end_0 = const()[name = string("op_11917_end_0"), val = tensor<int32, [4]>([1, 16, 7, 128])];
            tensor<bool, [4]> var_11917_end_mask_0 = const()[name = string("op_11917_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_11917_squeeze_mask_0 = const()[name = string("op_11917_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_11917_cast_fp16 = slice_by_index(begin = var_11917_begin_0, end = var_11917_end_0, end_mask = var_11917_end_mask_0, squeeze_mask = var_11917_squeeze_mask_0, x = _inversed_query_71_cast_fp16)[name = string("op_11917_cast_fp16")];
            tensor<int32, [1]> var_11918_axes_0 = const()[name = string("op_11918_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_11918_cast_fp16 = expand_dims(axes = var_11918_axes_0, x = var_11917_cast_fp16)[name = string("op_11918_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_11919_cast_fp16 = mul(x = state_859_cast_fp16, y = var_11918_cast_fp16)[name = string("op_11919_cast_fp16")];
            tensor<int32, [1]> var_11921_axes_0 = const()[name = string("op_11921_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_11921_keep_dims_0 = const()[name = string("op_11921_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_11921_cast_fp16 = reduce_sum(axes = var_11921_axes_0, keep_dims = var_11921_keep_dims_0, x = var_11919_cast_fp16)[name = string("op_11921_cast_fp16")];
            tensor<int32, [3]> var_11924_begin_0 = const()[name = string("op_11924_begin_0"), val = tensor<int32, [3]>([0, 0, 7])];
            tensor<int32, [3]> var_11924_end_0 = const()[name = string("op_11924_end_0"), val = tensor<int32, [3]>([1, 16, 8])];
            tensor<bool, [3]> var_11924_end_mask_0 = const()[name = string("op_11924_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_11924_squeeze_mask_0 = const()[name = string("op_11924_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_11924_cast_fp16 = slice_by_index(begin = var_11924_begin_0, end = var_11924_end_0, end_mask = var_11924_end_mask_0, squeeze_mask = var_11924_squeeze_mask_0, x = decay_27_cast_fp16)[name = string("op_11924_cast_fp16")];
            tensor<fp16, [1, 16]> var_11925_cast_fp16 = exp(x = var_11924_cast_fp16)[name = string("op_11925_cast_fp16")];
            tensor<int32, [1]> var_11926_axes_0 = const()[name = string("op_11926_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_11926_cast_fp16 = expand_dims(axes = var_11926_axes_0, x = var_11925_cast_fp16)[name = string("op_11926_cast_fp16")];
            tensor<int32, [1]> var_11927_axes_0 = const()[name = string("op_11927_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_11927_cast_fp16 = expand_dims(axes = var_11927_axes_0, x = var_11926_cast_fp16)[name = string("op_11927_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_861_cast_fp16 = mul(x = state_859_cast_fp16, y = var_11927_cast_fp16)[name = string("state_861_cast_fp16")];
            tensor<int32, [4]> key_token_431_begin_0 = const()[name = string("key_token_431_begin_0"), val = tensor<int32, [4]>([0, 0, 7, 0])];
            tensor<int32, [4]> key_token_431_end_0 = const()[name = string("key_token_431_end_0"), val = tensor<int32, [4]>([1, 16, 8, 128])];
            tensor<bool, [4]> key_token_431_end_mask_0 = const()[name = string("key_token_431_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_431_squeeze_mask_0 = const()[name = string("key_token_431_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_431_cast_fp16 = slice_by_index(begin = key_token_431_begin_0, end = key_token_431_end_0, end_mask = key_token_431_end_mask_0, squeeze_mask = key_token_431_squeeze_mask_0, x = key_71_cast_fp16)[name = string("key_token_431_cast_fp16")];
            tensor<int32, [4]> value_token_431_begin_0 = const()[name = string("value_token_431_begin_0"), val = tensor<int32, [4]>([0, 0, 7, 0])];
            tensor<int32, [4]> value_token_431_end_0 = const()[name = string("value_token_431_end_0"), val = tensor<int32, [4]>([1, 16, 8, 128])];
            tensor<bool, [4]> value_token_431_end_mask_0 = const()[name = string("value_token_431_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_431_squeeze_mask_0 = const()[name = string("value_token_431_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_431_cast_fp16 = slice_by_index(begin = value_token_431_begin_0, end = value_token_431_end_0, end_mask = value_token_431_end_mask_0, squeeze_mask = value_token_431_squeeze_mask_0, x = value_257_cast_fp16)[name = string("value_token_431_cast_fp16")];
            tensor<int32, [1]> var_11935_axes_0 = const()[name = string("op_11935_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_11935_cast_fp16 = expand_dims(axes = var_11935_axes_0, x = key_token_431_cast_fp16)[name = string("op_11935_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_11936_cast_fp16 = mul(x = state_861_cast_fp16, y = var_11935_cast_fp16)[name = string("op_11936_cast_fp16")];
            tensor<int32, [1]> memory_431_axes_0 = const()[name = string("memory_431_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_431_keep_dims_0 = const()[name = string("memory_431_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_431_cast_fp16 = reduce_sum(axes = memory_431_axes_0, keep_dims = memory_431_keep_dims_0, x = var_11936_cast_fp16)[name = string("memory_431_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_11939_cast_fp16 = sub(x = value_token_431_cast_fp16, y = memory_431_cast_fp16)[name = string("op_11939_cast_fp16")];
            tensor<int32, [3]> var_11942_begin_0 = const()[name = string("op_11942_begin_0"), val = tensor<int32, [3]>([0, 0, 7])];
            tensor<int32, [3]> var_11942_end_0 = const()[name = string("op_11942_end_0"), val = tensor<int32, [3]>([1, 16, 8])];
            tensor<bool, [3]> var_11942_end_mask_0 = const()[name = string("op_11942_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_11942_squeeze_mask_0 = const()[name = string("op_11942_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_11942_cast_fp16 = slice_by_index(begin = var_11942_begin_0, end = var_11942_end_0, end_mask = var_11942_end_mask_0, squeeze_mask = var_11942_squeeze_mask_0, x = beta_27_cast_fp16)[name = string("op_11942_cast_fp16")];
            tensor<int32, [1]> var_11943_axes_0 = const()[name = string("op_11943_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_11943_cast_fp16 = expand_dims(axes = var_11943_axes_0, x = var_11942_cast_fp16)[name = string("op_11943_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_431_cast_fp16 = mul(x = var_11939_cast_fp16, y = var_11943_cast_fp16)[name = string("delta_431_cast_fp16")];
            tensor<int32, [1]> var_11946_axes_0 = const()[name = string("op_11946_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_11946_cast_fp16 = expand_dims(axes = var_11946_axes_0, x = delta_431_cast_fp16)[name = string("op_11946_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_11947_cast_fp16 = mul(x = var_11935_cast_fp16, y = var_11946_cast_fp16)[name = string("op_11947_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_863_cast_fp16 = add(x = state_861_cast_fp16, y = var_11947_cast_fp16)[name = string("state_863_cast_fp16")];
            tensor<int32, [4]> var_11951_begin_0 = const()[name = string("op_11951_begin_0"), val = tensor<int32, [4]>([0, 0, 7, 0])];
            tensor<int32, [4]> var_11951_end_0 = const()[name = string("op_11951_end_0"), val = tensor<int32, [4]>([1, 16, 8, 128])];
            tensor<bool, [4]> var_11951_end_mask_0 = const()[name = string("op_11951_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_11951_squeeze_mask_0 = const()[name = string("op_11951_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_11951_cast_fp16 = slice_by_index(begin = var_11951_begin_0, end = var_11951_end_0, end_mask = var_11951_end_mask_0, squeeze_mask = var_11951_squeeze_mask_0, x = _inversed_query_71_cast_fp16)[name = string("op_11951_cast_fp16")];
            tensor<int32, [1]> var_11952_axes_0 = const()[name = string("op_11952_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_11952_cast_fp16 = expand_dims(axes = var_11952_axes_0, x = var_11951_cast_fp16)[name = string("op_11952_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_11953_cast_fp16 = mul(x = state_863_cast_fp16, y = var_11952_cast_fp16)[name = string("op_11953_cast_fp16")];
            tensor<int32, [1]> var_11955_axes_0 = const()[name = string("op_11955_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_11955_keep_dims_0 = const()[name = string("op_11955_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_11955_cast_fp16 = reduce_sum(axes = var_11955_axes_0, keep_dims = var_11955_keep_dims_0, x = var_11953_cast_fp16)[name = string("op_11955_cast_fp16")];
            tensor<int32, [3]> var_11958_begin_0 = const()[name = string("op_11958_begin_0"), val = tensor<int32, [3]>([0, 0, 8])];
            tensor<int32, [3]> var_11958_end_0 = const()[name = string("op_11958_end_0"), val = tensor<int32, [3]>([1, 16, 9])];
            tensor<bool, [3]> var_11958_end_mask_0 = const()[name = string("op_11958_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_11958_squeeze_mask_0 = const()[name = string("op_11958_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_11958_cast_fp16 = slice_by_index(begin = var_11958_begin_0, end = var_11958_end_0, end_mask = var_11958_end_mask_0, squeeze_mask = var_11958_squeeze_mask_0, x = decay_27_cast_fp16)[name = string("op_11958_cast_fp16")];
            tensor<fp16, [1, 16]> var_11959_cast_fp16 = exp(x = var_11958_cast_fp16)[name = string("op_11959_cast_fp16")];
            tensor<int32, [1]> var_11960_axes_0 = const()[name = string("op_11960_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_11960_cast_fp16 = expand_dims(axes = var_11960_axes_0, x = var_11959_cast_fp16)[name = string("op_11960_cast_fp16")];
            tensor<int32, [1]> var_11961_axes_0 = const()[name = string("op_11961_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_11961_cast_fp16 = expand_dims(axes = var_11961_axes_0, x = var_11960_cast_fp16)[name = string("op_11961_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_865_cast_fp16 = mul(x = state_863_cast_fp16, y = var_11961_cast_fp16)[name = string("state_865_cast_fp16")];
            tensor<int32, [4]> key_token_433_begin_0 = const()[name = string("key_token_433_begin_0"), val = tensor<int32, [4]>([0, 0, 8, 0])];
            tensor<int32, [4]> key_token_433_end_0 = const()[name = string("key_token_433_end_0"), val = tensor<int32, [4]>([1, 16, 9, 128])];
            tensor<bool, [4]> key_token_433_end_mask_0 = const()[name = string("key_token_433_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_433_squeeze_mask_0 = const()[name = string("key_token_433_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_433_cast_fp16 = slice_by_index(begin = key_token_433_begin_0, end = key_token_433_end_0, end_mask = key_token_433_end_mask_0, squeeze_mask = key_token_433_squeeze_mask_0, x = key_71_cast_fp16)[name = string("key_token_433_cast_fp16")];
            tensor<int32, [4]> value_token_433_begin_0 = const()[name = string("value_token_433_begin_0"), val = tensor<int32, [4]>([0, 0, 8, 0])];
            tensor<int32, [4]> value_token_433_end_0 = const()[name = string("value_token_433_end_0"), val = tensor<int32, [4]>([1, 16, 9, 128])];
            tensor<bool, [4]> value_token_433_end_mask_0 = const()[name = string("value_token_433_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_433_squeeze_mask_0 = const()[name = string("value_token_433_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_433_cast_fp16 = slice_by_index(begin = value_token_433_begin_0, end = value_token_433_end_0, end_mask = value_token_433_end_mask_0, squeeze_mask = value_token_433_squeeze_mask_0, x = value_257_cast_fp16)[name = string("value_token_433_cast_fp16")];
            tensor<int32, [1]> var_11969_axes_0 = const()[name = string("op_11969_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_11969_cast_fp16 = expand_dims(axes = var_11969_axes_0, x = key_token_433_cast_fp16)[name = string("op_11969_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_11970_cast_fp16 = mul(x = state_865_cast_fp16, y = var_11969_cast_fp16)[name = string("op_11970_cast_fp16")];
            tensor<int32, [1]> memory_433_axes_0 = const()[name = string("memory_433_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_433_keep_dims_0 = const()[name = string("memory_433_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_433_cast_fp16 = reduce_sum(axes = memory_433_axes_0, keep_dims = memory_433_keep_dims_0, x = var_11970_cast_fp16)[name = string("memory_433_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_11973_cast_fp16 = sub(x = value_token_433_cast_fp16, y = memory_433_cast_fp16)[name = string("op_11973_cast_fp16")];
            tensor<int32, [3]> var_11976_begin_0 = const()[name = string("op_11976_begin_0"), val = tensor<int32, [3]>([0, 0, 8])];
            tensor<int32, [3]> var_11976_end_0 = const()[name = string("op_11976_end_0"), val = tensor<int32, [3]>([1, 16, 9])];
            tensor<bool, [3]> var_11976_end_mask_0 = const()[name = string("op_11976_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_11976_squeeze_mask_0 = const()[name = string("op_11976_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_11976_cast_fp16 = slice_by_index(begin = var_11976_begin_0, end = var_11976_end_0, end_mask = var_11976_end_mask_0, squeeze_mask = var_11976_squeeze_mask_0, x = beta_27_cast_fp16)[name = string("op_11976_cast_fp16")];
            tensor<int32, [1]> var_11977_axes_0 = const()[name = string("op_11977_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_11977_cast_fp16 = expand_dims(axes = var_11977_axes_0, x = var_11976_cast_fp16)[name = string("op_11977_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_433_cast_fp16 = mul(x = var_11973_cast_fp16, y = var_11977_cast_fp16)[name = string("delta_433_cast_fp16")];
            tensor<int32, [1]> var_11980_axes_0 = const()[name = string("op_11980_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_11980_cast_fp16 = expand_dims(axes = var_11980_axes_0, x = delta_433_cast_fp16)[name = string("op_11980_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_11981_cast_fp16 = mul(x = var_11969_cast_fp16, y = var_11980_cast_fp16)[name = string("op_11981_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_867_cast_fp16 = add(x = state_865_cast_fp16, y = var_11981_cast_fp16)[name = string("state_867_cast_fp16")];
            tensor<int32, [4]> var_11985_begin_0 = const()[name = string("op_11985_begin_0"), val = tensor<int32, [4]>([0, 0, 8, 0])];
            tensor<int32, [4]> var_11985_end_0 = const()[name = string("op_11985_end_0"), val = tensor<int32, [4]>([1, 16, 9, 128])];
            tensor<bool, [4]> var_11985_end_mask_0 = const()[name = string("op_11985_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_11985_squeeze_mask_0 = const()[name = string("op_11985_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_11985_cast_fp16 = slice_by_index(begin = var_11985_begin_0, end = var_11985_end_0, end_mask = var_11985_end_mask_0, squeeze_mask = var_11985_squeeze_mask_0, x = _inversed_query_71_cast_fp16)[name = string("op_11985_cast_fp16")];
            tensor<int32, [1]> var_11986_axes_0 = const()[name = string("op_11986_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_11986_cast_fp16 = expand_dims(axes = var_11986_axes_0, x = var_11985_cast_fp16)[name = string("op_11986_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_11987_cast_fp16 = mul(x = state_867_cast_fp16, y = var_11986_cast_fp16)[name = string("op_11987_cast_fp16")];
            tensor<int32, [1]> var_11989_axes_0 = const()[name = string("op_11989_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_11989_keep_dims_0 = const()[name = string("op_11989_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_11989_cast_fp16 = reduce_sum(axes = var_11989_axes_0, keep_dims = var_11989_keep_dims_0, x = var_11987_cast_fp16)[name = string("op_11989_cast_fp16")];
            tensor<int32, [3]> var_11992_begin_0 = const()[name = string("op_11992_begin_0"), val = tensor<int32, [3]>([0, 0, 9])];
            tensor<int32, [3]> var_11992_end_0 = const()[name = string("op_11992_end_0"), val = tensor<int32, [3]>([1, 16, 10])];
            tensor<bool, [3]> var_11992_end_mask_0 = const()[name = string("op_11992_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_11992_squeeze_mask_0 = const()[name = string("op_11992_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_11992_cast_fp16 = slice_by_index(begin = var_11992_begin_0, end = var_11992_end_0, end_mask = var_11992_end_mask_0, squeeze_mask = var_11992_squeeze_mask_0, x = decay_27_cast_fp16)[name = string("op_11992_cast_fp16")];
            tensor<fp16, [1, 16]> var_11993_cast_fp16 = exp(x = var_11992_cast_fp16)[name = string("op_11993_cast_fp16")];
            tensor<int32, [1]> var_11994_axes_0 = const()[name = string("op_11994_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_11994_cast_fp16 = expand_dims(axes = var_11994_axes_0, x = var_11993_cast_fp16)[name = string("op_11994_cast_fp16")];
            tensor<int32, [1]> var_11995_axes_0 = const()[name = string("op_11995_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_11995_cast_fp16 = expand_dims(axes = var_11995_axes_0, x = var_11994_cast_fp16)[name = string("op_11995_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_869_cast_fp16 = mul(x = state_867_cast_fp16, y = var_11995_cast_fp16)[name = string("state_869_cast_fp16")];
            tensor<int32, [4]> key_token_435_begin_0 = const()[name = string("key_token_435_begin_0"), val = tensor<int32, [4]>([0, 0, 9, 0])];
            tensor<int32, [4]> key_token_435_end_0 = const()[name = string("key_token_435_end_0"), val = tensor<int32, [4]>([1, 16, 10, 128])];
            tensor<bool, [4]> key_token_435_end_mask_0 = const()[name = string("key_token_435_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_435_squeeze_mask_0 = const()[name = string("key_token_435_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_435_cast_fp16 = slice_by_index(begin = key_token_435_begin_0, end = key_token_435_end_0, end_mask = key_token_435_end_mask_0, squeeze_mask = key_token_435_squeeze_mask_0, x = key_71_cast_fp16)[name = string("key_token_435_cast_fp16")];
            tensor<int32, [4]> value_token_435_begin_0 = const()[name = string("value_token_435_begin_0"), val = tensor<int32, [4]>([0, 0, 9, 0])];
            tensor<int32, [4]> value_token_435_end_0 = const()[name = string("value_token_435_end_0"), val = tensor<int32, [4]>([1, 16, 10, 128])];
            tensor<bool, [4]> value_token_435_end_mask_0 = const()[name = string("value_token_435_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_435_squeeze_mask_0 = const()[name = string("value_token_435_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_435_cast_fp16 = slice_by_index(begin = value_token_435_begin_0, end = value_token_435_end_0, end_mask = value_token_435_end_mask_0, squeeze_mask = value_token_435_squeeze_mask_0, x = value_257_cast_fp16)[name = string("value_token_435_cast_fp16")];
            tensor<int32, [1]> var_12003_axes_0 = const()[name = string("op_12003_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_12003_cast_fp16 = expand_dims(axes = var_12003_axes_0, x = key_token_435_cast_fp16)[name = string("op_12003_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_12004_cast_fp16 = mul(x = state_869_cast_fp16, y = var_12003_cast_fp16)[name = string("op_12004_cast_fp16")];
            tensor<int32, [1]> memory_435_axes_0 = const()[name = string("memory_435_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_435_keep_dims_0 = const()[name = string("memory_435_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_435_cast_fp16 = reduce_sum(axes = memory_435_axes_0, keep_dims = memory_435_keep_dims_0, x = var_12004_cast_fp16)[name = string("memory_435_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_12007_cast_fp16 = sub(x = value_token_435_cast_fp16, y = memory_435_cast_fp16)[name = string("op_12007_cast_fp16")];
            tensor<int32, [3]> var_12010_begin_0 = const()[name = string("op_12010_begin_0"), val = tensor<int32, [3]>([0, 0, 9])];
            tensor<int32, [3]> var_12010_end_0 = const()[name = string("op_12010_end_0"), val = tensor<int32, [3]>([1, 16, 10])];
            tensor<bool, [3]> var_12010_end_mask_0 = const()[name = string("op_12010_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_12010_squeeze_mask_0 = const()[name = string("op_12010_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_12010_cast_fp16 = slice_by_index(begin = var_12010_begin_0, end = var_12010_end_0, end_mask = var_12010_end_mask_0, squeeze_mask = var_12010_squeeze_mask_0, x = beta_27_cast_fp16)[name = string("op_12010_cast_fp16")];
            tensor<int32, [1]> var_12011_axes_0 = const()[name = string("op_12011_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_12011_cast_fp16 = expand_dims(axes = var_12011_axes_0, x = var_12010_cast_fp16)[name = string("op_12011_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_435_cast_fp16 = mul(x = var_12007_cast_fp16, y = var_12011_cast_fp16)[name = string("delta_435_cast_fp16")];
            tensor<int32, [1]> var_12014_axes_0 = const()[name = string("op_12014_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_12014_cast_fp16 = expand_dims(axes = var_12014_axes_0, x = delta_435_cast_fp16)[name = string("op_12014_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_12015_cast_fp16 = mul(x = var_12003_cast_fp16, y = var_12014_cast_fp16)[name = string("op_12015_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_871_cast_fp16 = add(x = state_869_cast_fp16, y = var_12015_cast_fp16)[name = string("state_871_cast_fp16")];
            tensor<int32, [4]> var_12019_begin_0 = const()[name = string("op_12019_begin_0"), val = tensor<int32, [4]>([0, 0, 9, 0])];
            tensor<int32, [4]> var_12019_end_0 = const()[name = string("op_12019_end_0"), val = tensor<int32, [4]>([1, 16, 10, 128])];
            tensor<bool, [4]> var_12019_end_mask_0 = const()[name = string("op_12019_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_12019_squeeze_mask_0 = const()[name = string("op_12019_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_12019_cast_fp16 = slice_by_index(begin = var_12019_begin_0, end = var_12019_end_0, end_mask = var_12019_end_mask_0, squeeze_mask = var_12019_squeeze_mask_0, x = _inversed_query_71_cast_fp16)[name = string("op_12019_cast_fp16")];
            tensor<int32, [1]> var_12020_axes_0 = const()[name = string("op_12020_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_12020_cast_fp16 = expand_dims(axes = var_12020_axes_0, x = var_12019_cast_fp16)[name = string("op_12020_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_12021_cast_fp16 = mul(x = state_871_cast_fp16, y = var_12020_cast_fp16)[name = string("op_12021_cast_fp16")];
            tensor<int32, [1]> var_12023_axes_0 = const()[name = string("op_12023_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_12023_keep_dims_0 = const()[name = string("op_12023_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_12023_cast_fp16 = reduce_sum(axes = var_12023_axes_0, keep_dims = var_12023_keep_dims_0, x = var_12021_cast_fp16)[name = string("op_12023_cast_fp16")];
            tensor<int32, [3]> var_12026_begin_0 = const()[name = string("op_12026_begin_0"), val = tensor<int32, [3]>([0, 0, 10])];
            tensor<int32, [3]> var_12026_end_0 = const()[name = string("op_12026_end_0"), val = tensor<int32, [3]>([1, 16, 11])];
            tensor<bool, [3]> var_12026_end_mask_0 = const()[name = string("op_12026_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_12026_squeeze_mask_0 = const()[name = string("op_12026_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_12026_cast_fp16 = slice_by_index(begin = var_12026_begin_0, end = var_12026_end_0, end_mask = var_12026_end_mask_0, squeeze_mask = var_12026_squeeze_mask_0, x = decay_27_cast_fp16)[name = string("op_12026_cast_fp16")];
            tensor<fp16, [1, 16]> var_12027_cast_fp16 = exp(x = var_12026_cast_fp16)[name = string("op_12027_cast_fp16")];
            tensor<int32, [1]> var_12028_axes_0 = const()[name = string("op_12028_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_12028_cast_fp16 = expand_dims(axes = var_12028_axes_0, x = var_12027_cast_fp16)[name = string("op_12028_cast_fp16")];
            tensor<int32, [1]> var_12029_axes_0 = const()[name = string("op_12029_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_12029_cast_fp16 = expand_dims(axes = var_12029_axes_0, x = var_12028_cast_fp16)[name = string("op_12029_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_873_cast_fp16 = mul(x = state_871_cast_fp16, y = var_12029_cast_fp16)[name = string("state_873_cast_fp16")];
            tensor<int32, [4]> key_token_437_begin_0 = const()[name = string("key_token_437_begin_0"), val = tensor<int32, [4]>([0, 0, 10, 0])];
            tensor<int32, [4]> key_token_437_end_0 = const()[name = string("key_token_437_end_0"), val = tensor<int32, [4]>([1, 16, 11, 128])];
            tensor<bool, [4]> key_token_437_end_mask_0 = const()[name = string("key_token_437_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_437_squeeze_mask_0 = const()[name = string("key_token_437_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_437_cast_fp16 = slice_by_index(begin = key_token_437_begin_0, end = key_token_437_end_0, end_mask = key_token_437_end_mask_0, squeeze_mask = key_token_437_squeeze_mask_0, x = key_71_cast_fp16)[name = string("key_token_437_cast_fp16")];
            tensor<int32, [4]> value_token_437_begin_0 = const()[name = string("value_token_437_begin_0"), val = tensor<int32, [4]>([0, 0, 10, 0])];
            tensor<int32, [4]> value_token_437_end_0 = const()[name = string("value_token_437_end_0"), val = tensor<int32, [4]>([1, 16, 11, 128])];
            tensor<bool, [4]> value_token_437_end_mask_0 = const()[name = string("value_token_437_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_437_squeeze_mask_0 = const()[name = string("value_token_437_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_437_cast_fp16 = slice_by_index(begin = value_token_437_begin_0, end = value_token_437_end_0, end_mask = value_token_437_end_mask_0, squeeze_mask = value_token_437_squeeze_mask_0, x = value_257_cast_fp16)[name = string("value_token_437_cast_fp16")];
            tensor<int32, [1]> var_12037_axes_0 = const()[name = string("op_12037_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_12037_cast_fp16 = expand_dims(axes = var_12037_axes_0, x = key_token_437_cast_fp16)[name = string("op_12037_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_12038_cast_fp16 = mul(x = state_873_cast_fp16, y = var_12037_cast_fp16)[name = string("op_12038_cast_fp16")];
            tensor<int32, [1]> memory_437_axes_0 = const()[name = string("memory_437_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_437_keep_dims_0 = const()[name = string("memory_437_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_437_cast_fp16 = reduce_sum(axes = memory_437_axes_0, keep_dims = memory_437_keep_dims_0, x = var_12038_cast_fp16)[name = string("memory_437_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_12041_cast_fp16 = sub(x = value_token_437_cast_fp16, y = memory_437_cast_fp16)[name = string("op_12041_cast_fp16")];
            tensor<int32, [3]> var_12044_begin_0 = const()[name = string("op_12044_begin_0"), val = tensor<int32, [3]>([0, 0, 10])];
            tensor<int32, [3]> var_12044_end_0 = const()[name = string("op_12044_end_0"), val = tensor<int32, [3]>([1, 16, 11])];
            tensor<bool, [3]> var_12044_end_mask_0 = const()[name = string("op_12044_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_12044_squeeze_mask_0 = const()[name = string("op_12044_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_12044_cast_fp16 = slice_by_index(begin = var_12044_begin_0, end = var_12044_end_0, end_mask = var_12044_end_mask_0, squeeze_mask = var_12044_squeeze_mask_0, x = beta_27_cast_fp16)[name = string("op_12044_cast_fp16")];
            tensor<int32, [1]> var_12045_axes_0 = const()[name = string("op_12045_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_12045_cast_fp16 = expand_dims(axes = var_12045_axes_0, x = var_12044_cast_fp16)[name = string("op_12045_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_437_cast_fp16 = mul(x = var_12041_cast_fp16, y = var_12045_cast_fp16)[name = string("delta_437_cast_fp16")];
            tensor<int32, [1]> var_12048_axes_0 = const()[name = string("op_12048_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_12048_cast_fp16 = expand_dims(axes = var_12048_axes_0, x = delta_437_cast_fp16)[name = string("op_12048_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_12049_cast_fp16 = mul(x = var_12037_cast_fp16, y = var_12048_cast_fp16)[name = string("op_12049_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_875_cast_fp16 = add(x = state_873_cast_fp16, y = var_12049_cast_fp16)[name = string("state_875_cast_fp16")];
            tensor<int32, [4]> var_12053_begin_0 = const()[name = string("op_12053_begin_0"), val = tensor<int32, [4]>([0, 0, 10, 0])];
            tensor<int32, [4]> var_12053_end_0 = const()[name = string("op_12053_end_0"), val = tensor<int32, [4]>([1, 16, 11, 128])];
            tensor<bool, [4]> var_12053_end_mask_0 = const()[name = string("op_12053_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_12053_squeeze_mask_0 = const()[name = string("op_12053_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_12053_cast_fp16 = slice_by_index(begin = var_12053_begin_0, end = var_12053_end_0, end_mask = var_12053_end_mask_0, squeeze_mask = var_12053_squeeze_mask_0, x = _inversed_query_71_cast_fp16)[name = string("op_12053_cast_fp16")];
            tensor<int32, [1]> var_12054_axes_0 = const()[name = string("op_12054_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_12054_cast_fp16 = expand_dims(axes = var_12054_axes_0, x = var_12053_cast_fp16)[name = string("op_12054_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_12055_cast_fp16 = mul(x = state_875_cast_fp16, y = var_12054_cast_fp16)[name = string("op_12055_cast_fp16")];
            tensor<int32, [1]> var_12057_axes_0 = const()[name = string("op_12057_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_12057_keep_dims_0 = const()[name = string("op_12057_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_12057_cast_fp16 = reduce_sum(axes = var_12057_axes_0, keep_dims = var_12057_keep_dims_0, x = var_12055_cast_fp16)[name = string("op_12057_cast_fp16")];
            tensor<int32, [3]> var_12060_begin_0 = const()[name = string("op_12060_begin_0"), val = tensor<int32, [3]>([0, 0, 11])];
            tensor<int32, [3]> var_12060_end_0 = const()[name = string("op_12060_end_0"), val = tensor<int32, [3]>([1, 16, 12])];
            tensor<bool, [3]> var_12060_end_mask_0 = const()[name = string("op_12060_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_12060_squeeze_mask_0 = const()[name = string("op_12060_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_12060_cast_fp16 = slice_by_index(begin = var_12060_begin_0, end = var_12060_end_0, end_mask = var_12060_end_mask_0, squeeze_mask = var_12060_squeeze_mask_0, x = decay_27_cast_fp16)[name = string("op_12060_cast_fp16")];
            tensor<fp16, [1, 16]> var_12061_cast_fp16 = exp(x = var_12060_cast_fp16)[name = string("op_12061_cast_fp16")];
            tensor<int32, [1]> var_12062_axes_0 = const()[name = string("op_12062_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_12062_cast_fp16 = expand_dims(axes = var_12062_axes_0, x = var_12061_cast_fp16)[name = string("op_12062_cast_fp16")];
            tensor<int32, [1]> var_12063_axes_0 = const()[name = string("op_12063_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_12063_cast_fp16 = expand_dims(axes = var_12063_axes_0, x = var_12062_cast_fp16)[name = string("op_12063_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_877_cast_fp16 = mul(x = state_875_cast_fp16, y = var_12063_cast_fp16)[name = string("state_877_cast_fp16")];
            tensor<int32, [4]> key_token_439_begin_0 = const()[name = string("key_token_439_begin_0"), val = tensor<int32, [4]>([0, 0, 11, 0])];
            tensor<int32, [4]> key_token_439_end_0 = const()[name = string("key_token_439_end_0"), val = tensor<int32, [4]>([1, 16, 12, 128])];
            tensor<bool, [4]> key_token_439_end_mask_0 = const()[name = string("key_token_439_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_439_squeeze_mask_0 = const()[name = string("key_token_439_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_439_cast_fp16 = slice_by_index(begin = key_token_439_begin_0, end = key_token_439_end_0, end_mask = key_token_439_end_mask_0, squeeze_mask = key_token_439_squeeze_mask_0, x = key_71_cast_fp16)[name = string("key_token_439_cast_fp16")];
            tensor<int32, [4]> value_token_439_begin_0 = const()[name = string("value_token_439_begin_0"), val = tensor<int32, [4]>([0, 0, 11, 0])];
            tensor<int32, [4]> value_token_439_end_0 = const()[name = string("value_token_439_end_0"), val = tensor<int32, [4]>([1, 16, 12, 128])];
            tensor<bool, [4]> value_token_439_end_mask_0 = const()[name = string("value_token_439_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_439_squeeze_mask_0 = const()[name = string("value_token_439_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_439_cast_fp16 = slice_by_index(begin = value_token_439_begin_0, end = value_token_439_end_0, end_mask = value_token_439_end_mask_0, squeeze_mask = value_token_439_squeeze_mask_0, x = value_257_cast_fp16)[name = string("value_token_439_cast_fp16")];
            tensor<int32, [1]> var_12071_axes_0 = const()[name = string("op_12071_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_12071_cast_fp16 = expand_dims(axes = var_12071_axes_0, x = key_token_439_cast_fp16)[name = string("op_12071_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_12072_cast_fp16 = mul(x = state_877_cast_fp16, y = var_12071_cast_fp16)[name = string("op_12072_cast_fp16")];
            tensor<int32, [1]> memory_439_axes_0 = const()[name = string("memory_439_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_439_keep_dims_0 = const()[name = string("memory_439_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_439_cast_fp16 = reduce_sum(axes = memory_439_axes_0, keep_dims = memory_439_keep_dims_0, x = var_12072_cast_fp16)[name = string("memory_439_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_12075_cast_fp16 = sub(x = value_token_439_cast_fp16, y = memory_439_cast_fp16)[name = string("op_12075_cast_fp16")];
            tensor<int32, [3]> var_12078_begin_0 = const()[name = string("op_12078_begin_0"), val = tensor<int32, [3]>([0, 0, 11])];
            tensor<int32, [3]> var_12078_end_0 = const()[name = string("op_12078_end_0"), val = tensor<int32, [3]>([1, 16, 12])];
            tensor<bool, [3]> var_12078_end_mask_0 = const()[name = string("op_12078_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_12078_squeeze_mask_0 = const()[name = string("op_12078_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_12078_cast_fp16 = slice_by_index(begin = var_12078_begin_0, end = var_12078_end_0, end_mask = var_12078_end_mask_0, squeeze_mask = var_12078_squeeze_mask_0, x = beta_27_cast_fp16)[name = string("op_12078_cast_fp16")];
            tensor<int32, [1]> var_12079_axes_0 = const()[name = string("op_12079_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_12079_cast_fp16 = expand_dims(axes = var_12079_axes_0, x = var_12078_cast_fp16)[name = string("op_12079_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_439_cast_fp16 = mul(x = var_12075_cast_fp16, y = var_12079_cast_fp16)[name = string("delta_439_cast_fp16")];
            tensor<int32, [1]> var_12082_axes_0 = const()[name = string("op_12082_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_12082_cast_fp16 = expand_dims(axes = var_12082_axes_0, x = delta_439_cast_fp16)[name = string("op_12082_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_12083_cast_fp16 = mul(x = var_12071_cast_fp16, y = var_12082_cast_fp16)[name = string("op_12083_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_879_cast_fp16 = add(x = state_877_cast_fp16, y = var_12083_cast_fp16)[name = string("state_879_cast_fp16")];
            tensor<int32, [4]> var_12087_begin_0 = const()[name = string("op_12087_begin_0"), val = tensor<int32, [4]>([0, 0, 11, 0])];
            tensor<int32, [4]> var_12087_end_0 = const()[name = string("op_12087_end_0"), val = tensor<int32, [4]>([1, 16, 12, 128])];
            tensor<bool, [4]> var_12087_end_mask_0 = const()[name = string("op_12087_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_12087_squeeze_mask_0 = const()[name = string("op_12087_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_12087_cast_fp16 = slice_by_index(begin = var_12087_begin_0, end = var_12087_end_0, end_mask = var_12087_end_mask_0, squeeze_mask = var_12087_squeeze_mask_0, x = _inversed_query_71_cast_fp16)[name = string("op_12087_cast_fp16")];
            tensor<int32, [1]> var_12088_axes_0 = const()[name = string("op_12088_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_12088_cast_fp16 = expand_dims(axes = var_12088_axes_0, x = var_12087_cast_fp16)[name = string("op_12088_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_12089_cast_fp16 = mul(x = state_879_cast_fp16, y = var_12088_cast_fp16)[name = string("op_12089_cast_fp16")];
            tensor<int32, [1]> var_12091_axes_0 = const()[name = string("op_12091_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_12091_keep_dims_0 = const()[name = string("op_12091_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_12091_cast_fp16 = reduce_sum(axes = var_12091_axes_0, keep_dims = var_12091_keep_dims_0, x = var_12089_cast_fp16)[name = string("op_12091_cast_fp16")];
            tensor<int32, [3]> var_12094_begin_0 = const()[name = string("op_12094_begin_0"), val = tensor<int32, [3]>([0, 0, 12])];
            tensor<int32, [3]> var_12094_end_0 = const()[name = string("op_12094_end_0"), val = tensor<int32, [3]>([1, 16, 13])];
            tensor<bool, [3]> var_12094_end_mask_0 = const()[name = string("op_12094_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_12094_squeeze_mask_0 = const()[name = string("op_12094_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_12094_cast_fp16 = slice_by_index(begin = var_12094_begin_0, end = var_12094_end_0, end_mask = var_12094_end_mask_0, squeeze_mask = var_12094_squeeze_mask_0, x = decay_27_cast_fp16)[name = string("op_12094_cast_fp16")];
            tensor<fp16, [1, 16]> var_12095_cast_fp16 = exp(x = var_12094_cast_fp16)[name = string("op_12095_cast_fp16")];
            tensor<int32, [1]> var_12096_axes_0 = const()[name = string("op_12096_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_12096_cast_fp16 = expand_dims(axes = var_12096_axes_0, x = var_12095_cast_fp16)[name = string("op_12096_cast_fp16")];
            tensor<int32, [1]> var_12097_axes_0 = const()[name = string("op_12097_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_12097_cast_fp16 = expand_dims(axes = var_12097_axes_0, x = var_12096_cast_fp16)[name = string("op_12097_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_881_cast_fp16 = mul(x = state_879_cast_fp16, y = var_12097_cast_fp16)[name = string("state_881_cast_fp16")];
            tensor<int32, [4]> key_token_441_begin_0 = const()[name = string("key_token_441_begin_0"), val = tensor<int32, [4]>([0, 0, 12, 0])];
            tensor<int32, [4]> key_token_441_end_0 = const()[name = string("key_token_441_end_0"), val = tensor<int32, [4]>([1, 16, 13, 128])];
            tensor<bool, [4]> key_token_441_end_mask_0 = const()[name = string("key_token_441_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_441_squeeze_mask_0 = const()[name = string("key_token_441_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_441_cast_fp16 = slice_by_index(begin = key_token_441_begin_0, end = key_token_441_end_0, end_mask = key_token_441_end_mask_0, squeeze_mask = key_token_441_squeeze_mask_0, x = key_71_cast_fp16)[name = string("key_token_441_cast_fp16")];
            tensor<int32, [4]> value_token_441_begin_0 = const()[name = string("value_token_441_begin_0"), val = tensor<int32, [4]>([0, 0, 12, 0])];
            tensor<int32, [4]> value_token_441_end_0 = const()[name = string("value_token_441_end_0"), val = tensor<int32, [4]>([1, 16, 13, 128])];
            tensor<bool, [4]> value_token_441_end_mask_0 = const()[name = string("value_token_441_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_441_squeeze_mask_0 = const()[name = string("value_token_441_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_441_cast_fp16 = slice_by_index(begin = value_token_441_begin_0, end = value_token_441_end_0, end_mask = value_token_441_end_mask_0, squeeze_mask = value_token_441_squeeze_mask_0, x = value_257_cast_fp16)[name = string("value_token_441_cast_fp16")];
            tensor<int32, [1]> var_12105_axes_0 = const()[name = string("op_12105_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_12105_cast_fp16 = expand_dims(axes = var_12105_axes_0, x = key_token_441_cast_fp16)[name = string("op_12105_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_12106_cast_fp16 = mul(x = state_881_cast_fp16, y = var_12105_cast_fp16)[name = string("op_12106_cast_fp16")];
            tensor<int32, [1]> memory_441_axes_0 = const()[name = string("memory_441_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_441_keep_dims_0 = const()[name = string("memory_441_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_441_cast_fp16 = reduce_sum(axes = memory_441_axes_0, keep_dims = memory_441_keep_dims_0, x = var_12106_cast_fp16)[name = string("memory_441_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_12109_cast_fp16 = sub(x = value_token_441_cast_fp16, y = memory_441_cast_fp16)[name = string("op_12109_cast_fp16")];
            tensor<int32, [3]> var_12112_begin_0 = const()[name = string("op_12112_begin_0"), val = tensor<int32, [3]>([0, 0, 12])];
            tensor<int32, [3]> var_12112_end_0 = const()[name = string("op_12112_end_0"), val = tensor<int32, [3]>([1, 16, 13])];
            tensor<bool, [3]> var_12112_end_mask_0 = const()[name = string("op_12112_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_12112_squeeze_mask_0 = const()[name = string("op_12112_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_12112_cast_fp16 = slice_by_index(begin = var_12112_begin_0, end = var_12112_end_0, end_mask = var_12112_end_mask_0, squeeze_mask = var_12112_squeeze_mask_0, x = beta_27_cast_fp16)[name = string("op_12112_cast_fp16")];
            tensor<int32, [1]> var_12113_axes_0 = const()[name = string("op_12113_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_12113_cast_fp16 = expand_dims(axes = var_12113_axes_0, x = var_12112_cast_fp16)[name = string("op_12113_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_441_cast_fp16 = mul(x = var_12109_cast_fp16, y = var_12113_cast_fp16)[name = string("delta_441_cast_fp16")];
            tensor<int32, [1]> var_12116_axes_0 = const()[name = string("op_12116_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_12116_cast_fp16 = expand_dims(axes = var_12116_axes_0, x = delta_441_cast_fp16)[name = string("op_12116_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_12117_cast_fp16 = mul(x = var_12105_cast_fp16, y = var_12116_cast_fp16)[name = string("op_12117_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_883_cast_fp16 = add(x = state_881_cast_fp16, y = var_12117_cast_fp16)[name = string("state_883_cast_fp16")];
            tensor<int32, [4]> var_12121_begin_0 = const()[name = string("op_12121_begin_0"), val = tensor<int32, [4]>([0, 0, 12, 0])];
            tensor<int32, [4]> var_12121_end_0 = const()[name = string("op_12121_end_0"), val = tensor<int32, [4]>([1, 16, 13, 128])];
            tensor<bool, [4]> var_12121_end_mask_0 = const()[name = string("op_12121_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_12121_squeeze_mask_0 = const()[name = string("op_12121_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_12121_cast_fp16 = slice_by_index(begin = var_12121_begin_0, end = var_12121_end_0, end_mask = var_12121_end_mask_0, squeeze_mask = var_12121_squeeze_mask_0, x = _inversed_query_71_cast_fp16)[name = string("op_12121_cast_fp16")];
            tensor<int32, [1]> var_12122_axes_0 = const()[name = string("op_12122_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_12122_cast_fp16 = expand_dims(axes = var_12122_axes_0, x = var_12121_cast_fp16)[name = string("op_12122_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_12123_cast_fp16 = mul(x = state_883_cast_fp16, y = var_12122_cast_fp16)[name = string("op_12123_cast_fp16")];
            tensor<int32, [1]> var_12125_axes_0 = const()[name = string("op_12125_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_12125_keep_dims_0 = const()[name = string("op_12125_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_12125_cast_fp16 = reduce_sum(axes = var_12125_axes_0, keep_dims = var_12125_keep_dims_0, x = var_12123_cast_fp16)[name = string("op_12125_cast_fp16")];
            tensor<int32, [3]> var_12128_begin_0 = const()[name = string("op_12128_begin_0"), val = tensor<int32, [3]>([0, 0, 13])];
            tensor<int32, [3]> var_12128_end_0 = const()[name = string("op_12128_end_0"), val = tensor<int32, [3]>([1, 16, 14])];
            tensor<bool, [3]> var_12128_end_mask_0 = const()[name = string("op_12128_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_12128_squeeze_mask_0 = const()[name = string("op_12128_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_12128_cast_fp16 = slice_by_index(begin = var_12128_begin_0, end = var_12128_end_0, end_mask = var_12128_end_mask_0, squeeze_mask = var_12128_squeeze_mask_0, x = decay_27_cast_fp16)[name = string("op_12128_cast_fp16")];
            tensor<fp16, [1, 16]> var_12129_cast_fp16 = exp(x = var_12128_cast_fp16)[name = string("op_12129_cast_fp16")];
            tensor<int32, [1]> var_12130_axes_0 = const()[name = string("op_12130_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_12130_cast_fp16 = expand_dims(axes = var_12130_axes_0, x = var_12129_cast_fp16)[name = string("op_12130_cast_fp16")];
            tensor<int32, [1]> var_12131_axes_0 = const()[name = string("op_12131_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_12131_cast_fp16 = expand_dims(axes = var_12131_axes_0, x = var_12130_cast_fp16)[name = string("op_12131_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_885_cast_fp16 = mul(x = state_883_cast_fp16, y = var_12131_cast_fp16)[name = string("state_885_cast_fp16")];
            tensor<int32, [4]> key_token_443_begin_0 = const()[name = string("key_token_443_begin_0"), val = tensor<int32, [4]>([0, 0, 13, 0])];
            tensor<int32, [4]> key_token_443_end_0 = const()[name = string("key_token_443_end_0"), val = tensor<int32, [4]>([1, 16, 14, 128])];
            tensor<bool, [4]> key_token_443_end_mask_0 = const()[name = string("key_token_443_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_443_squeeze_mask_0 = const()[name = string("key_token_443_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_443_cast_fp16 = slice_by_index(begin = key_token_443_begin_0, end = key_token_443_end_0, end_mask = key_token_443_end_mask_0, squeeze_mask = key_token_443_squeeze_mask_0, x = key_71_cast_fp16)[name = string("key_token_443_cast_fp16")];
            tensor<int32, [4]> value_token_443_begin_0 = const()[name = string("value_token_443_begin_0"), val = tensor<int32, [4]>([0, 0, 13, 0])];
            tensor<int32, [4]> value_token_443_end_0 = const()[name = string("value_token_443_end_0"), val = tensor<int32, [4]>([1, 16, 14, 128])];
            tensor<bool, [4]> value_token_443_end_mask_0 = const()[name = string("value_token_443_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_443_squeeze_mask_0 = const()[name = string("value_token_443_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_443_cast_fp16 = slice_by_index(begin = value_token_443_begin_0, end = value_token_443_end_0, end_mask = value_token_443_end_mask_0, squeeze_mask = value_token_443_squeeze_mask_0, x = value_257_cast_fp16)[name = string("value_token_443_cast_fp16")];
            tensor<int32, [1]> var_12139_axes_0 = const()[name = string("op_12139_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_12139_cast_fp16 = expand_dims(axes = var_12139_axes_0, x = key_token_443_cast_fp16)[name = string("op_12139_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_12140_cast_fp16 = mul(x = state_885_cast_fp16, y = var_12139_cast_fp16)[name = string("op_12140_cast_fp16")];
            tensor<int32, [1]> memory_443_axes_0 = const()[name = string("memory_443_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_443_keep_dims_0 = const()[name = string("memory_443_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_443_cast_fp16 = reduce_sum(axes = memory_443_axes_0, keep_dims = memory_443_keep_dims_0, x = var_12140_cast_fp16)[name = string("memory_443_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_12143_cast_fp16 = sub(x = value_token_443_cast_fp16, y = memory_443_cast_fp16)[name = string("op_12143_cast_fp16")];
            tensor<int32, [3]> var_12146_begin_0 = const()[name = string("op_12146_begin_0"), val = tensor<int32, [3]>([0, 0, 13])];
            tensor<int32, [3]> var_12146_end_0 = const()[name = string("op_12146_end_0"), val = tensor<int32, [3]>([1, 16, 14])];
            tensor<bool, [3]> var_12146_end_mask_0 = const()[name = string("op_12146_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_12146_squeeze_mask_0 = const()[name = string("op_12146_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_12146_cast_fp16 = slice_by_index(begin = var_12146_begin_0, end = var_12146_end_0, end_mask = var_12146_end_mask_0, squeeze_mask = var_12146_squeeze_mask_0, x = beta_27_cast_fp16)[name = string("op_12146_cast_fp16")];
            tensor<int32, [1]> var_12147_axes_0 = const()[name = string("op_12147_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_12147_cast_fp16 = expand_dims(axes = var_12147_axes_0, x = var_12146_cast_fp16)[name = string("op_12147_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_443_cast_fp16 = mul(x = var_12143_cast_fp16, y = var_12147_cast_fp16)[name = string("delta_443_cast_fp16")];
            tensor<int32, [1]> var_12150_axes_0 = const()[name = string("op_12150_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_12150_cast_fp16 = expand_dims(axes = var_12150_axes_0, x = delta_443_cast_fp16)[name = string("op_12150_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_12151_cast_fp16 = mul(x = var_12139_cast_fp16, y = var_12150_cast_fp16)[name = string("op_12151_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_887_cast_fp16 = add(x = state_885_cast_fp16, y = var_12151_cast_fp16)[name = string("state_887_cast_fp16")];
            tensor<int32, [4]> var_12155_begin_0 = const()[name = string("op_12155_begin_0"), val = tensor<int32, [4]>([0, 0, 13, 0])];
            tensor<int32, [4]> var_12155_end_0 = const()[name = string("op_12155_end_0"), val = tensor<int32, [4]>([1, 16, 14, 128])];
            tensor<bool, [4]> var_12155_end_mask_0 = const()[name = string("op_12155_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_12155_squeeze_mask_0 = const()[name = string("op_12155_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_12155_cast_fp16 = slice_by_index(begin = var_12155_begin_0, end = var_12155_end_0, end_mask = var_12155_end_mask_0, squeeze_mask = var_12155_squeeze_mask_0, x = _inversed_query_71_cast_fp16)[name = string("op_12155_cast_fp16")];
            tensor<int32, [1]> var_12156_axes_0 = const()[name = string("op_12156_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_12156_cast_fp16 = expand_dims(axes = var_12156_axes_0, x = var_12155_cast_fp16)[name = string("op_12156_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_12157_cast_fp16 = mul(x = state_887_cast_fp16, y = var_12156_cast_fp16)[name = string("op_12157_cast_fp16")];
            tensor<int32, [1]> var_12159_axes_0 = const()[name = string("op_12159_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_12159_keep_dims_0 = const()[name = string("op_12159_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_12159_cast_fp16 = reduce_sum(axes = var_12159_axes_0, keep_dims = var_12159_keep_dims_0, x = var_12157_cast_fp16)[name = string("op_12159_cast_fp16")];
            tensor<int32, [3]> var_12162_begin_0 = const()[name = string("op_12162_begin_0"), val = tensor<int32, [3]>([0, 0, 14])];
            tensor<int32, [3]> var_12162_end_0 = const()[name = string("op_12162_end_0"), val = tensor<int32, [3]>([1, 16, 15])];
            tensor<bool, [3]> var_12162_end_mask_0 = const()[name = string("op_12162_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_12162_squeeze_mask_0 = const()[name = string("op_12162_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_12162_cast_fp16 = slice_by_index(begin = var_12162_begin_0, end = var_12162_end_0, end_mask = var_12162_end_mask_0, squeeze_mask = var_12162_squeeze_mask_0, x = decay_27_cast_fp16)[name = string("op_12162_cast_fp16")];
            tensor<fp16, [1, 16]> var_12163_cast_fp16 = exp(x = var_12162_cast_fp16)[name = string("op_12163_cast_fp16")];
            tensor<int32, [1]> var_12164_axes_0 = const()[name = string("op_12164_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_12164_cast_fp16 = expand_dims(axes = var_12164_axes_0, x = var_12163_cast_fp16)[name = string("op_12164_cast_fp16")];
            tensor<int32, [1]> var_12165_axes_0 = const()[name = string("op_12165_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_12165_cast_fp16 = expand_dims(axes = var_12165_axes_0, x = var_12164_cast_fp16)[name = string("op_12165_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_889_cast_fp16 = mul(x = state_887_cast_fp16, y = var_12165_cast_fp16)[name = string("state_889_cast_fp16")];
            tensor<int32, [4]> key_token_445_begin_0 = const()[name = string("key_token_445_begin_0"), val = tensor<int32, [4]>([0, 0, 14, 0])];
            tensor<int32, [4]> key_token_445_end_0 = const()[name = string("key_token_445_end_0"), val = tensor<int32, [4]>([1, 16, 15, 128])];
            tensor<bool, [4]> key_token_445_end_mask_0 = const()[name = string("key_token_445_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_445_squeeze_mask_0 = const()[name = string("key_token_445_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_445_cast_fp16 = slice_by_index(begin = key_token_445_begin_0, end = key_token_445_end_0, end_mask = key_token_445_end_mask_0, squeeze_mask = key_token_445_squeeze_mask_0, x = key_71_cast_fp16)[name = string("key_token_445_cast_fp16")];
            tensor<int32, [4]> value_token_445_begin_0 = const()[name = string("value_token_445_begin_0"), val = tensor<int32, [4]>([0, 0, 14, 0])];
            tensor<int32, [4]> value_token_445_end_0 = const()[name = string("value_token_445_end_0"), val = tensor<int32, [4]>([1, 16, 15, 128])];
            tensor<bool, [4]> value_token_445_end_mask_0 = const()[name = string("value_token_445_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_445_squeeze_mask_0 = const()[name = string("value_token_445_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_445_cast_fp16 = slice_by_index(begin = value_token_445_begin_0, end = value_token_445_end_0, end_mask = value_token_445_end_mask_0, squeeze_mask = value_token_445_squeeze_mask_0, x = value_257_cast_fp16)[name = string("value_token_445_cast_fp16")];
            tensor<int32, [1]> var_12173_axes_0 = const()[name = string("op_12173_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_12173_cast_fp16 = expand_dims(axes = var_12173_axes_0, x = key_token_445_cast_fp16)[name = string("op_12173_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_12174_cast_fp16 = mul(x = state_889_cast_fp16, y = var_12173_cast_fp16)[name = string("op_12174_cast_fp16")];
            tensor<int32, [1]> memory_445_axes_0 = const()[name = string("memory_445_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_445_keep_dims_0 = const()[name = string("memory_445_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_445_cast_fp16 = reduce_sum(axes = memory_445_axes_0, keep_dims = memory_445_keep_dims_0, x = var_12174_cast_fp16)[name = string("memory_445_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_12177_cast_fp16 = sub(x = value_token_445_cast_fp16, y = memory_445_cast_fp16)[name = string("op_12177_cast_fp16")];
            tensor<int32, [3]> var_12180_begin_0 = const()[name = string("op_12180_begin_0"), val = tensor<int32, [3]>([0, 0, 14])];
            tensor<int32, [3]> var_12180_end_0 = const()[name = string("op_12180_end_0"), val = tensor<int32, [3]>([1, 16, 15])];
            tensor<bool, [3]> var_12180_end_mask_0 = const()[name = string("op_12180_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_12180_squeeze_mask_0 = const()[name = string("op_12180_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_12180_cast_fp16 = slice_by_index(begin = var_12180_begin_0, end = var_12180_end_0, end_mask = var_12180_end_mask_0, squeeze_mask = var_12180_squeeze_mask_0, x = beta_27_cast_fp16)[name = string("op_12180_cast_fp16")];
            tensor<int32, [1]> var_12181_axes_0 = const()[name = string("op_12181_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_12181_cast_fp16 = expand_dims(axes = var_12181_axes_0, x = var_12180_cast_fp16)[name = string("op_12181_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_445_cast_fp16 = mul(x = var_12177_cast_fp16, y = var_12181_cast_fp16)[name = string("delta_445_cast_fp16")];
            tensor<int32, [1]> var_12184_axes_0 = const()[name = string("op_12184_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_12184_cast_fp16 = expand_dims(axes = var_12184_axes_0, x = delta_445_cast_fp16)[name = string("op_12184_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_12185_cast_fp16 = mul(x = var_12173_cast_fp16, y = var_12184_cast_fp16)[name = string("op_12185_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_891_cast_fp16 = add(x = state_889_cast_fp16, y = var_12185_cast_fp16)[name = string("state_891_cast_fp16")];
            tensor<int32, [4]> var_12189_begin_0 = const()[name = string("op_12189_begin_0"), val = tensor<int32, [4]>([0, 0, 14, 0])];
            tensor<int32, [4]> var_12189_end_0 = const()[name = string("op_12189_end_0"), val = tensor<int32, [4]>([1, 16, 15, 128])];
            tensor<bool, [4]> var_12189_end_mask_0 = const()[name = string("op_12189_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_12189_squeeze_mask_0 = const()[name = string("op_12189_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_12189_cast_fp16 = slice_by_index(begin = var_12189_begin_0, end = var_12189_end_0, end_mask = var_12189_end_mask_0, squeeze_mask = var_12189_squeeze_mask_0, x = _inversed_query_71_cast_fp16)[name = string("op_12189_cast_fp16")];
            tensor<int32, [1]> var_12190_axes_0 = const()[name = string("op_12190_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_12190_cast_fp16 = expand_dims(axes = var_12190_axes_0, x = var_12189_cast_fp16)[name = string("op_12190_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_12191_cast_fp16 = mul(x = state_891_cast_fp16, y = var_12190_cast_fp16)[name = string("op_12191_cast_fp16")];
            tensor<int32, [1]> var_12193_axes_0 = const()[name = string("op_12193_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_12193_keep_dims_0 = const()[name = string("op_12193_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_12193_cast_fp16 = reduce_sum(axes = var_12193_axes_0, keep_dims = var_12193_keep_dims_0, x = var_12191_cast_fp16)[name = string("op_12193_cast_fp16")];
            tensor<int32, [3]> var_12196_begin_0 = const()[name = string("op_12196_begin_0"), val = tensor<int32, [3]>([0, 0, 15])];
            tensor<int32, [3]> var_12196_end_0 = const()[name = string("op_12196_end_0"), val = tensor<int32, [3]>([1, 16, 16])];
            tensor<bool, [3]> var_12196_end_mask_0 = const()[name = string("op_12196_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_12196_squeeze_mask_0 = const()[name = string("op_12196_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_12196_cast_fp16 = slice_by_index(begin = var_12196_begin_0, end = var_12196_end_0, end_mask = var_12196_end_mask_0, squeeze_mask = var_12196_squeeze_mask_0, x = decay_27_cast_fp16)[name = string("op_12196_cast_fp16")];
            tensor<fp16, [1, 16]> var_12197_cast_fp16 = exp(x = var_12196_cast_fp16)[name = string("op_12197_cast_fp16")];
            tensor<int32, [1]> var_12198_axes_0 = const()[name = string("op_12198_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_12198_cast_fp16 = expand_dims(axes = var_12198_axes_0, x = var_12197_cast_fp16)[name = string("op_12198_cast_fp16")];
            tensor<int32, [1]> var_12199_axes_0 = const()[name = string("op_12199_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_12199_cast_fp16 = expand_dims(axes = var_12199_axes_0, x = var_12198_cast_fp16)[name = string("op_12199_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_893_cast_fp16 = mul(x = state_891_cast_fp16, y = var_12199_cast_fp16)[name = string("state_893_cast_fp16")];
            tensor<int32, [4]> key_token_447_begin_0 = const()[name = string("key_token_447_begin_0"), val = tensor<int32, [4]>([0, 0, 15, 0])];
            tensor<int32, [4]> key_token_447_end_0 = const()[name = string("key_token_447_end_0"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<bool, [4]> key_token_447_end_mask_0 = const()[name = string("key_token_447_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_447_squeeze_mask_0 = const()[name = string("key_token_447_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_447_cast_fp16 = slice_by_index(begin = key_token_447_begin_0, end = key_token_447_end_0, end_mask = key_token_447_end_mask_0, squeeze_mask = key_token_447_squeeze_mask_0, x = key_71_cast_fp16)[name = string("key_token_447_cast_fp16")];
            tensor<int32, [4]> value_token_447_begin_0 = const()[name = string("value_token_447_begin_0"), val = tensor<int32, [4]>([0, 0, 15, 0])];
            tensor<int32, [4]> value_token_447_end_0 = const()[name = string("value_token_447_end_0"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<bool, [4]> value_token_447_end_mask_0 = const()[name = string("value_token_447_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_447_squeeze_mask_0 = const()[name = string("value_token_447_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_447_cast_fp16 = slice_by_index(begin = value_token_447_begin_0, end = value_token_447_end_0, end_mask = value_token_447_end_mask_0, squeeze_mask = value_token_447_squeeze_mask_0, x = value_257_cast_fp16)[name = string("value_token_447_cast_fp16")];
            tensor<int32, [1]> var_12207_axes_0 = const()[name = string("op_12207_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_12207_cast_fp16 = expand_dims(axes = var_12207_axes_0, x = key_token_447_cast_fp16)[name = string("op_12207_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_12208_cast_fp16 = mul(x = state_893_cast_fp16, y = var_12207_cast_fp16)[name = string("op_12208_cast_fp16")];
            tensor<int32, [1]> memory_447_axes_0 = const()[name = string("memory_447_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_447_keep_dims_0 = const()[name = string("memory_447_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_447_cast_fp16 = reduce_sum(axes = memory_447_axes_0, keep_dims = memory_447_keep_dims_0, x = var_12208_cast_fp16)[name = string("memory_447_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_12211_cast_fp16 = sub(x = value_token_447_cast_fp16, y = memory_447_cast_fp16)[name = string("op_12211_cast_fp16")];
            tensor<int32, [3]> var_12214_begin_0 = const()[name = string("op_12214_begin_0"), val = tensor<int32, [3]>([0, 0, 15])];
            tensor<int32, [3]> var_12214_end_0 = const()[name = string("op_12214_end_0"), val = tensor<int32, [3]>([1, 16, 16])];
            tensor<bool, [3]> var_12214_end_mask_0 = const()[name = string("op_12214_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_12214_squeeze_mask_0 = const()[name = string("op_12214_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_12214_cast_fp16 = slice_by_index(begin = var_12214_begin_0, end = var_12214_end_0, end_mask = var_12214_end_mask_0, squeeze_mask = var_12214_squeeze_mask_0, x = beta_27_cast_fp16)[name = string("op_12214_cast_fp16")];
            tensor<int32, [1]> var_12215_axes_0 = const()[name = string("op_12215_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_12215_cast_fp16 = expand_dims(axes = var_12215_axes_0, x = var_12214_cast_fp16)[name = string("op_12215_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_447_cast_fp16 = mul(x = var_12211_cast_fp16, y = var_12215_cast_fp16)[name = string("delta_447_cast_fp16")];
            tensor<int32, [1]> var_12218_axes_0 = const()[name = string("op_12218_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_12218_cast_fp16 = expand_dims(axes = var_12218_axes_0, x = delta_447_cast_fp16)[name = string("op_12218_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_12219_cast_fp16 = mul(x = var_12207_cast_fp16, y = var_12218_cast_fp16)[name = string("op_12219_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> rec17_out = add(x = state_893_cast_fp16, y = var_12219_cast_fp16)[name = string("state_895_cast_fp16")];
            tensor<int32, [4]> var_12223_begin_0 = const()[name = string("op_12223_begin_0"), val = tensor<int32, [4]>([0, 0, 15, 0])];
            tensor<int32, [4]> var_12223_end_0 = const()[name = string("op_12223_end_0"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<bool, [4]> var_12223_end_mask_0 = const()[name = string("op_12223_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_12223_squeeze_mask_0 = const()[name = string("op_12223_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_12223_cast_fp16 = slice_by_index(begin = var_12223_begin_0, end = var_12223_end_0, end_mask = var_12223_end_mask_0, squeeze_mask = var_12223_squeeze_mask_0, x = _inversed_query_71_cast_fp16)[name = string("op_12223_cast_fp16")];
            tensor<int32, [1]> var_12224_axes_0 = const()[name = string("op_12224_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_12224_cast_fp16 = expand_dims(axes = var_12224_axes_0, x = var_12223_cast_fp16)[name = string("op_12224_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_12225_cast_fp16 = mul(x = rec17_out, y = var_12224_cast_fp16)[name = string("op_12225_cast_fp16")];
            tensor<int32, [1]> var_12227_axes_0 = const()[name = string("op_12227_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_12227_keep_dims_0 = const()[name = string("op_12227_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_12227_cast_fp16 = reduce_sum(axes = var_12227_axes_0, keep_dims = var_12227_keep_dims_0, x = var_12225_cast_fp16)[name = string("op_12227_cast_fp16")];
            int32 attention_131_axis_0 = const()[name = string("attention_131_axis_0"), val = int32(1)];
            tensor<fp16, [1, 16, 16, 128]> attention_131_cast_fp16 = stack(axis = attention_131_axis_0, values = (var_11717_cast_fp16, var_11751_cast_fp16, var_11785_cast_fp16, var_11819_cast_fp16, var_11853_cast_fp16, var_11887_cast_fp16, var_11921_cast_fp16, var_11955_cast_fp16, var_11989_cast_fp16, var_12023_cast_fp16, var_12057_cast_fp16, var_12091_cast_fp16, var_12125_cast_fp16, var_12159_cast_fp16, var_12193_cast_fp16, var_12227_cast_fp16))[name = string("attention_131_cast_fp16")];
            tensor<int32, [2]> var_12230 = const()[name = string("op_12230"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [256, 128]> attention_133_cast_fp16 = reshape(shape = var_12230, x = attention_131_cast_fp16)[name = string("attention_133_cast_fp16")];
            tensor<int32, [2]> var_12232 = const()[name = string("op_12232"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [256, 128]> input_227_cast_fp16 = reshape(shape = var_12232, x = linear_135_cast_fp16)[name = string("input_227_cast_fp16")];
            tensor<fp16, [256, 128]> var_12234_cast_fp16 = mul(x = attention_133_cast_fp16, y = attention_133_cast_fp16)[name = string("op_12234_cast_fp16")];
            tensor<int32, [1]> variance_113_axes_0 = const()[name = string("variance_113_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_113_keep_dims_0 = const()[name = string("variance_113_keep_dims_0"), val = bool(true)];
            tensor<fp16, [256, 1]> variance_113_cast_fp16 = reduce_mean(axes = variance_113_axes_0, keep_dims = variance_113_keep_dims_0, x = var_12234_cast_fp16)[name = string("variance_113_cast_fp16")];
            fp16 var_12237_to_fp16 = const()[name = string("op_12237_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [256, 1]> var_12238_cast_fp16 = add(x = variance_113_cast_fp16, y = var_12237_to_fp16)[name = string("op_12238_cast_fp16")];
            fp32 var_12239_epsilon_0 = const()[name = string("op_12239_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [256, 1]> var_12239_cast_fp16 = rsqrt(epsilon = var_12239_epsilon_0, x = var_12238_cast_fp16)[name = string("op_12239_cast_fp16")];
            tensor<fp16, [256, 128]> attention_135_cast_fp16 = mul(x = attention_133_cast_fp16, y = var_12239_cast_fp16)[name = string("attention_135_cast_fp16")];
            tensor<fp16, [128]> final_group_1_gated_norm_promoted_to_fp16 = const()[name = string("final_group_1_gated_norm_promoted_to_fp16"), val = tensor<fp16, [128]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(274695232)))];
            tensor<fp16, [256, 128]> attention_137_cast_fp16 = mul(x = attention_135_cast_fp16, y = final_group_1_gated_norm_promoted_to_fp16)[name = string("attention_137_cast_fp16")];
            tensor<fp16, [256, 128]> var_12242_cast_fp16 = silu(x = input_227_cast_fp16)[name = string("op_12242_cast_fp16")];
            tensor<fp16, [256, 128]> attention_139_cast_fp16 = mul(x = attention_137_cast_fp16, y = var_12242_cast_fp16)[name = string("attention_139_cast_fp16")];
            tensor<int32, [2]> var_12244 = const()[name = string("op_12244"), val = tensor<int32, [2]>([16, 2048])];
            tensor<fp16, [16, 2048]> input_229_cast_fp16 = reshape(shape = var_12244, x = attention_139_cast_fp16)[name = string("input_229_cast_fp16")];
            tensor<fp16, [1024, 2048]> final_group_1_out_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 2048]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(274695552))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(276268480))))[name = string("final_group_1_out_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 1024]> linear_136_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = final_group_1_out_proj_weight_promoted_to_fp16_palettized, x = input_229_cast_fp16)[name = string("linear_136_cast_fp16")];
            tensor<fp16, [16, 1024]> value_259_cast_fp16 = add(x = value_247_cast_fp16, y = linear_136_cast_fp16)[name = string("value_259_cast_fp16")];
            tensor<fp16, [16, 1024]> var_12249_cast_fp16 = mul(x = value_259_cast_fp16, y = value_259_cast_fp16)[name = string("op_12249_cast_fp16")];
            tensor<int32, [1]> variance_115_axes_0 = const()[name = string("variance_115_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_115_keep_dims_0 = const()[name = string("variance_115_keep_dims_0"), val = bool(true)];
            tensor<fp16, [16, 1]> variance_115_cast_fp16 = reduce_mean(axes = variance_115_axes_0, keep_dims = variance_115_keep_dims_0, x = var_12249_cast_fp16)[name = string("variance_115_cast_fp16")];
            fp16 var_12252_to_fp16 = const()[name = string("op_12252_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [16, 1]> var_12253_cast_fp16 = add(x = variance_115_cast_fp16, y = var_12252_to_fp16)[name = string("op_12253_cast_fp16")];
            fp32 var_12254_epsilon_0 = const()[name = string("op_12254_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [16, 1]> var_12254_cast_fp16 = rsqrt(epsilon = var_12254_epsilon_0, x = var_12253_cast_fp16)[name = string("op_12254_cast_fp16")];
            tensor<fp16, [16, 1024]> var_12255_cast_fp16 = mul(x = value_259_cast_fp16, y = var_12254_cast_fp16)[name = string("op_12255_cast_fp16")];
            tensor<fp16, [1024]> var_12257_to_fp16 = const()[name = string("op_12257_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(276276736)))];
            tensor<fp16, [16, 1024]> input_231_cast_fp16 = mul(x = var_12255_cast_fp16, y = var_12257_to_fp16)[name = string("input_231_cast_fp16")];
            tensor<fp16, [3584, 1024]> final_group_1_gate_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(276278848))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(279031424))))[name = string("final_group_1_gate_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 3584]> linear_137_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = final_group_1_gate_proj_weight_promoted_to_fp16_palettized, x = input_231_cast_fp16)[name = string("linear_137_cast_fp16")];
            tensor<fp16, [16, 3584]> var_12261_cast_fp16 = silu(x = linear_137_cast_fp16)[name = string("op_12261_cast_fp16")];
            tensor<fp16, [3584, 1024]> final_group_1_up_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(279060160))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(281812736))))[name = string("final_group_1_up_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 3584]> linear_138_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = final_group_1_up_proj_weight_promoted_to_fp16_palettized, x = input_231_cast_fp16)[name = string("linear_138_cast_fp16")];
            tensor<fp16, [16, 3584]> input_235_cast_fp16 = mul(x = var_12261_cast_fp16, y = linear_138_cast_fp16)[name = string("input_235_cast_fp16")];
            tensor<fp16, [1024, 3584]> final_group_1_down_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 3584]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(281841472))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(284594048))))[name = string("final_group_1_down_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 1024]> linear_139_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = final_group_1_down_proj_weight_promoted_to_fp16_palettized, x = input_235_cast_fp16)[name = string("linear_139_cast_fp16")];
            tensor<fp16, [16, 1024]> value_261_cast_fp16 = add(x = value_259_cast_fp16, y = linear_139_cast_fp16)[name = string("value_261_cast_fp16")];
            int32 var_12299 = const()[name = string("op_12299"), val = int32(-1)];
            tensor<fp16, [16, 1024]> var_12314_cast_fp16 = mul(x = value_261_cast_fp16, y = value_261_cast_fp16)[name = string("op_12314_cast_fp16")];
            tensor<int32, [1]> variance_117_axes_0 = const()[name = string("variance_117_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_117_keep_dims_0 = const()[name = string("variance_117_keep_dims_0"), val = bool(true)];
            tensor<fp16, [16, 1]> variance_117_cast_fp16 = reduce_mean(axes = variance_117_axes_0, keep_dims = variance_117_keep_dims_0, x = var_12314_cast_fp16)[name = string("variance_117_cast_fp16")];
            fp16 var_12317_to_fp16 = const()[name = string("op_12317_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [16, 1]> var_12318_cast_fp16 = add(x = variance_117_cast_fp16, y = var_12317_to_fp16)[name = string("op_12318_cast_fp16")];
            fp32 var_12319_epsilon_0 = const()[name = string("op_12319_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [16, 1]> var_12319_cast_fp16 = rsqrt(epsilon = var_12319_epsilon_0, x = var_12318_cast_fp16)[name = string("op_12319_cast_fp16")];
            tensor<fp16, [16, 1024]> var_12320_cast_fp16 = mul(x = value_261_cast_fp16, y = var_12319_cast_fp16)[name = string("op_12320_cast_fp16")];
            tensor<fp16, [1024]> var_12322_to_fp16 = const()[name = string("op_12322_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(284602304)))];
            tensor<fp16, [16, 1024]> input_237_cast_fp16 = mul(x = var_12320_cast_fp16, y = var_12322_to_fp16)[name = string("input_237_cast_fp16")];
            tensor<fp16, [6144, 1024]> final_group_2_in_proj_qkv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(284604416))), lut = tensor<fp16, [384, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(289323072))))[name = string("final_group_2_in_proj_qkv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 6144]> linear_140_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = final_group_2_in_proj_qkv_weight_promoted_to_fp16_palettized, x = input_237_cast_fp16)[name = string("linear_140_cast_fp16")];
            tensor<int32, [2]> var_12326_perm_0 = const()[name = string("op_12326_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> mixed_axes_0 = const()[name = string("mixed_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [6144, 16]> var_12326_cast_fp16 = transpose(perm = var_12326_perm_0, x = linear_140_cast_fp16)[name = string("transpose_10")];
            tensor<fp16, [1, 6144, 16]> mixed_cast_fp16 = expand_dims(axes = mixed_axes_0, x = var_12326_cast_fp16)[name = string("mixed_cast_fp16")];
            bool convolution_input_interleave_0 = const()[name = string("convolution_input_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 6144, 19]> convolution_input_cast_fp16 = concat(axis = var_12299, interleave = convolution_input_interleave_0, values = (conv18, mixed_cast_fp16))[name = string("convolution_input_cast_fp16")];
            string input_239_pad_type_0 = const()[name = string("input_239_pad_type_0"), val = string("valid")];
            int32 input_239_groups_0 = const()[name = string("input_239_groups_0"), val = int32(6144)];
            tensor<int32, [1]> input_239_strides_0 = const()[name = string("input_239_strides_0"), val = tensor<int32, [1]>([1])];
            tensor<int32, [2]> input_239_pad_0 = const()[name = string("input_239_pad_0"), val = tensor<int32, [2]>([0, 0])];
            tensor<int32, [1]> input_239_dilations_0 = const()[name = string("input_239_dilations_0"), val = tensor<int32, [1]>([1])];
            tensor<fp16, [6144, 1, 4]> final_group_2_conv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1, 4]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(289372288))), lut = tensor<fp16, [384, 1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(289390784))))[name = string("final_group_2_conv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 6144, 16]> input_239_cast_fp16 = conv(dilations = input_239_dilations_0, groups = input_239_groups_0, pad = input_239_pad_0, pad_type = input_239_pad_type_0, strides = input_239_strides_0, weight = final_group_2_conv_weight_promoted_to_fp16_palettized, x = convolution_input_cast_fp16)[name = string("input_239_cast_fp16")];
            tensor<fp16, [1, 6144, 16]> var_12335_cast_fp16 = silu(x = input_239_cast_fp16)[name = string("op_12335_cast_fp16")];
            tensor<int32, [3]> var_12336_perm_0 = const()[name = string("op_12336_perm_0"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [3]> var_12339_begin_0 = const()[name = string("op_12339_begin_0"), val = tensor<int32, [3]>([0, 0, 16])];
            tensor<int32, [3]> var_12339_end_0 = const()[name = string("op_12339_end_0"), val = tensor<int32, [3]>([1, 6144, 19])];
            tensor<bool, [3]> var_12339_end_mask_0 = const()[name = string("op_12339_end_mask_0"), val = tensor<bool, [3]>([true, true, true])];
            tensor<fp16, [1, 6144, 3]> conv18_out = slice_by_index(begin = var_12339_begin_0, end = var_12339_end_0, end_mask = var_12339_end_mask_0, x = convolution_input_cast_fp16)[name = string("op_12339_cast_fp16")];
            tensor<int32, [3]> var_12340 = const()[name = string("op_12340"), val = tensor<int32, [3]>([2048, 2048, 2048])];
            int32 var_12341_axis_0 = const()[name = string("op_12341_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 16, 6144]> var_12336_cast_fp16 = transpose(perm = var_12336_perm_0, x = var_12335_cast_fp16)[name = string("transpose_9")];
            tensor<fp16, [1, 16, 2048]> var_12341_cast_fp16_0, tensor<fp16, [1, 16, 2048]> var_12341_cast_fp16_1, tensor<fp16, [1, 16, 2048]> var_12341_cast_fp16_2 = split(axis = var_12341_axis_0, split_sizes = var_12340, x = var_12336_cast_fp16)[name = string("op_12341_cast_fp16")];
            tensor<int32, [4]> var_12345 = const()[name = string("op_12345"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<fp16, [1, 16, 16, 128]> value_265_cast_fp16 = reshape(shape = var_12345, x = var_12341_cast_fp16_0)[name = string("value_265_cast_fp16")];
            tensor<int32, [4]> var_12347 = const()[name = string("op_12347"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<fp16, [1, 16, 16, 128]> value_267_cast_fp16 = reshape(shape = var_12347, x = var_12341_cast_fp16_1)[name = string("value_267_cast_fp16")];
            tensor<int32, [4]> var_12349 = const()[name = string("op_12349"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<fp16, [1, 16, 16, 128]> value_269_cast_fp16 = reshape(shape = var_12349, x = var_12341_cast_fp16_2)[name = string("value_269_cast_fp16")];
            tensor<fp16, [1, 16, 16, 128]> var_12351_cast_fp16 = mul(x = value_265_cast_fp16, y = value_265_cast_fp16)[name = string("op_12351_cast_fp16")];
            tensor<int32, [1]> var_12353_axes_0 = const()[name = string("op_12353_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_12353_keep_dims_0 = const()[name = string("op_12353_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 16, 16, 1]> var_12353_cast_fp16 = reduce_sum(axes = var_12353_axes_0, keep_dims = var_12353_keep_dims_0, x = var_12351_cast_fp16)[name = string("op_12353_cast_fp16")];
            fp16 var_12354_to_fp16 = const()[name = string("op_12354_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 16, 16, 1]> var_12355_cast_fp16 = add(x = var_12353_cast_fp16, y = var_12354_to_fp16)[name = string("op_12355_cast_fp16")];
            fp32 var_12356_epsilon_0 = const()[name = string("op_12356_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 16, 16, 1]> var_12356_cast_fp16 = rsqrt(epsilon = var_12356_epsilon_0, x = var_12355_cast_fp16)[name = string("op_12356_cast_fp16")];
            tensor<fp16, [1, 16, 16, 128]> var_12357_cast_fp16 = mul(x = value_265_cast_fp16, y = var_12356_cast_fp16)[name = string("op_12357_cast_fp16")];
            tensor<int32, [4]> var_12358_perm_0 = const()[name = string("op_12358_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            fp16 _inversed_query_75_y_0_to_fp16 = const()[name = string("_inversed_query_75_y_0_to_fp16"), val = fp16(0x1.6ap-4)];
            tensor<fp16, [1, 16, 16, 128]> var_12358_cast_fp16 = transpose(perm = var_12358_perm_0, x = var_12357_cast_fp16)[name = string("transpose_8")];
            tensor<fp16, [1, 16, 16, 128]> _inversed_query_75_cast_fp16 = mul(x = var_12358_cast_fp16, y = _inversed_query_75_y_0_to_fp16)[name = string("_inversed_query_75_cast_fp16")];
            tensor<fp16, [1, 16, 16, 128]> var_12361_cast_fp16 = mul(x = value_267_cast_fp16, y = value_267_cast_fp16)[name = string("op_12361_cast_fp16")];
            tensor<int32, [1]> var_12363_axes_0 = const()[name = string("op_12363_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_12363_keep_dims_0 = const()[name = string("op_12363_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 16, 16, 1]> var_12363_cast_fp16 = reduce_sum(axes = var_12363_axes_0, keep_dims = var_12363_keep_dims_0, x = var_12361_cast_fp16)[name = string("op_12363_cast_fp16")];
            fp16 var_12364_to_fp16 = const()[name = string("op_12364_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 16, 16, 1]> var_12365_cast_fp16 = add(x = var_12363_cast_fp16, y = var_12364_to_fp16)[name = string("op_12365_cast_fp16")];
            fp32 var_12366_epsilon_0 = const()[name = string("op_12366_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 16, 16, 1]> var_12366_cast_fp16 = rsqrt(epsilon = var_12366_epsilon_0, x = var_12365_cast_fp16)[name = string("op_12366_cast_fp16")];
            tensor<fp16, [1, 16, 16, 128]> var_12367_cast_fp16 = mul(x = value_267_cast_fp16, y = var_12366_cast_fp16)[name = string("op_12367_cast_fp16")];
            tensor<int32, [4]> key_75_perm_0 = const()[name = string("key_75_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<int32, [4]> value_271_perm_0 = const()[name = string("value_271_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<fp16, [16, 1024]> final_group_2_in_proj_b_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(289440000))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(289452352))))[name = string("final_group_2_in_proj_b_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16]> linear_141_bias_0_to_fp16 = const()[name = string("linear_141_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(4862656)))];
            tensor<fp16, [16, 16]> linear_141_cast_fp16 = linear(bias = linear_141_bias_0_to_fp16, weight = final_group_2_in_proj_b_weight_promoted_to_fp16_palettized, x = input_237_cast_fp16)[name = string("linear_141_cast_fp16")];
            tensor<fp16, [16, 16]> var_12372_cast_fp16 = sigmoid(x = linear_141_cast_fp16)[name = string("op_12372_cast_fp16")];
            tensor<int32, [2]> var_12373_perm_0 = const()[name = string("op_12373_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> beta_axes_0 = const()[name = string("beta_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 16]> var_12373_cast_fp16 = transpose(perm = var_12373_perm_0, x = var_12372_cast_fp16)[name = string("transpose_7")];
            tensor<fp16, [1, 16, 16]> beta_cast_fp16 = expand_dims(axes = beta_axes_0, x = var_12373_cast_fp16)[name = string("beta_cast_fp16")];
            tensor<fp16, [16, 1024]> op_12379_weight_0_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(289452544))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(289464896))))[name = string("op_12379_weight_0_to_fp16_palettized")];
            tensor<fp16, [16]> var_12379_bias_0_to_fp16 = const()[name = string("op_12379_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(289465088)))];
            tensor<fp16, [16, 16]> var_12379_cast_fp16 = linear(bias = var_12379_bias_0_to_fp16, weight = op_12379_weight_0_to_fp16_palettized, x = input_237_cast_fp16)[name = string("op_12379_cast_fp16")];
            tensor<fp16, [16, 16]> var_12380_cast_fp16 = softplus(x = var_12379_cast_fp16)[name = string("op_12380_cast_fp16")];
            tensor<fp16, [16]> var_12376_promoted_to_fp16 = const()[name = string("op_12376_promoted_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(289465216)))];
            tensor<fp16, [16, 16]> var_12381_cast_fp16 = mul(x = var_12376_promoted_to_fp16, y = var_12380_cast_fp16)[name = string("op_12381_cast_fp16")];
            tensor<int32, [2]> var_12382_perm_0 = const()[name = string("op_12382_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> decay_axes_0 = const()[name = string("decay_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 16]> var_12382_cast_fp16 = transpose(perm = var_12382_perm_0, x = var_12381_cast_fp16)[name = string("transpose_6")];
            tensor<fp16, [1, 16, 16]> decay_cast_fp16 = expand_dims(axes = decay_axes_0, x = var_12382_cast_fp16)[name = string("decay_cast_fp16")];
            tensor<fp16, [2048, 1024]> final_group_2_in_proj_z_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [2048, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(289465344))), lut = tensor<fp16, [128, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(291038272))))[name = string("final_group_2_in_proj_z_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 2048]> linear_143_cast_fp16 = linear(bias = linear_3_bias_0_to_fp16, weight = final_group_2_in_proj_z_weight_promoted_to_fp16_palettized, x = input_237_cast_fp16)[name = string("linear_143_cast_fp16")];
            tensor<int32, [3]> var_12390_begin_0 = const()[name = string("op_12390_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_12390_end_0 = const()[name = string("op_12390_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_12390_end_mask_0 = const()[name = string("op_12390_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_12390_squeeze_mask_0 = const()[name = string("op_12390_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_12390_cast_fp16 = slice_by_index(begin = var_12390_begin_0, end = var_12390_end_0, end_mask = var_12390_end_mask_0, squeeze_mask = var_12390_squeeze_mask_0, x = decay_cast_fp16)[name = string("op_12390_cast_fp16")];
            tensor<fp16, [1, 16]> var_12391_cast_fp16 = exp(x = var_12390_cast_fp16)[name = string("op_12391_cast_fp16")];
            tensor<int32, [1]> var_12392_axes_0 = const()[name = string("op_12392_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_12392_cast_fp16 = expand_dims(axes = var_12392_axes_0, x = var_12391_cast_fp16)[name = string("op_12392_cast_fp16")];
            tensor<int32, [1]> var_12393_axes_0 = const()[name = string("op_12393_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_12393_cast_fp16 = expand_dims(axes = var_12393_axes_0, x = var_12392_cast_fp16)[name = string("op_12393_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_897_cast_fp16 = mul(x = rec18, y = var_12393_cast_fp16)[name = string("state_897_cast_fp16")];
            tensor<int32, [4]> key_token_449_begin_0 = const()[name = string("key_token_449_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> key_token_449_end_0 = const()[name = string("key_token_449_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> key_token_449_end_mask_0 = const()[name = string("key_token_449_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_449_squeeze_mask_0 = const()[name = string("key_token_449_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 16, 128]> key_75_cast_fp16 = transpose(perm = key_75_perm_0, x = var_12367_cast_fp16)[name = string("transpose_5")];
            tensor<fp16, [1, 16, 128]> key_token_449_cast_fp16 = slice_by_index(begin = key_token_449_begin_0, end = key_token_449_end_0, end_mask = key_token_449_end_mask_0, squeeze_mask = key_token_449_squeeze_mask_0, x = key_75_cast_fp16)[name = string("key_token_449_cast_fp16")];
            tensor<int32, [4]> value_token_449_begin_0 = const()[name = string("value_token_449_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> value_token_449_end_0 = const()[name = string("value_token_449_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> value_token_449_end_mask_0 = const()[name = string("value_token_449_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_449_squeeze_mask_0 = const()[name = string("value_token_449_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 16, 128]> value_271_cast_fp16 = transpose(perm = value_271_perm_0, x = value_269_cast_fp16)[name = string("transpose_4")];
            tensor<fp16, [1, 16, 128]> value_token_449_cast_fp16 = slice_by_index(begin = value_token_449_begin_0, end = value_token_449_end_0, end_mask = value_token_449_end_mask_0, squeeze_mask = value_token_449_squeeze_mask_0, x = value_271_cast_fp16)[name = string("value_token_449_cast_fp16")];
            tensor<int32, [1]> var_12401_axes_0 = const()[name = string("op_12401_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_12401_cast_fp16 = expand_dims(axes = var_12401_axes_0, x = key_token_449_cast_fp16)[name = string("op_12401_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_12402_cast_fp16 = mul(x = state_897_cast_fp16, y = var_12401_cast_fp16)[name = string("op_12402_cast_fp16")];
            tensor<int32, [1]> memory_449_axes_0 = const()[name = string("memory_449_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_449_keep_dims_0 = const()[name = string("memory_449_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_449_cast_fp16 = reduce_sum(axes = memory_449_axes_0, keep_dims = memory_449_keep_dims_0, x = var_12402_cast_fp16)[name = string("memory_449_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_12405_cast_fp16 = sub(x = value_token_449_cast_fp16, y = memory_449_cast_fp16)[name = string("op_12405_cast_fp16")];
            tensor<int32, [3]> var_12408_begin_0 = const()[name = string("op_12408_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_12408_end_0 = const()[name = string("op_12408_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_12408_end_mask_0 = const()[name = string("op_12408_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_12408_squeeze_mask_0 = const()[name = string("op_12408_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_12408_cast_fp16 = slice_by_index(begin = var_12408_begin_0, end = var_12408_end_0, end_mask = var_12408_end_mask_0, squeeze_mask = var_12408_squeeze_mask_0, x = beta_cast_fp16)[name = string("op_12408_cast_fp16")];
            tensor<int32, [1]> var_12409_axes_0 = const()[name = string("op_12409_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_12409_cast_fp16 = expand_dims(axes = var_12409_axes_0, x = var_12408_cast_fp16)[name = string("op_12409_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_449_cast_fp16 = mul(x = var_12405_cast_fp16, y = var_12409_cast_fp16)[name = string("delta_449_cast_fp16")];
            tensor<int32, [1]> var_12412_axes_0 = const()[name = string("op_12412_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_12412_cast_fp16 = expand_dims(axes = var_12412_axes_0, x = delta_449_cast_fp16)[name = string("op_12412_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_12413_cast_fp16 = mul(x = var_12401_cast_fp16, y = var_12412_cast_fp16)[name = string("op_12413_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_899_cast_fp16 = add(x = state_897_cast_fp16, y = var_12413_cast_fp16)[name = string("state_899_cast_fp16")];
            tensor<int32, [4]> var_12417_begin_0 = const()[name = string("op_12417_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> var_12417_end_0 = const()[name = string("op_12417_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> var_12417_end_mask_0 = const()[name = string("op_12417_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_12417_squeeze_mask_0 = const()[name = string("op_12417_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_12417_cast_fp16 = slice_by_index(begin = var_12417_begin_0, end = var_12417_end_0, end_mask = var_12417_end_mask_0, squeeze_mask = var_12417_squeeze_mask_0, x = _inversed_query_75_cast_fp16)[name = string("op_12417_cast_fp16")];
            tensor<int32, [1]> var_12418_axes_0 = const()[name = string("op_12418_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_12418_cast_fp16 = expand_dims(axes = var_12418_axes_0, x = var_12417_cast_fp16)[name = string("op_12418_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_12419_cast_fp16 = mul(x = state_899_cast_fp16, y = var_12418_cast_fp16)[name = string("op_12419_cast_fp16")];
            tensor<int32, [1]> var_12421_axes_0 = const()[name = string("op_12421_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_12421_keep_dims_0 = const()[name = string("op_12421_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_12421_cast_fp16 = reduce_sum(axes = var_12421_axes_0, keep_dims = var_12421_keep_dims_0, x = var_12419_cast_fp16)[name = string("op_12421_cast_fp16")];
            tensor<int32, [3]> var_12424_begin_0 = const()[name = string("op_12424_begin_0"), val = tensor<int32, [3]>([0, 0, 1])];
            tensor<int32, [3]> var_12424_end_0 = const()[name = string("op_12424_end_0"), val = tensor<int32, [3]>([1, 16, 2])];
            tensor<bool, [3]> var_12424_end_mask_0 = const()[name = string("op_12424_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_12424_squeeze_mask_0 = const()[name = string("op_12424_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_12424_cast_fp16 = slice_by_index(begin = var_12424_begin_0, end = var_12424_end_0, end_mask = var_12424_end_mask_0, squeeze_mask = var_12424_squeeze_mask_0, x = decay_cast_fp16)[name = string("op_12424_cast_fp16")];
            tensor<fp16, [1, 16]> var_12425_cast_fp16 = exp(x = var_12424_cast_fp16)[name = string("op_12425_cast_fp16")];
            tensor<int32, [1]> var_12426_axes_0 = const()[name = string("op_12426_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_12426_cast_fp16 = expand_dims(axes = var_12426_axes_0, x = var_12425_cast_fp16)[name = string("op_12426_cast_fp16")];
            tensor<int32, [1]> var_12427_axes_0 = const()[name = string("op_12427_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_12427_cast_fp16 = expand_dims(axes = var_12427_axes_0, x = var_12426_cast_fp16)[name = string("op_12427_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_901_cast_fp16 = mul(x = state_899_cast_fp16, y = var_12427_cast_fp16)[name = string("state_901_cast_fp16")];
            tensor<int32, [4]> key_token_451_begin_0 = const()[name = string("key_token_451_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> key_token_451_end_0 = const()[name = string("key_token_451_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> key_token_451_end_mask_0 = const()[name = string("key_token_451_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_451_squeeze_mask_0 = const()[name = string("key_token_451_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_451_cast_fp16 = slice_by_index(begin = key_token_451_begin_0, end = key_token_451_end_0, end_mask = key_token_451_end_mask_0, squeeze_mask = key_token_451_squeeze_mask_0, x = key_75_cast_fp16)[name = string("key_token_451_cast_fp16")];
            tensor<int32, [4]> value_token_451_begin_0 = const()[name = string("value_token_451_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> value_token_451_end_0 = const()[name = string("value_token_451_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> value_token_451_end_mask_0 = const()[name = string("value_token_451_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_451_squeeze_mask_0 = const()[name = string("value_token_451_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_451_cast_fp16 = slice_by_index(begin = value_token_451_begin_0, end = value_token_451_end_0, end_mask = value_token_451_end_mask_0, squeeze_mask = value_token_451_squeeze_mask_0, x = value_271_cast_fp16)[name = string("value_token_451_cast_fp16")];
            tensor<int32, [1]> var_12435_axes_0 = const()[name = string("op_12435_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_12435_cast_fp16 = expand_dims(axes = var_12435_axes_0, x = key_token_451_cast_fp16)[name = string("op_12435_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_12436_cast_fp16 = mul(x = state_901_cast_fp16, y = var_12435_cast_fp16)[name = string("op_12436_cast_fp16")];
            tensor<int32, [1]> memory_451_axes_0 = const()[name = string("memory_451_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_451_keep_dims_0 = const()[name = string("memory_451_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_451_cast_fp16 = reduce_sum(axes = memory_451_axes_0, keep_dims = memory_451_keep_dims_0, x = var_12436_cast_fp16)[name = string("memory_451_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_12439_cast_fp16 = sub(x = value_token_451_cast_fp16, y = memory_451_cast_fp16)[name = string("op_12439_cast_fp16")];
            tensor<int32, [3]> var_12442_begin_0 = const()[name = string("op_12442_begin_0"), val = tensor<int32, [3]>([0, 0, 1])];
            tensor<int32, [3]> var_12442_end_0 = const()[name = string("op_12442_end_0"), val = tensor<int32, [3]>([1, 16, 2])];
            tensor<bool, [3]> var_12442_end_mask_0 = const()[name = string("op_12442_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_12442_squeeze_mask_0 = const()[name = string("op_12442_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_12442_cast_fp16 = slice_by_index(begin = var_12442_begin_0, end = var_12442_end_0, end_mask = var_12442_end_mask_0, squeeze_mask = var_12442_squeeze_mask_0, x = beta_cast_fp16)[name = string("op_12442_cast_fp16")];
            tensor<int32, [1]> var_12443_axes_0 = const()[name = string("op_12443_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_12443_cast_fp16 = expand_dims(axes = var_12443_axes_0, x = var_12442_cast_fp16)[name = string("op_12443_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_451_cast_fp16 = mul(x = var_12439_cast_fp16, y = var_12443_cast_fp16)[name = string("delta_451_cast_fp16")];
            tensor<int32, [1]> var_12446_axes_0 = const()[name = string("op_12446_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_12446_cast_fp16 = expand_dims(axes = var_12446_axes_0, x = delta_451_cast_fp16)[name = string("op_12446_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_12447_cast_fp16 = mul(x = var_12435_cast_fp16, y = var_12446_cast_fp16)[name = string("op_12447_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_903_cast_fp16 = add(x = state_901_cast_fp16, y = var_12447_cast_fp16)[name = string("state_903_cast_fp16")];
            tensor<int32, [4]> var_12451_begin_0 = const()[name = string("op_12451_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> var_12451_end_0 = const()[name = string("op_12451_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> var_12451_end_mask_0 = const()[name = string("op_12451_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_12451_squeeze_mask_0 = const()[name = string("op_12451_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_12451_cast_fp16 = slice_by_index(begin = var_12451_begin_0, end = var_12451_end_0, end_mask = var_12451_end_mask_0, squeeze_mask = var_12451_squeeze_mask_0, x = _inversed_query_75_cast_fp16)[name = string("op_12451_cast_fp16")];
            tensor<int32, [1]> var_12452_axes_0 = const()[name = string("op_12452_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_12452_cast_fp16 = expand_dims(axes = var_12452_axes_0, x = var_12451_cast_fp16)[name = string("op_12452_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_12453_cast_fp16 = mul(x = state_903_cast_fp16, y = var_12452_cast_fp16)[name = string("op_12453_cast_fp16")];
            tensor<int32, [1]> var_12455_axes_0 = const()[name = string("op_12455_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_12455_keep_dims_0 = const()[name = string("op_12455_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_12455_cast_fp16 = reduce_sum(axes = var_12455_axes_0, keep_dims = var_12455_keep_dims_0, x = var_12453_cast_fp16)[name = string("op_12455_cast_fp16")];
            tensor<int32, [3]> var_12458_begin_0 = const()[name = string("op_12458_begin_0"), val = tensor<int32, [3]>([0, 0, 2])];
            tensor<int32, [3]> var_12458_end_0 = const()[name = string("op_12458_end_0"), val = tensor<int32, [3]>([1, 16, 3])];
            tensor<bool, [3]> var_12458_end_mask_0 = const()[name = string("op_12458_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_12458_squeeze_mask_0 = const()[name = string("op_12458_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_12458_cast_fp16 = slice_by_index(begin = var_12458_begin_0, end = var_12458_end_0, end_mask = var_12458_end_mask_0, squeeze_mask = var_12458_squeeze_mask_0, x = decay_cast_fp16)[name = string("op_12458_cast_fp16")];
            tensor<fp16, [1, 16]> var_12459_cast_fp16 = exp(x = var_12458_cast_fp16)[name = string("op_12459_cast_fp16")];
            tensor<int32, [1]> var_12460_axes_0 = const()[name = string("op_12460_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_12460_cast_fp16 = expand_dims(axes = var_12460_axes_0, x = var_12459_cast_fp16)[name = string("op_12460_cast_fp16")];
            tensor<int32, [1]> var_12461_axes_0 = const()[name = string("op_12461_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_12461_cast_fp16 = expand_dims(axes = var_12461_axes_0, x = var_12460_cast_fp16)[name = string("op_12461_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_905_cast_fp16 = mul(x = state_903_cast_fp16, y = var_12461_cast_fp16)[name = string("state_905_cast_fp16")];
            tensor<int32, [4]> key_token_453_begin_0 = const()[name = string("key_token_453_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> key_token_453_end_0 = const()[name = string("key_token_453_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> key_token_453_end_mask_0 = const()[name = string("key_token_453_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_453_squeeze_mask_0 = const()[name = string("key_token_453_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_453_cast_fp16 = slice_by_index(begin = key_token_453_begin_0, end = key_token_453_end_0, end_mask = key_token_453_end_mask_0, squeeze_mask = key_token_453_squeeze_mask_0, x = key_75_cast_fp16)[name = string("key_token_453_cast_fp16")];
            tensor<int32, [4]> value_token_453_begin_0 = const()[name = string("value_token_453_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> value_token_453_end_0 = const()[name = string("value_token_453_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> value_token_453_end_mask_0 = const()[name = string("value_token_453_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_453_squeeze_mask_0 = const()[name = string("value_token_453_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_453_cast_fp16 = slice_by_index(begin = value_token_453_begin_0, end = value_token_453_end_0, end_mask = value_token_453_end_mask_0, squeeze_mask = value_token_453_squeeze_mask_0, x = value_271_cast_fp16)[name = string("value_token_453_cast_fp16")];
            tensor<int32, [1]> var_12469_axes_0 = const()[name = string("op_12469_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_12469_cast_fp16 = expand_dims(axes = var_12469_axes_0, x = key_token_453_cast_fp16)[name = string("op_12469_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_12470_cast_fp16 = mul(x = state_905_cast_fp16, y = var_12469_cast_fp16)[name = string("op_12470_cast_fp16")];
            tensor<int32, [1]> memory_453_axes_0 = const()[name = string("memory_453_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_453_keep_dims_0 = const()[name = string("memory_453_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_453_cast_fp16 = reduce_sum(axes = memory_453_axes_0, keep_dims = memory_453_keep_dims_0, x = var_12470_cast_fp16)[name = string("memory_453_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_12473_cast_fp16 = sub(x = value_token_453_cast_fp16, y = memory_453_cast_fp16)[name = string("op_12473_cast_fp16")];
            tensor<int32, [3]> var_12476_begin_0 = const()[name = string("op_12476_begin_0"), val = tensor<int32, [3]>([0, 0, 2])];
            tensor<int32, [3]> var_12476_end_0 = const()[name = string("op_12476_end_0"), val = tensor<int32, [3]>([1, 16, 3])];
            tensor<bool, [3]> var_12476_end_mask_0 = const()[name = string("op_12476_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_12476_squeeze_mask_0 = const()[name = string("op_12476_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_12476_cast_fp16 = slice_by_index(begin = var_12476_begin_0, end = var_12476_end_0, end_mask = var_12476_end_mask_0, squeeze_mask = var_12476_squeeze_mask_0, x = beta_cast_fp16)[name = string("op_12476_cast_fp16")];
            tensor<int32, [1]> var_12477_axes_0 = const()[name = string("op_12477_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_12477_cast_fp16 = expand_dims(axes = var_12477_axes_0, x = var_12476_cast_fp16)[name = string("op_12477_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_453_cast_fp16 = mul(x = var_12473_cast_fp16, y = var_12477_cast_fp16)[name = string("delta_453_cast_fp16")];
            tensor<int32, [1]> var_12480_axes_0 = const()[name = string("op_12480_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_12480_cast_fp16 = expand_dims(axes = var_12480_axes_0, x = delta_453_cast_fp16)[name = string("op_12480_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_12481_cast_fp16 = mul(x = var_12469_cast_fp16, y = var_12480_cast_fp16)[name = string("op_12481_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_907_cast_fp16 = add(x = state_905_cast_fp16, y = var_12481_cast_fp16)[name = string("state_907_cast_fp16")];
            tensor<int32, [4]> var_12485_begin_0 = const()[name = string("op_12485_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> var_12485_end_0 = const()[name = string("op_12485_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> var_12485_end_mask_0 = const()[name = string("op_12485_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_12485_squeeze_mask_0 = const()[name = string("op_12485_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_12485_cast_fp16 = slice_by_index(begin = var_12485_begin_0, end = var_12485_end_0, end_mask = var_12485_end_mask_0, squeeze_mask = var_12485_squeeze_mask_0, x = _inversed_query_75_cast_fp16)[name = string("op_12485_cast_fp16")];
            tensor<int32, [1]> var_12486_axes_0 = const()[name = string("op_12486_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_12486_cast_fp16 = expand_dims(axes = var_12486_axes_0, x = var_12485_cast_fp16)[name = string("op_12486_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_12487_cast_fp16 = mul(x = state_907_cast_fp16, y = var_12486_cast_fp16)[name = string("op_12487_cast_fp16")];
            tensor<int32, [1]> var_12489_axes_0 = const()[name = string("op_12489_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_12489_keep_dims_0 = const()[name = string("op_12489_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_12489_cast_fp16 = reduce_sum(axes = var_12489_axes_0, keep_dims = var_12489_keep_dims_0, x = var_12487_cast_fp16)[name = string("op_12489_cast_fp16")];
            tensor<int32, [3]> var_12492_begin_0 = const()[name = string("op_12492_begin_0"), val = tensor<int32, [3]>([0, 0, 3])];
            tensor<int32, [3]> var_12492_end_0 = const()[name = string("op_12492_end_0"), val = tensor<int32, [3]>([1, 16, 4])];
            tensor<bool, [3]> var_12492_end_mask_0 = const()[name = string("op_12492_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_12492_squeeze_mask_0 = const()[name = string("op_12492_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_12492_cast_fp16 = slice_by_index(begin = var_12492_begin_0, end = var_12492_end_0, end_mask = var_12492_end_mask_0, squeeze_mask = var_12492_squeeze_mask_0, x = decay_cast_fp16)[name = string("op_12492_cast_fp16")];
            tensor<fp16, [1, 16]> var_12493_cast_fp16 = exp(x = var_12492_cast_fp16)[name = string("op_12493_cast_fp16")];
            tensor<int32, [1]> var_12494_axes_0 = const()[name = string("op_12494_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_12494_cast_fp16 = expand_dims(axes = var_12494_axes_0, x = var_12493_cast_fp16)[name = string("op_12494_cast_fp16")];
            tensor<int32, [1]> var_12495_axes_0 = const()[name = string("op_12495_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_12495_cast_fp16 = expand_dims(axes = var_12495_axes_0, x = var_12494_cast_fp16)[name = string("op_12495_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_909_cast_fp16 = mul(x = state_907_cast_fp16, y = var_12495_cast_fp16)[name = string("state_909_cast_fp16")];
            tensor<int32, [4]> key_token_455_begin_0 = const()[name = string("key_token_455_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> key_token_455_end_0 = const()[name = string("key_token_455_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> key_token_455_end_mask_0 = const()[name = string("key_token_455_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_455_squeeze_mask_0 = const()[name = string("key_token_455_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_455_cast_fp16 = slice_by_index(begin = key_token_455_begin_0, end = key_token_455_end_0, end_mask = key_token_455_end_mask_0, squeeze_mask = key_token_455_squeeze_mask_0, x = key_75_cast_fp16)[name = string("key_token_455_cast_fp16")];
            tensor<int32, [4]> value_token_455_begin_0 = const()[name = string("value_token_455_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> value_token_455_end_0 = const()[name = string("value_token_455_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> value_token_455_end_mask_0 = const()[name = string("value_token_455_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_455_squeeze_mask_0 = const()[name = string("value_token_455_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_455_cast_fp16 = slice_by_index(begin = value_token_455_begin_0, end = value_token_455_end_0, end_mask = value_token_455_end_mask_0, squeeze_mask = value_token_455_squeeze_mask_0, x = value_271_cast_fp16)[name = string("value_token_455_cast_fp16")];
            tensor<int32, [1]> var_12503_axes_0 = const()[name = string("op_12503_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_12503_cast_fp16 = expand_dims(axes = var_12503_axes_0, x = key_token_455_cast_fp16)[name = string("op_12503_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_12504_cast_fp16 = mul(x = state_909_cast_fp16, y = var_12503_cast_fp16)[name = string("op_12504_cast_fp16")];
            tensor<int32, [1]> memory_455_axes_0 = const()[name = string("memory_455_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_455_keep_dims_0 = const()[name = string("memory_455_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_455_cast_fp16 = reduce_sum(axes = memory_455_axes_0, keep_dims = memory_455_keep_dims_0, x = var_12504_cast_fp16)[name = string("memory_455_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_12507_cast_fp16 = sub(x = value_token_455_cast_fp16, y = memory_455_cast_fp16)[name = string("op_12507_cast_fp16")];
            tensor<int32, [3]> var_12510_begin_0 = const()[name = string("op_12510_begin_0"), val = tensor<int32, [3]>([0, 0, 3])];
            tensor<int32, [3]> var_12510_end_0 = const()[name = string("op_12510_end_0"), val = tensor<int32, [3]>([1, 16, 4])];
            tensor<bool, [3]> var_12510_end_mask_0 = const()[name = string("op_12510_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_12510_squeeze_mask_0 = const()[name = string("op_12510_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_12510_cast_fp16 = slice_by_index(begin = var_12510_begin_0, end = var_12510_end_0, end_mask = var_12510_end_mask_0, squeeze_mask = var_12510_squeeze_mask_0, x = beta_cast_fp16)[name = string("op_12510_cast_fp16")];
            tensor<int32, [1]> var_12511_axes_0 = const()[name = string("op_12511_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_12511_cast_fp16 = expand_dims(axes = var_12511_axes_0, x = var_12510_cast_fp16)[name = string("op_12511_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_455_cast_fp16 = mul(x = var_12507_cast_fp16, y = var_12511_cast_fp16)[name = string("delta_455_cast_fp16")];
            tensor<int32, [1]> var_12514_axes_0 = const()[name = string("op_12514_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_12514_cast_fp16 = expand_dims(axes = var_12514_axes_0, x = delta_455_cast_fp16)[name = string("op_12514_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_12515_cast_fp16 = mul(x = var_12503_cast_fp16, y = var_12514_cast_fp16)[name = string("op_12515_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_911_cast_fp16 = add(x = state_909_cast_fp16, y = var_12515_cast_fp16)[name = string("state_911_cast_fp16")];
            tensor<int32, [4]> var_12519_begin_0 = const()[name = string("op_12519_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> var_12519_end_0 = const()[name = string("op_12519_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> var_12519_end_mask_0 = const()[name = string("op_12519_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_12519_squeeze_mask_0 = const()[name = string("op_12519_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_12519_cast_fp16 = slice_by_index(begin = var_12519_begin_0, end = var_12519_end_0, end_mask = var_12519_end_mask_0, squeeze_mask = var_12519_squeeze_mask_0, x = _inversed_query_75_cast_fp16)[name = string("op_12519_cast_fp16")];
            tensor<int32, [1]> var_12520_axes_0 = const()[name = string("op_12520_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_12520_cast_fp16 = expand_dims(axes = var_12520_axes_0, x = var_12519_cast_fp16)[name = string("op_12520_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_12521_cast_fp16 = mul(x = state_911_cast_fp16, y = var_12520_cast_fp16)[name = string("op_12521_cast_fp16")];
            tensor<int32, [1]> var_12523_axes_0 = const()[name = string("op_12523_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_12523_keep_dims_0 = const()[name = string("op_12523_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_12523_cast_fp16 = reduce_sum(axes = var_12523_axes_0, keep_dims = var_12523_keep_dims_0, x = var_12521_cast_fp16)[name = string("op_12523_cast_fp16")];
            tensor<int32, [3]> var_12526_begin_0 = const()[name = string("op_12526_begin_0"), val = tensor<int32, [3]>([0, 0, 4])];
            tensor<int32, [3]> var_12526_end_0 = const()[name = string("op_12526_end_0"), val = tensor<int32, [3]>([1, 16, 5])];
            tensor<bool, [3]> var_12526_end_mask_0 = const()[name = string("op_12526_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_12526_squeeze_mask_0 = const()[name = string("op_12526_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_12526_cast_fp16 = slice_by_index(begin = var_12526_begin_0, end = var_12526_end_0, end_mask = var_12526_end_mask_0, squeeze_mask = var_12526_squeeze_mask_0, x = decay_cast_fp16)[name = string("op_12526_cast_fp16")];
            tensor<fp16, [1, 16]> var_12527_cast_fp16 = exp(x = var_12526_cast_fp16)[name = string("op_12527_cast_fp16")];
            tensor<int32, [1]> var_12528_axes_0 = const()[name = string("op_12528_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_12528_cast_fp16 = expand_dims(axes = var_12528_axes_0, x = var_12527_cast_fp16)[name = string("op_12528_cast_fp16")];
            tensor<int32, [1]> var_12529_axes_0 = const()[name = string("op_12529_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_12529_cast_fp16 = expand_dims(axes = var_12529_axes_0, x = var_12528_cast_fp16)[name = string("op_12529_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_913_cast_fp16 = mul(x = state_911_cast_fp16, y = var_12529_cast_fp16)[name = string("state_913_cast_fp16")];
            tensor<int32, [4]> key_token_457_begin_0 = const()[name = string("key_token_457_begin_0"), val = tensor<int32, [4]>([0, 0, 4, 0])];
            tensor<int32, [4]> key_token_457_end_0 = const()[name = string("key_token_457_end_0"), val = tensor<int32, [4]>([1, 16, 5, 128])];
            tensor<bool, [4]> key_token_457_end_mask_0 = const()[name = string("key_token_457_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_457_squeeze_mask_0 = const()[name = string("key_token_457_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_457_cast_fp16 = slice_by_index(begin = key_token_457_begin_0, end = key_token_457_end_0, end_mask = key_token_457_end_mask_0, squeeze_mask = key_token_457_squeeze_mask_0, x = key_75_cast_fp16)[name = string("key_token_457_cast_fp16")];
            tensor<int32, [4]> value_token_457_begin_0 = const()[name = string("value_token_457_begin_0"), val = tensor<int32, [4]>([0, 0, 4, 0])];
            tensor<int32, [4]> value_token_457_end_0 = const()[name = string("value_token_457_end_0"), val = tensor<int32, [4]>([1, 16, 5, 128])];
            tensor<bool, [4]> value_token_457_end_mask_0 = const()[name = string("value_token_457_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_457_squeeze_mask_0 = const()[name = string("value_token_457_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_457_cast_fp16 = slice_by_index(begin = value_token_457_begin_0, end = value_token_457_end_0, end_mask = value_token_457_end_mask_0, squeeze_mask = value_token_457_squeeze_mask_0, x = value_271_cast_fp16)[name = string("value_token_457_cast_fp16")];
            tensor<int32, [1]> var_12537_axes_0 = const()[name = string("op_12537_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_12537_cast_fp16 = expand_dims(axes = var_12537_axes_0, x = key_token_457_cast_fp16)[name = string("op_12537_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_12538_cast_fp16 = mul(x = state_913_cast_fp16, y = var_12537_cast_fp16)[name = string("op_12538_cast_fp16")];
            tensor<int32, [1]> memory_457_axes_0 = const()[name = string("memory_457_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_457_keep_dims_0 = const()[name = string("memory_457_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_457_cast_fp16 = reduce_sum(axes = memory_457_axes_0, keep_dims = memory_457_keep_dims_0, x = var_12538_cast_fp16)[name = string("memory_457_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_12541_cast_fp16 = sub(x = value_token_457_cast_fp16, y = memory_457_cast_fp16)[name = string("op_12541_cast_fp16")];
            tensor<int32, [3]> var_12544_begin_0 = const()[name = string("op_12544_begin_0"), val = tensor<int32, [3]>([0, 0, 4])];
            tensor<int32, [3]> var_12544_end_0 = const()[name = string("op_12544_end_0"), val = tensor<int32, [3]>([1, 16, 5])];
            tensor<bool, [3]> var_12544_end_mask_0 = const()[name = string("op_12544_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_12544_squeeze_mask_0 = const()[name = string("op_12544_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_12544_cast_fp16 = slice_by_index(begin = var_12544_begin_0, end = var_12544_end_0, end_mask = var_12544_end_mask_0, squeeze_mask = var_12544_squeeze_mask_0, x = beta_cast_fp16)[name = string("op_12544_cast_fp16")];
            tensor<int32, [1]> var_12545_axes_0 = const()[name = string("op_12545_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_12545_cast_fp16 = expand_dims(axes = var_12545_axes_0, x = var_12544_cast_fp16)[name = string("op_12545_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_457_cast_fp16 = mul(x = var_12541_cast_fp16, y = var_12545_cast_fp16)[name = string("delta_457_cast_fp16")];
            tensor<int32, [1]> var_12548_axes_0 = const()[name = string("op_12548_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_12548_cast_fp16 = expand_dims(axes = var_12548_axes_0, x = delta_457_cast_fp16)[name = string("op_12548_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_12549_cast_fp16 = mul(x = var_12537_cast_fp16, y = var_12548_cast_fp16)[name = string("op_12549_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_915_cast_fp16 = add(x = state_913_cast_fp16, y = var_12549_cast_fp16)[name = string("state_915_cast_fp16")];
            tensor<int32, [4]> var_12553_begin_0 = const()[name = string("op_12553_begin_0"), val = tensor<int32, [4]>([0, 0, 4, 0])];
            tensor<int32, [4]> var_12553_end_0 = const()[name = string("op_12553_end_0"), val = tensor<int32, [4]>([1, 16, 5, 128])];
            tensor<bool, [4]> var_12553_end_mask_0 = const()[name = string("op_12553_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_12553_squeeze_mask_0 = const()[name = string("op_12553_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_12553_cast_fp16 = slice_by_index(begin = var_12553_begin_0, end = var_12553_end_0, end_mask = var_12553_end_mask_0, squeeze_mask = var_12553_squeeze_mask_0, x = _inversed_query_75_cast_fp16)[name = string("op_12553_cast_fp16")];
            tensor<int32, [1]> var_12554_axes_0 = const()[name = string("op_12554_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_12554_cast_fp16 = expand_dims(axes = var_12554_axes_0, x = var_12553_cast_fp16)[name = string("op_12554_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_12555_cast_fp16 = mul(x = state_915_cast_fp16, y = var_12554_cast_fp16)[name = string("op_12555_cast_fp16")];
            tensor<int32, [1]> var_12557_axes_0 = const()[name = string("op_12557_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_12557_keep_dims_0 = const()[name = string("op_12557_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_12557_cast_fp16 = reduce_sum(axes = var_12557_axes_0, keep_dims = var_12557_keep_dims_0, x = var_12555_cast_fp16)[name = string("op_12557_cast_fp16")];
            tensor<int32, [3]> var_12560_begin_0 = const()[name = string("op_12560_begin_0"), val = tensor<int32, [3]>([0, 0, 5])];
            tensor<int32, [3]> var_12560_end_0 = const()[name = string("op_12560_end_0"), val = tensor<int32, [3]>([1, 16, 6])];
            tensor<bool, [3]> var_12560_end_mask_0 = const()[name = string("op_12560_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_12560_squeeze_mask_0 = const()[name = string("op_12560_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_12560_cast_fp16 = slice_by_index(begin = var_12560_begin_0, end = var_12560_end_0, end_mask = var_12560_end_mask_0, squeeze_mask = var_12560_squeeze_mask_0, x = decay_cast_fp16)[name = string("op_12560_cast_fp16")];
            tensor<fp16, [1, 16]> var_12561_cast_fp16 = exp(x = var_12560_cast_fp16)[name = string("op_12561_cast_fp16")];
            tensor<int32, [1]> var_12562_axes_0 = const()[name = string("op_12562_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_12562_cast_fp16 = expand_dims(axes = var_12562_axes_0, x = var_12561_cast_fp16)[name = string("op_12562_cast_fp16")];
            tensor<int32, [1]> var_12563_axes_0 = const()[name = string("op_12563_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_12563_cast_fp16 = expand_dims(axes = var_12563_axes_0, x = var_12562_cast_fp16)[name = string("op_12563_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_917_cast_fp16 = mul(x = state_915_cast_fp16, y = var_12563_cast_fp16)[name = string("state_917_cast_fp16")];
            tensor<int32, [4]> key_token_459_begin_0 = const()[name = string("key_token_459_begin_0"), val = tensor<int32, [4]>([0, 0, 5, 0])];
            tensor<int32, [4]> key_token_459_end_0 = const()[name = string("key_token_459_end_0"), val = tensor<int32, [4]>([1, 16, 6, 128])];
            tensor<bool, [4]> key_token_459_end_mask_0 = const()[name = string("key_token_459_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_459_squeeze_mask_0 = const()[name = string("key_token_459_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_459_cast_fp16 = slice_by_index(begin = key_token_459_begin_0, end = key_token_459_end_0, end_mask = key_token_459_end_mask_0, squeeze_mask = key_token_459_squeeze_mask_0, x = key_75_cast_fp16)[name = string("key_token_459_cast_fp16")];
            tensor<int32, [4]> value_token_459_begin_0 = const()[name = string("value_token_459_begin_0"), val = tensor<int32, [4]>([0, 0, 5, 0])];
            tensor<int32, [4]> value_token_459_end_0 = const()[name = string("value_token_459_end_0"), val = tensor<int32, [4]>([1, 16, 6, 128])];
            tensor<bool, [4]> value_token_459_end_mask_0 = const()[name = string("value_token_459_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_459_squeeze_mask_0 = const()[name = string("value_token_459_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_459_cast_fp16 = slice_by_index(begin = value_token_459_begin_0, end = value_token_459_end_0, end_mask = value_token_459_end_mask_0, squeeze_mask = value_token_459_squeeze_mask_0, x = value_271_cast_fp16)[name = string("value_token_459_cast_fp16")];
            tensor<int32, [1]> var_12571_axes_0 = const()[name = string("op_12571_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_12571_cast_fp16 = expand_dims(axes = var_12571_axes_0, x = key_token_459_cast_fp16)[name = string("op_12571_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_12572_cast_fp16 = mul(x = state_917_cast_fp16, y = var_12571_cast_fp16)[name = string("op_12572_cast_fp16")];
            tensor<int32, [1]> memory_459_axes_0 = const()[name = string("memory_459_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_459_keep_dims_0 = const()[name = string("memory_459_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_459_cast_fp16 = reduce_sum(axes = memory_459_axes_0, keep_dims = memory_459_keep_dims_0, x = var_12572_cast_fp16)[name = string("memory_459_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_12575_cast_fp16 = sub(x = value_token_459_cast_fp16, y = memory_459_cast_fp16)[name = string("op_12575_cast_fp16")];
            tensor<int32, [3]> var_12578_begin_0 = const()[name = string("op_12578_begin_0"), val = tensor<int32, [3]>([0, 0, 5])];
            tensor<int32, [3]> var_12578_end_0 = const()[name = string("op_12578_end_0"), val = tensor<int32, [3]>([1, 16, 6])];
            tensor<bool, [3]> var_12578_end_mask_0 = const()[name = string("op_12578_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_12578_squeeze_mask_0 = const()[name = string("op_12578_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_12578_cast_fp16 = slice_by_index(begin = var_12578_begin_0, end = var_12578_end_0, end_mask = var_12578_end_mask_0, squeeze_mask = var_12578_squeeze_mask_0, x = beta_cast_fp16)[name = string("op_12578_cast_fp16")];
            tensor<int32, [1]> var_12579_axes_0 = const()[name = string("op_12579_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_12579_cast_fp16 = expand_dims(axes = var_12579_axes_0, x = var_12578_cast_fp16)[name = string("op_12579_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_459_cast_fp16 = mul(x = var_12575_cast_fp16, y = var_12579_cast_fp16)[name = string("delta_459_cast_fp16")];
            tensor<int32, [1]> var_12582_axes_0 = const()[name = string("op_12582_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_12582_cast_fp16 = expand_dims(axes = var_12582_axes_0, x = delta_459_cast_fp16)[name = string("op_12582_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_12583_cast_fp16 = mul(x = var_12571_cast_fp16, y = var_12582_cast_fp16)[name = string("op_12583_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_919_cast_fp16 = add(x = state_917_cast_fp16, y = var_12583_cast_fp16)[name = string("state_919_cast_fp16")];
            tensor<int32, [4]> var_12587_begin_0 = const()[name = string("op_12587_begin_0"), val = tensor<int32, [4]>([0, 0, 5, 0])];
            tensor<int32, [4]> var_12587_end_0 = const()[name = string("op_12587_end_0"), val = tensor<int32, [4]>([1, 16, 6, 128])];
            tensor<bool, [4]> var_12587_end_mask_0 = const()[name = string("op_12587_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_12587_squeeze_mask_0 = const()[name = string("op_12587_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_12587_cast_fp16 = slice_by_index(begin = var_12587_begin_0, end = var_12587_end_0, end_mask = var_12587_end_mask_0, squeeze_mask = var_12587_squeeze_mask_0, x = _inversed_query_75_cast_fp16)[name = string("op_12587_cast_fp16")];
            tensor<int32, [1]> var_12588_axes_0 = const()[name = string("op_12588_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_12588_cast_fp16 = expand_dims(axes = var_12588_axes_0, x = var_12587_cast_fp16)[name = string("op_12588_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_12589_cast_fp16 = mul(x = state_919_cast_fp16, y = var_12588_cast_fp16)[name = string("op_12589_cast_fp16")];
            tensor<int32, [1]> var_12591_axes_0 = const()[name = string("op_12591_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_12591_keep_dims_0 = const()[name = string("op_12591_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_12591_cast_fp16 = reduce_sum(axes = var_12591_axes_0, keep_dims = var_12591_keep_dims_0, x = var_12589_cast_fp16)[name = string("op_12591_cast_fp16")];
            tensor<int32, [3]> var_12594_begin_0 = const()[name = string("op_12594_begin_0"), val = tensor<int32, [3]>([0, 0, 6])];
            tensor<int32, [3]> var_12594_end_0 = const()[name = string("op_12594_end_0"), val = tensor<int32, [3]>([1, 16, 7])];
            tensor<bool, [3]> var_12594_end_mask_0 = const()[name = string("op_12594_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_12594_squeeze_mask_0 = const()[name = string("op_12594_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_12594_cast_fp16 = slice_by_index(begin = var_12594_begin_0, end = var_12594_end_0, end_mask = var_12594_end_mask_0, squeeze_mask = var_12594_squeeze_mask_0, x = decay_cast_fp16)[name = string("op_12594_cast_fp16")];
            tensor<fp16, [1, 16]> var_12595_cast_fp16 = exp(x = var_12594_cast_fp16)[name = string("op_12595_cast_fp16")];
            tensor<int32, [1]> var_12596_axes_0 = const()[name = string("op_12596_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_12596_cast_fp16 = expand_dims(axes = var_12596_axes_0, x = var_12595_cast_fp16)[name = string("op_12596_cast_fp16")];
            tensor<int32, [1]> var_12597_axes_0 = const()[name = string("op_12597_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_12597_cast_fp16 = expand_dims(axes = var_12597_axes_0, x = var_12596_cast_fp16)[name = string("op_12597_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_921_cast_fp16 = mul(x = state_919_cast_fp16, y = var_12597_cast_fp16)[name = string("state_921_cast_fp16")];
            tensor<int32, [4]> key_token_461_begin_0 = const()[name = string("key_token_461_begin_0"), val = tensor<int32, [4]>([0, 0, 6, 0])];
            tensor<int32, [4]> key_token_461_end_0 = const()[name = string("key_token_461_end_0"), val = tensor<int32, [4]>([1, 16, 7, 128])];
            tensor<bool, [4]> key_token_461_end_mask_0 = const()[name = string("key_token_461_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_461_squeeze_mask_0 = const()[name = string("key_token_461_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_461_cast_fp16 = slice_by_index(begin = key_token_461_begin_0, end = key_token_461_end_0, end_mask = key_token_461_end_mask_0, squeeze_mask = key_token_461_squeeze_mask_0, x = key_75_cast_fp16)[name = string("key_token_461_cast_fp16")];
            tensor<int32, [4]> value_token_461_begin_0 = const()[name = string("value_token_461_begin_0"), val = tensor<int32, [4]>([0, 0, 6, 0])];
            tensor<int32, [4]> value_token_461_end_0 = const()[name = string("value_token_461_end_0"), val = tensor<int32, [4]>([1, 16, 7, 128])];
            tensor<bool, [4]> value_token_461_end_mask_0 = const()[name = string("value_token_461_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_461_squeeze_mask_0 = const()[name = string("value_token_461_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_461_cast_fp16 = slice_by_index(begin = value_token_461_begin_0, end = value_token_461_end_0, end_mask = value_token_461_end_mask_0, squeeze_mask = value_token_461_squeeze_mask_0, x = value_271_cast_fp16)[name = string("value_token_461_cast_fp16")];
            tensor<int32, [1]> var_12605_axes_0 = const()[name = string("op_12605_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_12605_cast_fp16 = expand_dims(axes = var_12605_axes_0, x = key_token_461_cast_fp16)[name = string("op_12605_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_12606_cast_fp16 = mul(x = state_921_cast_fp16, y = var_12605_cast_fp16)[name = string("op_12606_cast_fp16")];
            tensor<int32, [1]> memory_461_axes_0 = const()[name = string("memory_461_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_461_keep_dims_0 = const()[name = string("memory_461_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_461_cast_fp16 = reduce_sum(axes = memory_461_axes_0, keep_dims = memory_461_keep_dims_0, x = var_12606_cast_fp16)[name = string("memory_461_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_12609_cast_fp16 = sub(x = value_token_461_cast_fp16, y = memory_461_cast_fp16)[name = string("op_12609_cast_fp16")];
            tensor<int32, [3]> var_12612_begin_0 = const()[name = string("op_12612_begin_0"), val = tensor<int32, [3]>([0, 0, 6])];
            tensor<int32, [3]> var_12612_end_0 = const()[name = string("op_12612_end_0"), val = tensor<int32, [3]>([1, 16, 7])];
            tensor<bool, [3]> var_12612_end_mask_0 = const()[name = string("op_12612_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_12612_squeeze_mask_0 = const()[name = string("op_12612_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_12612_cast_fp16 = slice_by_index(begin = var_12612_begin_0, end = var_12612_end_0, end_mask = var_12612_end_mask_0, squeeze_mask = var_12612_squeeze_mask_0, x = beta_cast_fp16)[name = string("op_12612_cast_fp16")];
            tensor<int32, [1]> var_12613_axes_0 = const()[name = string("op_12613_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_12613_cast_fp16 = expand_dims(axes = var_12613_axes_0, x = var_12612_cast_fp16)[name = string("op_12613_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_461_cast_fp16 = mul(x = var_12609_cast_fp16, y = var_12613_cast_fp16)[name = string("delta_461_cast_fp16")];
            tensor<int32, [1]> var_12616_axes_0 = const()[name = string("op_12616_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_12616_cast_fp16 = expand_dims(axes = var_12616_axes_0, x = delta_461_cast_fp16)[name = string("op_12616_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_12617_cast_fp16 = mul(x = var_12605_cast_fp16, y = var_12616_cast_fp16)[name = string("op_12617_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_923_cast_fp16 = add(x = state_921_cast_fp16, y = var_12617_cast_fp16)[name = string("state_923_cast_fp16")];
            tensor<int32, [4]> var_12621_begin_0 = const()[name = string("op_12621_begin_0"), val = tensor<int32, [4]>([0, 0, 6, 0])];
            tensor<int32, [4]> var_12621_end_0 = const()[name = string("op_12621_end_0"), val = tensor<int32, [4]>([1, 16, 7, 128])];
            tensor<bool, [4]> var_12621_end_mask_0 = const()[name = string("op_12621_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_12621_squeeze_mask_0 = const()[name = string("op_12621_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_12621_cast_fp16 = slice_by_index(begin = var_12621_begin_0, end = var_12621_end_0, end_mask = var_12621_end_mask_0, squeeze_mask = var_12621_squeeze_mask_0, x = _inversed_query_75_cast_fp16)[name = string("op_12621_cast_fp16")];
            tensor<int32, [1]> var_12622_axes_0 = const()[name = string("op_12622_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_12622_cast_fp16 = expand_dims(axes = var_12622_axes_0, x = var_12621_cast_fp16)[name = string("op_12622_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_12623_cast_fp16 = mul(x = state_923_cast_fp16, y = var_12622_cast_fp16)[name = string("op_12623_cast_fp16")];
            tensor<int32, [1]> var_12625_axes_0 = const()[name = string("op_12625_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_12625_keep_dims_0 = const()[name = string("op_12625_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_12625_cast_fp16 = reduce_sum(axes = var_12625_axes_0, keep_dims = var_12625_keep_dims_0, x = var_12623_cast_fp16)[name = string("op_12625_cast_fp16")];
            tensor<int32, [3]> var_12628_begin_0 = const()[name = string("op_12628_begin_0"), val = tensor<int32, [3]>([0, 0, 7])];
            tensor<int32, [3]> var_12628_end_0 = const()[name = string("op_12628_end_0"), val = tensor<int32, [3]>([1, 16, 8])];
            tensor<bool, [3]> var_12628_end_mask_0 = const()[name = string("op_12628_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_12628_squeeze_mask_0 = const()[name = string("op_12628_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_12628_cast_fp16 = slice_by_index(begin = var_12628_begin_0, end = var_12628_end_0, end_mask = var_12628_end_mask_0, squeeze_mask = var_12628_squeeze_mask_0, x = decay_cast_fp16)[name = string("op_12628_cast_fp16")];
            tensor<fp16, [1, 16]> var_12629_cast_fp16 = exp(x = var_12628_cast_fp16)[name = string("op_12629_cast_fp16")];
            tensor<int32, [1]> var_12630_axes_0 = const()[name = string("op_12630_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_12630_cast_fp16 = expand_dims(axes = var_12630_axes_0, x = var_12629_cast_fp16)[name = string("op_12630_cast_fp16")];
            tensor<int32, [1]> var_12631_axes_0 = const()[name = string("op_12631_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_12631_cast_fp16 = expand_dims(axes = var_12631_axes_0, x = var_12630_cast_fp16)[name = string("op_12631_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_925_cast_fp16 = mul(x = state_923_cast_fp16, y = var_12631_cast_fp16)[name = string("state_925_cast_fp16")];
            tensor<int32, [4]> key_token_463_begin_0 = const()[name = string("key_token_463_begin_0"), val = tensor<int32, [4]>([0, 0, 7, 0])];
            tensor<int32, [4]> key_token_463_end_0 = const()[name = string("key_token_463_end_0"), val = tensor<int32, [4]>([1, 16, 8, 128])];
            tensor<bool, [4]> key_token_463_end_mask_0 = const()[name = string("key_token_463_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_463_squeeze_mask_0 = const()[name = string("key_token_463_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_463_cast_fp16 = slice_by_index(begin = key_token_463_begin_0, end = key_token_463_end_0, end_mask = key_token_463_end_mask_0, squeeze_mask = key_token_463_squeeze_mask_0, x = key_75_cast_fp16)[name = string("key_token_463_cast_fp16")];
            tensor<int32, [4]> value_token_463_begin_0 = const()[name = string("value_token_463_begin_0"), val = tensor<int32, [4]>([0, 0, 7, 0])];
            tensor<int32, [4]> value_token_463_end_0 = const()[name = string("value_token_463_end_0"), val = tensor<int32, [4]>([1, 16, 8, 128])];
            tensor<bool, [4]> value_token_463_end_mask_0 = const()[name = string("value_token_463_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_463_squeeze_mask_0 = const()[name = string("value_token_463_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_463_cast_fp16 = slice_by_index(begin = value_token_463_begin_0, end = value_token_463_end_0, end_mask = value_token_463_end_mask_0, squeeze_mask = value_token_463_squeeze_mask_0, x = value_271_cast_fp16)[name = string("value_token_463_cast_fp16")];
            tensor<int32, [1]> var_12639_axes_0 = const()[name = string("op_12639_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_12639_cast_fp16 = expand_dims(axes = var_12639_axes_0, x = key_token_463_cast_fp16)[name = string("op_12639_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_12640_cast_fp16 = mul(x = state_925_cast_fp16, y = var_12639_cast_fp16)[name = string("op_12640_cast_fp16")];
            tensor<int32, [1]> memory_463_axes_0 = const()[name = string("memory_463_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_463_keep_dims_0 = const()[name = string("memory_463_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_463_cast_fp16 = reduce_sum(axes = memory_463_axes_0, keep_dims = memory_463_keep_dims_0, x = var_12640_cast_fp16)[name = string("memory_463_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_12643_cast_fp16 = sub(x = value_token_463_cast_fp16, y = memory_463_cast_fp16)[name = string("op_12643_cast_fp16")];
            tensor<int32, [3]> var_12646_begin_0 = const()[name = string("op_12646_begin_0"), val = tensor<int32, [3]>([0, 0, 7])];
            tensor<int32, [3]> var_12646_end_0 = const()[name = string("op_12646_end_0"), val = tensor<int32, [3]>([1, 16, 8])];
            tensor<bool, [3]> var_12646_end_mask_0 = const()[name = string("op_12646_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_12646_squeeze_mask_0 = const()[name = string("op_12646_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_12646_cast_fp16 = slice_by_index(begin = var_12646_begin_0, end = var_12646_end_0, end_mask = var_12646_end_mask_0, squeeze_mask = var_12646_squeeze_mask_0, x = beta_cast_fp16)[name = string("op_12646_cast_fp16")];
            tensor<int32, [1]> var_12647_axes_0 = const()[name = string("op_12647_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_12647_cast_fp16 = expand_dims(axes = var_12647_axes_0, x = var_12646_cast_fp16)[name = string("op_12647_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_463_cast_fp16 = mul(x = var_12643_cast_fp16, y = var_12647_cast_fp16)[name = string("delta_463_cast_fp16")];
            tensor<int32, [1]> var_12650_axes_0 = const()[name = string("op_12650_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_12650_cast_fp16 = expand_dims(axes = var_12650_axes_0, x = delta_463_cast_fp16)[name = string("op_12650_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_12651_cast_fp16 = mul(x = var_12639_cast_fp16, y = var_12650_cast_fp16)[name = string("op_12651_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_927_cast_fp16 = add(x = state_925_cast_fp16, y = var_12651_cast_fp16)[name = string("state_927_cast_fp16")];
            tensor<int32, [4]> var_12655_begin_0 = const()[name = string("op_12655_begin_0"), val = tensor<int32, [4]>([0, 0, 7, 0])];
            tensor<int32, [4]> var_12655_end_0 = const()[name = string("op_12655_end_0"), val = tensor<int32, [4]>([1, 16, 8, 128])];
            tensor<bool, [4]> var_12655_end_mask_0 = const()[name = string("op_12655_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_12655_squeeze_mask_0 = const()[name = string("op_12655_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_12655_cast_fp16 = slice_by_index(begin = var_12655_begin_0, end = var_12655_end_0, end_mask = var_12655_end_mask_0, squeeze_mask = var_12655_squeeze_mask_0, x = _inversed_query_75_cast_fp16)[name = string("op_12655_cast_fp16")];
            tensor<int32, [1]> var_12656_axes_0 = const()[name = string("op_12656_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_12656_cast_fp16 = expand_dims(axes = var_12656_axes_0, x = var_12655_cast_fp16)[name = string("op_12656_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_12657_cast_fp16 = mul(x = state_927_cast_fp16, y = var_12656_cast_fp16)[name = string("op_12657_cast_fp16")];
            tensor<int32, [1]> var_12659_axes_0 = const()[name = string("op_12659_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_12659_keep_dims_0 = const()[name = string("op_12659_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_12659_cast_fp16 = reduce_sum(axes = var_12659_axes_0, keep_dims = var_12659_keep_dims_0, x = var_12657_cast_fp16)[name = string("op_12659_cast_fp16")];
            tensor<int32, [3]> var_12662_begin_0 = const()[name = string("op_12662_begin_0"), val = tensor<int32, [3]>([0, 0, 8])];
            tensor<int32, [3]> var_12662_end_0 = const()[name = string("op_12662_end_0"), val = tensor<int32, [3]>([1, 16, 9])];
            tensor<bool, [3]> var_12662_end_mask_0 = const()[name = string("op_12662_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_12662_squeeze_mask_0 = const()[name = string("op_12662_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_12662_cast_fp16 = slice_by_index(begin = var_12662_begin_0, end = var_12662_end_0, end_mask = var_12662_end_mask_0, squeeze_mask = var_12662_squeeze_mask_0, x = decay_cast_fp16)[name = string("op_12662_cast_fp16")];
            tensor<fp16, [1, 16]> var_12663_cast_fp16 = exp(x = var_12662_cast_fp16)[name = string("op_12663_cast_fp16")];
            tensor<int32, [1]> var_12664_axes_0 = const()[name = string("op_12664_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_12664_cast_fp16 = expand_dims(axes = var_12664_axes_0, x = var_12663_cast_fp16)[name = string("op_12664_cast_fp16")];
            tensor<int32, [1]> var_12665_axes_0 = const()[name = string("op_12665_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_12665_cast_fp16 = expand_dims(axes = var_12665_axes_0, x = var_12664_cast_fp16)[name = string("op_12665_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_929_cast_fp16 = mul(x = state_927_cast_fp16, y = var_12665_cast_fp16)[name = string("state_929_cast_fp16")];
            tensor<int32, [4]> key_token_465_begin_0 = const()[name = string("key_token_465_begin_0"), val = tensor<int32, [4]>([0, 0, 8, 0])];
            tensor<int32, [4]> key_token_465_end_0 = const()[name = string("key_token_465_end_0"), val = tensor<int32, [4]>([1, 16, 9, 128])];
            tensor<bool, [4]> key_token_465_end_mask_0 = const()[name = string("key_token_465_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_465_squeeze_mask_0 = const()[name = string("key_token_465_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_465_cast_fp16 = slice_by_index(begin = key_token_465_begin_0, end = key_token_465_end_0, end_mask = key_token_465_end_mask_0, squeeze_mask = key_token_465_squeeze_mask_0, x = key_75_cast_fp16)[name = string("key_token_465_cast_fp16")];
            tensor<int32, [4]> value_token_465_begin_0 = const()[name = string("value_token_465_begin_0"), val = tensor<int32, [4]>([0, 0, 8, 0])];
            tensor<int32, [4]> value_token_465_end_0 = const()[name = string("value_token_465_end_0"), val = tensor<int32, [4]>([1, 16, 9, 128])];
            tensor<bool, [4]> value_token_465_end_mask_0 = const()[name = string("value_token_465_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_465_squeeze_mask_0 = const()[name = string("value_token_465_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_465_cast_fp16 = slice_by_index(begin = value_token_465_begin_0, end = value_token_465_end_0, end_mask = value_token_465_end_mask_0, squeeze_mask = value_token_465_squeeze_mask_0, x = value_271_cast_fp16)[name = string("value_token_465_cast_fp16")];
            tensor<int32, [1]> var_12673_axes_0 = const()[name = string("op_12673_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_12673_cast_fp16 = expand_dims(axes = var_12673_axes_0, x = key_token_465_cast_fp16)[name = string("op_12673_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_12674_cast_fp16 = mul(x = state_929_cast_fp16, y = var_12673_cast_fp16)[name = string("op_12674_cast_fp16")];
            tensor<int32, [1]> memory_465_axes_0 = const()[name = string("memory_465_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_465_keep_dims_0 = const()[name = string("memory_465_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_465_cast_fp16 = reduce_sum(axes = memory_465_axes_0, keep_dims = memory_465_keep_dims_0, x = var_12674_cast_fp16)[name = string("memory_465_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_12677_cast_fp16 = sub(x = value_token_465_cast_fp16, y = memory_465_cast_fp16)[name = string("op_12677_cast_fp16")];
            tensor<int32, [3]> var_12680_begin_0 = const()[name = string("op_12680_begin_0"), val = tensor<int32, [3]>([0, 0, 8])];
            tensor<int32, [3]> var_12680_end_0 = const()[name = string("op_12680_end_0"), val = tensor<int32, [3]>([1, 16, 9])];
            tensor<bool, [3]> var_12680_end_mask_0 = const()[name = string("op_12680_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_12680_squeeze_mask_0 = const()[name = string("op_12680_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_12680_cast_fp16 = slice_by_index(begin = var_12680_begin_0, end = var_12680_end_0, end_mask = var_12680_end_mask_0, squeeze_mask = var_12680_squeeze_mask_0, x = beta_cast_fp16)[name = string("op_12680_cast_fp16")];
            tensor<int32, [1]> var_12681_axes_0 = const()[name = string("op_12681_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_12681_cast_fp16 = expand_dims(axes = var_12681_axes_0, x = var_12680_cast_fp16)[name = string("op_12681_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_465_cast_fp16 = mul(x = var_12677_cast_fp16, y = var_12681_cast_fp16)[name = string("delta_465_cast_fp16")];
            tensor<int32, [1]> var_12684_axes_0 = const()[name = string("op_12684_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_12684_cast_fp16 = expand_dims(axes = var_12684_axes_0, x = delta_465_cast_fp16)[name = string("op_12684_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_12685_cast_fp16 = mul(x = var_12673_cast_fp16, y = var_12684_cast_fp16)[name = string("op_12685_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_931_cast_fp16 = add(x = state_929_cast_fp16, y = var_12685_cast_fp16)[name = string("state_931_cast_fp16")];
            tensor<int32, [4]> var_12689_begin_0 = const()[name = string("op_12689_begin_0"), val = tensor<int32, [4]>([0, 0, 8, 0])];
            tensor<int32, [4]> var_12689_end_0 = const()[name = string("op_12689_end_0"), val = tensor<int32, [4]>([1, 16, 9, 128])];
            tensor<bool, [4]> var_12689_end_mask_0 = const()[name = string("op_12689_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_12689_squeeze_mask_0 = const()[name = string("op_12689_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_12689_cast_fp16 = slice_by_index(begin = var_12689_begin_0, end = var_12689_end_0, end_mask = var_12689_end_mask_0, squeeze_mask = var_12689_squeeze_mask_0, x = _inversed_query_75_cast_fp16)[name = string("op_12689_cast_fp16")];
            tensor<int32, [1]> var_12690_axes_0 = const()[name = string("op_12690_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_12690_cast_fp16 = expand_dims(axes = var_12690_axes_0, x = var_12689_cast_fp16)[name = string("op_12690_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_12691_cast_fp16 = mul(x = state_931_cast_fp16, y = var_12690_cast_fp16)[name = string("op_12691_cast_fp16")];
            tensor<int32, [1]> var_12693_axes_0 = const()[name = string("op_12693_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_12693_keep_dims_0 = const()[name = string("op_12693_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_12693_cast_fp16 = reduce_sum(axes = var_12693_axes_0, keep_dims = var_12693_keep_dims_0, x = var_12691_cast_fp16)[name = string("op_12693_cast_fp16")];
            tensor<int32, [3]> var_12696_begin_0 = const()[name = string("op_12696_begin_0"), val = tensor<int32, [3]>([0, 0, 9])];
            tensor<int32, [3]> var_12696_end_0 = const()[name = string("op_12696_end_0"), val = tensor<int32, [3]>([1, 16, 10])];
            tensor<bool, [3]> var_12696_end_mask_0 = const()[name = string("op_12696_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_12696_squeeze_mask_0 = const()[name = string("op_12696_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_12696_cast_fp16 = slice_by_index(begin = var_12696_begin_0, end = var_12696_end_0, end_mask = var_12696_end_mask_0, squeeze_mask = var_12696_squeeze_mask_0, x = decay_cast_fp16)[name = string("op_12696_cast_fp16")];
            tensor<fp16, [1, 16]> var_12697_cast_fp16 = exp(x = var_12696_cast_fp16)[name = string("op_12697_cast_fp16")];
            tensor<int32, [1]> var_12698_axes_0 = const()[name = string("op_12698_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_12698_cast_fp16 = expand_dims(axes = var_12698_axes_0, x = var_12697_cast_fp16)[name = string("op_12698_cast_fp16")];
            tensor<int32, [1]> var_12699_axes_0 = const()[name = string("op_12699_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_12699_cast_fp16 = expand_dims(axes = var_12699_axes_0, x = var_12698_cast_fp16)[name = string("op_12699_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_933_cast_fp16 = mul(x = state_931_cast_fp16, y = var_12699_cast_fp16)[name = string("state_933_cast_fp16")];
            tensor<int32, [4]> key_token_467_begin_0 = const()[name = string("key_token_467_begin_0"), val = tensor<int32, [4]>([0, 0, 9, 0])];
            tensor<int32, [4]> key_token_467_end_0 = const()[name = string("key_token_467_end_0"), val = tensor<int32, [4]>([1, 16, 10, 128])];
            tensor<bool, [4]> key_token_467_end_mask_0 = const()[name = string("key_token_467_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_467_squeeze_mask_0 = const()[name = string("key_token_467_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_467_cast_fp16 = slice_by_index(begin = key_token_467_begin_0, end = key_token_467_end_0, end_mask = key_token_467_end_mask_0, squeeze_mask = key_token_467_squeeze_mask_0, x = key_75_cast_fp16)[name = string("key_token_467_cast_fp16")];
            tensor<int32, [4]> value_token_467_begin_0 = const()[name = string("value_token_467_begin_0"), val = tensor<int32, [4]>([0, 0, 9, 0])];
            tensor<int32, [4]> value_token_467_end_0 = const()[name = string("value_token_467_end_0"), val = tensor<int32, [4]>([1, 16, 10, 128])];
            tensor<bool, [4]> value_token_467_end_mask_0 = const()[name = string("value_token_467_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_467_squeeze_mask_0 = const()[name = string("value_token_467_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_467_cast_fp16 = slice_by_index(begin = value_token_467_begin_0, end = value_token_467_end_0, end_mask = value_token_467_end_mask_0, squeeze_mask = value_token_467_squeeze_mask_0, x = value_271_cast_fp16)[name = string("value_token_467_cast_fp16")];
            tensor<int32, [1]> var_12707_axes_0 = const()[name = string("op_12707_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_12707_cast_fp16 = expand_dims(axes = var_12707_axes_0, x = key_token_467_cast_fp16)[name = string("op_12707_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_12708_cast_fp16 = mul(x = state_933_cast_fp16, y = var_12707_cast_fp16)[name = string("op_12708_cast_fp16")];
            tensor<int32, [1]> memory_467_axes_0 = const()[name = string("memory_467_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_467_keep_dims_0 = const()[name = string("memory_467_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_467_cast_fp16 = reduce_sum(axes = memory_467_axes_0, keep_dims = memory_467_keep_dims_0, x = var_12708_cast_fp16)[name = string("memory_467_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_12711_cast_fp16 = sub(x = value_token_467_cast_fp16, y = memory_467_cast_fp16)[name = string("op_12711_cast_fp16")];
            tensor<int32, [3]> var_12714_begin_0 = const()[name = string("op_12714_begin_0"), val = tensor<int32, [3]>([0, 0, 9])];
            tensor<int32, [3]> var_12714_end_0 = const()[name = string("op_12714_end_0"), val = tensor<int32, [3]>([1, 16, 10])];
            tensor<bool, [3]> var_12714_end_mask_0 = const()[name = string("op_12714_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_12714_squeeze_mask_0 = const()[name = string("op_12714_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_12714_cast_fp16 = slice_by_index(begin = var_12714_begin_0, end = var_12714_end_0, end_mask = var_12714_end_mask_0, squeeze_mask = var_12714_squeeze_mask_0, x = beta_cast_fp16)[name = string("op_12714_cast_fp16")];
            tensor<int32, [1]> var_12715_axes_0 = const()[name = string("op_12715_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_12715_cast_fp16 = expand_dims(axes = var_12715_axes_0, x = var_12714_cast_fp16)[name = string("op_12715_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_467_cast_fp16 = mul(x = var_12711_cast_fp16, y = var_12715_cast_fp16)[name = string("delta_467_cast_fp16")];
            tensor<int32, [1]> var_12718_axes_0 = const()[name = string("op_12718_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_12718_cast_fp16 = expand_dims(axes = var_12718_axes_0, x = delta_467_cast_fp16)[name = string("op_12718_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_12719_cast_fp16 = mul(x = var_12707_cast_fp16, y = var_12718_cast_fp16)[name = string("op_12719_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_935_cast_fp16 = add(x = state_933_cast_fp16, y = var_12719_cast_fp16)[name = string("state_935_cast_fp16")];
            tensor<int32, [4]> var_12723_begin_0 = const()[name = string("op_12723_begin_0"), val = tensor<int32, [4]>([0, 0, 9, 0])];
            tensor<int32, [4]> var_12723_end_0 = const()[name = string("op_12723_end_0"), val = tensor<int32, [4]>([1, 16, 10, 128])];
            tensor<bool, [4]> var_12723_end_mask_0 = const()[name = string("op_12723_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_12723_squeeze_mask_0 = const()[name = string("op_12723_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_12723_cast_fp16 = slice_by_index(begin = var_12723_begin_0, end = var_12723_end_0, end_mask = var_12723_end_mask_0, squeeze_mask = var_12723_squeeze_mask_0, x = _inversed_query_75_cast_fp16)[name = string("op_12723_cast_fp16")];
            tensor<int32, [1]> var_12724_axes_0 = const()[name = string("op_12724_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_12724_cast_fp16 = expand_dims(axes = var_12724_axes_0, x = var_12723_cast_fp16)[name = string("op_12724_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_12725_cast_fp16 = mul(x = state_935_cast_fp16, y = var_12724_cast_fp16)[name = string("op_12725_cast_fp16")];
            tensor<int32, [1]> var_12727_axes_0 = const()[name = string("op_12727_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_12727_keep_dims_0 = const()[name = string("op_12727_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_12727_cast_fp16 = reduce_sum(axes = var_12727_axes_0, keep_dims = var_12727_keep_dims_0, x = var_12725_cast_fp16)[name = string("op_12727_cast_fp16")];
            tensor<int32, [3]> var_12730_begin_0 = const()[name = string("op_12730_begin_0"), val = tensor<int32, [3]>([0, 0, 10])];
            tensor<int32, [3]> var_12730_end_0 = const()[name = string("op_12730_end_0"), val = tensor<int32, [3]>([1, 16, 11])];
            tensor<bool, [3]> var_12730_end_mask_0 = const()[name = string("op_12730_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_12730_squeeze_mask_0 = const()[name = string("op_12730_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_12730_cast_fp16 = slice_by_index(begin = var_12730_begin_0, end = var_12730_end_0, end_mask = var_12730_end_mask_0, squeeze_mask = var_12730_squeeze_mask_0, x = decay_cast_fp16)[name = string("op_12730_cast_fp16")];
            tensor<fp16, [1, 16]> var_12731_cast_fp16 = exp(x = var_12730_cast_fp16)[name = string("op_12731_cast_fp16")];
            tensor<int32, [1]> var_12732_axes_0 = const()[name = string("op_12732_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_12732_cast_fp16 = expand_dims(axes = var_12732_axes_0, x = var_12731_cast_fp16)[name = string("op_12732_cast_fp16")];
            tensor<int32, [1]> var_12733_axes_0 = const()[name = string("op_12733_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_12733_cast_fp16 = expand_dims(axes = var_12733_axes_0, x = var_12732_cast_fp16)[name = string("op_12733_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_937_cast_fp16 = mul(x = state_935_cast_fp16, y = var_12733_cast_fp16)[name = string("state_937_cast_fp16")];
            tensor<int32, [4]> key_token_469_begin_0 = const()[name = string("key_token_469_begin_0"), val = tensor<int32, [4]>([0, 0, 10, 0])];
            tensor<int32, [4]> key_token_469_end_0 = const()[name = string("key_token_469_end_0"), val = tensor<int32, [4]>([1, 16, 11, 128])];
            tensor<bool, [4]> key_token_469_end_mask_0 = const()[name = string("key_token_469_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_469_squeeze_mask_0 = const()[name = string("key_token_469_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_469_cast_fp16 = slice_by_index(begin = key_token_469_begin_0, end = key_token_469_end_0, end_mask = key_token_469_end_mask_0, squeeze_mask = key_token_469_squeeze_mask_0, x = key_75_cast_fp16)[name = string("key_token_469_cast_fp16")];
            tensor<int32, [4]> value_token_469_begin_0 = const()[name = string("value_token_469_begin_0"), val = tensor<int32, [4]>([0, 0, 10, 0])];
            tensor<int32, [4]> value_token_469_end_0 = const()[name = string("value_token_469_end_0"), val = tensor<int32, [4]>([1, 16, 11, 128])];
            tensor<bool, [4]> value_token_469_end_mask_0 = const()[name = string("value_token_469_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_469_squeeze_mask_0 = const()[name = string("value_token_469_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_469_cast_fp16 = slice_by_index(begin = value_token_469_begin_0, end = value_token_469_end_0, end_mask = value_token_469_end_mask_0, squeeze_mask = value_token_469_squeeze_mask_0, x = value_271_cast_fp16)[name = string("value_token_469_cast_fp16")];
            tensor<int32, [1]> var_12741_axes_0 = const()[name = string("op_12741_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_12741_cast_fp16 = expand_dims(axes = var_12741_axes_0, x = key_token_469_cast_fp16)[name = string("op_12741_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_12742_cast_fp16 = mul(x = state_937_cast_fp16, y = var_12741_cast_fp16)[name = string("op_12742_cast_fp16")];
            tensor<int32, [1]> memory_469_axes_0 = const()[name = string("memory_469_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_469_keep_dims_0 = const()[name = string("memory_469_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_469_cast_fp16 = reduce_sum(axes = memory_469_axes_0, keep_dims = memory_469_keep_dims_0, x = var_12742_cast_fp16)[name = string("memory_469_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_12745_cast_fp16 = sub(x = value_token_469_cast_fp16, y = memory_469_cast_fp16)[name = string("op_12745_cast_fp16")];
            tensor<int32, [3]> var_12748_begin_0 = const()[name = string("op_12748_begin_0"), val = tensor<int32, [3]>([0, 0, 10])];
            tensor<int32, [3]> var_12748_end_0 = const()[name = string("op_12748_end_0"), val = tensor<int32, [3]>([1, 16, 11])];
            tensor<bool, [3]> var_12748_end_mask_0 = const()[name = string("op_12748_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_12748_squeeze_mask_0 = const()[name = string("op_12748_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_12748_cast_fp16 = slice_by_index(begin = var_12748_begin_0, end = var_12748_end_0, end_mask = var_12748_end_mask_0, squeeze_mask = var_12748_squeeze_mask_0, x = beta_cast_fp16)[name = string("op_12748_cast_fp16")];
            tensor<int32, [1]> var_12749_axes_0 = const()[name = string("op_12749_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_12749_cast_fp16 = expand_dims(axes = var_12749_axes_0, x = var_12748_cast_fp16)[name = string("op_12749_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_469_cast_fp16 = mul(x = var_12745_cast_fp16, y = var_12749_cast_fp16)[name = string("delta_469_cast_fp16")];
            tensor<int32, [1]> var_12752_axes_0 = const()[name = string("op_12752_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_12752_cast_fp16 = expand_dims(axes = var_12752_axes_0, x = delta_469_cast_fp16)[name = string("op_12752_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_12753_cast_fp16 = mul(x = var_12741_cast_fp16, y = var_12752_cast_fp16)[name = string("op_12753_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_939_cast_fp16 = add(x = state_937_cast_fp16, y = var_12753_cast_fp16)[name = string("state_939_cast_fp16")];
            tensor<int32, [4]> var_12757_begin_0 = const()[name = string("op_12757_begin_0"), val = tensor<int32, [4]>([0, 0, 10, 0])];
            tensor<int32, [4]> var_12757_end_0 = const()[name = string("op_12757_end_0"), val = tensor<int32, [4]>([1, 16, 11, 128])];
            tensor<bool, [4]> var_12757_end_mask_0 = const()[name = string("op_12757_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_12757_squeeze_mask_0 = const()[name = string("op_12757_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_12757_cast_fp16 = slice_by_index(begin = var_12757_begin_0, end = var_12757_end_0, end_mask = var_12757_end_mask_0, squeeze_mask = var_12757_squeeze_mask_0, x = _inversed_query_75_cast_fp16)[name = string("op_12757_cast_fp16")];
            tensor<int32, [1]> var_12758_axes_0 = const()[name = string("op_12758_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_12758_cast_fp16 = expand_dims(axes = var_12758_axes_0, x = var_12757_cast_fp16)[name = string("op_12758_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_12759_cast_fp16 = mul(x = state_939_cast_fp16, y = var_12758_cast_fp16)[name = string("op_12759_cast_fp16")];
            tensor<int32, [1]> var_12761_axes_0 = const()[name = string("op_12761_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_12761_keep_dims_0 = const()[name = string("op_12761_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_12761_cast_fp16 = reduce_sum(axes = var_12761_axes_0, keep_dims = var_12761_keep_dims_0, x = var_12759_cast_fp16)[name = string("op_12761_cast_fp16")];
            tensor<int32, [3]> var_12764_begin_0 = const()[name = string("op_12764_begin_0"), val = tensor<int32, [3]>([0, 0, 11])];
            tensor<int32, [3]> var_12764_end_0 = const()[name = string("op_12764_end_0"), val = tensor<int32, [3]>([1, 16, 12])];
            tensor<bool, [3]> var_12764_end_mask_0 = const()[name = string("op_12764_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_12764_squeeze_mask_0 = const()[name = string("op_12764_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_12764_cast_fp16 = slice_by_index(begin = var_12764_begin_0, end = var_12764_end_0, end_mask = var_12764_end_mask_0, squeeze_mask = var_12764_squeeze_mask_0, x = decay_cast_fp16)[name = string("op_12764_cast_fp16")];
            tensor<fp16, [1, 16]> var_12765_cast_fp16 = exp(x = var_12764_cast_fp16)[name = string("op_12765_cast_fp16")];
            tensor<int32, [1]> var_12766_axes_0 = const()[name = string("op_12766_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_12766_cast_fp16 = expand_dims(axes = var_12766_axes_0, x = var_12765_cast_fp16)[name = string("op_12766_cast_fp16")];
            tensor<int32, [1]> var_12767_axes_0 = const()[name = string("op_12767_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_12767_cast_fp16 = expand_dims(axes = var_12767_axes_0, x = var_12766_cast_fp16)[name = string("op_12767_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_941_cast_fp16 = mul(x = state_939_cast_fp16, y = var_12767_cast_fp16)[name = string("state_941_cast_fp16")];
            tensor<int32, [4]> key_token_471_begin_0 = const()[name = string("key_token_471_begin_0"), val = tensor<int32, [4]>([0, 0, 11, 0])];
            tensor<int32, [4]> key_token_471_end_0 = const()[name = string("key_token_471_end_0"), val = tensor<int32, [4]>([1, 16, 12, 128])];
            tensor<bool, [4]> key_token_471_end_mask_0 = const()[name = string("key_token_471_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_471_squeeze_mask_0 = const()[name = string("key_token_471_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_471_cast_fp16 = slice_by_index(begin = key_token_471_begin_0, end = key_token_471_end_0, end_mask = key_token_471_end_mask_0, squeeze_mask = key_token_471_squeeze_mask_0, x = key_75_cast_fp16)[name = string("key_token_471_cast_fp16")];
            tensor<int32, [4]> value_token_471_begin_0 = const()[name = string("value_token_471_begin_0"), val = tensor<int32, [4]>([0, 0, 11, 0])];
            tensor<int32, [4]> value_token_471_end_0 = const()[name = string("value_token_471_end_0"), val = tensor<int32, [4]>([1, 16, 12, 128])];
            tensor<bool, [4]> value_token_471_end_mask_0 = const()[name = string("value_token_471_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_471_squeeze_mask_0 = const()[name = string("value_token_471_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_471_cast_fp16 = slice_by_index(begin = value_token_471_begin_0, end = value_token_471_end_0, end_mask = value_token_471_end_mask_0, squeeze_mask = value_token_471_squeeze_mask_0, x = value_271_cast_fp16)[name = string("value_token_471_cast_fp16")];
            tensor<int32, [1]> var_12775_axes_0 = const()[name = string("op_12775_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_12775_cast_fp16 = expand_dims(axes = var_12775_axes_0, x = key_token_471_cast_fp16)[name = string("op_12775_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_12776_cast_fp16 = mul(x = state_941_cast_fp16, y = var_12775_cast_fp16)[name = string("op_12776_cast_fp16")];
            tensor<int32, [1]> memory_471_axes_0 = const()[name = string("memory_471_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_471_keep_dims_0 = const()[name = string("memory_471_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_471_cast_fp16 = reduce_sum(axes = memory_471_axes_0, keep_dims = memory_471_keep_dims_0, x = var_12776_cast_fp16)[name = string("memory_471_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_12779_cast_fp16 = sub(x = value_token_471_cast_fp16, y = memory_471_cast_fp16)[name = string("op_12779_cast_fp16")];
            tensor<int32, [3]> var_12782_begin_0 = const()[name = string("op_12782_begin_0"), val = tensor<int32, [3]>([0, 0, 11])];
            tensor<int32, [3]> var_12782_end_0 = const()[name = string("op_12782_end_0"), val = tensor<int32, [3]>([1, 16, 12])];
            tensor<bool, [3]> var_12782_end_mask_0 = const()[name = string("op_12782_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_12782_squeeze_mask_0 = const()[name = string("op_12782_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_12782_cast_fp16 = slice_by_index(begin = var_12782_begin_0, end = var_12782_end_0, end_mask = var_12782_end_mask_0, squeeze_mask = var_12782_squeeze_mask_0, x = beta_cast_fp16)[name = string("op_12782_cast_fp16")];
            tensor<int32, [1]> var_12783_axes_0 = const()[name = string("op_12783_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_12783_cast_fp16 = expand_dims(axes = var_12783_axes_0, x = var_12782_cast_fp16)[name = string("op_12783_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_471_cast_fp16 = mul(x = var_12779_cast_fp16, y = var_12783_cast_fp16)[name = string("delta_471_cast_fp16")];
            tensor<int32, [1]> var_12786_axes_0 = const()[name = string("op_12786_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_12786_cast_fp16 = expand_dims(axes = var_12786_axes_0, x = delta_471_cast_fp16)[name = string("op_12786_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_12787_cast_fp16 = mul(x = var_12775_cast_fp16, y = var_12786_cast_fp16)[name = string("op_12787_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_943_cast_fp16 = add(x = state_941_cast_fp16, y = var_12787_cast_fp16)[name = string("state_943_cast_fp16")];
            tensor<int32, [4]> var_12791_begin_0 = const()[name = string("op_12791_begin_0"), val = tensor<int32, [4]>([0, 0, 11, 0])];
            tensor<int32, [4]> var_12791_end_0 = const()[name = string("op_12791_end_0"), val = tensor<int32, [4]>([1, 16, 12, 128])];
            tensor<bool, [4]> var_12791_end_mask_0 = const()[name = string("op_12791_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_12791_squeeze_mask_0 = const()[name = string("op_12791_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_12791_cast_fp16 = slice_by_index(begin = var_12791_begin_0, end = var_12791_end_0, end_mask = var_12791_end_mask_0, squeeze_mask = var_12791_squeeze_mask_0, x = _inversed_query_75_cast_fp16)[name = string("op_12791_cast_fp16")];
            tensor<int32, [1]> var_12792_axes_0 = const()[name = string("op_12792_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_12792_cast_fp16 = expand_dims(axes = var_12792_axes_0, x = var_12791_cast_fp16)[name = string("op_12792_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_12793_cast_fp16 = mul(x = state_943_cast_fp16, y = var_12792_cast_fp16)[name = string("op_12793_cast_fp16")];
            tensor<int32, [1]> var_12795_axes_0 = const()[name = string("op_12795_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_12795_keep_dims_0 = const()[name = string("op_12795_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_12795_cast_fp16 = reduce_sum(axes = var_12795_axes_0, keep_dims = var_12795_keep_dims_0, x = var_12793_cast_fp16)[name = string("op_12795_cast_fp16")];
            tensor<int32, [3]> var_12798_begin_0 = const()[name = string("op_12798_begin_0"), val = tensor<int32, [3]>([0, 0, 12])];
            tensor<int32, [3]> var_12798_end_0 = const()[name = string("op_12798_end_0"), val = tensor<int32, [3]>([1, 16, 13])];
            tensor<bool, [3]> var_12798_end_mask_0 = const()[name = string("op_12798_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_12798_squeeze_mask_0 = const()[name = string("op_12798_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_12798_cast_fp16 = slice_by_index(begin = var_12798_begin_0, end = var_12798_end_0, end_mask = var_12798_end_mask_0, squeeze_mask = var_12798_squeeze_mask_0, x = decay_cast_fp16)[name = string("op_12798_cast_fp16")];
            tensor<fp16, [1, 16]> var_12799_cast_fp16 = exp(x = var_12798_cast_fp16)[name = string("op_12799_cast_fp16")];
            tensor<int32, [1]> var_12800_axes_0 = const()[name = string("op_12800_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_12800_cast_fp16 = expand_dims(axes = var_12800_axes_0, x = var_12799_cast_fp16)[name = string("op_12800_cast_fp16")];
            tensor<int32, [1]> var_12801_axes_0 = const()[name = string("op_12801_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_12801_cast_fp16 = expand_dims(axes = var_12801_axes_0, x = var_12800_cast_fp16)[name = string("op_12801_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_945_cast_fp16 = mul(x = state_943_cast_fp16, y = var_12801_cast_fp16)[name = string("state_945_cast_fp16")];
            tensor<int32, [4]> key_token_473_begin_0 = const()[name = string("key_token_473_begin_0"), val = tensor<int32, [4]>([0, 0, 12, 0])];
            tensor<int32, [4]> key_token_473_end_0 = const()[name = string("key_token_473_end_0"), val = tensor<int32, [4]>([1, 16, 13, 128])];
            tensor<bool, [4]> key_token_473_end_mask_0 = const()[name = string("key_token_473_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_473_squeeze_mask_0 = const()[name = string("key_token_473_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_473_cast_fp16 = slice_by_index(begin = key_token_473_begin_0, end = key_token_473_end_0, end_mask = key_token_473_end_mask_0, squeeze_mask = key_token_473_squeeze_mask_0, x = key_75_cast_fp16)[name = string("key_token_473_cast_fp16")];
            tensor<int32, [4]> value_token_473_begin_0 = const()[name = string("value_token_473_begin_0"), val = tensor<int32, [4]>([0, 0, 12, 0])];
            tensor<int32, [4]> value_token_473_end_0 = const()[name = string("value_token_473_end_0"), val = tensor<int32, [4]>([1, 16, 13, 128])];
            tensor<bool, [4]> value_token_473_end_mask_0 = const()[name = string("value_token_473_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_473_squeeze_mask_0 = const()[name = string("value_token_473_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_473_cast_fp16 = slice_by_index(begin = value_token_473_begin_0, end = value_token_473_end_0, end_mask = value_token_473_end_mask_0, squeeze_mask = value_token_473_squeeze_mask_0, x = value_271_cast_fp16)[name = string("value_token_473_cast_fp16")];
            tensor<int32, [1]> var_12809_axes_0 = const()[name = string("op_12809_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_12809_cast_fp16 = expand_dims(axes = var_12809_axes_0, x = key_token_473_cast_fp16)[name = string("op_12809_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_12810_cast_fp16 = mul(x = state_945_cast_fp16, y = var_12809_cast_fp16)[name = string("op_12810_cast_fp16")];
            tensor<int32, [1]> memory_473_axes_0 = const()[name = string("memory_473_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_473_keep_dims_0 = const()[name = string("memory_473_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_473_cast_fp16 = reduce_sum(axes = memory_473_axes_0, keep_dims = memory_473_keep_dims_0, x = var_12810_cast_fp16)[name = string("memory_473_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_12813_cast_fp16 = sub(x = value_token_473_cast_fp16, y = memory_473_cast_fp16)[name = string("op_12813_cast_fp16")];
            tensor<int32, [3]> var_12816_begin_0 = const()[name = string("op_12816_begin_0"), val = tensor<int32, [3]>([0, 0, 12])];
            tensor<int32, [3]> var_12816_end_0 = const()[name = string("op_12816_end_0"), val = tensor<int32, [3]>([1, 16, 13])];
            tensor<bool, [3]> var_12816_end_mask_0 = const()[name = string("op_12816_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_12816_squeeze_mask_0 = const()[name = string("op_12816_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_12816_cast_fp16 = slice_by_index(begin = var_12816_begin_0, end = var_12816_end_0, end_mask = var_12816_end_mask_0, squeeze_mask = var_12816_squeeze_mask_0, x = beta_cast_fp16)[name = string("op_12816_cast_fp16")];
            tensor<int32, [1]> var_12817_axes_0 = const()[name = string("op_12817_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_12817_cast_fp16 = expand_dims(axes = var_12817_axes_0, x = var_12816_cast_fp16)[name = string("op_12817_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_473_cast_fp16 = mul(x = var_12813_cast_fp16, y = var_12817_cast_fp16)[name = string("delta_473_cast_fp16")];
            tensor<int32, [1]> var_12820_axes_0 = const()[name = string("op_12820_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_12820_cast_fp16 = expand_dims(axes = var_12820_axes_0, x = delta_473_cast_fp16)[name = string("op_12820_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_12821_cast_fp16 = mul(x = var_12809_cast_fp16, y = var_12820_cast_fp16)[name = string("op_12821_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_947_cast_fp16 = add(x = state_945_cast_fp16, y = var_12821_cast_fp16)[name = string("state_947_cast_fp16")];
            tensor<int32, [4]> var_12825_begin_0 = const()[name = string("op_12825_begin_0"), val = tensor<int32, [4]>([0, 0, 12, 0])];
            tensor<int32, [4]> var_12825_end_0 = const()[name = string("op_12825_end_0"), val = tensor<int32, [4]>([1, 16, 13, 128])];
            tensor<bool, [4]> var_12825_end_mask_0 = const()[name = string("op_12825_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_12825_squeeze_mask_0 = const()[name = string("op_12825_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_12825_cast_fp16 = slice_by_index(begin = var_12825_begin_0, end = var_12825_end_0, end_mask = var_12825_end_mask_0, squeeze_mask = var_12825_squeeze_mask_0, x = _inversed_query_75_cast_fp16)[name = string("op_12825_cast_fp16")];
            tensor<int32, [1]> var_12826_axes_0 = const()[name = string("op_12826_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_12826_cast_fp16 = expand_dims(axes = var_12826_axes_0, x = var_12825_cast_fp16)[name = string("op_12826_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_12827_cast_fp16 = mul(x = state_947_cast_fp16, y = var_12826_cast_fp16)[name = string("op_12827_cast_fp16")];
            tensor<int32, [1]> var_12829_axes_0 = const()[name = string("op_12829_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_12829_keep_dims_0 = const()[name = string("op_12829_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_12829_cast_fp16 = reduce_sum(axes = var_12829_axes_0, keep_dims = var_12829_keep_dims_0, x = var_12827_cast_fp16)[name = string("op_12829_cast_fp16")];
            tensor<int32, [3]> var_12832_begin_0 = const()[name = string("op_12832_begin_0"), val = tensor<int32, [3]>([0, 0, 13])];
            tensor<int32, [3]> var_12832_end_0 = const()[name = string("op_12832_end_0"), val = tensor<int32, [3]>([1, 16, 14])];
            tensor<bool, [3]> var_12832_end_mask_0 = const()[name = string("op_12832_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_12832_squeeze_mask_0 = const()[name = string("op_12832_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_12832_cast_fp16 = slice_by_index(begin = var_12832_begin_0, end = var_12832_end_0, end_mask = var_12832_end_mask_0, squeeze_mask = var_12832_squeeze_mask_0, x = decay_cast_fp16)[name = string("op_12832_cast_fp16")];
            tensor<fp16, [1, 16]> var_12833_cast_fp16 = exp(x = var_12832_cast_fp16)[name = string("op_12833_cast_fp16")];
            tensor<int32, [1]> var_12834_axes_0 = const()[name = string("op_12834_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_12834_cast_fp16 = expand_dims(axes = var_12834_axes_0, x = var_12833_cast_fp16)[name = string("op_12834_cast_fp16")];
            tensor<int32, [1]> var_12835_axes_0 = const()[name = string("op_12835_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_12835_cast_fp16 = expand_dims(axes = var_12835_axes_0, x = var_12834_cast_fp16)[name = string("op_12835_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_949_cast_fp16 = mul(x = state_947_cast_fp16, y = var_12835_cast_fp16)[name = string("state_949_cast_fp16")];
            tensor<int32, [4]> key_token_475_begin_0 = const()[name = string("key_token_475_begin_0"), val = tensor<int32, [4]>([0, 0, 13, 0])];
            tensor<int32, [4]> key_token_475_end_0 = const()[name = string("key_token_475_end_0"), val = tensor<int32, [4]>([1, 16, 14, 128])];
            tensor<bool, [4]> key_token_475_end_mask_0 = const()[name = string("key_token_475_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_475_squeeze_mask_0 = const()[name = string("key_token_475_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_475_cast_fp16 = slice_by_index(begin = key_token_475_begin_0, end = key_token_475_end_0, end_mask = key_token_475_end_mask_0, squeeze_mask = key_token_475_squeeze_mask_0, x = key_75_cast_fp16)[name = string("key_token_475_cast_fp16")];
            tensor<int32, [4]> value_token_475_begin_0 = const()[name = string("value_token_475_begin_0"), val = tensor<int32, [4]>([0, 0, 13, 0])];
            tensor<int32, [4]> value_token_475_end_0 = const()[name = string("value_token_475_end_0"), val = tensor<int32, [4]>([1, 16, 14, 128])];
            tensor<bool, [4]> value_token_475_end_mask_0 = const()[name = string("value_token_475_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_475_squeeze_mask_0 = const()[name = string("value_token_475_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_475_cast_fp16 = slice_by_index(begin = value_token_475_begin_0, end = value_token_475_end_0, end_mask = value_token_475_end_mask_0, squeeze_mask = value_token_475_squeeze_mask_0, x = value_271_cast_fp16)[name = string("value_token_475_cast_fp16")];
            tensor<int32, [1]> var_12843_axes_0 = const()[name = string("op_12843_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_12843_cast_fp16 = expand_dims(axes = var_12843_axes_0, x = key_token_475_cast_fp16)[name = string("op_12843_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_12844_cast_fp16 = mul(x = state_949_cast_fp16, y = var_12843_cast_fp16)[name = string("op_12844_cast_fp16")];
            tensor<int32, [1]> memory_475_axes_0 = const()[name = string("memory_475_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_475_keep_dims_0 = const()[name = string("memory_475_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_475_cast_fp16 = reduce_sum(axes = memory_475_axes_0, keep_dims = memory_475_keep_dims_0, x = var_12844_cast_fp16)[name = string("memory_475_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_12847_cast_fp16 = sub(x = value_token_475_cast_fp16, y = memory_475_cast_fp16)[name = string("op_12847_cast_fp16")];
            tensor<int32, [3]> var_12850_begin_0 = const()[name = string("op_12850_begin_0"), val = tensor<int32, [3]>([0, 0, 13])];
            tensor<int32, [3]> var_12850_end_0 = const()[name = string("op_12850_end_0"), val = tensor<int32, [3]>([1, 16, 14])];
            tensor<bool, [3]> var_12850_end_mask_0 = const()[name = string("op_12850_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_12850_squeeze_mask_0 = const()[name = string("op_12850_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_12850_cast_fp16 = slice_by_index(begin = var_12850_begin_0, end = var_12850_end_0, end_mask = var_12850_end_mask_0, squeeze_mask = var_12850_squeeze_mask_0, x = beta_cast_fp16)[name = string("op_12850_cast_fp16")];
            tensor<int32, [1]> var_12851_axes_0 = const()[name = string("op_12851_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_12851_cast_fp16 = expand_dims(axes = var_12851_axes_0, x = var_12850_cast_fp16)[name = string("op_12851_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_475_cast_fp16 = mul(x = var_12847_cast_fp16, y = var_12851_cast_fp16)[name = string("delta_475_cast_fp16")];
            tensor<int32, [1]> var_12854_axes_0 = const()[name = string("op_12854_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_12854_cast_fp16 = expand_dims(axes = var_12854_axes_0, x = delta_475_cast_fp16)[name = string("op_12854_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_12855_cast_fp16 = mul(x = var_12843_cast_fp16, y = var_12854_cast_fp16)[name = string("op_12855_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_951_cast_fp16 = add(x = state_949_cast_fp16, y = var_12855_cast_fp16)[name = string("state_951_cast_fp16")];
            tensor<int32, [4]> var_12859_begin_0 = const()[name = string("op_12859_begin_0"), val = tensor<int32, [4]>([0, 0, 13, 0])];
            tensor<int32, [4]> var_12859_end_0 = const()[name = string("op_12859_end_0"), val = tensor<int32, [4]>([1, 16, 14, 128])];
            tensor<bool, [4]> var_12859_end_mask_0 = const()[name = string("op_12859_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_12859_squeeze_mask_0 = const()[name = string("op_12859_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_12859_cast_fp16 = slice_by_index(begin = var_12859_begin_0, end = var_12859_end_0, end_mask = var_12859_end_mask_0, squeeze_mask = var_12859_squeeze_mask_0, x = _inversed_query_75_cast_fp16)[name = string("op_12859_cast_fp16")];
            tensor<int32, [1]> var_12860_axes_0 = const()[name = string("op_12860_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_12860_cast_fp16 = expand_dims(axes = var_12860_axes_0, x = var_12859_cast_fp16)[name = string("op_12860_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_12861_cast_fp16 = mul(x = state_951_cast_fp16, y = var_12860_cast_fp16)[name = string("op_12861_cast_fp16")];
            tensor<int32, [1]> var_12863_axes_0 = const()[name = string("op_12863_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_12863_keep_dims_0 = const()[name = string("op_12863_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_12863_cast_fp16 = reduce_sum(axes = var_12863_axes_0, keep_dims = var_12863_keep_dims_0, x = var_12861_cast_fp16)[name = string("op_12863_cast_fp16")];
            tensor<int32, [3]> var_12866_begin_0 = const()[name = string("op_12866_begin_0"), val = tensor<int32, [3]>([0, 0, 14])];
            tensor<int32, [3]> var_12866_end_0 = const()[name = string("op_12866_end_0"), val = tensor<int32, [3]>([1, 16, 15])];
            tensor<bool, [3]> var_12866_end_mask_0 = const()[name = string("op_12866_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_12866_squeeze_mask_0 = const()[name = string("op_12866_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_12866_cast_fp16 = slice_by_index(begin = var_12866_begin_0, end = var_12866_end_0, end_mask = var_12866_end_mask_0, squeeze_mask = var_12866_squeeze_mask_0, x = decay_cast_fp16)[name = string("op_12866_cast_fp16")];
            tensor<fp16, [1, 16]> var_12867_cast_fp16 = exp(x = var_12866_cast_fp16)[name = string("op_12867_cast_fp16")];
            tensor<int32, [1]> var_12868_axes_0 = const()[name = string("op_12868_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_12868_cast_fp16 = expand_dims(axes = var_12868_axes_0, x = var_12867_cast_fp16)[name = string("op_12868_cast_fp16")];
            tensor<int32, [1]> var_12869_axes_0 = const()[name = string("op_12869_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_12869_cast_fp16 = expand_dims(axes = var_12869_axes_0, x = var_12868_cast_fp16)[name = string("op_12869_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_953_cast_fp16 = mul(x = state_951_cast_fp16, y = var_12869_cast_fp16)[name = string("state_953_cast_fp16")];
            tensor<int32, [4]> key_token_477_begin_0 = const()[name = string("key_token_477_begin_0"), val = tensor<int32, [4]>([0, 0, 14, 0])];
            tensor<int32, [4]> key_token_477_end_0 = const()[name = string("key_token_477_end_0"), val = tensor<int32, [4]>([1, 16, 15, 128])];
            tensor<bool, [4]> key_token_477_end_mask_0 = const()[name = string("key_token_477_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_477_squeeze_mask_0 = const()[name = string("key_token_477_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_477_cast_fp16 = slice_by_index(begin = key_token_477_begin_0, end = key_token_477_end_0, end_mask = key_token_477_end_mask_0, squeeze_mask = key_token_477_squeeze_mask_0, x = key_75_cast_fp16)[name = string("key_token_477_cast_fp16")];
            tensor<int32, [4]> value_token_477_begin_0 = const()[name = string("value_token_477_begin_0"), val = tensor<int32, [4]>([0, 0, 14, 0])];
            tensor<int32, [4]> value_token_477_end_0 = const()[name = string("value_token_477_end_0"), val = tensor<int32, [4]>([1, 16, 15, 128])];
            tensor<bool, [4]> value_token_477_end_mask_0 = const()[name = string("value_token_477_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_477_squeeze_mask_0 = const()[name = string("value_token_477_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_477_cast_fp16 = slice_by_index(begin = value_token_477_begin_0, end = value_token_477_end_0, end_mask = value_token_477_end_mask_0, squeeze_mask = value_token_477_squeeze_mask_0, x = value_271_cast_fp16)[name = string("value_token_477_cast_fp16")];
            tensor<int32, [1]> var_12877_axes_0 = const()[name = string("op_12877_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_12877_cast_fp16 = expand_dims(axes = var_12877_axes_0, x = key_token_477_cast_fp16)[name = string("op_12877_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_12878_cast_fp16 = mul(x = state_953_cast_fp16, y = var_12877_cast_fp16)[name = string("op_12878_cast_fp16")];
            tensor<int32, [1]> memory_477_axes_0 = const()[name = string("memory_477_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_477_keep_dims_0 = const()[name = string("memory_477_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_477_cast_fp16 = reduce_sum(axes = memory_477_axes_0, keep_dims = memory_477_keep_dims_0, x = var_12878_cast_fp16)[name = string("memory_477_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_12881_cast_fp16 = sub(x = value_token_477_cast_fp16, y = memory_477_cast_fp16)[name = string("op_12881_cast_fp16")];
            tensor<int32, [3]> var_12884_begin_0 = const()[name = string("op_12884_begin_0"), val = tensor<int32, [3]>([0, 0, 14])];
            tensor<int32, [3]> var_12884_end_0 = const()[name = string("op_12884_end_0"), val = tensor<int32, [3]>([1, 16, 15])];
            tensor<bool, [3]> var_12884_end_mask_0 = const()[name = string("op_12884_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_12884_squeeze_mask_0 = const()[name = string("op_12884_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_12884_cast_fp16 = slice_by_index(begin = var_12884_begin_0, end = var_12884_end_0, end_mask = var_12884_end_mask_0, squeeze_mask = var_12884_squeeze_mask_0, x = beta_cast_fp16)[name = string("op_12884_cast_fp16")];
            tensor<int32, [1]> var_12885_axes_0 = const()[name = string("op_12885_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_12885_cast_fp16 = expand_dims(axes = var_12885_axes_0, x = var_12884_cast_fp16)[name = string("op_12885_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_477_cast_fp16 = mul(x = var_12881_cast_fp16, y = var_12885_cast_fp16)[name = string("delta_477_cast_fp16")];
            tensor<int32, [1]> var_12888_axes_0 = const()[name = string("op_12888_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_12888_cast_fp16 = expand_dims(axes = var_12888_axes_0, x = delta_477_cast_fp16)[name = string("op_12888_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_12889_cast_fp16 = mul(x = var_12877_cast_fp16, y = var_12888_cast_fp16)[name = string("op_12889_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_955_cast_fp16 = add(x = state_953_cast_fp16, y = var_12889_cast_fp16)[name = string("state_955_cast_fp16")];
            tensor<int32, [4]> var_12893_begin_0 = const()[name = string("op_12893_begin_0"), val = tensor<int32, [4]>([0, 0, 14, 0])];
            tensor<int32, [4]> var_12893_end_0 = const()[name = string("op_12893_end_0"), val = tensor<int32, [4]>([1, 16, 15, 128])];
            tensor<bool, [4]> var_12893_end_mask_0 = const()[name = string("op_12893_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_12893_squeeze_mask_0 = const()[name = string("op_12893_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_12893_cast_fp16 = slice_by_index(begin = var_12893_begin_0, end = var_12893_end_0, end_mask = var_12893_end_mask_0, squeeze_mask = var_12893_squeeze_mask_0, x = _inversed_query_75_cast_fp16)[name = string("op_12893_cast_fp16")];
            tensor<int32, [1]> var_12894_axes_0 = const()[name = string("op_12894_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_12894_cast_fp16 = expand_dims(axes = var_12894_axes_0, x = var_12893_cast_fp16)[name = string("op_12894_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_12895_cast_fp16 = mul(x = state_955_cast_fp16, y = var_12894_cast_fp16)[name = string("op_12895_cast_fp16")];
            tensor<int32, [1]> var_12897_axes_0 = const()[name = string("op_12897_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_12897_keep_dims_0 = const()[name = string("op_12897_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_12897_cast_fp16 = reduce_sum(axes = var_12897_axes_0, keep_dims = var_12897_keep_dims_0, x = var_12895_cast_fp16)[name = string("op_12897_cast_fp16")];
            tensor<int32, [3]> var_12900_begin_0 = const()[name = string("op_12900_begin_0"), val = tensor<int32, [3]>([0, 0, 15])];
            tensor<int32, [3]> var_12900_end_0 = const()[name = string("op_12900_end_0"), val = tensor<int32, [3]>([1, 16, 16])];
            tensor<bool, [3]> var_12900_end_mask_0 = const()[name = string("op_12900_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_12900_squeeze_mask_0 = const()[name = string("op_12900_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_12900_cast_fp16 = slice_by_index(begin = var_12900_begin_0, end = var_12900_end_0, end_mask = var_12900_end_mask_0, squeeze_mask = var_12900_squeeze_mask_0, x = decay_cast_fp16)[name = string("op_12900_cast_fp16")];
            tensor<fp16, [1, 16]> var_12901_cast_fp16 = exp(x = var_12900_cast_fp16)[name = string("op_12901_cast_fp16")];
            tensor<int32, [1]> var_12902_axes_0 = const()[name = string("op_12902_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_12902_cast_fp16 = expand_dims(axes = var_12902_axes_0, x = var_12901_cast_fp16)[name = string("op_12902_cast_fp16")];
            tensor<int32, [1]> var_12903_axes_0 = const()[name = string("op_12903_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_12903_cast_fp16 = expand_dims(axes = var_12903_axes_0, x = var_12902_cast_fp16)[name = string("op_12903_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_957_cast_fp16 = mul(x = state_955_cast_fp16, y = var_12903_cast_fp16)[name = string("state_957_cast_fp16")];
            tensor<int32, [4]> key_token_begin_0 = const()[name = string("key_token_begin_0"), val = tensor<int32, [4]>([0, 0, 15, 0])];
            tensor<int32, [4]> key_token_end_0 = const()[name = string("key_token_end_0"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<bool, [4]> key_token_end_mask_0 = const()[name = string("key_token_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_squeeze_mask_0 = const()[name = string("key_token_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_cast_fp16 = slice_by_index(begin = key_token_begin_0, end = key_token_end_0, end_mask = key_token_end_mask_0, squeeze_mask = key_token_squeeze_mask_0, x = key_75_cast_fp16)[name = string("key_token_cast_fp16")];
            tensor<int32, [4]> value_token_begin_0 = const()[name = string("value_token_begin_0"), val = tensor<int32, [4]>([0, 0, 15, 0])];
            tensor<int32, [4]> value_token_end_0 = const()[name = string("value_token_end_0"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<bool, [4]> value_token_end_mask_0 = const()[name = string("value_token_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_squeeze_mask_0 = const()[name = string("value_token_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_cast_fp16 = slice_by_index(begin = value_token_begin_0, end = value_token_end_0, end_mask = value_token_end_mask_0, squeeze_mask = value_token_squeeze_mask_0, x = value_271_cast_fp16)[name = string("value_token_cast_fp16")];
            tensor<int32, [1]> var_12911_axes_0 = const()[name = string("op_12911_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_12911_cast_fp16 = expand_dims(axes = var_12911_axes_0, x = key_token_cast_fp16)[name = string("op_12911_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_12912_cast_fp16 = mul(x = state_957_cast_fp16, y = var_12911_cast_fp16)[name = string("op_12912_cast_fp16")];
            tensor<int32, [1]> memory_axes_0 = const()[name = string("memory_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_keep_dims_0 = const()[name = string("memory_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_cast_fp16 = reduce_sum(axes = memory_axes_0, keep_dims = memory_keep_dims_0, x = var_12912_cast_fp16)[name = string("memory_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_12915_cast_fp16 = sub(x = value_token_cast_fp16, y = memory_cast_fp16)[name = string("op_12915_cast_fp16")];
            tensor<int32, [3]> var_12918_begin_0 = const()[name = string("op_12918_begin_0"), val = tensor<int32, [3]>([0, 0, 15])];
            tensor<int32, [3]> var_12918_end_0 = const()[name = string("op_12918_end_0"), val = tensor<int32, [3]>([1, 16, 16])];
            tensor<bool, [3]> var_12918_end_mask_0 = const()[name = string("op_12918_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_12918_squeeze_mask_0 = const()[name = string("op_12918_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_12918_cast_fp16 = slice_by_index(begin = var_12918_begin_0, end = var_12918_end_0, end_mask = var_12918_end_mask_0, squeeze_mask = var_12918_squeeze_mask_0, x = beta_cast_fp16)[name = string("op_12918_cast_fp16")];
            tensor<int32, [1]> var_12919_axes_0 = const()[name = string("op_12919_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_12919_cast_fp16 = expand_dims(axes = var_12919_axes_0, x = var_12918_cast_fp16)[name = string("op_12919_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_cast_fp16 = mul(x = var_12915_cast_fp16, y = var_12919_cast_fp16)[name = string("delta_cast_fp16")];
            tensor<int32, [1]> var_12922_axes_0 = const()[name = string("op_12922_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_12922_cast_fp16 = expand_dims(axes = var_12922_axes_0, x = delta_cast_fp16)[name = string("op_12922_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_12923_cast_fp16 = mul(x = var_12911_cast_fp16, y = var_12922_cast_fp16)[name = string("op_12923_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> rec18_out = add(x = state_957_cast_fp16, y = var_12923_cast_fp16)[name = string("state_cast_fp16")];
            tensor<int32, [4]> var_12927_begin_0 = const()[name = string("op_12927_begin_0"), val = tensor<int32, [4]>([0, 0, 15, 0])];
            tensor<int32, [4]> var_12927_end_0 = const()[name = string("op_12927_end_0"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<bool, [4]> var_12927_end_mask_0 = const()[name = string("op_12927_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_12927_squeeze_mask_0 = const()[name = string("op_12927_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_12927_cast_fp16 = slice_by_index(begin = var_12927_begin_0, end = var_12927_end_0, end_mask = var_12927_end_mask_0, squeeze_mask = var_12927_squeeze_mask_0, x = _inversed_query_75_cast_fp16)[name = string("op_12927_cast_fp16")];
            tensor<int32, [1]> var_12928_axes_0 = const()[name = string("op_12928_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_12928_cast_fp16 = expand_dims(axes = var_12928_axes_0, x = var_12927_cast_fp16)[name = string("op_12928_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_12929_cast_fp16 = mul(x = rec18_out, y = var_12928_cast_fp16)[name = string("op_12929_cast_fp16")];
            tensor<int32, [1]> var_12931_axes_0 = const()[name = string("op_12931_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_12931_keep_dims_0 = const()[name = string("op_12931_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_12931_cast_fp16 = reduce_sum(axes = var_12931_axes_0, keep_dims = var_12931_keep_dims_0, x = var_12929_cast_fp16)[name = string("op_12931_cast_fp16")];
            int32 attention_141_axis_0 = const()[name = string("attention_141_axis_0"), val = int32(1)];
            tensor<fp16, [1, 16, 16, 128]> attention_141_cast_fp16 = stack(axis = attention_141_axis_0, values = (var_12421_cast_fp16, var_12455_cast_fp16, var_12489_cast_fp16, var_12523_cast_fp16, var_12557_cast_fp16, var_12591_cast_fp16, var_12625_cast_fp16, var_12659_cast_fp16, var_12693_cast_fp16, var_12727_cast_fp16, var_12761_cast_fp16, var_12795_cast_fp16, var_12829_cast_fp16, var_12863_cast_fp16, var_12897_cast_fp16, var_12931_cast_fp16))[name = string("attention_141_cast_fp16")];
            tensor<int32, [2]> var_12934 = const()[name = string("op_12934"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [256, 128]> attention_143_cast_fp16 = reshape(shape = var_12934, x = attention_141_cast_fp16)[name = string("attention_143_cast_fp16")];
            tensor<int32, [2]> var_12936 = const()[name = string("op_12936"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [256, 128]> input_241_cast_fp16 = reshape(shape = var_12936, x = linear_143_cast_fp16)[name = string("input_241_cast_fp16")];
            tensor<fp16, [256, 128]> var_12938_cast_fp16 = mul(x = attention_143_cast_fp16, y = attention_143_cast_fp16)[name = string("op_12938_cast_fp16")];
            tensor<int32, [1]> variance_119_axes_0 = const()[name = string("variance_119_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_119_keep_dims_0 = const()[name = string("variance_119_keep_dims_0"), val = bool(true)];
            tensor<fp16, [256, 1]> variance_119_cast_fp16 = reduce_mean(axes = variance_119_axes_0, keep_dims = variance_119_keep_dims_0, x = var_12938_cast_fp16)[name = string("variance_119_cast_fp16")];
            fp16 var_12941_to_fp16 = const()[name = string("op_12941_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [256, 1]> var_12942_cast_fp16 = add(x = variance_119_cast_fp16, y = var_12941_to_fp16)[name = string("op_12942_cast_fp16")];
            fp32 var_12943_epsilon_0 = const()[name = string("op_12943_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [256, 1]> var_12943_cast_fp16 = rsqrt(epsilon = var_12943_epsilon_0, x = var_12942_cast_fp16)[name = string("op_12943_cast_fp16")];
            tensor<fp16, [256, 128]> attention_145_cast_fp16 = mul(x = attention_143_cast_fp16, y = var_12943_cast_fp16)[name = string("attention_145_cast_fp16")];
            tensor<fp16, [128]> final_group_2_gated_norm_promoted_to_fp16 = const()[name = string("final_group_2_gated_norm_promoted_to_fp16"), val = tensor<fp16, [128]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(291054720)))];
            tensor<fp16, [256, 128]> attention_147_cast_fp16 = mul(x = attention_145_cast_fp16, y = final_group_2_gated_norm_promoted_to_fp16)[name = string("attention_147_cast_fp16")];
            tensor<fp16, [256, 128]> var_12946_cast_fp16 = silu(x = input_241_cast_fp16)[name = string("op_12946_cast_fp16")];
            tensor<fp16, [256, 128]> attention_149_cast_fp16 = mul(x = attention_147_cast_fp16, y = var_12946_cast_fp16)[name = string("attention_149_cast_fp16")];
            tensor<int32, [2]> var_12948 = const()[name = string("op_12948"), val = tensor<int32, [2]>([16, 2048])];
            tensor<fp16, [16, 2048]> input_243_cast_fp16 = reshape(shape = var_12948, x = attention_149_cast_fp16)[name = string("input_243_cast_fp16")];
            tensor<fp16, [1024, 2048]> final_group_2_out_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 2048]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(291055040))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(292627968))))[name = string("final_group_2_out_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 1024]> linear_144_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = final_group_2_out_proj_weight_promoted_to_fp16_palettized, x = input_243_cast_fp16)[name = string("linear_144_cast_fp16")];
            tensor<fp16, [16, 1024]> value_273_cast_fp16 = add(x = value_261_cast_fp16, y = linear_144_cast_fp16)[name = string("value_273_cast_fp16")];
            tensor<fp16, [16, 1024]> var_12953_cast_fp16 = mul(x = value_273_cast_fp16, y = value_273_cast_fp16)[name = string("op_12953_cast_fp16")];
            tensor<int32, [1]> variance_121_axes_0 = const()[name = string("variance_121_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_121_keep_dims_0 = const()[name = string("variance_121_keep_dims_0"), val = bool(true)];
            tensor<fp16, [16, 1]> variance_121_cast_fp16 = reduce_mean(axes = variance_121_axes_0, keep_dims = variance_121_keep_dims_0, x = var_12953_cast_fp16)[name = string("variance_121_cast_fp16")];
            fp16 var_12956_to_fp16 = const()[name = string("op_12956_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [16, 1]> var_12957_cast_fp16 = add(x = variance_121_cast_fp16, y = var_12956_to_fp16)[name = string("op_12957_cast_fp16")];
            fp32 var_12958_epsilon_0 = const()[name = string("op_12958_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [16, 1]> var_12958_cast_fp16 = rsqrt(epsilon = var_12958_epsilon_0, x = var_12957_cast_fp16)[name = string("op_12958_cast_fp16")];
            tensor<fp16, [16, 1024]> var_12959_cast_fp16 = mul(x = value_273_cast_fp16, y = var_12958_cast_fp16)[name = string("op_12959_cast_fp16")];
            tensor<fp16, [1024]> var_12961_to_fp16 = const()[name = string("op_12961_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(292636224)))];
            tensor<fp16, [16, 1024]> input_245_cast_fp16 = mul(x = var_12959_cast_fp16, y = var_12961_to_fp16)[name = string("input_245_cast_fp16")];
            tensor<fp16, [3584, 1024]> final_group_2_gate_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(292638336))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(295390912))))[name = string("final_group_2_gate_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 3584]> linear_145_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = final_group_2_gate_proj_weight_promoted_to_fp16_palettized, x = input_245_cast_fp16)[name = string("linear_145_cast_fp16")];
            tensor<fp16, [16, 3584]> var_12965_cast_fp16 = silu(x = linear_145_cast_fp16)[name = string("op_12965_cast_fp16")];
            tensor<fp16, [3584, 1024]> final_group_2_up_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(295419648))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(298172224))))[name = string("final_group_2_up_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 3584]> linear_146_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = final_group_2_up_proj_weight_promoted_to_fp16_palettized, x = input_245_cast_fp16)[name = string("linear_146_cast_fp16")];
            tensor<fp16, [16, 3584]> input_249_cast_fp16 = mul(x = var_12965_cast_fp16, y = linear_146_cast_fp16)[name = string("input_249_cast_fp16")];
            tensor<fp16, [1024, 3584]> final_group_2_down_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 3584]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(298200960))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(300953536))))[name = string("final_group_2_down_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 1024]> linear_147_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = final_group_2_down_proj_weight_promoted_to_fp16_palettized, x = input_249_cast_fp16)[name = string("linear_147_cast_fp16")];
            tensor<fp16, [16, 1024]> hidden_out = add(x = value_273_cast_fp16, y = linear_147_cast_fp16)[name = string("value_275_cast_fp16")];
            int32 var_12990 = const()[name = string("op_12990"), val = int32(-1)];
            tensor<fp16, [16, 1024]> var_12997_cast_fp16 = mul(x = hidden_out, y = hidden_out)[name = string("op_12997_cast_fp16")];
            tensor<int32, [1]> variance_123_axes_0 = const()[name = string("variance_123_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_123_keep_dims_0 = const()[name = string("variance_123_keep_dims_0"), val = bool(true)];
            tensor<fp16, [16, 1]> variance_123_cast_fp16 = reduce_mean(axes = variance_123_axes_0, keep_dims = variance_123_keep_dims_0, x = var_12997_cast_fp16)[name = string("variance_123_cast_fp16")];
            fp16 var_13000_to_fp16 = const()[name = string("op_13000_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [16, 1]> var_13001_cast_fp16 = add(x = variance_123_cast_fp16, y = var_13000_to_fp16)[name = string("op_13001_cast_fp16")];
            fp32 var_13002_epsilon_0 = const()[name = string("op_13002_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [16, 1]> var_13002_cast_fp16 = rsqrt(epsilon = var_13002_epsilon_0, x = var_13001_cast_fp16)[name = string("op_13002_cast_fp16")];
            tensor<fp16, [16, 1024]> var_13003_cast_fp16 = mul(x = hidden_out, y = var_13002_cast_fp16)[name = string("op_13003_cast_fp16")];
            tensor<fp16, [1024]> var_13005_to_fp16 = const()[name = string("op_13005_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(300961792)))];
            tensor<fp16, [16, 1024]> input_cast_fp16 = mul(x = var_13003_cast_fp16, y = var_13005_to_fp16)[name = string("input_cast_fp16")];
            tensor<fp16, [4096, 1024]> projection19_q_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [4096, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(300963904))), lut = tensor<fp16, [256, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(304109696))))[name = string("projection19_q_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 4096]> linear_148_cast_fp16 = linear(bias = linear_24_bias_0_to_fp16, weight = projection19_q_proj_weight_promoted_to_fp16_palettized, x = input_cast_fp16)[name = string("linear_148_cast_fp16")];
            tensor<int32, [3]> var_13009 = const()[name = string("op_13009"), val = tensor<int32, [3]>([16, 8, 512])];
            tensor<fp16, [16, 8, 512]> query_and_gate_cast_fp16 = reshape(shape = var_13009, x = linear_148_cast_fp16)[name = string("query_and_gate_cast_fp16")];
            tensor<int32, [2]> var_13011_split_sizes_0 = const()[name = string("op_13011_split_sizes_0"), val = tensor<int32, [2]>([256, 256])];
            int32 var_13011_axis_0 = const()[name = string("op_13011_axis_0"), val = int32(-1)];
            tensor<fp16, [16, 8, 256]> var_13011_cast_fp16_0, tensor<fp16, [16, 8, 256]> var_13011_cast_fp16_1 = split(axis = var_13011_axis_0, split_sizes = var_13011_split_sizes_0, x = query_and_gate_cast_fp16)[name = string("op_13011_cast_fp16")];
            tensor<fp16, [512, 1024]> projection19_k_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [512, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(304142528))), lut = tensor<fp16, [32, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(304535808))))[name = string("projection19_k_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 512]> linear_149_cast_fp16 = linear(bias = linear_25_bias_0_to_fp16, weight = projection19_k_proj_weight_promoted_to_fp16_palettized, x = input_cast_fp16)[name = string("linear_149_cast_fp16")];
            tensor<int32, [3]> var_13015 = const()[name = string("op_13015"), val = tensor<int32, [3]>([16, 2, 256])];
            tensor<fp16, [16, 2, 256]> value_279_cast_fp16 = reshape(shape = var_13015, x = linear_149_cast_fp16)[name = string("value_279_cast_fp16")];
            tensor<fp16, [512, 1024]> projection19_v_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [512, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(304539968))), lut = tensor<fp16, [32, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(304933248))))[name = string("projection19_v_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 512]> linear_150_cast_fp16 = linear(bias = linear_25_bias_0_to_fp16, weight = projection19_v_proj_weight_promoted_to_fp16_palettized, x = input_cast_fp16)[name = string("linear_150_cast_fp16")];
            tensor<int32, [3]> var_13019 = const()[name = string("op_13019"), val = tensor<int32, [3]>([16, 2, 256])];
            tensor<fp16, [16, 2, 256]> value_cast_fp16 = reshape(shape = var_13019, x = linear_150_cast_fp16)[name = string("value_cast_fp16")];
            tensor<fp16, [16, 8, 256]> var_13021_cast_fp16 = mul(x = var_13011_cast_fp16_0, y = var_13011_cast_fp16_0)[name = string("op_13021_cast_fp16")];
            tensor<int32, [1]> variance_125_axes_0 = const()[name = string("variance_125_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_125_keep_dims_0 = const()[name = string("variance_125_keep_dims_0"), val = bool(true)];
            tensor<fp16, [16, 8, 1]> variance_125_cast_fp16 = reduce_mean(axes = variance_125_axes_0, keep_dims = variance_125_keep_dims_0, x = var_13021_cast_fp16)[name = string("variance_125_cast_fp16")];
            fp16 var_13024_to_fp16 = const()[name = string("op_13024_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [16, 8, 1]> var_13025_cast_fp16 = add(x = variance_125_cast_fp16, y = var_13024_to_fp16)[name = string("op_13025_cast_fp16")];
            fp32 var_13026_epsilon_0 = const()[name = string("op_13026_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [16, 8, 1]> var_13026_cast_fp16 = rsqrt(epsilon = var_13026_epsilon_0, x = var_13025_cast_fp16)[name = string("op_13026_cast_fp16")];
            tensor<fp16, [16, 8, 256]> var_13027_cast_fp16 = mul(x = var_13011_cast_fp16_0, y = var_13026_cast_fp16)[name = string("op_13027_cast_fp16")];
            tensor<fp16, [256]> var_13029_to_fp16 = const()[name = string("op_13029_to_fp16"), val = tensor<fp16, [256]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(304937408)))];
            tensor<fp16, [16, 8, 256]> var_13030_cast_fp16 = mul(x = var_13027_cast_fp16, y = var_13029_to_fp16)[name = string("op_13030_cast_fp16")];
            tensor<int32, [3]> var_13031 = const()[name = string("op_13031"), val = tensor<int32, [3]>([1, 0, 2])];
            tensor<int32, [1]> value_281_axes_0 = const()[name = string("value_281_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [8, 16, 256]> var_13032_cast_fp16 = transpose(perm = var_13031, x = var_13030_cast_fp16)[name = string("transpose_3")];
            tensor<fp16, [1, 8, 16, 256]> value_281_cast_fp16 = expand_dims(axes = value_281_axes_0, x = var_13032_cast_fp16)[name = string("value_281_cast_fp16")];
            tensor<fp16, [16, 2, 256]> var_13034_cast_fp16 = mul(x = value_279_cast_fp16, y = value_279_cast_fp16)[name = string("op_13034_cast_fp16")];
            tensor<int32, [1]> variance_axes_0 = const()[name = string("variance_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_keep_dims_0 = const()[name = string("variance_keep_dims_0"), val = bool(true)];
            tensor<fp16, [16, 2, 1]> variance_cast_fp16 = reduce_mean(axes = variance_axes_0, keep_dims = variance_keep_dims_0, x = var_13034_cast_fp16)[name = string("variance_cast_fp16")];
            fp16 var_13037_to_fp16 = const()[name = string("op_13037_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [16, 2, 1]> var_13038_cast_fp16 = add(x = variance_cast_fp16, y = var_13037_to_fp16)[name = string("op_13038_cast_fp16")];
            fp32 var_13039_epsilon_0 = const()[name = string("op_13039_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [16, 2, 1]> var_13039_cast_fp16 = rsqrt(epsilon = var_13039_epsilon_0, x = var_13038_cast_fp16)[name = string("op_13039_cast_fp16")];
            tensor<fp16, [16, 2, 256]> var_13040_cast_fp16 = mul(x = value_279_cast_fp16, y = var_13039_cast_fp16)[name = string("op_13040_cast_fp16")];
            tensor<fp16, [256]> var_13042_to_fp16 = const()[name = string("op_13042_to_fp16"), val = tensor<fp16, [256]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(304937984)))];
            tensor<fp16, [16, 2, 256]> var_13043_cast_fp16 = mul(x = var_13040_cast_fp16, y = var_13042_to_fp16)[name = string("op_13043_cast_fp16")];
            tensor<int32, [3]> var_13044 = const()[name = string("op_13044"), val = tensor<int32, [3]>([1, 0, 2])];
            tensor<int32, [1]> value_283_axes_0 = const()[name = string("value_283_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [2, 16, 256]> var_13045_cast_fp16 = transpose(perm = var_13044, x = var_13043_cast_fp16)[name = string("transpose_2")];
            tensor<fp16, [1, 2, 16, 256]> value_283_cast_fp16 = expand_dims(axes = value_283_axes_0, x = var_13045_cast_fp16)[name = string("value_283_cast_fp16")];
            tensor<int32, [4]> rotated_17_begin_0 = const()[name = string("rotated_17_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> rotated_17_end_0 = const()[name = string("rotated_17_end_0"), val = tensor<int32, [4]>([1, 8, 16, 64])];
            tensor<bool, [4]> rotated_17_end_mask_0 = const()[name = string("rotated_17_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
            tensor<fp16, [1, 8, 16, 64]> rotated_17_cast_fp16 = slice_by_index(begin = rotated_17_begin_0, end = rotated_17_end_0, end_mask = rotated_17_end_mask_0, x = value_281_cast_fp16)[name = string("rotated_17_cast_fp16")];
            tensor<int32, [4]> passthrough_17_begin_0 = const()[name = string("passthrough_17_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 64])];
            tensor<int32, [4]> passthrough_17_end_0 = const()[name = string("passthrough_17_end_0"), val = tensor<int32, [4]>([1, 8, 16, 256])];
            tensor<bool, [4]> passthrough_17_end_mask_0 = const()[name = string("passthrough_17_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
            tensor<fp16, [1, 8, 16, 192]> passthrough_17_cast_fp16 = slice_by_index(begin = passthrough_17_begin_0, end = passthrough_17_end_0, end_mask = passthrough_17_end_mask_0, x = value_281_cast_fp16)[name = string("passthrough_17_cast_fp16")];
            tensor<int32, [2]> var_13049_split_sizes_0 = const()[name = string("op_13049_split_sizes_0"), val = tensor<int32, [2]>([32, 32])];
            int32 var_13049_axis_0 = const()[name = string("op_13049_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 8, 16, 32]> var_13049_cast_fp16_0, tensor<fp16, [1, 8, 16, 32]> var_13049_cast_fp16_1 = split(axis = var_13049_axis_0, split_sizes = var_13049_split_sizes_0, x = rotated_17_cast_fp16)[name = string("op_13049_cast_fp16")];
            fp16 const_175_promoted_to_fp16 = const()[name = string("const_175_promoted_to_fp16"), val = fp16(-0x1p+0)];
            tensor<fp16, [1, 8, 16, 32]> var_13051_cast_fp16 = mul(x = var_13049_cast_fp16_1, y = const_175_promoted_to_fp16)[name = string("op_13051_cast_fp16")];
            bool rotated_half_17_interleave_0 = const()[name = string("rotated_half_17_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 8, 16, 64]> rotated_half_17_cast_fp16 = concat(axis = var_12990, interleave = rotated_half_17_interleave_0, values = (var_13051_cast_fp16, var_13049_cast_fp16_0))[name = string("rotated_half_17_cast_fp16")];
            tensor<fp16, [1, 8, 16, 64]> var_13054_cast_fp16 = mul(x = rotated_17_cast_fp16, y = cos)[name = string("op_13054_cast_fp16")];
            tensor<fp16, [1, 8, 16, 64]> var_13055_cast_fp16 = mul(x = rotated_half_17_cast_fp16, y = sin)[name = string("op_13055_cast_fp16")];
            tensor<fp16, [1, 8, 16, 64]> var_13056_cast_fp16 = add(x = var_13054_cast_fp16, y = var_13055_cast_fp16)[name = string("op_13056_cast_fp16")];
            bool query_interleave_0 = const()[name = string("query_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 8, 16, 256]> query_cast_fp16 = concat(axis = var_12990, interleave = query_interleave_0, values = (var_13056_cast_fp16, passthrough_17_cast_fp16))[name = string("query_cast_fp16")];
            tensor<int32, [4]> rotated_begin_0 = const()[name = string("rotated_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> rotated_end_0 = const()[name = string("rotated_end_0"), val = tensor<int32, [4]>([1, 2, 16, 64])];
            tensor<bool, [4]> rotated_end_mask_0 = const()[name = string("rotated_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
            tensor<fp16, [1, 2, 16, 64]> rotated_cast_fp16 = slice_by_index(begin = rotated_begin_0, end = rotated_end_0, end_mask = rotated_end_mask_0, x = value_283_cast_fp16)[name = string("rotated_cast_fp16")];
            tensor<int32, [4]> passthrough_begin_0 = const()[name = string("passthrough_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 64])];
            tensor<int32, [4]> passthrough_end_0 = const()[name = string("passthrough_end_0"), val = tensor<int32, [4]>([1, 2, 16, 256])];
            tensor<bool, [4]> passthrough_end_mask_0 = const()[name = string("passthrough_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
            tensor<fp16, [1, 2, 16, 192]> passthrough_cast_fp16 = slice_by_index(begin = passthrough_begin_0, end = passthrough_end_0, end_mask = passthrough_end_mask_0, x = value_283_cast_fp16)[name = string("passthrough_cast_fp16")];
            tensor<int32, [2]> var_13061_split_sizes_0 = const()[name = string("op_13061_split_sizes_0"), val = tensor<int32, [2]>([32, 32])];
            int32 var_13061_axis_0 = const()[name = string("op_13061_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 2, 16, 32]> var_13061_cast_fp16_0, tensor<fp16, [1, 2, 16, 32]> var_13061_cast_fp16_1 = split(axis = var_13061_axis_0, split_sizes = var_13061_split_sizes_0, x = rotated_cast_fp16)[name = string("op_13061_cast_fp16")];
            fp16 const_176_promoted_to_fp16 = const()[name = string("const_176_promoted_to_fp16"), val = fp16(-0x1p+0)];
            tensor<fp16, [1, 2, 16, 32]> var_13063_cast_fp16 = mul(x = var_13061_cast_fp16_1, y = const_176_promoted_to_fp16)[name = string("op_13063_cast_fp16")];
            bool rotated_half_interleave_0 = const()[name = string("rotated_half_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 2, 16, 64]> rotated_half_cast_fp16 = concat(axis = var_12990, interleave = rotated_half_interleave_0, values = (var_13063_cast_fp16, var_13061_cast_fp16_0))[name = string("rotated_half_cast_fp16")];
            tensor<fp16, [1, 2, 16, 64]> var_13066_cast_fp16 = mul(x = rotated_cast_fp16, y = cos)[name = string("op_13066_cast_fp16")];
            tensor<fp16, [1, 2, 16, 64]> var_13067_cast_fp16 = mul(x = rotated_half_cast_fp16, y = sin)[name = string("op_13067_cast_fp16")];
            tensor<fp16, [1, 2, 16, 64]> var_13068_cast_fp16 = add(x = var_13066_cast_fp16, y = var_13067_cast_fp16)[name = string("op_13068_cast_fp16")];
            bool key_interleave_0 = const()[name = string("key_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 2, 16, 256]> key_cast_fp16 = concat(axis = var_12990, interleave = key_interleave_0, values = (var_13068_cast_fp16, passthrough_cast_fp16))[name = string("key_cast_fp16")];
            tensor<int32, [4]> var_13071 = const()[name = string("op_13071"), val = tensor<int32, [4]>([2, 4, 16, 256])];
            tensor<fp16, [2, 4, 16, 256]> var_13072_cast_fp16 = reshape(shape = var_13071, x = query_cast_fp16)[name = string("op_13072_cast_fp16")];
            tensor<int32, [4]> var_13073 = const()[name = string("op_13073"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<int32, [3]> var_13075 = const()[name = string("op_13075"), val = tensor<int32, [3]>([2, 16, 256])];
            tensor<fp16, [2, 16, 256]> k19 = reshape(shape = var_13075, x = key_cast_fp16)[name = string("op_13076_cast_fp16")];
            tensor<int32, [3]> var_13077 = const()[name = string("op_13077"), val = tensor<int32, [3]>([1, 0, 2])];
            tensor<int32, [2]> var_13079 = const()[name = string("op_13079"), val = tensor<int32, [2]>([16, 2048])];
            tensor<fp16, [16, 2048]> gate19 = reshape(shape = var_13079, x = var_13011_cast_fp16_1)[name = string("op_13080_cast_fp16")];
            tensor<fp16, [2, 16, 4, 256]> q19 = transpose(perm = var_13073, x = var_13072_cast_fp16)[name = string("transpose_0")];
            tensor<fp16, [2, 16, 256]> v19 = transpose(perm = var_13077, x = value_cast_fp16)[name = string("transpose_1")];
        } -> (hidden_out, conv0_out, rec0_out, conv1_out, rec1_out, conv2_out, rec2_out, conv4_out, rec4_out, conv5_out, rec5_out, conv6_out, rec6_out, conv8_out, rec8_out, conv9_out, rec9_out, conv10_out, rec10_out, conv12_out, rec12_out, conv13_out, rec13_out, conv14_out, rec14_out, conv16_out, rec16_out, conv17_out, rec17_out, conv18_out, rec18_out, q19, k19, v19, gate19);
    func prefill4<ios18>(tensor<fp16, [1, 6144, 3]> conv0, tensor<fp16, [1, 6144, 3]> conv1, tensor<fp16, [1, 6144, 3]> conv10, tensor<fp16, [1, 6144, 3]> conv12, tensor<fp16, [1, 6144, 3]> conv13, tensor<fp16, [1, 6144, 3]> conv14, tensor<fp16, [1, 6144, 3]> conv16, tensor<fp16, [1, 6144, 3]> conv17, tensor<fp16, [1, 6144, 3]> conv18, tensor<fp16, [1, 6144, 3]> conv2, tensor<fp16, [1, 6144, 3]> conv4, tensor<fp16, [1, 6144, 3]> conv5, tensor<fp16, [1, 6144, 3]> conv6, tensor<fp16, [1, 6144, 3]> conv8, tensor<fp16, [1, 6144, 3]> conv9, tensor<fp16, [1, 1, 4, 64]> cos, tensor<int32, [1]> current_pos, tensor<fp16, [4, 1024]> hidden, state<tensor<fp16, [4, 2, 2, 2048, 256]>> kv_cache, tensor<fp16, [1, 4, 2048]> mask3, tensor<fp16, [1, 16, 128, 128]> rec0, tensor<fp16, [1, 16, 128, 128]> rec1, tensor<fp16, [1, 16, 128, 128]> rec10, tensor<fp16, [1, 16, 128, 128]> rec12, tensor<fp16, [1, 16, 128, 128]> rec13, tensor<fp16, [1, 16, 128, 128]> rec14, tensor<fp16, [1, 16, 128, 128]> rec16, tensor<fp16, [1, 16, 128, 128]> rec17, tensor<fp16, [1, 16, 128, 128]> rec18, tensor<fp16, [1, 16, 128, 128]> rec2, tensor<fp16, [1, 16, 128, 128]> rec4, tensor<fp16, [1, 16, 128, 128]> rec5, tensor<fp16, [1, 16, 128, 128]> rec6, tensor<fp16, [1, 16, 128, 128]> rec8, tensor<fp16, [1, 16, 128, 128]> rec9, tensor<fp16, [1, 1, 4, 64]> sin) {
            int32 var_120 = const()[name = string("op_120"), val = int32(-1)];
            tensor<fp16, [4, 1024]> var_135_cast_fp16 = mul(x = hidden, y = hidden)[name = string("op_135_cast_fp16")];
            tensor<int32, [1]> variance_1_axes_0 = const()[name = string("variance_1_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_1_keep_dims_0 = const()[name = string("variance_1_keep_dims_0"), val = bool(true)];
            tensor<fp16, [4, 1]> variance_1_cast_fp16 = reduce_mean(axes = variance_1_axes_0, keep_dims = variance_1_keep_dims_0, x = var_135_cast_fp16)[name = string("variance_1_cast_fp16")];
            fp16 var_138_to_fp16 = const()[name = string("op_138_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [4, 1]> var_139_cast_fp16 = add(x = variance_1_cast_fp16, y = var_138_to_fp16)[name = string("op_139_cast_fp16")];
            fp32 var_140_epsilon_0 = const()[name = string("op_140_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [4, 1]> var_140_cast_fp16 = rsqrt(epsilon = var_140_epsilon_0, x = var_139_cast_fp16)[name = string("op_140_cast_fp16")];
            tensor<fp16, [4, 1024]> var_141_cast_fp16 = mul(x = hidden, y = var_140_cast_fp16)[name = string("op_141_cast_fp16")];
            tensor<fp16, [1024]> var_143_to_fp16 = const()[name = string("op_143_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(64)))];
            tensor<fp16, [4, 1024]> input_1_cast_fp16 = mul(x = var_141_cast_fp16, y = var_143_to_fp16)[name = string("input_1_cast_fp16")];
            tensor<fp16, [6144, 1024]> groups_0_0_in_proj_qkv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(2176))), lut = tensor<fp16, [384, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(4720832))))[name = string("groups_0_0_in_proj_qkv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [6144]> linear_0_bias_0_to_fp16 = const()[name = string("linear_0_bias_0_to_fp16"), val = tensor<fp16, [6144]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(4770048)))];
            tensor<fp16, [4, 6144]> linear_0_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = groups_0_0_in_proj_qkv_weight_promoted_to_fp16_palettized, x = input_1_cast_fp16)[name = string("linear_0_cast_fp16")];
            tensor<int32, [2]> var_147_perm_0 = const()[name = string("op_147_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> mixed_1_axes_0 = const()[name = string("mixed_1_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [6144, 4]> var_147_cast_fp16 = transpose(perm = var_147_perm_0, x = linear_0_cast_fp16)[name = string("transpose_141")];
            tensor<fp16, [1, 6144, 4]> mixed_1_cast_fp16 = expand_dims(axes = mixed_1_axes_0, x = var_147_cast_fp16)[name = string("mixed_1_cast_fp16")];
            bool convolution_input_1_interleave_0 = const()[name = string("convolution_input_1_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 6144, 7]> convolution_input_1_cast_fp16 = concat(axis = var_120, interleave = convolution_input_1_interleave_0, values = (conv0, mixed_1_cast_fp16))[name = string("convolution_input_1_cast_fp16")];
            string input_3_pad_type_0 = const()[name = string("input_3_pad_type_0"), val = string("valid")];
            int32 input_3_groups_0 = const()[name = string("input_3_groups_0"), val = int32(6144)];
            tensor<int32, [1]> input_3_strides_0 = const()[name = string("input_3_strides_0"), val = tensor<int32, [1]>([1])];
            tensor<int32, [2]> input_3_pad_0 = const()[name = string("input_3_pad_0"), val = tensor<int32, [2]>([0, 0])];
            tensor<int32, [1]> input_3_dilations_0 = const()[name = string("input_3_dilations_0"), val = tensor<int32, [1]>([1])];
            tensor<fp16, [6144, 1, 4]> groups_0_0_conv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1, 4]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(4782400))), lut = tensor<fp16, [384, 1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(4800896))))[name = string("groups_0_0_conv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 6144, 4]> input_3_cast_fp16 = conv(dilations = input_3_dilations_0, groups = input_3_groups_0, pad = input_3_pad_0, pad_type = input_3_pad_type_0, strides = input_3_strides_0, weight = groups_0_0_conv_weight_promoted_to_fp16_palettized, x = convolution_input_1_cast_fp16)[name = string("input_3_cast_fp16")];
            tensor<fp16, [1, 6144, 4]> var_156_cast_fp16 = silu(x = input_3_cast_fp16)[name = string("op_156_cast_fp16")];
            tensor<int32, [3]> var_157_perm_0 = const()[name = string("op_157_perm_0"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [3]> var_160_begin_0 = const()[name = string("op_160_begin_0"), val = tensor<int32, [3]>([0, 0, 4])];
            tensor<int32, [3]> var_160_end_0 = const()[name = string("op_160_end_0"), val = tensor<int32, [3]>([1, 6144, 7])];
            tensor<bool, [3]> var_160_end_mask_0 = const()[name = string("op_160_end_mask_0"), val = tensor<bool, [3]>([true, true, true])];
            tensor<fp16, [1, 6144, 3]> conv0_out = slice_by_index(begin = var_160_begin_0, end = var_160_end_0, end_mask = var_160_end_mask_0, x = convolution_input_1_cast_fp16)[name = string("op_160_cast_fp16")];
            tensor<int32, [3]> var_161 = const()[name = string("op_161"), val = tensor<int32, [3]>([2048, 2048, 2048])];
            int32 var_162_axis_0 = const()[name = string("op_162_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 4, 6144]> var_157_cast_fp16 = transpose(perm = var_157_perm_0, x = var_156_cast_fp16)[name = string("transpose_140")];
            tensor<fp16, [1, 4, 2048]> var_162_cast_fp16_0, tensor<fp16, [1, 4, 2048]> var_162_cast_fp16_1, tensor<fp16, [1, 4, 2048]> var_162_cast_fp16_2 = split(axis = var_162_axis_0, split_sizes = var_161, x = var_157_cast_fp16)[name = string("op_162_cast_fp16")];
            tensor<int32, [4]> var_166 = const()[name = string("op_166"), val = tensor<int32, [4]>([1, 4, 16, 128])];
            tensor<fp16, [1, 4, 16, 128]> value_5_cast_fp16 = reshape(shape = var_166, x = var_162_cast_fp16_0)[name = string("value_5_cast_fp16")];
            tensor<int32, [4]> var_168 = const()[name = string("op_168"), val = tensor<int32, [4]>([1, 4, 16, 128])];
            tensor<fp16, [1, 4, 16, 128]> value_7_cast_fp16 = reshape(shape = var_168, x = var_162_cast_fp16_1)[name = string("value_7_cast_fp16")];
            tensor<int32, [4]> var_170 = const()[name = string("op_170"), val = tensor<int32, [4]>([1, 4, 16, 128])];
            tensor<fp16, [1, 4, 16, 128]> value_9_cast_fp16 = reshape(shape = var_170, x = var_162_cast_fp16_2)[name = string("value_9_cast_fp16")];
            tensor<fp16, [1, 4, 16, 128]> var_172_cast_fp16 = mul(x = value_5_cast_fp16, y = value_5_cast_fp16)[name = string("op_172_cast_fp16")];
            tensor<int32, [1]> var_174_axes_0 = const()[name = string("op_174_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_174_keep_dims_0 = const()[name = string("op_174_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 4, 16, 1]> var_174_cast_fp16 = reduce_sum(axes = var_174_axes_0, keep_dims = var_174_keep_dims_0, x = var_172_cast_fp16)[name = string("op_174_cast_fp16")];
            fp16 var_175_to_fp16 = const()[name = string("op_175_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 4, 16, 1]> var_176_cast_fp16 = add(x = var_174_cast_fp16, y = var_175_to_fp16)[name = string("op_176_cast_fp16")];
            fp32 var_177_epsilon_0 = const()[name = string("op_177_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 4, 16, 1]> var_177_cast_fp16 = rsqrt(epsilon = var_177_epsilon_0, x = var_176_cast_fp16)[name = string("op_177_cast_fp16")];
            tensor<fp16, [1, 4, 16, 128]> var_178_cast_fp16 = mul(x = value_5_cast_fp16, y = var_177_cast_fp16)[name = string("op_178_cast_fp16")];
            tensor<int32, [4]> var_179_perm_0 = const()[name = string("op_179_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            fp16 _inversed_query_3_y_0_to_fp16 = const()[name = string("_inversed_query_3_y_0_to_fp16"), val = fp16(0x1.6ap-4)];
            tensor<fp16, [1, 16, 4, 128]> var_179_cast_fp16 = transpose(perm = var_179_perm_0, x = var_178_cast_fp16)[name = string("transpose_139")];
            tensor<fp16, [1, 16, 4, 128]> _inversed_query_3_cast_fp16 = mul(x = var_179_cast_fp16, y = _inversed_query_3_y_0_to_fp16)[name = string("_inversed_query_3_cast_fp16")];
            tensor<fp16, [1, 4, 16, 128]> var_182_cast_fp16 = mul(x = value_7_cast_fp16, y = value_7_cast_fp16)[name = string("op_182_cast_fp16")];
            tensor<int32, [1]> var_184_axes_0 = const()[name = string("op_184_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_184_keep_dims_0 = const()[name = string("op_184_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 4, 16, 1]> var_184_cast_fp16 = reduce_sum(axes = var_184_axes_0, keep_dims = var_184_keep_dims_0, x = var_182_cast_fp16)[name = string("op_184_cast_fp16")];
            fp16 var_185_to_fp16 = const()[name = string("op_185_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 4, 16, 1]> var_186_cast_fp16 = add(x = var_184_cast_fp16, y = var_185_to_fp16)[name = string("op_186_cast_fp16")];
            fp32 var_187_epsilon_0 = const()[name = string("op_187_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 4, 16, 1]> var_187_cast_fp16 = rsqrt(epsilon = var_187_epsilon_0, x = var_186_cast_fp16)[name = string("op_187_cast_fp16")];
            tensor<fp16, [1, 4, 16, 128]> var_188_cast_fp16 = mul(x = value_7_cast_fp16, y = var_187_cast_fp16)[name = string("op_188_cast_fp16")];
            tensor<int32, [4]> key_3_perm_0 = const()[name = string("key_3_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<int32, [4]> value_11_perm_0 = const()[name = string("value_11_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<fp16, [16, 1024]> groups_0_0_in_proj_b_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(4850112))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(4862464))))[name = string("groups_0_0_in_proj_b_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16]> linear_1_bias_0_to_fp16 = const()[name = string("linear_1_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(4862656)))];
            tensor<fp16, [4, 16]> linear_1_cast_fp16 = linear(bias = linear_1_bias_0_to_fp16, weight = groups_0_0_in_proj_b_weight_promoted_to_fp16_palettized, x = input_1_cast_fp16)[name = string("linear_1_cast_fp16")];
            tensor<fp16, [4, 16]> var_193_cast_fp16 = sigmoid(x = linear_1_cast_fp16)[name = string("op_193_cast_fp16")];
            tensor<int32, [2]> var_194_perm_0 = const()[name = string("op_194_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> beta_1_axes_0 = const()[name = string("beta_1_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 4]> var_194_cast_fp16 = transpose(perm = var_194_perm_0, x = var_193_cast_fp16)[name = string("transpose_138")];
            tensor<fp16, [1, 16, 4]> beta_1_cast_fp16 = expand_dims(axes = beta_1_axes_0, x = var_194_cast_fp16)[name = string("beta_1_cast_fp16")];
            tensor<fp16, [16, 1024]> op_200_weight_0_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(4862784))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(4875136))))[name = string("op_200_weight_0_to_fp16_palettized")];
            tensor<fp16, [16]> var_200_bias_0_to_fp16 = const()[name = string("op_200_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(4875328)))];
            tensor<fp16, [4, 16]> var_200_cast_fp16 = linear(bias = var_200_bias_0_to_fp16, weight = op_200_weight_0_to_fp16_palettized, x = input_1_cast_fp16)[name = string("op_200_cast_fp16")];
            tensor<fp16, [4, 16]> var_201_cast_fp16 = softplus(x = var_200_cast_fp16)[name = string("op_201_cast_fp16")];
            tensor<fp16, [16]> var_197_promoted_to_fp16 = const()[name = string("op_197_promoted_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(4875456)))];
            tensor<fp16, [4, 16]> var_202_cast_fp16 = mul(x = var_197_promoted_to_fp16, y = var_201_cast_fp16)[name = string("op_202_cast_fp16")];
            tensor<int32, [2]> var_203_perm_0 = const()[name = string("op_203_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> decay_1_axes_0 = const()[name = string("decay_1_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 4]> var_203_cast_fp16 = transpose(perm = var_203_perm_0, x = var_202_cast_fp16)[name = string("transpose_137")];
            tensor<fp16, [1, 16, 4]> decay_1_cast_fp16 = expand_dims(axes = decay_1_axes_0, x = var_203_cast_fp16)[name = string("decay_1_cast_fp16")];
            tensor<fp16, [2048, 1024]> groups_0_0_in_proj_z_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [2048, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(4875584))), lut = tensor<fp16, [128, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(6448512))))[name = string("groups_0_0_in_proj_z_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [2048]> linear_3_bias_0_to_fp16 = const()[name = string("linear_3_bias_0_to_fp16"), val = tensor<fp16, [2048]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(6464960)))];
            tensor<fp16, [4, 2048]> linear_3_cast_fp16 = linear(bias = linear_3_bias_0_to_fp16, weight = groups_0_0_in_proj_z_weight_promoted_to_fp16_palettized, x = input_1_cast_fp16)[name = string("linear_3_cast_fp16")];
            tensor<int32, [3]> var_211_begin_0 = const()[name = string("op_211_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_211_end_0 = const()[name = string("op_211_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_211_end_mask_0 = const()[name = string("op_211_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_211_squeeze_mask_0 = const()[name = string("op_211_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_211_cast_fp16 = slice_by_index(begin = var_211_begin_0, end = var_211_end_0, end_mask = var_211_end_mask_0, squeeze_mask = var_211_squeeze_mask_0, x = decay_1_cast_fp16)[name = string("op_211_cast_fp16")];
            tensor<fp16, [1, 16]> var_212_cast_fp16 = exp(x = var_211_cast_fp16)[name = string("op_212_cast_fp16")];
            tensor<int32, [1]> var_213_axes_0 = const()[name = string("op_213_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_213_cast_fp16 = expand_dims(axes = var_213_axes_0, x = var_212_cast_fp16)[name = string("op_213_cast_fp16")];
            tensor<int32, [1]> var_214_axes_0 = const()[name = string("op_214_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_214_cast_fp16 = expand_dims(axes = var_214_axes_0, x = var_213_cast_fp16)[name = string("op_214_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_1_cast_fp16 = mul(x = rec0, y = var_214_cast_fp16)[name = string("state_1_cast_fp16")];
            tensor<int32, [4]> key_token_1_begin_0 = const()[name = string("key_token_1_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> key_token_1_end_0 = const()[name = string("key_token_1_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> key_token_1_end_mask_0 = const()[name = string("key_token_1_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_1_squeeze_mask_0 = const()[name = string("key_token_1_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 4, 128]> key_3_cast_fp16 = transpose(perm = key_3_perm_0, x = var_188_cast_fp16)[name = string("transpose_136")];
            tensor<fp16, [1, 16, 128]> key_token_1_cast_fp16 = slice_by_index(begin = key_token_1_begin_0, end = key_token_1_end_0, end_mask = key_token_1_end_mask_0, squeeze_mask = key_token_1_squeeze_mask_0, x = key_3_cast_fp16)[name = string("key_token_1_cast_fp16")];
            tensor<int32, [4]> value_token_1_begin_0 = const()[name = string("value_token_1_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> value_token_1_end_0 = const()[name = string("value_token_1_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> value_token_1_end_mask_0 = const()[name = string("value_token_1_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_1_squeeze_mask_0 = const()[name = string("value_token_1_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 4, 128]> value_11_cast_fp16 = transpose(perm = value_11_perm_0, x = value_9_cast_fp16)[name = string("transpose_135")];
            tensor<fp16, [1, 16, 128]> value_token_1_cast_fp16 = slice_by_index(begin = value_token_1_begin_0, end = value_token_1_end_0, end_mask = value_token_1_end_mask_0, squeeze_mask = value_token_1_squeeze_mask_0, x = value_11_cast_fp16)[name = string("value_token_1_cast_fp16")];
            tensor<int32, [1]> var_222_axes_0 = const()[name = string("op_222_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_222_cast_fp16 = expand_dims(axes = var_222_axes_0, x = key_token_1_cast_fp16)[name = string("op_222_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_223_cast_fp16 = mul(x = state_1_cast_fp16, y = var_222_cast_fp16)[name = string("op_223_cast_fp16")];
            tensor<int32, [1]> memory_1_axes_0 = const()[name = string("memory_1_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_1_keep_dims_0 = const()[name = string("memory_1_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_1_cast_fp16 = reduce_sum(axes = memory_1_axes_0, keep_dims = memory_1_keep_dims_0, x = var_223_cast_fp16)[name = string("memory_1_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_226_cast_fp16 = sub(x = value_token_1_cast_fp16, y = memory_1_cast_fp16)[name = string("op_226_cast_fp16")];
            tensor<int32, [3]> var_229_begin_0 = const()[name = string("op_229_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_229_end_0 = const()[name = string("op_229_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_229_end_mask_0 = const()[name = string("op_229_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_229_squeeze_mask_0 = const()[name = string("op_229_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_229_cast_fp16 = slice_by_index(begin = var_229_begin_0, end = var_229_end_0, end_mask = var_229_end_mask_0, squeeze_mask = var_229_squeeze_mask_0, x = beta_1_cast_fp16)[name = string("op_229_cast_fp16")];
            tensor<int32, [1]> var_230_axes_0 = const()[name = string("op_230_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_230_cast_fp16 = expand_dims(axes = var_230_axes_0, x = var_229_cast_fp16)[name = string("op_230_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_1_cast_fp16 = mul(x = var_226_cast_fp16, y = var_230_cast_fp16)[name = string("delta_1_cast_fp16")];
            tensor<int32, [1]> var_233_axes_0 = const()[name = string("op_233_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_233_cast_fp16 = expand_dims(axes = var_233_axes_0, x = delta_1_cast_fp16)[name = string("op_233_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_234_cast_fp16 = mul(x = var_222_cast_fp16, y = var_233_cast_fp16)[name = string("op_234_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_3_cast_fp16 = add(x = state_1_cast_fp16, y = var_234_cast_fp16)[name = string("state_3_cast_fp16")];
            tensor<int32, [4]> var_238_begin_0 = const()[name = string("op_238_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> var_238_end_0 = const()[name = string("op_238_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> var_238_end_mask_0 = const()[name = string("op_238_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_238_squeeze_mask_0 = const()[name = string("op_238_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_238_cast_fp16 = slice_by_index(begin = var_238_begin_0, end = var_238_end_0, end_mask = var_238_end_mask_0, squeeze_mask = var_238_squeeze_mask_0, x = _inversed_query_3_cast_fp16)[name = string("op_238_cast_fp16")];
            tensor<int32, [1]> var_239_axes_0 = const()[name = string("op_239_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_239_cast_fp16 = expand_dims(axes = var_239_axes_0, x = var_238_cast_fp16)[name = string("op_239_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_240_cast_fp16 = mul(x = state_3_cast_fp16, y = var_239_cast_fp16)[name = string("op_240_cast_fp16")];
            tensor<int32, [1]> var_242_axes_0 = const()[name = string("op_242_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_242_keep_dims_0 = const()[name = string("op_242_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_242_cast_fp16 = reduce_sum(axes = var_242_axes_0, keep_dims = var_242_keep_dims_0, x = var_240_cast_fp16)[name = string("op_242_cast_fp16")];
            tensor<int32, [3]> var_245_begin_0 = const()[name = string("op_245_begin_0"), val = tensor<int32, [3]>([0, 0, 1])];
            tensor<int32, [3]> var_245_end_0 = const()[name = string("op_245_end_0"), val = tensor<int32, [3]>([1, 16, 2])];
            tensor<bool, [3]> var_245_end_mask_0 = const()[name = string("op_245_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_245_squeeze_mask_0 = const()[name = string("op_245_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_245_cast_fp16 = slice_by_index(begin = var_245_begin_0, end = var_245_end_0, end_mask = var_245_end_mask_0, squeeze_mask = var_245_squeeze_mask_0, x = decay_1_cast_fp16)[name = string("op_245_cast_fp16")];
            tensor<fp16, [1, 16]> var_246_cast_fp16 = exp(x = var_245_cast_fp16)[name = string("op_246_cast_fp16")];
            tensor<int32, [1]> var_247_axes_0 = const()[name = string("op_247_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_247_cast_fp16 = expand_dims(axes = var_247_axes_0, x = var_246_cast_fp16)[name = string("op_247_cast_fp16")];
            tensor<int32, [1]> var_248_axes_0 = const()[name = string("op_248_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_248_cast_fp16 = expand_dims(axes = var_248_axes_0, x = var_247_cast_fp16)[name = string("op_248_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_5_cast_fp16 = mul(x = state_3_cast_fp16, y = var_248_cast_fp16)[name = string("state_5_cast_fp16")];
            tensor<int32, [4]> key_token_3_begin_0 = const()[name = string("key_token_3_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> key_token_3_end_0 = const()[name = string("key_token_3_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> key_token_3_end_mask_0 = const()[name = string("key_token_3_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_3_squeeze_mask_0 = const()[name = string("key_token_3_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_3_cast_fp16 = slice_by_index(begin = key_token_3_begin_0, end = key_token_3_end_0, end_mask = key_token_3_end_mask_0, squeeze_mask = key_token_3_squeeze_mask_0, x = key_3_cast_fp16)[name = string("key_token_3_cast_fp16")];
            tensor<int32, [4]> value_token_3_begin_0 = const()[name = string("value_token_3_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> value_token_3_end_0 = const()[name = string("value_token_3_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> value_token_3_end_mask_0 = const()[name = string("value_token_3_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_3_squeeze_mask_0 = const()[name = string("value_token_3_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_3_cast_fp16 = slice_by_index(begin = value_token_3_begin_0, end = value_token_3_end_0, end_mask = value_token_3_end_mask_0, squeeze_mask = value_token_3_squeeze_mask_0, x = value_11_cast_fp16)[name = string("value_token_3_cast_fp16")];
            tensor<int32, [1]> var_256_axes_0 = const()[name = string("op_256_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_256_cast_fp16 = expand_dims(axes = var_256_axes_0, x = key_token_3_cast_fp16)[name = string("op_256_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_257_cast_fp16 = mul(x = state_5_cast_fp16, y = var_256_cast_fp16)[name = string("op_257_cast_fp16")];
            tensor<int32, [1]> memory_3_axes_0 = const()[name = string("memory_3_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_3_keep_dims_0 = const()[name = string("memory_3_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_3_cast_fp16 = reduce_sum(axes = memory_3_axes_0, keep_dims = memory_3_keep_dims_0, x = var_257_cast_fp16)[name = string("memory_3_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_260_cast_fp16 = sub(x = value_token_3_cast_fp16, y = memory_3_cast_fp16)[name = string("op_260_cast_fp16")];
            tensor<int32, [3]> var_263_begin_0 = const()[name = string("op_263_begin_0"), val = tensor<int32, [3]>([0, 0, 1])];
            tensor<int32, [3]> var_263_end_0 = const()[name = string("op_263_end_0"), val = tensor<int32, [3]>([1, 16, 2])];
            tensor<bool, [3]> var_263_end_mask_0 = const()[name = string("op_263_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_263_squeeze_mask_0 = const()[name = string("op_263_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_263_cast_fp16 = slice_by_index(begin = var_263_begin_0, end = var_263_end_0, end_mask = var_263_end_mask_0, squeeze_mask = var_263_squeeze_mask_0, x = beta_1_cast_fp16)[name = string("op_263_cast_fp16")];
            tensor<int32, [1]> var_264_axes_0 = const()[name = string("op_264_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_264_cast_fp16 = expand_dims(axes = var_264_axes_0, x = var_263_cast_fp16)[name = string("op_264_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_3_cast_fp16 = mul(x = var_260_cast_fp16, y = var_264_cast_fp16)[name = string("delta_3_cast_fp16")];
            tensor<int32, [1]> var_267_axes_0 = const()[name = string("op_267_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_267_cast_fp16 = expand_dims(axes = var_267_axes_0, x = delta_3_cast_fp16)[name = string("op_267_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_268_cast_fp16 = mul(x = var_256_cast_fp16, y = var_267_cast_fp16)[name = string("op_268_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_7_cast_fp16 = add(x = state_5_cast_fp16, y = var_268_cast_fp16)[name = string("state_7_cast_fp16")];
            tensor<int32, [4]> var_272_begin_0 = const()[name = string("op_272_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> var_272_end_0 = const()[name = string("op_272_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> var_272_end_mask_0 = const()[name = string("op_272_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_272_squeeze_mask_0 = const()[name = string("op_272_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_272_cast_fp16 = slice_by_index(begin = var_272_begin_0, end = var_272_end_0, end_mask = var_272_end_mask_0, squeeze_mask = var_272_squeeze_mask_0, x = _inversed_query_3_cast_fp16)[name = string("op_272_cast_fp16")];
            tensor<int32, [1]> var_273_axes_0 = const()[name = string("op_273_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_273_cast_fp16 = expand_dims(axes = var_273_axes_0, x = var_272_cast_fp16)[name = string("op_273_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_274_cast_fp16 = mul(x = state_7_cast_fp16, y = var_273_cast_fp16)[name = string("op_274_cast_fp16")];
            tensor<int32, [1]> var_276_axes_0 = const()[name = string("op_276_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_276_keep_dims_0 = const()[name = string("op_276_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_276_cast_fp16 = reduce_sum(axes = var_276_axes_0, keep_dims = var_276_keep_dims_0, x = var_274_cast_fp16)[name = string("op_276_cast_fp16")];
            tensor<int32, [3]> var_279_begin_0 = const()[name = string("op_279_begin_0"), val = tensor<int32, [3]>([0, 0, 2])];
            tensor<int32, [3]> var_279_end_0 = const()[name = string("op_279_end_0"), val = tensor<int32, [3]>([1, 16, 3])];
            tensor<bool, [3]> var_279_end_mask_0 = const()[name = string("op_279_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_279_squeeze_mask_0 = const()[name = string("op_279_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_279_cast_fp16 = slice_by_index(begin = var_279_begin_0, end = var_279_end_0, end_mask = var_279_end_mask_0, squeeze_mask = var_279_squeeze_mask_0, x = decay_1_cast_fp16)[name = string("op_279_cast_fp16")];
            tensor<fp16, [1, 16]> var_280_cast_fp16 = exp(x = var_279_cast_fp16)[name = string("op_280_cast_fp16")];
            tensor<int32, [1]> var_281_axes_0 = const()[name = string("op_281_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_281_cast_fp16 = expand_dims(axes = var_281_axes_0, x = var_280_cast_fp16)[name = string("op_281_cast_fp16")];
            tensor<int32, [1]> var_282_axes_0 = const()[name = string("op_282_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_282_cast_fp16 = expand_dims(axes = var_282_axes_0, x = var_281_cast_fp16)[name = string("op_282_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_9_cast_fp16 = mul(x = state_7_cast_fp16, y = var_282_cast_fp16)[name = string("state_9_cast_fp16")];
            tensor<int32, [4]> key_token_5_begin_0 = const()[name = string("key_token_5_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> key_token_5_end_0 = const()[name = string("key_token_5_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> key_token_5_end_mask_0 = const()[name = string("key_token_5_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_5_squeeze_mask_0 = const()[name = string("key_token_5_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_5_cast_fp16 = slice_by_index(begin = key_token_5_begin_0, end = key_token_5_end_0, end_mask = key_token_5_end_mask_0, squeeze_mask = key_token_5_squeeze_mask_0, x = key_3_cast_fp16)[name = string("key_token_5_cast_fp16")];
            tensor<int32, [4]> value_token_5_begin_0 = const()[name = string("value_token_5_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> value_token_5_end_0 = const()[name = string("value_token_5_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> value_token_5_end_mask_0 = const()[name = string("value_token_5_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_5_squeeze_mask_0 = const()[name = string("value_token_5_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_5_cast_fp16 = slice_by_index(begin = value_token_5_begin_0, end = value_token_5_end_0, end_mask = value_token_5_end_mask_0, squeeze_mask = value_token_5_squeeze_mask_0, x = value_11_cast_fp16)[name = string("value_token_5_cast_fp16")];
            tensor<int32, [1]> var_290_axes_0 = const()[name = string("op_290_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_290_cast_fp16 = expand_dims(axes = var_290_axes_0, x = key_token_5_cast_fp16)[name = string("op_290_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_291_cast_fp16 = mul(x = state_9_cast_fp16, y = var_290_cast_fp16)[name = string("op_291_cast_fp16")];
            tensor<int32, [1]> memory_5_axes_0 = const()[name = string("memory_5_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_5_keep_dims_0 = const()[name = string("memory_5_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_5_cast_fp16 = reduce_sum(axes = memory_5_axes_0, keep_dims = memory_5_keep_dims_0, x = var_291_cast_fp16)[name = string("memory_5_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_294_cast_fp16 = sub(x = value_token_5_cast_fp16, y = memory_5_cast_fp16)[name = string("op_294_cast_fp16")];
            tensor<int32, [3]> var_297_begin_0 = const()[name = string("op_297_begin_0"), val = tensor<int32, [3]>([0, 0, 2])];
            tensor<int32, [3]> var_297_end_0 = const()[name = string("op_297_end_0"), val = tensor<int32, [3]>([1, 16, 3])];
            tensor<bool, [3]> var_297_end_mask_0 = const()[name = string("op_297_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_297_squeeze_mask_0 = const()[name = string("op_297_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_297_cast_fp16 = slice_by_index(begin = var_297_begin_0, end = var_297_end_0, end_mask = var_297_end_mask_0, squeeze_mask = var_297_squeeze_mask_0, x = beta_1_cast_fp16)[name = string("op_297_cast_fp16")];
            tensor<int32, [1]> var_298_axes_0 = const()[name = string("op_298_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_298_cast_fp16 = expand_dims(axes = var_298_axes_0, x = var_297_cast_fp16)[name = string("op_298_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_5_cast_fp16 = mul(x = var_294_cast_fp16, y = var_298_cast_fp16)[name = string("delta_5_cast_fp16")];
            tensor<int32, [1]> var_301_axes_0 = const()[name = string("op_301_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_301_cast_fp16 = expand_dims(axes = var_301_axes_0, x = delta_5_cast_fp16)[name = string("op_301_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_302_cast_fp16 = mul(x = var_290_cast_fp16, y = var_301_cast_fp16)[name = string("op_302_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_11_cast_fp16 = add(x = state_9_cast_fp16, y = var_302_cast_fp16)[name = string("state_11_cast_fp16")];
            tensor<int32, [4]> var_306_begin_0 = const()[name = string("op_306_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> var_306_end_0 = const()[name = string("op_306_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> var_306_end_mask_0 = const()[name = string("op_306_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_306_squeeze_mask_0 = const()[name = string("op_306_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_306_cast_fp16 = slice_by_index(begin = var_306_begin_0, end = var_306_end_0, end_mask = var_306_end_mask_0, squeeze_mask = var_306_squeeze_mask_0, x = _inversed_query_3_cast_fp16)[name = string("op_306_cast_fp16")];
            tensor<int32, [1]> var_307_axes_0 = const()[name = string("op_307_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_307_cast_fp16 = expand_dims(axes = var_307_axes_0, x = var_306_cast_fp16)[name = string("op_307_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_308_cast_fp16 = mul(x = state_11_cast_fp16, y = var_307_cast_fp16)[name = string("op_308_cast_fp16")];
            tensor<int32, [1]> var_310_axes_0 = const()[name = string("op_310_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_310_keep_dims_0 = const()[name = string("op_310_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_310_cast_fp16 = reduce_sum(axes = var_310_axes_0, keep_dims = var_310_keep_dims_0, x = var_308_cast_fp16)[name = string("op_310_cast_fp16")];
            tensor<int32, [3]> var_313_begin_0 = const()[name = string("op_313_begin_0"), val = tensor<int32, [3]>([0, 0, 3])];
            tensor<int32, [3]> var_313_end_0 = const()[name = string("op_313_end_0"), val = tensor<int32, [3]>([1, 16, 4])];
            tensor<bool, [3]> var_313_end_mask_0 = const()[name = string("op_313_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_313_squeeze_mask_0 = const()[name = string("op_313_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_313_cast_fp16 = slice_by_index(begin = var_313_begin_0, end = var_313_end_0, end_mask = var_313_end_mask_0, squeeze_mask = var_313_squeeze_mask_0, x = decay_1_cast_fp16)[name = string("op_313_cast_fp16")];
            tensor<fp16, [1, 16]> var_314_cast_fp16 = exp(x = var_313_cast_fp16)[name = string("op_314_cast_fp16")];
            tensor<int32, [1]> var_315_axes_0 = const()[name = string("op_315_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_315_cast_fp16 = expand_dims(axes = var_315_axes_0, x = var_314_cast_fp16)[name = string("op_315_cast_fp16")];
            tensor<int32, [1]> var_316_axes_0 = const()[name = string("op_316_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_316_cast_fp16 = expand_dims(axes = var_316_axes_0, x = var_315_cast_fp16)[name = string("op_316_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_13_cast_fp16 = mul(x = state_11_cast_fp16, y = var_316_cast_fp16)[name = string("state_13_cast_fp16")];
            tensor<int32, [4]> key_token_7_begin_0 = const()[name = string("key_token_7_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> key_token_7_end_0 = const()[name = string("key_token_7_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> key_token_7_end_mask_0 = const()[name = string("key_token_7_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_7_squeeze_mask_0 = const()[name = string("key_token_7_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_7_cast_fp16 = slice_by_index(begin = key_token_7_begin_0, end = key_token_7_end_0, end_mask = key_token_7_end_mask_0, squeeze_mask = key_token_7_squeeze_mask_0, x = key_3_cast_fp16)[name = string("key_token_7_cast_fp16")];
            tensor<int32, [4]> value_token_7_begin_0 = const()[name = string("value_token_7_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> value_token_7_end_0 = const()[name = string("value_token_7_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> value_token_7_end_mask_0 = const()[name = string("value_token_7_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_7_squeeze_mask_0 = const()[name = string("value_token_7_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_7_cast_fp16 = slice_by_index(begin = value_token_7_begin_0, end = value_token_7_end_0, end_mask = value_token_7_end_mask_0, squeeze_mask = value_token_7_squeeze_mask_0, x = value_11_cast_fp16)[name = string("value_token_7_cast_fp16")];
            tensor<int32, [1]> var_324_axes_0 = const()[name = string("op_324_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_324_cast_fp16 = expand_dims(axes = var_324_axes_0, x = key_token_7_cast_fp16)[name = string("op_324_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_325_cast_fp16 = mul(x = state_13_cast_fp16, y = var_324_cast_fp16)[name = string("op_325_cast_fp16")];
            tensor<int32, [1]> memory_7_axes_0 = const()[name = string("memory_7_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_7_keep_dims_0 = const()[name = string("memory_7_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_7_cast_fp16 = reduce_sum(axes = memory_7_axes_0, keep_dims = memory_7_keep_dims_0, x = var_325_cast_fp16)[name = string("memory_7_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_328_cast_fp16 = sub(x = value_token_7_cast_fp16, y = memory_7_cast_fp16)[name = string("op_328_cast_fp16")];
            tensor<int32, [3]> var_331_begin_0 = const()[name = string("op_331_begin_0"), val = tensor<int32, [3]>([0, 0, 3])];
            tensor<int32, [3]> var_331_end_0 = const()[name = string("op_331_end_0"), val = tensor<int32, [3]>([1, 16, 4])];
            tensor<bool, [3]> var_331_end_mask_0 = const()[name = string("op_331_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_331_squeeze_mask_0 = const()[name = string("op_331_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_331_cast_fp16 = slice_by_index(begin = var_331_begin_0, end = var_331_end_0, end_mask = var_331_end_mask_0, squeeze_mask = var_331_squeeze_mask_0, x = beta_1_cast_fp16)[name = string("op_331_cast_fp16")];
            tensor<int32, [1]> var_332_axes_0 = const()[name = string("op_332_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_332_cast_fp16 = expand_dims(axes = var_332_axes_0, x = var_331_cast_fp16)[name = string("op_332_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_7_cast_fp16 = mul(x = var_328_cast_fp16, y = var_332_cast_fp16)[name = string("delta_7_cast_fp16")];
            tensor<int32, [1]> var_335_axes_0 = const()[name = string("op_335_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_335_cast_fp16 = expand_dims(axes = var_335_axes_0, x = delta_7_cast_fp16)[name = string("op_335_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_336_cast_fp16 = mul(x = var_324_cast_fp16, y = var_335_cast_fp16)[name = string("op_336_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> rec0_out = add(x = state_13_cast_fp16, y = var_336_cast_fp16)[name = string("state_15_cast_fp16")];
            tensor<int32, [4]> var_340_begin_0 = const()[name = string("op_340_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> var_340_end_0 = const()[name = string("op_340_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> var_340_end_mask_0 = const()[name = string("op_340_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_340_squeeze_mask_0 = const()[name = string("op_340_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_340_cast_fp16 = slice_by_index(begin = var_340_begin_0, end = var_340_end_0, end_mask = var_340_end_mask_0, squeeze_mask = var_340_squeeze_mask_0, x = _inversed_query_3_cast_fp16)[name = string("op_340_cast_fp16")];
            tensor<int32, [1]> var_341_axes_0 = const()[name = string("op_341_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_341_cast_fp16 = expand_dims(axes = var_341_axes_0, x = var_340_cast_fp16)[name = string("op_341_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_342_cast_fp16 = mul(x = rec0_out, y = var_341_cast_fp16)[name = string("op_342_cast_fp16")];
            tensor<int32, [1]> var_344_axes_0 = const()[name = string("op_344_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_344_keep_dims_0 = const()[name = string("op_344_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_344_cast_fp16 = reduce_sum(axes = var_344_axes_0, keep_dims = var_344_keep_dims_0, x = var_342_cast_fp16)[name = string("op_344_cast_fp16")];
            int32 attention_1_axis_0 = const()[name = string("attention_1_axis_0"), val = int32(1)];
            tensor<fp16, [1, 4, 16, 128]> attention_1_cast_fp16 = stack(axis = attention_1_axis_0, values = (var_242_cast_fp16, var_276_cast_fp16, var_310_cast_fp16, var_344_cast_fp16))[name = string("attention_1_cast_fp16")];
            tensor<int32, [2]> var_347 = const()[name = string("op_347"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [64, 128]> attention_3_cast_fp16 = reshape(shape = var_347, x = attention_1_cast_fp16)[name = string("attention_3_cast_fp16")];
            tensor<int32, [2]> var_349 = const()[name = string("op_349"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [64, 128]> input_5_cast_fp16 = reshape(shape = var_349, x = linear_3_cast_fp16)[name = string("input_5_cast_fp16")];
            tensor<fp16, [64, 128]> var_351_cast_fp16 = mul(x = attention_3_cast_fp16, y = attention_3_cast_fp16)[name = string("op_351_cast_fp16")];
            tensor<int32, [1]> variance_3_axes_0 = const()[name = string("variance_3_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_3_keep_dims_0 = const()[name = string("variance_3_keep_dims_0"), val = bool(true)];
            tensor<fp16, [64, 1]> variance_3_cast_fp16 = reduce_mean(axes = variance_3_axes_0, keep_dims = variance_3_keep_dims_0, x = var_351_cast_fp16)[name = string("variance_3_cast_fp16")];
            fp16 var_354_to_fp16 = const()[name = string("op_354_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [64, 1]> var_355_cast_fp16 = add(x = variance_3_cast_fp16, y = var_354_to_fp16)[name = string("op_355_cast_fp16")];
            fp32 var_356_epsilon_0 = const()[name = string("op_356_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [64, 1]> var_356_cast_fp16 = rsqrt(epsilon = var_356_epsilon_0, x = var_355_cast_fp16)[name = string("op_356_cast_fp16")];
            tensor<fp16, [64, 128]> attention_5_cast_fp16 = mul(x = attention_3_cast_fp16, y = var_356_cast_fp16)[name = string("attention_5_cast_fp16")];
            tensor<fp16, [128]> groups_0_0_gated_norm_promoted_to_fp16 = const()[name = string("groups_0_0_gated_norm_promoted_to_fp16"), val = tensor<fp16, [128]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(6469120)))];
            tensor<fp16, [64, 128]> attention_7_cast_fp16 = mul(x = attention_5_cast_fp16, y = groups_0_0_gated_norm_promoted_to_fp16)[name = string("attention_7_cast_fp16")];
            tensor<fp16, [64, 128]> var_359_cast_fp16 = silu(x = input_5_cast_fp16)[name = string("op_359_cast_fp16")];
            tensor<fp16, [64, 128]> attention_9_cast_fp16 = mul(x = attention_7_cast_fp16, y = var_359_cast_fp16)[name = string("attention_9_cast_fp16")];
            tensor<int32, [2]> var_361 = const()[name = string("op_361"), val = tensor<int32, [2]>([4, 2048])];
            tensor<fp16, [4, 2048]> input_7_cast_fp16 = reshape(shape = var_361, x = attention_9_cast_fp16)[name = string("input_7_cast_fp16")];
            tensor<fp16, [1024, 2048]> groups_0_0_out_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 2048]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(6469440))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(8042368))))[name = string("groups_0_0_out_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1024]> linear_4_bias_0_to_fp16 = const()[name = string("linear_4_bias_0_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(8050624)))];
            tensor<fp16, [4, 1024]> linear_4_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = groups_0_0_out_proj_weight_promoted_to_fp16_palettized, x = input_7_cast_fp16)[name = string("linear_4_cast_fp16")];
            tensor<fp16, [4, 1024]> value_13_cast_fp16 = add(x = hidden, y = linear_4_cast_fp16)[name = string("value_13_cast_fp16")];
            tensor<fp16, [4, 1024]> var_366_cast_fp16 = mul(x = value_13_cast_fp16, y = value_13_cast_fp16)[name = string("op_366_cast_fp16")];
            tensor<int32, [1]> variance_5_axes_0 = const()[name = string("variance_5_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_5_keep_dims_0 = const()[name = string("variance_5_keep_dims_0"), val = bool(true)];
            tensor<fp16, [4, 1]> variance_5_cast_fp16 = reduce_mean(axes = variance_5_axes_0, keep_dims = variance_5_keep_dims_0, x = var_366_cast_fp16)[name = string("variance_5_cast_fp16")];
            fp16 var_369_to_fp16 = const()[name = string("op_369_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [4, 1]> var_370_cast_fp16 = add(x = variance_5_cast_fp16, y = var_369_to_fp16)[name = string("op_370_cast_fp16")];
            fp32 var_371_epsilon_0 = const()[name = string("op_371_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [4, 1]> var_371_cast_fp16 = rsqrt(epsilon = var_371_epsilon_0, x = var_370_cast_fp16)[name = string("op_371_cast_fp16")];
            tensor<fp16, [4, 1024]> var_372_cast_fp16 = mul(x = value_13_cast_fp16, y = var_371_cast_fp16)[name = string("op_372_cast_fp16")];
            tensor<fp16, [1024]> var_374_to_fp16 = const()[name = string("op_374_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(8052736)))];
            tensor<fp16, [4, 1024]> input_9_cast_fp16 = mul(x = var_372_cast_fp16, y = var_374_to_fp16)[name = string("input_9_cast_fp16")];
            tensor<fp16, [3584, 1024]> groups_0_0_gate_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(8054848))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(10807424))))[name = string("groups_0_0_gate_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [3584]> linear_5_bias_0_to_fp16 = const()[name = string("linear_5_bias_0_to_fp16"), val = tensor<fp16, [3584]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(10836160)))];
            tensor<fp16, [4, 3584]> linear_5_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = groups_0_0_gate_proj_weight_promoted_to_fp16_palettized, x = input_9_cast_fp16)[name = string("linear_5_cast_fp16")];
            tensor<fp16, [4, 3584]> var_378_cast_fp16 = silu(x = linear_5_cast_fp16)[name = string("op_378_cast_fp16")];
            tensor<fp16, [3584, 1024]> groups_0_0_up_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(10843392))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(13595968))))[name = string("groups_0_0_up_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 3584]> linear_6_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = groups_0_0_up_proj_weight_promoted_to_fp16_palettized, x = input_9_cast_fp16)[name = string("linear_6_cast_fp16")];
            tensor<fp16, [4, 3584]> input_13_cast_fp16 = mul(x = var_378_cast_fp16, y = linear_6_cast_fp16)[name = string("input_13_cast_fp16")];
            tensor<fp16, [1024, 3584]> groups_0_0_down_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 3584]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(13624704))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(16377280))))[name = string("groups_0_0_down_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 1024]> linear_7_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = groups_0_0_down_proj_weight_promoted_to_fp16_palettized, x = input_13_cast_fp16)[name = string("linear_7_cast_fp16")];
            tensor<fp16, [4, 1024]> value_15_cast_fp16 = add(x = value_13_cast_fp16, y = linear_7_cast_fp16)[name = string("value_15_cast_fp16")];
            int32 var_405 = const()[name = string("op_405"), val = int32(-1)];
            tensor<fp16, [4, 1024]> var_420_cast_fp16 = mul(x = value_15_cast_fp16, y = value_15_cast_fp16)[name = string("op_420_cast_fp16")];
            tensor<int32, [1]> variance_7_axes_0 = const()[name = string("variance_7_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_7_keep_dims_0 = const()[name = string("variance_7_keep_dims_0"), val = bool(true)];
            tensor<fp16, [4, 1]> variance_7_cast_fp16 = reduce_mean(axes = variance_7_axes_0, keep_dims = variance_7_keep_dims_0, x = var_420_cast_fp16)[name = string("variance_7_cast_fp16")];
            fp16 var_423_to_fp16 = const()[name = string("op_423_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [4, 1]> var_424_cast_fp16 = add(x = variance_7_cast_fp16, y = var_423_to_fp16)[name = string("op_424_cast_fp16")];
            fp32 var_425_epsilon_0 = const()[name = string("op_425_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [4, 1]> var_425_cast_fp16 = rsqrt(epsilon = var_425_epsilon_0, x = var_424_cast_fp16)[name = string("op_425_cast_fp16")];
            tensor<fp16, [4, 1024]> var_426_cast_fp16 = mul(x = value_15_cast_fp16, y = var_425_cast_fp16)[name = string("op_426_cast_fp16")];
            tensor<fp16, [1024]> var_428_to_fp16 = const()[name = string("op_428_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(16385536)))];
            tensor<fp16, [4, 1024]> input_15_cast_fp16 = mul(x = var_426_cast_fp16, y = var_428_to_fp16)[name = string("input_15_cast_fp16")];
            tensor<fp16, [6144, 1024]> groups_0_1_in_proj_qkv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(16387648))), lut = tensor<fp16, [384, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(21106304))))[name = string("groups_0_1_in_proj_qkv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 6144]> linear_8_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = groups_0_1_in_proj_qkv_weight_promoted_to_fp16_palettized, x = input_15_cast_fp16)[name = string("linear_8_cast_fp16")];
            tensor<int32, [2]> var_432_perm_0 = const()[name = string("op_432_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> mixed_3_axes_0 = const()[name = string("mixed_3_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [6144, 4]> var_432_cast_fp16 = transpose(perm = var_432_perm_0, x = linear_8_cast_fp16)[name = string("transpose_134")];
            tensor<fp16, [1, 6144, 4]> mixed_3_cast_fp16 = expand_dims(axes = mixed_3_axes_0, x = var_432_cast_fp16)[name = string("mixed_3_cast_fp16")];
            bool convolution_input_3_interleave_0 = const()[name = string("convolution_input_3_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 6144, 7]> convolution_input_3_cast_fp16 = concat(axis = var_405, interleave = convolution_input_3_interleave_0, values = (conv1, mixed_3_cast_fp16))[name = string("convolution_input_3_cast_fp16")];
            string input_17_pad_type_0 = const()[name = string("input_17_pad_type_0"), val = string("valid")];
            int32 input_17_groups_0 = const()[name = string("input_17_groups_0"), val = int32(6144)];
            tensor<int32, [1]> input_17_strides_0 = const()[name = string("input_17_strides_0"), val = tensor<int32, [1]>([1])];
            tensor<int32, [2]> input_17_pad_0 = const()[name = string("input_17_pad_0"), val = tensor<int32, [2]>([0, 0])];
            tensor<int32, [1]> input_17_dilations_0 = const()[name = string("input_17_dilations_0"), val = tensor<int32, [1]>([1])];
            tensor<fp16, [6144, 1, 4]> groups_0_1_conv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1, 4]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(21155520))), lut = tensor<fp16, [384, 1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(21174016))))[name = string("groups_0_1_conv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 6144, 4]> input_17_cast_fp16 = conv(dilations = input_17_dilations_0, groups = input_17_groups_0, pad = input_17_pad_0, pad_type = input_17_pad_type_0, strides = input_17_strides_0, weight = groups_0_1_conv_weight_promoted_to_fp16_palettized, x = convolution_input_3_cast_fp16)[name = string("input_17_cast_fp16")];
            tensor<fp16, [1, 6144, 4]> var_441_cast_fp16 = silu(x = input_17_cast_fp16)[name = string("op_441_cast_fp16")];
            tensor<int32, [3]> var_442_perm_0 = const()[name = string("op_442_perm_0"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [3]> var_445_begin_0 = const()[name = string("op_445_begin_0"), val = tensor<int32, [3]>([0, 0, 4])];
            tensor<int32, [3]> var_445_end_0 = const()[name = string("op_445_end_0"), val = tensor<int32, [3]>([1, 6144, 7])];
            tensor<bool, [3]> var_445_end_mask_0 = const()[name = string("op_445_end_mask_0"), val = tensor<bool, [3]>([true, true, true])];
            tensor<fp16, [1, 6144, 3]> conv1_out = slice_by_index(begin = var_445_begin_0, end = var_445_end_0, end_mask = var_445_end_mask_0, x = convolution_input_3_cast_fp16)[name = string("op_445_cast_fp16")];
            tensor<int32, [3]> var_446 = const()[name = string("op_446"), val = tensor<int32, [3]>([2048, 2048, 2048])];
            int32 var_447_axis_0 = const()[name = string("op_447_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 4, 6144]> var_442_cast_fp16 = transpose(perm = var_442_perm_0, x = var_441_cast_fp16)[name = string("transpose_133")];
            tensor<fp16, [1, 4, 2048]> var_447_cast_fp16_0, tensor<fp16, [1, 4, 2048]> var_447_cast_fp16_1, tensor<fp16, [1, 4, 2048]> var_447_cast_fp16_2 = split(axis = var_447_axis_0, split_sizes = var_446, x = var_442_cast_fp16)[name = string("op_447_cast_fp16")];
            tensor<int32, [4]> var_451 = const()[name = string("op_451"), val = tensor<int32, [4]>([1, 4, 16, 128])];
            tensor<fp16, [1, 4, 16, 128]> value_19_cast_fp16 = reshape(shape = var_451, x = var_447_cast_fp16_0)[name = string("value_19_cast_fp16")];
            tensor<int32, [4]> var_453 = const()[name = string("op_453"), val = tensor<int32, [4]>([1, 4, 16, 128])];
            tensor<fp16, [1, 4, 16, 128]> value_21_cast_fp16 = reshape(shape = var_453, x = var_447_cast_fp16_1)[name = string("value_21_cast_fp16")];
            tensor<int32, [4]> var_455 = const()[name = string("op_455"), val = tensor<int32, [4]>([1, 4, 16, 128])];
            tensor<fp16, [1, 4, 16, 128]> value_23_cast_fp16 = reshape(shape = var_455, x = var_447_cast_fp16_2)[name = string("value_23_cast_fp16")];
            tensor<fp16, [1, 4, 16, 128]> var_457_cast_fp16 = mul(x = value_19_cast_fp16, y = value_19_cast_fp16)[name = string("op_457_cast_fp16")];
            tensor<int32, [1]> var_459_axes_0 = const()[name = string("op_459_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_459_keep_dims_0 = const()[name = string("op_459_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 4, 16, 1]> var_459_cast_fp16 = reduce_sum(axes = var_459_axes_0, keep_dims = var_459_keep_dims_0, x = var_457_cast_fp16)[name = string("op_459_cast_fp16")];
            fp16 var_460_to_fp16 = const()[name = string("op_460_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 4, 16, 1]> var_461_cast_fp16 = add(x = var_459_cast_fp16, y = var_460_to_fp16)[name = string("op_461_cast_fp16")];
            fp32 var_462_epsilon_0 = const()[name = string("op_462_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 4, 16, 1]> var_462_cast_fp16 = rsqrt(epsilon = var_462_epsilon_0, x = var_461_cast_fp16)[name = string("op_462_cast_fp16")];
            tensor<fp16, [1, 4, 16, 128]> var_463_cast_fp16 = mul(x = value_19_cast_fp16, y = var_462_cast_fp16)[name = string("op_463_cast_fp16")];
            tensor<int32, [4]> var_464_perm_0 = const()[name = string("op_464_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            fp16 _inversed_query_7_y_0_to_fp16 = const()[name = string("_inversed_query_7_y_0_to_fp16"), val = fp16(0x1.6ap-4)];
            tensor<fp16, [1, 16, 4, 128]> var_464_cast_fp16 = transpose(perm = var_464_perm_0, x = var_463_cast_fp16)[name = string("transpose_132")];
            tensor<fp16, [1, 16, 4, 128]> _inversed_query_7_cast_fp16 = mul(x = var_464_cast_fp16, y = _inversed_query_7_y_0_to_fp16)[name = string("_inversed_query_7_cast_fp16")];
            tensor<fp16, [1, 4, 16, 128]> var_467_cast_fp16 = mul(x = value_21_cast_fp16, y = value_21_cast_fp16)[name = string("op_467_cast_fp16")];
            tensor<int32, [1]> var_469_axes_0 = const()[name = string("op_469_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_469_keep_dims_0 = const()[name = string("op_469_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 4, 16, 1]> var_469_cast_fp16 = reduce_sum(axes = var_469_axes_0, keep_dims = var_469_keep_dims_0, x = var_467_cast_fp16)[name = string("op_469_cast_fp16")];
            fp16 var_470_to_fp16 = const()[name = string("op_470_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 4, 16, 1]> var_471_cast_fp16 = add(x = var_469_cast_fp16, y = var_470_to_fp16)[name = string("op_471_cast_fp16")];
            fp32 var_472_epsilon_0 = const()[name = string("op_472_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 4, 16, 1]> var_472_cast_fp16 = rsqrt(epsilon = var_472_epsilon_0, x = var_471_cast_fp16)[name = string("op_472_cast_fp16")];
            tensor<fp16, [1, 4, 16, 128]> var_473_cast_fp16 = mul(x = value_21_cast_fp16, y = var_472_cast_fp16)[name = string("op_473_cast_fp16")];
            tensor<int32, [4]> key_7_perm_0 = const()[name = string("key_7_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<int32, [4]> value_25_perm_0 = const()[name = string("value_25_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<fp16, [16, 1024]> groups_0_1_in_proj_b_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(21223232))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(21235584))))[name = string("groups_0_1_in_proj_b_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16]> linear_9_bias_0_to_fp16 = const()[name = string("linear_9_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(4862656)))];
            tensor<fp16, [4, 16]> linear_9_cast_fp16 = linear(bias = linear_9_bias_0_to_fp16, weight = groups_0_1_in_proj_b_weight_promoted_to_fp16_palettized, x = input_15_cast_fp16)[name = string("linear_9_cast_fp16")];
            tensor<fp16, [4, 16]> var_478_cast_fp16 = sigmoid(x = linear_9_cast_fp16)[name = string("op_478_cast_fp16")];
            tensor<int32, [2]> var_479_perm_0 = const()[name = string("op_479_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> beta_3_axes_0 = const()[name = string("beta_3_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 4]> var_479_cast_fp16 = transpose(perm = var_479_perm_0, x = var_478_cast_fp16)[name = string("transpose_131")];
            tensor<fp16, [1, 16, 4]> beta_3_cast_fp16 = expand_dims(axes = beta_3_axes_0, x = var_479_cast_fp16)[name = string("beta_3_cast_fp16")];
            tensor<fp16, [16, 1024]> op_485_weight_0_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(21235776))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(21248128))))[name = string("op_485_weight_0_to_fp16_palettized")];
            tensor<fp16, [16]> var_485_bias_0_to_fp16 = const()[name = string("op_485_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(21248320)))];
            tensor<fp16, [4, 16]> var_485_cast_fp16 = linear(bias = var_485_bias_0_to_fp16, weight = op_485_weight_0_to_fp16_palettized, x = input_15_cast_fp16)[name = string("op_485_cast_fp16")];
            tensor<fp16, [4, 16]> var_486_cast_fp16 = softplus(x = var_485_cast_fp16)[name = string("op_486_cast_fp16")];
            tensor<fp16, [16]> var_482_promoted_to_fp16 = const()[name = string("op_482_promoted_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(21248448)))];
            tensor<fp16, [4, 16]> var_487_cast_fp16 = mul(x = var_482_promoted_to_fp16, y = var_486_cast_fp16)[name = string("op_487_cast_fp16")];
            tensor<int32, [2]> var_488_perm_0 = const()[name = string("op_488_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> decay_3_axes_0 = const()[name = string("decay_3_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 4]> var_488_cast_fp16 = transpose(perm = var_488_perm_0, x = var_487_cast_fp16)[name = string("transpose_130")];
            tensor<fp16, [1, 16, 4]> decay_3_cast_fp16 = expand_dims(axes = decay_3_axes_0, x = var_488_cast_fp16)[name = string("decay_3_cast_fp16")];
            tensor<fp16, [2048, 1024]> groups_0_1_in_proj_z_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [2048, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(21248576))), lut = tensor<fp16, [128, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(22821504))))[name = string("groups_0_1_in_proj_z_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 2048]> linear_11_cast_fp16 = linear(bias = linear_3_bias_0_to_fp16, weight = groups_0_1_in_proj_z_weight_promoted_to_fp16_palettized, x = input_15_cast_fp16)[name = string("linear_11_cast_fp16")];
            tensor<int32, [3]> var_496_begin_0 = const()[name = string("op_496_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_496_end_0 = const()[name = string("op_496_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_496_end_mask_0 = const()[name = string("op_496_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_496_squeeze_mask_0 = const()[name = string("op_496_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_496_cast_fp16 = slice_by_index(begin = var_496_begin_0, end = var_496_end_0, end_mask = var_496_end_mask_0, squeeze_mask = var_496_squeeze_mask_0, x = decay_3_cast_fp16)[name = string("op_496_cast_fp16")];
            tensor<fp16, [1, 16]> var_497_cast_fp16 = exp(x = var_496_cast_fp16)[name = string("op_497_cast_fp16")];
            tensor<int32, [1]> var_498_axes_0 = const()[name = string("op_498_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_498_cast_fp16 = expand_dims(axes = var_498_axes_0, x = var_497_cast_fp16)[name = string("op_498_cast_fp16")];
            tensor<int32, [1]> var_499_axes_0 = const()[name = string("op_499_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_499_cast_fp16 = expand_dims(axes = var_499_axes_0, x = var_498_cast_fp16)[name = string("op_499_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_17_cast_fp16 = mul(x = rec1, y = var_499_cast_fp16)[name = string("state_17_cast_fp16")];
            tensor<int32, [4]> key_token_9_begin_0 = const()[name = string("key_token_9_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> key_token_9_end_0 = const()[name = string("key_token_9_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> key_token_9_end_mask_0 = const()[name = string("key_token_9_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_9_squeeze_mask_0 = const()[name = string("key_token_9_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 4, 128]> key_7_cast_fp16 = transpose(perm = key_7_perm_0, x = var_473_cast_fp16)[name = string("transpose_129")];
            tensor<fp16, [1, 16, 128]> key_token_9_cast_fp16 = slice_by_index(begin = key_token_9_begin_0, end = key_token_9_end_0, end_mask = key_token_9_end_mask_0, squeeze_mask = key_token_9_squeeze_mask_0, x = key_7_cast_fp16)[name = string("key_token_9_cast_fp16")];
            tensor<int32, [4]> value_token_9_begin_0 = const()[name = string("value_token_9_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> value_token_9_end_0 = const()[name = string("value_token_9_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> value_token_9_end_mask_0 = const()[name = string("value_token_9_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_9_squeeze_mask_0 = const()[name = string("value_token_9_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 4, 128]> value_25_cast_fp16 = transpose(perm = value_25_perm_0, x = value_23_cast_fp16)[name = string("transpose_128")];
            tensor<fp16, [1, 16, 128]> value_token_9_cast_fp16 = slice_by_index(begin = value_token_9_begin_0, end = value_token_9_end_0, end_mask = value_token_9_end_mask_0, squeeze_mask = value_token_9_squeeze_mask_0, x = value_25_cast_fp16)[name = string("value_token_9_cast_fp16")];
            tensor<int32, [1]> var_507_axes_0 = const()[name = string("op_507_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_507_cast_fp16 = expand_dims(axes = var_507_axes_0, x = key_token_9_cast_fp16)[name = string("op_507_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_508_cast_fp16 = mul(x = state_17_cast_fp16, y = var_507_cast_fp16)[name = string("op_508_cast_fp16")];
            tensor<int32, [1]> memory_9_axes_0 = const()[name = string("memory_9_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_9_keep_dims_0 = const()[name = string("memory_9_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_9_cast_fp16 = reduce_sum(axes = memory_9_axes_0, keep_dims = memory_9_keep_dims_0, x = var_508_cast_fp16)[name = string("memory_9_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_511_cast_fp16 = sub(x = value_token_9_cast_fp16, y = memory_9_cast_fp16)[name = string("op_511_cast_fp16")];
            tensor<int32, [3]> var_514_begin_0 = const()[name = string("op_514_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_514_end_0 = const()[name = string("op_514_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_514_end_mask_0 = const()[name = string("op_514_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_514_squeeze_mask_0 = const()[name = string("op_514_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_514_cast_fp16 = slice_by_index(begin = var_514_begin_0, end = var_514_end_0, end_mask = var_514_end_mask_0, squeeze_mask = var_514_squeeze_mask_0, x = beta_3_cast_fp16)[name = string("op_514_cast_fp16")];
            tensor<int32, [1]> var_515_axes_0 = const()[name = string("op_515_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_515_cast_fp16 = expand_dims(axes = var_515_axes_0, x = var_514_cast_fp16)[name = string("op_515_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_9_cast_fp16 = mul(x = var_511_cast_fp16, y = var_515_cast_fp16)[name = string("delta_9_cast_fp16")];
            tensor<int32, [1]> var_518_axes_0 = const()[name = string("op_518_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_518_cast_fp16 = expand_dims(axes = var_518_axes_0, x = delta_9_cast_fp16)[name = string("op_518_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_519_cast_fp16 = mul(x = var_507_cast_fp16, y = var_518_cast_fp16)[name = string("op_519_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_19_cast_fp16 = add(x = state_17_cast_fp16, y = var_519_cast_fp16)[name = string("state_19_cast_fp16")];
            tensor<int32, [4]> var_523_begin_0 = const()[name = string("op_523_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> var_523_end_0 = const()[name = string("op_523_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> var_523_end_mask_0 = const()[name = string("op_523_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_523_squeeze_mask_0 = const()[name = string("op_523_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_523_cast_fp16 = slice_by_index(begin = var_523_begin_0, end = var_523_end_0, end_mask = var_523_end_mask_0, squeeze_mask = var_523_squeeze_mask_0, x = _inversed_query_7_cast_fp16)[name = string("op_523_cast_fp16")];
            tensor<int32, [1]> var_524_axes_0 = const()[name = string("op_524_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_524_cast_fp16 = expand_dims(axes = var_524_axes_0, x = var_523_cast_fp16)[name = string("op_524_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_525_cast_fp16 = mul(x = state_19_cast_fp16, y = var_524_cast_fp16)[name = string("op_525_cast_fp16")];
            tensor<int32, [1]> var_527_axes_0 = const()[name = string("op_527_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_527_keep_dims_0 = const()[name = string("op_527_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_527_cast_fp16 = reduce_sum(axes = var_527_axes_0, keep_dims = var_527_keep_dims_0, x = var_525_cast_fp16)[name = string("op_527_cast_fp16")];
            tensor<int32, [3]> var_530_begin_0 = const()[name = string("op_530_begin_0"), val = tensor<int32, [3]>([0, 0, 1])];
            tensor<int32, [3]> var_530_end_0 = const()[name = string("op_530_end_0"), val = tensor<int32, [3]>([1, 16, 2])];
            tensor<bool, [3]> var_530_end_mask_0 = const()[name = string("op_530_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_530_squeeze_mask_0 = const()[name = string("op_530_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_530_cast_fp16 = slice_by_index(begin = var_530_begin_0, end = var_530_end_0, end_mask = var_530_end_mask_0, squeeze_mask = var_530_squeeze_mask_0, x = decay_3_cast_fp16)[name = string("op_530_cast_fp16")];
            tensor<fp16, [1, 16]> var_531_cast_fp16 = exp(x = var_530_cast_fp16)[name = string("op_531_cast_fp16")];
            tensor<int32, [1]> var_532_axes_0 = const()[name = string("op_532_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_532_cast_fp16 = expand_dims(axes = var_532_axes_0, x = var_531_cast_fp16)[name = string("op_532_cast_fp16")];
            tensor<int32, [1]> var_533_axes_0 = const()[name = string("op_533_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_533_cast_fp16 = expand_dims(axes = var_533_axes_0, x = var_532_cast_fp16)[name = string("op_533_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_21_cast_fp16 = mul(x = state_19_cast_fp16, y = var_533_cast_fp16)[name = string("state_21_cast_fp16")];
            tensor<int32, [4]> key_token_11_begin_0 = const()[name = string("key_token_11_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> key_token_11_end_0 = const()[name = string("key_token_11_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> key_token_11_end_mask_0 = const()[name = string("key_token_11_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_11_squeeze_mask_0 = const()[name = string("key_token_11_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_11_cast_fp16 = slice_by_index(begin = key_token_11_begin_0, end = key_token_11_end_0, end_mask = key_token_11_end_mask_0, squeeze_mask = key_token_11_squeeze_mask_0, x = key_7_cast_fp16)[name = string("key_token_11_cast_fp16")];
            tensor<int32, [4]> value_token_11_begin_0 = const()[name = string("value_token_11_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> value_token_11_end_0 = const()[name = string("value_token_11_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> value_token_11_end_mask_0 = const()[name = string("value_token_11_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_11_squeeze_mask_0 = const()[name = string("value_token_11_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_11_cast_fp16 = slice_by_index(begin = value_token_11_begin_0, end = value_token_11_end_0, end_mask = value_token_11_end_mask_0, squeeze_mask = value_token_11_squeeze_mask_0, x = value_25_cast_fp16)[name = string("value_token_11_cast_fp16")];
            tensor<int32, [1]> var_541_axes_0 = const()[name = string("op_541_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_541_cast_fp16 = expand_dims(axes = var_541_axes_0, x = key_token_11_cast_fp16)[name = string("op_541_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_542_cast_fp16 = mul(x = state_21_cast_fp16, y = var_541_cast_fp16)[name = string("op_542_cast_fp16")];
            tensor<int32, [1]> memory_11_axes_0 = const()[name = string("memory_11_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_11_keep_dims_0 = const()[name = string("memory_11_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_11_cast_fp16 = reduce_sum(axes = memory_11_axes_0, keep_dims = memory_11_keep_dims_0, x = var_542_cast_fp16)[name = string("memory_11_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_545_cast_fp16 = sub(x = value_token_11_cast_fp16, y = memory_11_cast_fp16)[name = string("op_545_cast_fp16")];
            tensor<int32, [3]> var_548_begin_0 = const()[name = string("op_548_begin_0"), val = tensor<int32, [3]>([0, 0, 1])];
            tensor<int32, [3]> var_548_end_0 = const()[name = string("op_548_end_0"), val = tensor<int32, [3]>([1, 16, 2])];
            tensor<bool, [3]> var_548_end_mask_0 = const()[name = string("op_548_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_548_squeeze_mask_0 = const()[name = string("op_548_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_548_cast_fp16 = slice_by_index(begin = var_548_begin_0, end = var_548_end_0, end_mask = var_548_end_mask_0, squeeze_mask = var_548_squeeze_mask_0, x = beta_3_cast_fp16)[name = string("op_548_cast_fp16")];
            tensor<int32, [1]> var_549_axes_0 = const()[name = string("op_549_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_549_cast_fp16 = expand_dims(axes = var_549_axes_0, x = var_548_cast_fp16)[name = string("op_549_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_11_cast_fp16 = mul(x = var_545_cast_fp16, y = var_549_cast_fp16)[name = string("delta_11_cast_fp16")];
            tensor<int32, [1]> var_552_axes_0 = const()[name = string("op_552_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_552_cast_fp16 = expand_dims(axes = var_552_axes_0, x = delta_11_cast_fp16)[name = string("op_552_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_553_cast_fp16 = mul(x = var_541_cast_fp16, y = var_552_cast_fp16)[name = string("op_553_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_23_cast_fp16 = add(x = state_21_cast_fp16, y = var_553_cast_fp16)[name = string("state_23_cast_fp16")];
            tensor<int32, [4]> var_557_begin_0 = const()[name = string("op_557_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> var_557_end_0 = const()[name = string("op_557_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> var_557_end_mask_0 = const()[name = string("op_557_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_557_squeeze_mask_0 = const()[name = string("op_557_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_557_cast_fp16 = slice_by_index(begin = var_557_begin_0, end = var_557_end_0, end_mask = var_557_end_mask_0, squeeze_mask = var_557_squeeze_mask_0, x = _inversed_query_7_cast_fp16)[name = string("op_557_cast_fp16")];
            tensor<int32, [1]> var_558_axes_0 = const()[name = string("op_558_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_558_cast_fp16 = expand_dims(axes = var_558_axes_0, x = var_557_cast_fp16)[name = string("op_558_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_559_cast_fp16 = mul(x = state_23_cast_fp16, y = var_558_cast_fp16)[name = string("op_559_cast_fp16")];
            tensor<int32, [1]> var_561_axes_0 = const()[name = string("op_561_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_561_keep_dims_0 = const()[name = string("op_561_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_561_cast_fp16 = reduce_sum(axes = var_561_axes_0, keep_dims = var_561_keep_dims_0, x = var_559_cast_fp16)[name = string("op_561_cast_fp16")];
            tensor<int32, [3]> var_564_begin_0 = const()[name = string("op_564_begin_0"), val = tensor<int32, [3]>([0, 0, 2])];
            tensor<int32, [3]> var_564_end_0 = const()[name = string("op_564_end_0"), val = tensor<int32, [3]>([1, 16, 3])];
            tensor<bool, [3]> var_564_end_mask_0 = const()[name = string("op_564_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_564_squeeze_mask_0 = const()[name = string("op_564_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_564_cast_fp16 = slice_by_index(begin = var_564_begin_0, end = var_564_end_0, end_mask = var_564_end_mask_0, squeeze_mask = var_564_squeeze_mask_0, x = decay_3_cast_fp16)[name = string("op_564_cast_fp16")];
            tensor<fp16, [1, 16]> var_565_cast_fp16 = exp(x = var_564_cast_fp16)[name = string("op_565_cast_fp16")];
            tensor<int32, [1]> var_566_axes_0 = const()[name = string("op_566_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_566_cast_fp16 = expand_dims(axes = var_566_axes_0, x = var_565_cast_fp16)[name = string("op_566_cast_fp16")];
            tensor<int32, [1]> var_567_axes_0 = const()[name = string("op_567_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_567_cast_fp16 = expand_dims(axes = var_567_axes_0, x = var_566_cast_fp16)[name = string("op_567_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_25_cast_fp16 = mul(x = state_23_cast_fp16, y = var_567_cast_fp16)[name = string("state_25_cast_fp16")];
            tensor<int32, [4]> key_token_13_begin_0 = const()[name = string("key_token_13_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> key_token_13_end_0 = const()[name = string("key_token_13_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> key_token_13_end_mask_0 = const()[name = string("key_token_13_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_13_squeeze_mask_0 = const()[name = string("key_token_13_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_13_cast_fp16 = slice_by_index(begin = key_token_13_begin_0, end = key_token_13_end_0, end_mask = key_token_13_end_mask_0, squeeze_mask = key_token_13_squeeze_mask_0, x = key_7_cast_fp16)[name = string("key_token_13_cast_fp16")];
            tensor<int32, [4]> value_token_13_begin_0 = const()[name = string("value_token_13_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> value_token_13_end_0 = const()[name = string("value_token_13_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> value_token_13_end_mask_0 = const()[name = string("value_token_13_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_13_squeeze_mask_0 = const()[name = string("value_token_13_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_13_cast_fp16 = slice_by_index(begin = value_token_13_begin_0, end = value_token_13_end_0, end_mask = value_token_13_end_mask_0, squeeze_mask = value_token_13_squeeze_mask_0, x = value_25_cast_fp16)[name = string("value_token_13_cast_fp16")];
            tensor<int32, [1]> var_575_axes_0 = const()[name = string("op_575_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_575_cast_fp16 = expand_dims(axes = var_575_axes_0, x = key_token_13_cast_fp16)[name = string("op_575_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_576_cast_fp16 = mul(x = state_25_cast_fp16, y = var_575_cast_fp16)[name = string("op_576_cast_fp16")];
            tensor<int32, [1]> memory_13_axes_0 = const()[name = string("memory_13_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_13_keep_dims_0 = const()[name = string("memory_13_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_13_cast_fp16 = reduce_sum(axes = memory_13_axes_0, keep_dims = memory_13_keep_dims_0, x = var_576_cast_fp16)[name = string("memory_13_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_579_cast_fp16 = sub(x = value_token_13_cast_fp16, y = memory_13_cast_fp16)[name = string("op_579_cast_fp16")];
            tensor<int32, [3]> var_582_begin_0 = const()[name = string("op_582_begin_0"), val = tensor<int32, [3]>([0, 0, 2])];
            tensor<int32, [3]> var_582_end_0 = const()[name = string("op_582_end_0"), val = tensor<int32, [3]>([1, 16, 3])];
            tensor<bool, [3]> var_582_end_mask_0 = const()[name = string("op_582_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_582_squeeze_mask_0 = const()[name = string("op_582_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_582_cast_fp16 = slice_by_index(begin = var_582_begin_0, end = var_582_end_0, end_mask = var_582_end_mask_0, squeeze_mask = var_582_squeeze_mask_0, x = beta_3_cast_fp16)[name = string("op_582_cast_fp16")];
            tensor<int32, [1]> var_583_axes_0 = const()[name = string("op_583_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_583_cast_fp16 = expand_dims(axes = var_583_axes_0, x = var_582_cast_fp16)[name = string("op_583_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_13_cast_fp16 = mul(x = var_579_cast_fp16, y = var_583_cast_fp16)[name = string("delta_13_cast_fp16")];
            tensor<int32, [1]> var_586_axes_0 = const()[name = string("op_586_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_586_cast_fp16 = expand_dims(axes = var_586_axes_0, x = delta_13_cast_fp16)[name = string("op_586_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_587_cast_fp16 = mul(x = var_575_cast_fp16, y = var_586_cast_fp16)[name = string("op_587_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_27_cast_fp16 = add(x = state_25_cast_fp16, y = var_587_cast_fp16)[name = string("state_27_cast_fp16")];
            tensor<int32, [4]> var_591_begin_0 = const()[name = string("op_591_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> var_591_end_0 = const()[name = string("op_591_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> var_591_end_mask_0 = const()[name = string("op_591_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_591_squeeze_mask_0 = const()[name = string("op_591_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_591_cast_fp16 = slice_by_index(begin = var_591_begin_0, end = var_591_end_0, end_mask = var_591_end_mask_0, squeeze_mask = var_591_squeeze_mask_0, x = _inversed_query_7_cast_fp16)[name = string("op_591_cast_fp16")];
            tensor<int32, [1]> var_592_axes_0 = const()[name = string("op_592_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_592_cast_fp16 = expand_dims(axes = var_592_axes_0, x = var_591_cast_fp16)[name = string("op_592_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_593_cast_fp16 = mul(x = state_27_cast_fp16, y = var_592_cast_fp16)[name = string("op_593_cast_fp16")];
            tensor<int32, [1]> var_595_axes_0 = const()[name = string("op_595_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_595_keep_dims_0 = const()[name = string("op_595_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_595_cast_fp16 = reduce_sum(axes = var_595_axes_0, keep_dims = var_595_keep_dims_0, x = var_593_cast_fp16)[name = string("op_595_cast_fp16")];
            tensor<int32, [3]> var_598_begin_0 = const()[name = string("op_598_begin_0"), val = tensor<int32, [3]>([0, 0, 3])];
            tensor<int32, [3]> var_598_end_0 = const()[name = string("op_598_end_0"), val = tensor<int32, [3]>([1, 16, 4])];
            tensor<bool, [3]> var_598_end_mask_0 = const()[name = string("op_598_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_598_squeeze_mask_0 = const()[name = string("op_598_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_598_cast_fp16 = slice_by_index(begin = var_598_begin_0, end = var_598_end_0, end_mask = var_598_end_mask_0, squeeze_mask = var_598_squeeze_mask_0, x = decay_3_cast_fp16)[name = string("op_598_cast_fp16")];
            tensor<fp16, [1, 16]> var_599_cast_fp16 = exp(x = var_598_cast_fp16)[name = string("op_599_cast_fp16")];
            tensor<int32, [1]> var_600_axes_0 = const()[name = string("op_600_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_600_cast_fp16 = expand_dims(axes = var_600_axes_0, x = var_599_cast_fp16)[name = string("op_600_cast_fp16")];
            tensor<int32, [1]> var_601_axes_0 = const()[name = string("op_601_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_601_cast_fp16 = expand_dims(axes = var_601_axes_0, x = var_600_cast_fp16)[name = string("op_601_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_29_cast_fp16 = mul(x = state_27_cast_fp16, y = var_601_cast_fp16)[name = string("state_29_cast_fp16")];
            tensor<int32, [4]> key_token_15_begin_0 = const()[name = string("key_token_15_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> key_token_15_end_0 = const()[name = string("key_token_15_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> key_token_15_end_mask_0 = const()[name = string("key_token_15_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_15_squeeze_mask_0 = const()[name = string("key_token_15_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_15_cast_fp16 = slice_by_index(begin = key_token_15_begin_0, end = key_token_15_end_0, end_mask = key_token_15_end_mask_0, squeeze_mask = key_token_15_squeeze_mask_0, x = key_7_cast_fp16)[name = string("key_token_15_cast_fp16")];
            tensor<int32, [4]> value_token_15_begin_0 = const()[name = string("value_token_15_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> value_token_15_end_0 = const()[name = string("value_token_15_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> value_token_15_end_mask_0 = const()[name = string("value_token_15_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_15_squeeze_mask_0 = const()[name = string("value_token_15_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_15_cast_fp16 = slice_by_index(begin = value_token_15_begin_0, end = value_token_15_end_0, end_mask = value_token_15_end_mask_0, squeeze_mask = value_token_15_squeeze_mask_0, x = value_25_cast_fp16)[name = string("value_token_15_cast_fp16")];
            tensor<int32, [1]> var_609_axes_0 = const()[name = string("op_609_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_609_cast_fp16 = expand_dims(axes = var_609_axes_0, x = key_token_15_cast_fp16)[name = string("op_609_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_610_cast_fp16 = mul(x = state_29_cast_fp16, y = var_609_cast_fp16)[name = string("op_610_cast_fp16")];
            tensor<int32, [1]> memory_15_axes_0 = const()[name = string("memory_15_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_15_keep_dims_0 = const()[name = string("memory_15_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_15_cast_fp16 = reduce_sum(axes = memory_15_axes_0, keep_dims = memory_15_keep_dims_0, x = var_610_cast_fp16)[name = string("memory_15_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_613_cast_fp16 = sub(x = value_token_15_cast_fp16, y = memory_15_cast_fp16)[name = string("op_613_cast_fp16")];
            tensor<int32, [3]> var_616_begin_0 = const()[name = string("op_616_begin_0"), val = tensor<int32, [3]>([0, 0, 3])];
            tensor<int32, [3]> var_616_end_0 = const()[name = string("op_616_end_0"), val = tensor<int32, [3]>([1, 16, 4])];
            tensor<bool, [3]> var_616_end_mask_0 = const()[name = string("op_616_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_616_squeeze_mask_0 = const()[name = string("op_616_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_616_cast_fp16 = slice_by_index(begin = var_616_begin_0, end = var_616_end_0, end_mask = var_616_end_mask_0, squeeze_mask = var_616_squeeze_mask_0, x = beta_3_cast_fp16)[name = string("op_616_cast_fp16")];
            tensor<int32, [1]> var_617_axes_0 = const()[name = string("op_617_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_617_cast_fp16 = expand_dims(axes = var_617_axes_0, x = var_616_cast_fp16)[name = string("op_617_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_15_cast_fp16 = mul(x = var_613_cast_fp16, y = var_617_cast_fp16)[name = string("delta_15_cast_fp16")];
            tensor<int32, [1]> var_620_axes_0 = const()[name = string("op_620_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_620_cast_fp16 = expand_dims(axes = var_620_axes_0, x = delta_15_cast_fp16)[name = string("op_620_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_621_cast_fp16 = mul(x = var_609_cast_fp16, y = var_620_cast_fp16)[name = string("op_621_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> rec1_out = add(x = state_29_cast_fp16, y = var_621_cast_fp16)[name = string("state_31_cast_fp16")];
            tensor<int32, [4]> var_625_begin_0 = const()[name = string("op_625_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> var_625_end_0 = const()[name = string("op_625_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> var_625_end_mask_0 = const()[name = string("op_625_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_625_squeeze_mask_0 = const()[name = string("op_625_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_625_cast_fp16 = slice_by_index(begin = var_625_begin_0, end = var_625_end_0, end_mask = var_625_end_mask_0, squeeze_mask = var_625_squeeze_mask_0, x = _inversed_query_7_cast_fp16)[name = string("op_625_cast_fp16")];
            tensor<int32, [1]> var_626_axes_0 = const()[name = string("op_626_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_626_cast_fp16 = expand_dims(axes = var_626_axes_0, x = var_625_cast_fp16)[name = string("op_626_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_627_cast_fp16 = mul(x = rec1_out, y = var_626_cast_fp16)[name = string("op_627_cast_fp16")];
            tensor<int32, [1]> var_629_axes_0 = const()[name = string("op_629_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_629_keep_dims_0 = const()[name = string("op_629_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_629_cast_fp16 = reduce_sum(axes = var_629_axes_0, keep_dims = var_629_keep_dims_0, x = var_627_cast_fp16)[name = string("op_629_cast_fp16")];
            int32 attention_11_axis_0 = const()[name = string("attention_11_axis_0"), val = int32(1)];
            tensor<fp16, [1, 4, 16, 128]> attention_11_cast_fp16 = stack(axis = attention_11_axis_0, values = (var_527_cast_fp16, var_561_cast_fp16, var_595_cast_fp16, var_629_cast_fp16))[name = string("attention_11_cast_fp16")];
            tensor<int32, [2]> var_632 = const()[name = string("op_632"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [64, 128]> attention_13_cast_fp16 = reshape(shape = var_632, x = attention_11_cast_fp16)[name = string("attention_13_cast_fp16")];
            tensor<int32, [2]> var_634 = const()[name = string("op_634"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [64, 128]> input_19_cast_fp16 = reshape(shape = var_634, x = linear_11_cast_fp16)[name = string("input_19_cast_fp16")];
            tensor<fp16, [64, 128]> var_636_cast_fp16 = mul(x = attention_13_cast_fp16, y = attention_13_cast_fp16)[name = string("op_636_cast_fp16")];
            tensor<int32, [1]> variance_9_axes_0 = const()[name = string("variance_9_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_9_keep_dims_0 = const()[name = string("variance_9_keep_dims_0"), val = bool(true)];
            tensor<fp16, [64, 1]> variance_9_cast_fp16 = reduce_mean(axes = variance_9_axes_0, keep_dims = variance_9_keep_dims_0, x = var_636_cast_fp16)[name = string("variance_9_cast_fp16")];
            fp16 var_639_to_fp16 = const()[name = string("op_639_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [64, 1]> var_640_cast_fp16 = add(x = variance_9_cast_fp16, y = var_639_to_fp16)[name = string("op_640_cast_fp16")];
            fp32 var_641_epsilon_0 = const()[name = string("op_641_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [64, 1]> var_641_cast_fp16 = rsqrt(epsilon = var_641_epsilon_0, x = var_640_cast_fp16)[name = string("op_641_cast_fp16")];
            tensor<fp16, [64, 128]> attention_15_cast_fp16 = mul(x = attention_13_cast_fp16, y = var_641_cast_fp16)[name = string("attention_15_cast_fp16")];
            tensor<fp16, [128]> groups_0_1_gated_norm_promoted_to_fp16 = const()[name = string("groups_0_1_gated_norm_promoted_to_fp16"), val = tensor<fp16, [128]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(22837952)))];
            tensor<fp16, [64, 128]> attention_17_cast_fp16 = mul(x = attention_15_cast_fp16, y = groups_0_1_gated_norm_promoted_to_fp16)[name = string("attention_17_cast_fp16")];
            tensor<fp16, [64, 128]> var_644_cast_fp16 = silu(x = input_19_cast_fp16)[name = string("op_644_cast_fp16")];
            tensor<fp16, [64, 128]> attention_19_cast_fp16 = mul(x = attention_17_cast_fp16, y = var_644_cast_fp16)[name = string("attention_19_cast_fp16")];
            tensor<int32, [2]> var_646 = const()[name = string("op_646"), val = tensor<int32, [2]>([4, 2048])];
            tensor<fp16, [4, 2048]> input_21_cast_fp16 = reshape(shape = var_646, x = attention_19_cast_fp16)[name = string("input_21_cast_fp16")];
            tensor<fp16, [1024, 2048]> groups_0_1_out_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 2048]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(22838272))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(24411200))))[name = string("groups_0_1_out_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 1024]> linear_12_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = groups_0_1_out_proj_weight_promoted_to_fp16_palettized, x = input_21_cast_fp16)[name = string("linear_12_cast_fp16")];
            tensor<fp16, [4, 1024]> value_27_cast_fp16 = add(x = value_15_cast_fp16, y = linear_12_cast_fp16)[name = string("value_27_cast_fp16")];
            tensor<fp16, [4, 1024]> var_651_cast_fp16 = mul(x = value_27_cast_fp16, y = value_27_cast_fp16)[name = string("op_651_cast_fp16")];
            tensor<int32, [1]> variance_11_axes_0 = const()[name = string("variance_11_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_11_keep_dims_0 = const()[name = string("variance_11_keep_dims_0"), val = bool(true)];
            tensor<fp16, [4, 1]> variance_11_cast_fp16 = reduce_mean(axes = variance_11_axes_0, keep_dims = variance_11_keep_dims_0, x = var_651_cast_fp16)[name = string("variance_11_cast_fp16")];
            fp16 var_654_to_fp16 = const()[name = string("op_654_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [4, 1]> var_655_cast_fp16 = add(x = variance_11_cast_fp16, y = var_654_to_fp16)[name = string("op_655_cast_fp16")];
            fp32 var_656_epsilon_0 = const()[name = string("op_656_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [4, 1]> var_656_cast_fp16 = rsqrt(epsilon = var_656_epsilon_0, x = var_655_cast_fp16)[name = string("op_656_cast_fp16")];
            tensor<fp16, [4, 1024]> var_657_cast_fp16 = mul(x = value_27_cast_fp16, y = var_656_cast_fp16)[name = string("op_657_cast_fp16")];
            tensor<fp16, [1024]> var_659_to_fp16 = const()[name = string("op_659_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(24419456)))];
            tensor<fp16, [4, 1024]> input_23_cast_fp16 = mul(x = var_657_cast_fp16, y = var_659_to_fp16)[name = string("input_23_cast_fp16")];
            tensor<fp16, [3584, 1024]> groups_0_1_gate_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(24421568))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(27174144))))[name = string("groups_0_1_gate_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 3584]> linear_13_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = groups_0_1_gate_proj_weight_promoted_to_fp16_palettized, x = input_23_cast_fp16)[name = string("linear_13_cast_fp16")];
            tensor<fp16, [4, 3584]> var_663_cast_fp16 = silu(x = linear_13_cast_fp16)[name = string("op_663_cast_fp16")];
            tensor<fp16, [3584, 1024]> groups_0_1_up_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(27202880))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(29955456))))[name = string("groups_0_1_up_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 3584]> linear_14_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = groups_0_1_up_proj_weight_promoted_to_fp16_palettized, x = input_23_cast_fp16)[name = string("linear_14_cast_fp16")];
            tensor<fp16, [4, 3584]> input_27_cast_fp16 = mul(x = var_663_cast_fp16, y = linear_14_cast_fp16)[name = string("input_27_cast_fp16")];
            tensor<fp16, [1024, 3584]> groups_0_1_down_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 3584]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(29984192))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(32736768))))[name = string("groups_0_1_down_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 1024]> linear_15_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = groups_0_1_down_proj_weight_promoted_to_fp16_palettized, x = input_27_cast_fp16)[name = string("linear_15_cast_fp16")];
            tensor<fp16, [4, 1024]> value_29_cast_fp16 = add(x = value_27_cast_fp16, y = linear_15_cast_fp16)[name = string("value_29_cast_fp16")];
            int32 var_690 = const()[name = string("op_690"), val = int32(-1)];
            tensor<fp16, [4, 1024]> var_705_cast_fp16 = mul(x = value_29_cast_fp16, y = value_29_cast_fp16)[name = string("op_705_cast_fp16")];
            tensor<int32, [1]> variance_13_axes_0 = const()[name = string("variance_13_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_13_keep_dims_0 = const()[name = string("variance_13_keep_dims_0"), val = bool(true)];
            tensor<fp16, [4, 1]> variance_13_cast_fp16 = reduce_mean(axes = variance_13_axes_0, keep_dims = variance_13_keep_dims_0, x = var_705_cast_fp16)[name = string("variance_13_cast_fp16")];
            fp16 var_708_to_fp16 = const()[name = string("op_708_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [4, 1]> var_709_cast_fp16 = add(x = variance_13_cast_fp16, y = var_708_to_fp16)[name = string("op_709_cast_fp16")];
            fp32 var_710_epsilon_0 = const()[name = string("op_710_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [4, 1]> var_710_cast_fp16 = rsqrt(epsilon = var_710_epsilon_0, x = var_709_cast_fp16)[name = string("op_710_cast_fp16")];
            tensor<fp16, [4, 1024]> var_711_cast_fp16 = mul(x = value_29_cast_fp16, y = var_710_cast_fp16)[name = string("op_711_cast_fp16")];
            tensor<fp16, [1024]> var_713_to_fp16 = const()[name = string("op_713_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(32745024)))];
            tensor<fp16, [4, 1024]> input_29_cast_fp16 = mul(x = var_711_cast_fp16, y = var_713_to_fp16)[name = string("input_29_cast_fp16")];
            tensor<fp16, [6144, 1024]> groups_0_2_in_proj_qkv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(32747136))), lut = tensor<fp16, [384, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(37465792))))[name = string("groups_0_2_in_proj_qkv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 6144]> linear_16_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = groups_0_2_in_proj_qkv_weight_promoted_to_fp16_palettized, x = input_29_cast_fp16)[name = string("linear_16_cast_fp16")];
            tensor<int32, [2]> var_717_perm_0 = const()[name = string("op_717_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> mixed_5_axes_0 = const()[name = string("mixed_5_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [6144, 4]> var_717_cast_fp16 = transpose(perm = var_717_perm_0, x = linear_16_cast_fp16)[name = string("transpose_127")];
            tensor<fp16, [1, 6144, 4]> mixed_5_cast_fp16 = expand_dims(axes = mixed_5_axes_0, x = var_717_cast_fp16)[name = string("mixed_5_cast_fp16")];
            bool convolution_input_5_interleave_0 = const()[name = string("convolution_input_5_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 6144, 7]> convolution_input_5_cast_fp16 = concat(axis = var_690, interleave = convolution_input_5_interleave_0, values = (conv2, mixed_5_cast_fp16))[name = string("convolution_input_5_cast_fp16")];
            string input_31_pad_type_0 = const()[name = string("input_31_pad_type_0"), val = string("valid")];
            int32 input_31_groups_0 = const()[name = string("input_31_groups_0"), val = int32(6144)];
            tensor<int32, [1]> input_31_strides_0 = const()[name = string("input_31_strides_0"), val = tensor<int32, [1]>([1])];
            tensor<int32, [2]> input_31_pad_0 = const()[name = string("input_31_pad_0"), val = tensor<int32, [2]>([0, 0])];
            tensor<int32, [1]> input_31_dilations_0 = const()[name = string("input_31_dilations_0"), val = tensor<int32, [1]>([1])];
            tensor<fp16, [6144, 1, 4]> groups_0_2_conv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1, 4]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(37515008))), lut = tensor<fp16, [384, 1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(37533504))))[name = string("groups_0_2_conv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 6144, 4]> input_31_cast_fp16 = conv(dilations = input_31_dilations_0, groups = input_31_groups_0, pad = input_31_pad_0, pad_type = input_31_pad_type_0, strides = input_31_strides_0, weight = groups_0_2_conv_weight_promoted_to_fp16_palettized, x = convolution_input_5_cast_fp16)[name = string("input_31_cast_fp16")];
            tensor<fp16, [1, 6144, 4]> var_726_cast_fp16 = silu(x = input_31_cast_fp16)[name = string("op_726_cast_fp16")];
            tensor<int32, [3]> var_727_perm_0 = const()[name = string("op_727_perm_0"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [3]> var_730_begin_0 = const()[name = string("op_730_begin_0"), val = tensor<int32, [3]>([0, 0, 4])];
            tensor<int32, [3]> var_730_end_0 = const()[name = string("op_730_end_0"), val = tensor<int32, [3]>([1, 6144, 7])];
            tensor<bool, [3]> var_730_end_mask_0 = const()[name = string("op_730_end_mask_0"), val = tensor<bool, [3]>([true, true, true])];
            tensor<fp16, [1, 6144, 3]> conv2_out = slice_by_index(begin = var_730_begin_0, end = var_730_end_0, end_mask = var_730_end_mask_0, x = convolution_input_5_cast_fp16)[name = string("op_730_cast_fp16")];
            tensor<int32, [3]> var_731 = const()[name = string("op_731"), val = tensor<int32, [3]>([2048, 2048, 2048])];
            int32 var_732_axis_0 = const()[name = string("op_732_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 4, 6144]> var_727_cast_fp16 = transpose(perm = var_727_perm_0, x = var_726_cast_fp16)[name = string("transpose_126")];
            tensor<fp16, [1, 4, 2048]> var_732_cast_fp16_0, tensor<fp16, [1, 4, 2048]> var_732_cast_fp16_1, tensor<fp16, [1, 4, 2048]> var_732_cast_fp16_2 = split(axis = var_732_axis_0, split_sizes = var_731, x = var_727_cast_fp16)[name = string("op_732_cast_fp16")];
            tensor<int32, [4]> var_736 = const()[name = string("op_736"), val = tensor<int32, [4]>([1, 4, 16, 128])];
            tensor<fp16, [1, 4, 16, 128]> value_33_cast_fp16 = reshape(shape = var_736, x = var_732_cast_fp16_0)[name = string("value_33_cast_fp16")];
            tensor<int32, [4]> var_738 = const()[name = string("op_738"), val = tensor<int32, [4]>([1, 4, 16, 128])];
            tensor<fp16, [1, 4, 16, 128]> value_35_cast_fp16 = reshape(shape = var_738, x = var_732_cast_fp16_1)[name = string("value_35_cast_fp16")];
            tensor<int32, [4]> var_740 = const()[name = string("op_740"), val = tensor<int32, [4]>([1, 4, 16, 128])];
            tensor<fp16, [1, 4, 16, 128]> value_37_cast_fp16 = reshape(shape = var_740, x = var_732_cast_fp16_2)[name = string("value_37_cast_fp16")];
            tensor<fp16, [1, 4, 16, 128]> var_742_cast_fp16 = mul(x = value_33_cast_fp16, y = value_33_cast_fp16)[name = string("op_742_cast_fp16")];
            tensor<int32, [1]> var_744_axes_0 = const()[name = string("op_744_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_744_keep_dims_0 = const()[name = string("op_744_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 4, 16, 1]> var_744_cast_fp16 = reduce_sum(axes = var_744_axes_0, keep_dims = var_744_keep_dims_0, x = var_742_cast_fp16)[name = string("op_744_cast_fp16")];
            fp16 var_745_to_fp16 = const()[name = string("op_745_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 4, 16, 1]> var_746_cast_fp16 = add(x = var_744_cast_fp16, y = var_745_to_fp16)[name = string("op_746_cast_fp16")];
            fp32 var_747_epsilon_0 = const()[name = string("op_747_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 4, 16, 1]> var_747_cast_fp16 = rsqrt(epsilon = var_747_epsilon_0, x = var_746_cast_fp16)[name = string("op_747_cast_fp16")];
            tensor<fp16, [1, 4, 16, 128]> var_748_cast_fp16 = mul(x = value_33_cast_fp16, y = var_747_cast_fp16)[name = string("op_748_cast_fp16")];
            tensor<int32, [4]> var_749_perm_0 = const()[name = string("op_749_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            fp16 _inversed_query_11_y_0_to_fp16 = const()[name = string("_inversed_query_11_y_0_to_fp16"), val = fp16(0x1.6ap-4)];
            tensor<fp16, [1, 16, 4, 128]> var_749_cast_fp16 = transpose(perm = var_749_perm_0, x = var_748_cast_fp16)[name = string("transpose_125")];
            tensor<fp16, [1, 16, 4, 128]> _inversed_query_11_cast_fp16 = mul(x = var_749_cast_fp16, y = _inversed_query_11_y_0_to_fp16)[name = string("_inversed_query_11_cast_fp16")];
            tensor<fp16, [1, 4, 16, 128]> var_752_cast_fp16 = mul(x = value_35_cast_fp16, y = value_35_cast_fp16)[name = string("op_752_cast_fp16")];
            tensor<int32, [1]> var_754_axes_0 = const()[name = string("op_754_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_754_keep_dims_0 = const()[name = string("op_754_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 4, 16, 1]> var_754_cast_fp16 = reduce_sum(axes = var_754_axes_0, keep_dims = var_754_keep_dims_0, x = var_752_cast_fp16)[name = string("op_754_cast_fp16")];
            fp16 var_755_to_fp16 = const()[name = string("op_755_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 4, 16, 1]> var_756_cast_fp16 = add(x = var_754_cast_fp16, y = var_755_to_fp16)[name = string("op_756_cast_fp16")];
            fp32 var_757_epsilon_0 = const()[name = string("op_757_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 4, 16, 1]> var_757_cast_fp16 = rsqrt(epsilon = var_757_epsilon_0, x = var_756_cast_fp16)[name = string("op_757_cast_fp16")];
            tensor<fp16, [1, 4, 16, 128]> var_758_cast_fp16 = mul(x = value_35_cast_fp16, y = var_757_cast_fp16)[name = string("op_758_cast_fp16")];
            tensor<int32, [4]> key_11_perm_0 = const()[name = string("key_11_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<int32, [4]> value_39_perm_0 = const()[name = string("value_39_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<fp16, [16, 1024]> groups_0_2_in_proj_b_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(37582720))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(37595072))))[name = string("groups_0_2_in_proj_b_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16]> linear_17_bias_0_to_fp16 = const()[name = string("linear_17_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(4862656)))];
            tensor<fp16, [4, 16]> linear_17_cast_fp16 = linear(bias = linear_17_bias_0_to_fp16, weight = groups_0_2_in_proj_b_weight_promoted_to_fp16_palettized, x = input_29_cast_fp16)[name = string("linear_17_cast_fp16")];
            tensor<fp16, [4, 16]> var_763_cast_fp16 = sigmoid(x = linear_17_cast_fp16)[name = string("op_763_cast_fp16")];
            tensor<int32, [2]> var_764_perm_0 = const()[name = string("op_764_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> beta_5_axes_0 = const()[name = string("beta_5_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 4]> var_764_cast_fp16 = transpose(perm = var_764_perm_0, x = var_763_cast_fp16)[name = string("transpose_124")];
            tensor<fp16, [1, 16, 4]> beta_5_cast_fp16 = expand_dims(axes = beta_5_axes_0, x = var_764_cast_fp16)[name = string("beta_5_cast_fp16")];
            tensor<fp16, [16, 1024]> op_770_weight_0_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(37595264))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(37607616))))[name = string("op_770_weight_0_to_fp16_palettized")];
            tensor<fp16, [16]> var_770_bias_0_to_fp16 = const()[name = string("op_770_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(37607808)))];
            tensor<fp16, [4, 16]> var_770_cast_fp16 = linear(bias = var_770_bias_0_to_fp16, weight = op_770_weight_0_to_fp16_palettized, x = input_29_cast_fp16)[name = string("op_770_cast_fp16")];
            tensor<fp16, [4, 16]> var_771_cast_fp16 = softplus(x = var_770_cast_fp16)[name = string("op_771_cast_fp16")];
            tensor<fp16, [16]> var_767_promoted_to_fp16 = const()[name = string("op_767_promoted_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(37607936)))];
            tensor<fp16, [4, 16]> var_772_cast_fp16 = mul(x = var_767_promoted_to_fp16, y = var_771_cast_fp16)[name = string("op_772_cast_fp16")];
            tensor<int32, [2]> var_773_perm_0 = const()[name = string("op_773_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> decay_5_axes_0 = const()[name = string("decay_5_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 4]> var_773_cast_fp16 = transpose(perm = var_773_perm_0, x = var_772_cast_fp16)[name = string("transpose_123")];
            tensor<fp16, [1, 16, 4]> decay_5_cast_fp16 = expand_dims(axes = decay_5_axes_0, x = var_773_cast_fp16)[name = string("decay_5_cast_fp16")];
            tensor<fp16, [2048, 1024]> groups_0_2_in_proj_z_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [2048, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(37608064))), lut = tensor<fp16, [128, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(39180992))))[name = string("groups_0_2_in_proj_z_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 2048]> linear_19_cast_fp16 = linear(bias = linear_3_bias_0_to_fp16, weight = groups_0_2_in_proj_z_weight_promoted_to_fp16_palettized, x = input_29_cast_fp16)[name = string("linear_19_cast_fp16")];
            tensor<int32, [3]> var_781_begin_0 = const()[name = string("op_781_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_781_end_0 = const()[name = string("op_781_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_781_end_mask_0 = const()[name = string("op_781_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_781_squeeze_mask_0 = const()[name = string("op_781_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_781_cast_fp16 = slice_by_index(begin = var_781_begin_0, end = var_781_end_0, end_mask = var_781_end_mask_0, squeeze_mask = var_781_squeeze_mask_0, x = decay_5_cast_fp16)[name = string("op_781_cast_fp16")];
            tensor<fp16, [1, 16]> var_782_cast_fp16 = exp(x = var_781_cast_fp16)[name = string("op_782_cast_fp16")];
            tensor<int32, [1]> var_783_axes_0 = const()[name = string("op_783_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_783_cast_fp16 = expand_dims(axes = var_783_axes_0, x = var_782_cast_fp16)[name = string("op_783_cast_fp16")];
            tensor<int32, [1]> var_784_axes_0 = const()[name = string("op_784_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_784_cast_fp16 = expand_dims(axes = var_784_axes_0, x = var_783_cast_fp16)[name = string("op_784_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_33_cast_fp16 = mul(x = rec2, y = var_784_cast_fp16)[name = string("state_33_cast_fp16")];
            tensor<int32, [4]> key_token_17_begin_0 = const()[name = string("key_token_17_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> key_token_17_end_0 = const()[name = string("key_token_17_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> key_token_17_end_mask_0 = const()[name = string("key_token_17_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_17_squeeze_mask_0 = const()[name = string("key_token_17_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 4, 128]> key_11_cast_fp16 = transpose(perm = key_11_perm_0, x = var_758_cast_fp16)[name = string("transpose_122")];
            tensor<fp16, [1, 16, 128]> key_token_17_cast_fp16 = slice_by_index(begin = key_token_17_begin_0, end = key_token_17_end_0, end_mask = key_token_17_end_mask_0, squeeze_mask = key_token_17_squeeze_mask_0, x = key_11_cast_fp16)[name = string("key_token_17_cast_fp16")];
            tensor<int32, [4]> value_token_17_begin_0 = const()[name = string("value_token_17_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> value_token_17_end_0 = const()[name = string("value_token_17_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> value_token_17_end_mask_0 = const()[name = string("value_token_17_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_17_squeeze_mask_0 = const()[name = string("value_token_17_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 4, 128]> value_39_cast_fp16 = transpose(perm = value_39_perm_0, x = value_37_cast_fp16)[name = string("transpose_121")];
            tensor<fp16, [1, 16, 128]> value_token_17_cast_fp16 = slice_by_index(begin = value_token_17_begin_0, end = value_token_17_end_0, end_mask = value_token_17_end_mask_0, squeeze_mask = value_token_17_squeeze_mask_0, x = value_39_cast_fp16)[name = string("value_token_17_cast_fp16")];
            tensor<int32, [1]> var_792_axes_0 = const()[name = string("op_792_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_792_cast_fp16 = expand_dims(axes = var_792_axes_0, x = key_token_17_cast_fp16)[name = string("op_792_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_793_cast_fp16 = mul(x = state_33_cast_fp16, y = var_792_cast_fp16)[name = string("op_793_cast_fp16")];
            tensor<int32, [1]> memory_17_axes_0 = const()[name = string("memory_17_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_17_keep_dims_0 = const()[name = string("memory_17_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_17_cast_fp16 = reduce_sum(axes = memory_17_axes_0, keep_dims = memory_17_keep_dims_0, x = var_793_cast_fp16)[name = string("memory_17_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_796_cast_fp16 = sub(x = value_token_17_cast_fp16, y = memory_17_cast_fp16)[name = string("op_796_cast_fp16")];
            tensor<int32, [3]> var_799_begin_0 = const()[name = string("op_799_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_799_end_0 = const()[name = string("op_799_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_799_end_mask_0 = const()[name = string("op_799_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_799_squeeze_mask_0 = const()[name = string("op_799_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_799_cast_fp16 = slice_by_index(begin = var_799_begin_0, end = var_799_end_0, end_mask = var_799_end_mask_0, squeeze_mask = var_799_squeeze_mask_0, x = beta_5_cast_fp16)[name = string("op_799_cast_fp16")];
            tensor<int32, [1]> var_800_axes_0 = const()[name = string("op_800_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_800_cast_fp16 = expand_dims(axes = var_800_axes_0, x = var_799_cast_fp16)[name = string("op_800_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_17_cast_fp16 = mul(x = var_796_cast_fp16, y = var_800_cast_fp16)[name = string("delta_17_cast_fp16")];
            tensor<int32, [1]> var_803_axes_0 = const()[name = string("op_803_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_803_cast_fp16 = expand_dims(axes = var_803_axes_0, x = delta_17_cast_fp16)[name = string("op_803_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_804_cast_fp16 = mul(x = var_792_cast_fp16, y = var_803_cast_fp16)[name = string("op_804_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_35_cast_fp16 = add(x = state_33_cast_fp16, y = var_804_cast_fp16)[name = string("state_35_cast_fp16")];
            tensor<int32, [4]> var_808_begin_0 = const()[name = string("op_808_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> var_808_end_0 = const()[name = string("op_808_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> var_808_end_mask_0 = const()[name = string("op_808_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_808_squeeze_mask_0 = const()[name = string("op_808_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_808_cast_fp16 = slice_by_index(begin = var_808_begin_0, end = var_808_end_0, end_mask = var_808_end_mask_0, squeeze_mask = var_808_squeeze_mask_0, x = _inversed_query_11_cast_fp16)[name = string("op_808_cast_fp16")];
            tensor<int32, [1]> var_809_axes_0 = const()[name = string("op_809_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_809_cast_fp16 = expand_dims(axes = var_809_axes_0, x = var_808_cast_fp16)[name = string("op_809_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_810_cast_fp16 = mul(x = state_35_cast_fp16, y = var_809_cast_fp16)[name = string("op_810_cast_fp16")];
            tensor<int32, [1]> var_812_axes_0 = const()[name = string("op_812_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_812_keep_dims_0 = const()[name = string("op_812_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_812_cast_fp16 = reduce_sum(axes = var_812_axes_0, keep_dims = var_812_keep_dims_0, x = var_810_cast_fp16)[name = string("op_812_cast_fp16")];
            tensor<int32, [3]> var_815_begin_0 = const()[name = string("op_815_begin_0"), val = tensor<int32, [3]>([0, 0, 1])];
            tensor<int32, [3]> var_815_end_0 = const()[name = string("op_815_end_0"), val = tensor<int32, [3]>([1, 16, 2])];
            tensor<bool, [3]> var_815_end_mask_0 = const()[name = string("op_815_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_815_squeeze_mask_0 = const()[name = string("op_815_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_815_cast_fp16 = slice_by_index(begin = var_815_begin_0, end = var_815_end_0, end_mask = var_815_end_mask_0, squeeze_mask = var_815_squeeze_mask_0, x = decay_5_cast_fp16)[name = string("op_815_cast_fp16")];
            tensor<fp16, [1, 16]> var_816_cast_fp16 = exp(x = var_815_cast_fp16)[name = string("op_816_cast_fp16")];
            tensor<int32, [1]> var_817_axes_0 = const()[name = string("op_817_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_817_cast_fp16 = expand_dims(axes = var_817_axes_0, x = var_816_cast_fp16)[name = string("op_817_cast_fp16")];
            tensor<int32, [1]> var_818_axes_0 = const()[name = string("op_818_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_818_cast_fp16 = expand_dims(axes = var_818_axes_0, x = var_817_cast_fp16)[name = string("op_818_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_37_cast_fp16 = mul(x = state_35_cast_fp16, y = var_818_cast_fp16)[name = string("state_37_cast_fp16")];
            tensor<int32, [4]> key_token_19_begin_0 = const()[name = string("key_token_19_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> key_token_19_end_0 = const()[name = string("key_token_19_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> key_token_19_end_mask_0 = const()[name = string("key_token_19_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_19_squeeze_mask_0 = const()[name = string("key_token_19_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_19_cast_fp16 = slice_by_index(begin = key_token_19_begin_0, end = key_token_19_end_0, end_mask = key_token_19_end_mask_0, squeeze_mask = key_token_19_squeeze_mask_0, x = key_11_cast_fp16)[name = string("key_token_19_cast_fp16")];
            tensor<int32, [4]> value_token_19_begin_0 = const()[name = string("value_token_19_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> value_token_19_end_0 = const()[name = string("value_token_19_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> value_token_19_end_mask_0 = const()[name = string("value_token_19_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_19_squeeze_mask_0 = const()[name = string("value_token_19_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_19_cast_fp16 = slice_by_index(begin = value_token_19_begin_0, end = value_token_19_end_0, end_mask = value_token_19_end_mask_0, squeeze_mask = value_token_19_squeeze_mask_0, x = value_39_cast_fp16)[name = string("value_token_19_cast_fp16")];
            tensor<int32, [1]> var_826_axes_0 = const()[name = string("op_826_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_826_cast_fp16 = expand_dims(axes = var_826_axes_0, x = key_token_19_cast_fp16)[name = string("op_826_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_827_cast_fp16 = mul(x = state_37_cast_fp16, y = var_826_cast_fp16)[name = string("op_827_cast_fp16")];
            tensor<int32, [1]> memory_19_axes_0 = const()[name = string("memory_19_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_19_keep_dims_0 = const()[name = string("memory_19_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_19_cast_fp16 = reduce_sum(axes = memory_19_axes_0, keep_dims = memory_19_keep_dims_0, x = var_827_cast_fp16)[name = string("memory_19_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_830_cast_fp16 = sub(x = value_token_19_cast_fp16, y = memory_19_cast_fp16)[name = string("op_830_cast_fp16")];
            tensor<int32, [3]> var_833_begin_0 = const()[name = string("op_833_begin_0"), val = tensor<int32, [3]>([0, 0, 1])];
            tensor<int32, [3]> var_833_end_0 = const()[name = string("op_833_end_0"), val = tensor<int32, [3]>([1, 16, 2])];
            tensor<bool, [3]> var_833_end_mask_0 = const()[name = string("op_833_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_833_squeeze_mask_0 = const()[name = string("op_833_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_833_cast_fp16 = slice_by_index(begin = var_833_begin_0, end = var_833_end_0, end_mask = var_833_end_mask_0, squeeze_mask = var_833_squeeze_mask_0, x = beta_5_cast_fp16)[name = string("op_833_cast_fp16")];
            tensor<int32, [1]> var_834_axes_0 = const()[name = string("op_834_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_834_cast_fp16 = expand_dims(axes = var_834_axes_0, x = var_833_cast_fp16)[name = string("op_834_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_19_cast_fp16 = mul(x = var_830_cast_fp16, y = var_834_cast_fp16)[name = string("delta_19_cast_fp16")];
            tensor<int32, [1]> var_837_axes_0 = const()[name = string("op_837_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_837_cast_fp16 = expand_dims(axes = var_837_axes_0, x = delta_19_cast_fp16)[name = string("op_837_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_838_cast_fp16 = mul(x = var_826_cast_fp16, y = var_837_cast_fp16)[name = string("op_838_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_39_cast_fp16 = add(x = state_37_cast_fp16, y = var_838_cast_fp16)[name = string("state_39_cast_fp16")];
            tensor<int32, [4]> var_842_begin_0 = const()[name = string("op_842_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> var_842_end_0 = const()[name = string("op_842_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> var_842_end_mask_0 = const()[name = string("op_842_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_842_squeeze_mask_0 = const()[name = string("op_842_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_842_cast_fp16 = slice_by_index(begin = var_842_begin_0, end = var_842_end_0, end_mask = var_842_end_mask_0, squeeze_mask = var_842_squeeze_mask_0, x = _inversed_query_11_cast_fp16)[name = string("op_842_cast_fp16")];
            tensor<int32, [1]> var_843_axes_0 = const()[name = string("op_843_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_843_cast_fp16 = expand_dims(axes = var_843_axes_0, x = var_842_cast_fp16)[name = string("op_843_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_844_cast_fp16 = mul(x = state_39_cast_fp16, y = var_843_cast_fp16)[name = string("op_844_cast_fp16")];
            tensor<int32, [1]> var_846_axes_0 = const()[name = string("op_846_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_846_keep_dims_0 = const()[name = string("op_846_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_846_cast_fp16 = reduce_sum(axes = var_846_axes_0, keep_dims = var_846_keep_dims_0, x = var_844_cast_fp16)[name = string("op_846_cast_fp16")];
            tensor<int32, [3]> var_849_begin_0 = const()[name = string("op_849_begin_0"), val = tensor<int32, [3]>([0, 0, 2])];
            tensor<int32, [3]> var_849_end_0 = const()[name = string("op_849_end_0"), val = tensor<int32, [3]>([1, 16, 3])];
            tensor<bool, [3]> var_849_end_mask_0 = const()[name = string("op_849_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_849_squeeze_mask_0 = const()[name = string("op_849_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_849_cast_fp16 = slice_by_index(begin = var_849_begin_0, end = var_849_end_0, end_mask = var_849_end_mask_0, squeeze_mask = var_849_squeeze_mask_0, x = decay_5_cast_fp16)[name = string("op_849_cast_fp16")];
            tensor<fp16, [1, 16]> var_850_cast_fp16 = exp(x = var_849_cast_fp16)[name = string("op_850_cast_fp16")];
            tensor<int32, [1]> var_851_axes_0 = const()[name = string("op_851_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_851_cast_fp16 = expand_dims(axes = var_851_axes_0, x = var_850_cast_fp16)[name = string("op_851_cast_fp16")];
            tensor<int32, [1]> var_852_axes_0 = const()[name = string("op_852_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_852_cast_fp16 = expand_dims(axes = var_852_axes_0, x = var_851_cast_fp16)[name = string("op_852_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_41_cast_fp16 = mul(x = state_39_cast_fp16, y = var_852_cast_fp16)[name = string("state_41_cast_fp16")];
            tensor<int32, [4]> key_token_21_begin_0 = const()[name = string("key_token_21_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> key_token_21_end_0 = const()[name = string("key_token_21_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> key_token_21_end_mask_0 = const()[name = string("key_token_21_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_21_squeeze_mask_0 = const()[name = string("key_token_21_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_21_cast_fp16 = slice_by_index(begin = key_token_21_begin_0, end = key_token_21_end_0, end_mask = key_token_21_end_mask_0, squeeze_mask = key_token_21_squeeze_mask_0, x = key_11_cast_fp16)[name = string("key_token_21_cast_fp16")];
            tensor<int32, [4]> value_token_21_begin_0 = const()[name = string("value_token_21_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> value_token_21_end_0 = const()[name = string("value_token_21_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> value_token_21_end_mask_0 = const()[name = string("value_token_21_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_21_squeeze_mask_0 = const()[name = string("value_token_21_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_21_cast_fp16 = slice_by_index(begin = value_token_21_begin_0, end = value_token_21_end_0, end_mask = value_token_21_end_mask_0, squeeze_mask = value_token_21_squeeze_mask_0, x = value_39_cast_fp16)[name = string("value_token_21_cast_fp16")];
            tensor<int32, [1]> var_860_axes_0 = const()[name = string("op_860_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_860_cast_fp16 = expand_dims(axes = var_860_axes_0, x = key_token_21_cast_fp16)[name = string("op_860_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_861_cast_fp16 = mul(x = state_41_cast_fp16, y = var_860_cast_fp16)[name = string("op_861_cast_fp16")];
            tensor<int32, [1]> memory_21_axes_0 = const()[name = string("memory_21_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_21_keep_dims_0 = const()[name = string("memory_21_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_21_cast_fp16 = reduce_sum(axes = memory_21_axes_0, keep_dims = memory_21_keep_dims_0, x = var_861_cast_fp16)[name = string("memory_21_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_864_cast_fp16 = sub(x = value_token_21_cast_fp16, y = memory_21_cast_fp16)[name = string("op_864_cast_fp16")];
            tensor<int32, [3]> var_867_begin_0 = const()[name = string("op_867_begin_0"), val = tensor<int32, [3]>([0, 0, 2])];
            tensor<int32, [3]> var_867_end_0 = const()[name = string("op_867_end_0"), val = tensor<int32, [3]>([1, 16, 3])];
            tensor<bool, [3]> var_867_end_mask_0 = const()[name = string("op_867_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_867_squeeze_mask_0 = const()[name = string("op_867_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_867_cast_fp16 = slice_by_index(begin = var_867_begin_0, end = var_867_end_0, end_mask = var_867_end_mask_0, squeeze_mask = var_867_squeeze_mask_0, x = beta_5_cast_fp16)[name = string("op_867_cast_fp16")];
            tensor<int32, [1]> var_868_axes_0 = const()[name = string("op_868_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_868_cast_fp16 = expand_dims(axes = var_868_axes_0, x = var_867_cast_fp16)[name = string("op_868_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_21_cast_fp16 = mul(x = var_864_cast_fp16, y = var_868_cast_fp16)[name = string("delta_21_cast_fp16")];
            tensor<int32, [1]> var_871_axes_0 = const()[name = string("op_871_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_871_cast_fp16 = expand_dims(axes = var_871_axes_0, x = delta_21_cast_fp16)[name = string("op_871_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_872_cast_fp16 = mul(x = var_860_cast_fp16, y = var_871_cast_fp16)[name = string("op_872_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_43_cast_fp16 = add(x = state_41_cast_fp16, y = var_872_cast_fp16)[name = string("state_43_cast_fp16")];
            tensor<int32, [4]> var_876_begin_0 = const()[name = string("op_876_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> var_876_end_0 = const()[name = string("op_876_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> var_876_end_mask_0 = const()[name = string("op_876_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_876_squeeze_mask_0 = const()[name = string("op_876_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_876_cast_fp16 = slice_by_index(begin = var_876_begin_0, end = var_876_end_0, end_mask = var_876_end_mask_0, squeeze_mask = var_876_squeeze_mask_0, x = _inversed_query_11_cast_fp16)[name = string("op_876_cast_fp16")];
            tensor<int32, [1]> var_877_axes_0 = const()[name = string("op_877_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_877_cast_fp16 = expand_dims(axes = var_877_axes_0, x = var_876_cast_fp16)[name = string("op_877_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_878_cast_fp16 = mul(x = state_43_cast_fp16, y = var_877_cast_fp16)[name = string("op_878_cast_fp16")];
            tensor<int32, [1]> var_880_axes_0 = const()[name = string("op_880_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_880_keep_dims_0 = const()[name = string("op_880_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_880_cast_fp16 = reduce_sum(axes = var_880_axes_0, keep_dims = var_880_keep_dims_0, x = var_878_cast_fp16)[name = string("op_880_cast_fp16")];
            tensor<int32, [3]> var_883_begin_0 = const()[name = string("op_883_begin_0"), val = tensor<int32, [3]>([0, 0, 3])];
            tensor<int32, [3]> var_883_end_0 = const()[name = string("op_883_end_0"), val = tensor<int32, [3]>([1, 16, 4])];
            tensor<bool, [3]> var_883_end_mask_0 = const()[name = string("op_883_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_883_squeeze_mask_0 = const()[name = string("op_883_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_883_cast_fp16 = slice_by_index(begin = var_883_begin_0, end = var_883_end_0, end_mask = var_883_end_mask_0, squeeze_mask = var_883_squeeze_mask_0, x = decay_5_cast_fp16)[name = string("op_883_cast_fp16")];
            tensor<fp16, [1, 16]> var_884_cast_fp16 = exp(x = var_883_cast_fp16)[name = string("op_884_cast_fp16")];
            tensor<int32, [1]> var_885_axes_0 = const()[name = string("op_885_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_885_cast_fp16 = expand_dims(axes = var_885_axes_0, x = var_884_cast_fp16)[name = string("op_885_cast_fp16")];
            tensor<int32, [1]> var_886_axes_0 = const()[name = string("op_886_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_886_cast_fp16 = expand_dims(axes = var_886_axes_0, x = var_885_cast_fp16)[name = string("op_886_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_45_cast_fp16 = mul(x = state_43_cast_fp16, y = var_886_cast_fp16)[name = string("state_45_cast_fp16")];
            tensor<int32, [4]> key_token_23_begin_0 = const()[name = string("key_token_23_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> key_token_23_end_0 = const()[name = string("key_token_23_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> key_token_23_end_mask_0 = const()[name = string("key_token_23_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_23_squeeze_mask_0 = const()[name = string("key_token_23_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_23_cast_fp16 = slice_by_index(begin = key_token_23_begin_0, end = key_token_23_end_0, end_mask = key_token_23_end_mask_0, squeeze_mask = key_token_23_squeeze_mask_0, x = key_11_cast_fp16)[name = string("key_token_23_cast_fp16")];
            tensor<int32, [4]> value_token_23_begin_0 = const()[name = string("value_token_23_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> value_token_23_end_0 = const()[name = string("value_token_23_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> value_token_23_end_mask_0 = const()[name = string("value_token_23_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_23_squeeze_mask_0 = const()[name = string("value_token_23_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_23_cast_fp16 = slice_by_index(begin = value_token_23_begin_0, end = value_token_23_end_0, end_mask = value_token_23_end_mask_0, squeeze_mask = value_token_23_squeeze_mask_0, x = value_39_cast_fp16)[name = string("value_token_23_cast_fp16")];
            tensor<int32, [1]> var_894_axes_0 = const()[name = string("op_894_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_894_cast_fp16 = expand_dims(axes = var_894_axes_0, x = key_token_23_cast_fp16)[name = string("op_894_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_895_cast_fp16 = mul(x = state_45_cast_fp16, y = var_894_cast_fp16)[name = string("op_895_cast_fp16")];
            tensor<int32, [1]> memory_23_axes_0 = const()[name = string("memory_23_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_23_keep_dims_0 = const()[name = string("memory_23_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_23_cast_fp16 = reduce_sum(axes = memory_23_axes_0, keep_dims = memory_23_keep_dims_0, x = var_895_cast_fp16)[name = string("memory_23_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_898_cast_fp16 = sub(x = value_token_23_cast_fp16, y = memory_23_cast_fp16)[name = string("op_898_cast_fp16")];
            tensor<int32, [3]> var_901_begin_0 = const()[name = string("op_901_begin_0"), val = tensor<int32, [3]>([0, 0, 3])];
            tensor<int32, [3]> var_901_end_0 = const()[name = string("op_901_end_0"), val = tensor<int32, [3]>([1, 16, 4])];
            tensor<bool, [3]> var_901_end_mask_0 = const()[name = string("op_901_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_901_squeeze_mask_0 = const()[name = string("op_901_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_901_cast_fp16 = slice_by_index(begin = var_901_begin_0, end = var_901_end_0, end_mask = var_901_end_mask_0, squeeze_mask = var_901_squeeze_mask_0, x = beta_5_cast_fp16)[name = string("op_901_cast_fp16")];
            tensor<int32, [1]> var_902_axes_0 = const()[name = string("op_902_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_902_cast_fp16 = expand_dims(axes = var_902_axes_0, x = var_901_cast_fp16)[name = string("op_902_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_23_cast_fp16 = mul(x = var_898_cast_fp16, y = var_902_cast_fp16)[name = string("delta_23_cast_fp16")];
            tensor<int32, [1]> var_905_axes_0 = const()[name = string("op_905_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_905_cast_fp16 = expand_dims(axes = var_905_axes_0, x = delta_23_cast_fp16)[name = string("op_905_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_906_cast_fp16 = mul(x = var_894_cast_fp16, y = var_905_cast_fp16)[name = string("op_906_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> rec2_out = add(x = state_45_cast_fp16, y = var_906_cast_fp16)[name = string("state_47_cast_fp16")];
            tensor<int32, [4]> var_910_begin_0 = const()[name = string("op_910_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> var_910_end_0 = const()[name = string("op_910_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> var_910_end_mask_0 = const()[name = string("op_910_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_910_squeeze_mask_0 = const()[name = string("op_910_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_910_cast_fp16 = slice_by_index(begin = var_910_begin_0, end = var_910_end_0, end_mask = var_910_end_mask_0, squeeze_mask = var_910_squeeze_mask_0, x = _inversed_query_11_cast_fp16)[name = string("op_910_cast_fp16")];
            tensor<int32, [1]> var_911_axes_0 = const()[name = string("op_911_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_911_cast_fp16 = expand_dims(axes = var_911_axes_0, x = var_910_cast_fp16)[name = string("op_911_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_912_cast_fp16 = mul(x = rec2_out, y = var_911_cast_fp16)[name = string("op_912_cast_fp16")];
            tensor<int32, [1]> var_914_axes_0 = const()[name = string("op_914_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_914_keep_dims_0 = const()[name = string("op_914_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_914_cast_fp16 = reduce_sum(axes = var_914_axes_0, keep_dims = var_914_keep_dims_0, x = var_912_cast_fp16)[name = string("op_914_cast_fp16")];
            int32 attention_21_axis_0 = const()[name = string("attention_21_axis_0"), val = int32(1)];
            tensor<fp16, [1, 4, 16, 128]> attention_21_cast_fp16 = stack(axis = attention_21_axis_0, values = (var_812_cast_fp16, var_846_cast_fp16, var_880_cast_fp16, var_914_cast_fp16))[name = string("attention_21_cast_fp16")];
            tensor<int32, [2]> var_917 = const()[name = string("op_917"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [64, 128]> attention_23_cast_fp16 = reshape(shape = var_917, x = attention_21_cast_fp16)[name = string("attention_23_cast_fp16")];
            tensor<int32, [2]> var_919 = const()[name = string("op_919"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [64, 128]> input_33_cast_fp16 = reshape(shape = var_919, x = linear_19_cast_fp16)[name = string("input_33_cast_fp16")];
            tensor<fp16, [64, 128]> var_921_cast_fp16 = mul(x = attention_23_cast_fp16, y = attention_23_cast_fp16)[name = string("op_921_cast_fp16")];
            tensor<int32, [1]> variance_15_axes_0 = const()[name = string("variance_15_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_15_keep_dims_0 = const()[name = string("variance_15_keep_dims_0"), val = bool(true)];
            tensor<fp16, [64, 1]> variance_15_cast_fp16 = reduce_mean(axes = variance_15_axes_0, keep_dims = variance_15_keep_dims_0, x = var_921_cast_fp16)[name = string("variance_15_cast_fp16")];
            fp16 var_924_to_fp16 = const()[name = string("op_924_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [64, 1]> var_925_cast_fp16 = add(x = variance_15_cast_fp16, y = var_924_to_fp16)[name = string("op_925_cast_fp16")];
            fp32 var_926_epsilon_0 = const()[name = string("op_926_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [64, 1]> var_926_cast_fp16 = rsqrt(epsilon = var_926_epsilon_0, x = var_925_cast_fp16)[name = string("op_926_cast_fp16")];
            tensor<fp16, [64, 128]> attention_25_cast_fp16 = mul(x = attention_23_cast_fp16, y = var_926_cast_fp16)[name = string("attention_25_cast_fp16")];
            tensor<fp16, [128]> groups_0_2_gated_norm_promoted_to_fp16 = const()[name = string("groups_0_2_gated_norm_promoted_to_fp16"), val = tensor<fp16, [128]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(39197440)))];
            tensor<fp16, [64, 128]> attention_27_cast_fp16 = mul(x = attention_25_cast_fp16, y = groups_0_2_gated_norm_promoted_to_fp16)[name = string("attention_27_cast_fp16")];
            tensor<fp16, [64, 128]> var_929_cast_fp16 = silu(x = input_33_cast_fp16)[name = string("op_929_cast_fp16")];
            tensor<fp16, [64, 128]> attention_29_cast_fp16 = mul(x = attention_27_cast_fp16, y = var_929_cast_fp16)[name = string("attention_29_cast_fp16")];
            tensor<int32, [2]> var_931 = const()[name = string("op_931"), val = tensor<int32, [2]>([4, 2048])];
            tensor<fp16, [4, 2048]> input_35_cast_fp16 = reshape(shape = var_931, x = attention_29_cast_fp16)[name = string("input_35_cast_fp16")];
            tensor<fp16, [1024, 2048]> groups_0_2_out_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 2048]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(39197760))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(40770688))))[name = string("groups_0_2_out_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 1024]> linear_20_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = groups_0_2_out_proj_weight_promoted_to_fp16_palettized, x = input_35_cast_fp16)[name = string("linear_20_cast_fp16")];
            tensor<fp16, [4, 1024]> value_41_cast_fp16 = add(x = value_29_cast_fp16, y = linear_20_cast_fp16)[name = string("value_41_cast_fp16")];
            tensor<fp16, [4, 1024]> var_936_cast_fp16 = mul(x = value_41_cast_fp16, y = value_41_cast_fp16)[name = string("op_936_cast_fp16")];
            tensor<int32, [1]> variance_17_axes_0 = const()[name = string("variance_17_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_17_keep_dims_0 = const()[name = string("variance_17_keep_dims_0"), val = bool(true)];
            tensor<fp16, [4, 1]> variance_17_cast_fp16 = reduce_mean(axes = variance_17_axes_0, keep_dims = variance_17_keep_dims_0, x = var_936_cast_fp16)[name = string("variance_17_cast_fp16")];
            fp16 var_939_to_fp16 = const()[name = string("op_939_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [4, 1]> var_940_cast_fp16 = add(x = variance_17_cast_fp16, y = var_939_to_fp16)[name = string("op_940_cast_fp16")];
            fp32 var_941_epsilon_0 = const()[name = string("op_941_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [4, 1]> var_941_cast_fp16 = rsqrt(epsilon = var_941_epsilon_0, x = var_940_cast_fp16)[name = string("op_941_cast_fp16")];
            tensor<fp16, [4, 1024]> var_942_cast_fp16 = mul(x = value_41_cast_fp16, y = var_941_cast_fp16)[name = string("op_942_cast_fp16")];
            tensor<fp16, [1024]> var_944_to_fp16 = const()[name = string("op_944_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(40778944)))];
            tensor<fp16, [4, 1024]> input_37_cast_fp16 = mul(x = var_942_cast_fp16, y = var_944_to_fp16)[name = string("input_37_cast_fp16")];
            tensor<fp16, [3584, 1024]> groups_0_2_gate_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(40781056))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(43533632))))[name = string("groups_0_2_gate_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 3584]> linear_21_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = groups_0_2_gate_proj_weight_promoted_to_fp16_palettized, x = input_37_cast_fp16)[name = string("linear_21_cast_fp16")];
            tensor<fp16, [4, 3584]> var_948_cast_fp16 = silu(x = linear_21_cast_fp16)[name = string("op_948_cast_fp16")];
            tensor<fp16, [3584, 1024]> groups_0_2_up_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(43562368))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(46314944))))[name = string("groups_0_2_up_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 3584]> linear_22_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = groups_0_2_up_proj_weight_promoted_to_fp16_palettized, x = input_37_cast_fp16)[name = string("linear_22_cast_fp16")];
            tensor<fp16, [4, 3584]> input_41_cast_fp16 = mul(x = var_948_cast_fp16, y = linear_22_cast_fp16)[name = string("input_41_cast_fp16")];
            tensor<fp16, [1024, 3584]> groups_0_2_down_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 3584]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(46343680))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(49096256))))[name = string("groups_0_2_down_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 1024]> linear_23_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = groups_0_2_down_proj_weight_promoted_to_fp16_palettized, x = input_41_cast_fp16)[name = string("linear_23_cast_fp16")];
            tensor<fp16, [4, 1024]> value_43_cast_fp16 = add(x = value_41_cast_fp16, y = linear_23_cast_fp16)[name = string("value_43_cast_fp16")];
            int32 var_972 = const()[name = string("op_972"), val = int32(-1)];
            tensor<fp16, [4, 1024]> var_979_cast_fp16 = mul(x = value_43_cast_fp16, y = value_43_cast_fp16)[name = string("op_979_cast_fp16")];
            tensor<int32, [1]> variance_19_axes_0 = const()[name = string("variance_19_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_19_keep_dims_0 = const()[name = string("variance_19_keep_dims_0"), val = bool(true)];
            tensor<fp16, [4, 1]> variance_19_cast_fp16 = reduce_mean(axes = variance_19_axes_0, keep_dims = variance_19_keep_dims_0, x = var_979_cast_fp16)[name = string("variance_19_cast_fp16")];
            fp16 var_982_to_fp16 = const()[name = string("op_982_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [4, 1]> var_983_cast_fp16 = add(x = variance_19_cast_fp16, y = var_982_to_fp16)[name = string("op_983_cast_fp16")];
            fp32 var_984_epsilon_0 = const()[name = string("op_984_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [4, 1]> var_984_cast_fp16 = rsqrt(epsilon = var_984_epsilon_0, x = var_983_cast_fp16)[name = string("op_984_cast_fp16")];
            tensor<fp16, [4, 1024]> var_985_cast_fp16 = mul(x = value_43_cast_fp16, y = var_984_cast_fp16)[name = string("op_985_cast_fp16")];
            tensor<fp16, [1024]> var_987_to_fp16 = const()[name = string("op_987_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(49104512)))];
            tensor<fp16, [4, 1024]> input_43_cast_fp16 = mul(x = var_985_cast_fp16, y = var_987_to_fp16)[name = string("input_43_cast_fp16")];
            tensor<fp16, [4096, 1024]> projections_0_q_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [4096, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(49106624))), lut = tensor<fp16, [256, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(52252416))))[name = string("projections_0_q_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4096]> linear_24_bias_0_to_fp16 = const()[name = string("linear_24_bias_0_to_fp16"), val = tensor<fp16, [4096]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(52285248)))];
            tensor<fp16, [4, 4096]> linear_24_cast_fp16 = linear(bias = linear_24_bias_0_to_fp16, weight = projections_0_q_proj_weight_promoted_to_fp16_palettized, x = input_43_cast_fp16)[name = string("linear_24_cast_fp16")];
            tensor<int32, [3]> var_991 = const()[name = string("op_991"), val = tensor<int32, [3]>([4, 8, 512])];
            tensor<fp16, [4, 8, 512]> query_and_gate_1_cast_fp16 = reshape(shape = var_991, x = linear_24_cast_fp16)[name = string("query_and_gate_1_cast_fp16")];
            tensor<int32, [2]> var_993_split_sizes_0 = const()[name = string("op_993_split_sizes_0"), val = tensor<int32, [2]>([256, 256])];
            int32 var_993_axis_0 = const()[name = string("op_993_axis_0"), val = int32(-1)];
            tensor<fp16, [4, 8, 256]> var_993_cast_fp16_0, tensor<fp16, [4, 8, 256]> var_993_cast_fp16_1 = split(axis = var_993_axis_0, split_sizes = var_993_split_sizes_0, x = query_and_gate_1_cast_fp16)[name = string("op_993_cast_fp16")];
            tensor<fp16, [512, 1024]> projections_0_k_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [512, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(52293504))), lut = tensor<fp16, [32, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(52686784))))[name = string("projections_0_k_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [512]> linear_25_bias_0_to_fp16 = const()[name = string("linear_25_bias_0_to_fp16"), val = tensor<fp16, [512]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(52690944)))];
            tensor<fp16, [4, 512]> linear_25_cast_fp16 = linear(bias = linear_25_bias_0_to_fp16, weight = projections_0_k_proj_weight_promoted_to_fp16_palettized, x = input_43_cast_fp16)[name = string("linear_25_cast_fp16")];
            tensor<int32, [3]> var_997 = const()[name = string("op_997"), val = tensor<int32, [3]>([4, 2, 256])];
            tensor<fp16, [4, 2, 256]> value_47_cast_fp16 = reshape(shape = var_997, x = linear_25_cast_fp16)[name = string("value_47_cast_fp16")];
            tensor<fp16, [512, 1024]> projections_0_v_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [512, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(52692032))), lut = tensor<fp16, [32, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(53085312))))[name = string("projections_0_v_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 512]> linear_26_cast_fp16 = linear(bias = linear_25_bias_0_to_fp16, weight = projections_0_v_proj_weight_promoted_to_fp16_palettized, x = input_43_cast_fp16)[name = string("linear_26_cast_fp16")];
            tensor<int32, [3]> var_1001 = const()[name = string("op_1001"), val = tensor<int32, [3]>([4, 2, 256])];
            tensor<fp16, [4, 2, 256]> value_53_cast_fp16 = reshape(shape = var_1001, x = linear_26_cast_fp16)[name = string("value_53_cast_fp16")];
            tensor<fp16, [4, 8, 256]> var_1003_cast_fp16 = mul(x = var_993_cast_fp16_0, y = var_993_cast_fp16_0)[name = string("op_1003_cast_fp16")];
            tensor<int32, [1]> variance_21_axes_0 = const()[name = string("variance_21_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_21_keep_dims_0 = const()[name = string("variance_21_keep_dims_0"), val = bool(true)];
            tensor<fp16, [4, 8, 1]> variance_21_cast_fp16 = reduce_mean(axes = variance_21_axes_0, keep_dims = variance_21_keep_dims_0, x = var_1003_cast_fp16)[name = string("variance_21_cast_fp16")];
            fp16 var_1006_to_fp16 = const()[name = string("op_1006_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [4, 8, 1]> var_1007_cast_fp16 = add(x = variance_21_cast_fp16, y = var_1006_to_fp16)[name = string("op_1007_cast_fp16")];
            fp32 var_1008_epsilon_0 = const()[name = string("op_1008_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [4, 8, 1]> var_1008_cast_fp16 = rsqrt(epsilon = var_1008_epsilon_0, x = var_1007_cast_fp16)[name = string("op_1008_cast_fp16")];
            tensor<fp16, [4, 8, 256]> var_1009_cast_fp16 = mul(x = var_993_cast_fp16_0, y = var_1008_cast_fp16)[name = string("op_1009_cast_fp16")];
            tensor<fp16, [256]> var_1011_to_fp16 = const()[name = string("op_1011_to_fp16"), val = tensor<fp16, [256]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(53089472)))];
            tensor<fp16, [4, 8, 256]> var_1012_cast_fp16 = mul(x = var_1009_cast_fp16, y = var_1011_to_fp16)[name = string("op_1012_cast_fp16")];
            tensor<int32, [3]> var_1013 = const()[name = string("op_1013"), val = tensor<int32, [3]>([1, 0, 2])];
            tensor<int32, [1]> value_49_axes_0 = const()[name = string("value_49_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [8, 4, 256]> var_1014_cast_fp16 = transpose(perm = var_1013, x = var_1012_cast_fp16)[name = string("transpose_120")];
            tensor<fp16, [1, 8, 4, 256]> value_49_cast_fp16 = expand_dims(axes = value_49_axes_0, x = var_1014_cast_fp16)[name = string("value_49_cast_fp16")];
            tensor<fp16, [4, 2, 256]> var_1016_cast_fp16 = mul(x = value_47_cast_fp16, y = value_47_cast_fp16)[name = string("op_1016_cast_fp16")];
            tensor<int32, [1]> variance_23_axes_0 = const()[name = string("variance_23_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_23_keep_dims_0 = const()[name = string("variance_23_keep_dims_0"), val = bool(true)];
            tensor<fp16, [4, 2, 1]> variance_23_cast_fp16 = reduce_mean(axes = variance_23_axes_0, keep_dims = variance_23_keep_dims_0, x = var_1016_cast_fp16)[name = string("variance_23_cast_fp16")];
            fp16 var_1019_to_fp16 = const()[name = string("op_1019_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [4, 2, 1]> var_1020_cast_fp16 = add(x = variance_23_cast_fp16, y = var_1019_to_fp16)[name = string("op_1020_cast_fp16")];
            fp32 var_1021_epsilon_0 = const()[name = string("op_1021_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [4, 2, 1]> var_1021_cast_fp16 = rsqrt(epsilon = var_1021_epsilon_0, x = var_1020_cast_fp16)[name = string("op_1021_cast_fp16")];
            tensor<fp16, [4, 2, 256]> var_1022_cast_fp16 = mul(x = value_47_cast_fp16, y = var_1021_cast_fp16)[name = string("op_1022_cast_fp16")];
            tensor<fp16, [256]> var_1024_to_fp16 = const()[name = string("op_1024_to_fp16"), val = tensor<fp16, [256]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(53090048)))];
            tensor<fp16, [4, 2, 256]> var_1025_cast_fp16 = mul(x = var_1022_cast_fp16, y = var_1024_to_fp16)[name = string("op_1025_cast_fp16")];
            tensor<int32, [3]> var_1026 = const()[name = string("op_1026"), val = tensor<int32, [3]>([1, 0, 2])];
            tensor<int32, [1]> value_51_axes_0 = const()[name = string("value_51_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [2, 4, 256]> var_1027_cast_fp16 = transpose(perm = var_1026, x = var_1025_cast_fp16)[name = string("transpose_119")];
            tensor<fp16, [1, 2, 4, 256]> value_51_cast_fp16 = expand_dims(axes = value_51_axes_0, x = var_1027_cast_fp16)[name = string("value_51_cast_fp16")];
            tensor<int32, [4]> rotated_1_begin_0 = const()[name = string("rotated_1_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> rotated_1_end_0 = const()[name = string("rotated_1_end_0"), val = tensor<int32, [4]>([1, 8, 4, 64])];
            tensor<bool, [4]> rotated_1_end_mask_0 = const()[name = string("rotated_1_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
            tensor<fp16, [1, 8, 4, 64]> rotated_1_cast_fp16 = slice_by_index(begin = rotated_1_begin_0, end = rotated_1_end_0, end_mask = rotated_1_end_mask_0, x = value_49_cast_fp16)[name = string("rotated_1_cast_fp16")];
            tensor<int32, [4]> passthrough_1_begin_0 = const()[name = string("passthrough_1_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 64])];
            tensor<int32, [4]> passthrough_1_end_0 = const()[name = string("passthrough_1_end_0"), val = tensor<int32, [4]>([1, 8, 4, 256])];
            tensor<bool, [4]> passthrough_1_end_mask_0 = const()[name = string("passthrough_1_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
            tensor<fp16, [1, 8, 4, 192]> passthrough_1_cast_fp16 = slice_by_index(begin = passthrough_1_begin_0, end = passthrough_1_end_0, end_mask = passthrough_1_end_mask_0, x = value_49_cast_fp16)[name = string("passthrough_1_cast_fp16")];
            tensor<int32, [2]> var_1031_split_sizes_0 = const()[name = string("op_1031_split_sizes_0"), val = tensor<int32, [2]>([32, 32])];
            int32 var_1031_axis_0 = const()[name = string("op_1031_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 8, 4, 32]> var_1031_cast_fp16_0, tensor<fp16, [1, 8, 4, 32]> var_1031_cast_fp16_1 = split(axis = var_1031_axis_0, split_sizes = var_1031_split_sizes_0, x = rotated_1_cast_fp16)[name = string("op_1031_cast_fp16")];
            fp16 const_3_promoted_to_fp16 = const()[name = string("const_3_promoted_to_fp16"), val = fp16(-0x1p+0)];
            tensor<fp16, [1, 8, 4, 32]> var_1033_cast_fp16 = mul(x = var_1031_cast_fp16_1, y = const_3_promoted_to_fp16)[name = string("op_1033_cast_fp16")];
            bool rotated_half_1_interleave_0 = const()[name = string("rotated_half_1_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 8, 4, 64]> rotated_half_1_cast_fp16 = concat(axis = var_972, interleave = rotated_half_1_interleave_0, values = (var_1033_cast_fp16, var_1031_cast_fp16_0))[name = string("rotated_half_1_cast_fp16")];
            tensor<fp16, [1, 8, 4, 64]> var_1036_cast_fp16 = mul(x = rotated_1_cast_fp16, y = cos)[name = string("op_1036_cast_fp16")];
            tensor<fp16, [1, 8, 4, 64]> var_1037_cast_fp16 = mul(x = rotated_half_1_cast_fp16, y = sin)[name = string("op_1037_cast_fp16")];
            tensor<fp16, [1, 8, 4, 64]> var_1038_cast_fp16 = add(x = var_1036_cast_fp16, y = var_1037_cast_fp16)[name = string("op_1038_cast_fp16")];
            bool query_13_interleave_0 = const()[name = string("query_13_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 8, 4, 256]> query_13_cast_fp16 = concat(axis = var_972, interleave = query_13_interleave_0, values = (var_1038_cast_fp16, passthrough_1_cast_fp16))[name = string("query_13_cast_fp16")];
            tensor<int32, [4]> rotated_3_begin_0 = const()[name = string("rotated_3_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> rotated_3_end_0 = const()[name = string("rotated_3_end_0"), val = tensor<int32, [4]>([1, 2, 4, 64])];
            tensor<bool, [4]> rotated_3_end_mask_0 = const()[name = string("rotated_3_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
            tensor<fp16, [1, 2, 4, 64]> rotated_3_cast_fp16 = slice_by_index(begin = rotated_3_begin_0, end = rotated_3_end_0, end_mask = rotated_3_end_mask_0, x = value_51_cast_fp16)[name = string("rotated_3_cast_fp16")];
            tensor<int32, [4]> passthrough_3_begin_0 = const()[name = string("passthrough_3_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 64])];
            tensor<int32, [4]> passthrough_3_end_0 = const()[name = string("passthrough_3_end_0"), val = tensor<int32, [4]>([1, 2, 4, 256])];
            tensor<bool, [4]> passthrough_3_end_mask_0 = const()[name = string("passthrough_3_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
            tensor<fp16, [1, 2, 4, 192]> passthrough_3_cast_fp16 = slice_by_index(begin = passthrough_3_begin_0, end = passthrough_3_end_0, end_mask = passthrough_3_end_mask_0, x = value_51_cast_fp16)[name = string("passthrough_3_cast_fp16")];
            tensor<int32, [2]> var_1043_split_sizes_0 = const()[name = string("op_1043_split_sizes_0"), val = tensor<int32, [2]>([32, 32])];
            int32 var_1043_axis_0 = const()[name = string("op_1043_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 2, 4, 32]> var_1043_cast_fp16_0, tensor<fp16, [1, 2, 4, 32]> var_1043_cast_fp16_1 = split(axis = var_1043_axis_0, split_sizes = var_1043_split_sizes_0, x = rotated_3_cast_fp16)[name = string("op_1043_cast_fp16")];
            fp16 const_4_promoted_to_fp16 = const()[name = string("const_4_promoted_to_fp16"), val = fp16(-0x1p+0)];
            tensor<fp16, [1, 2, 4, 32]> var_1045_cast_fp16 = mul(x = var_1043_cast_fp16_1, y = const_4_promoted_to_fp16)[name = string("op_1045_cast_fp16")];
            bool rotated_half_3_interleave_0 = const()[name = string("rotated_half_3_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 2, 4, 64]> rotated_half_3_cast_fp16 = concat(axis = var_972, interleave = rotated_half_3_interleave_0, values = (var_1045_cast_fp16, var_1043_cast_fp16_0))[name = string("rotated_half_3_cast_fp16")];
            tensor<fp16, [1, 2, 4, 64]> var_1048_cast_fp16 = mul(x = rotated_3_cast_fp16, y = cos)[name = string("op_1048_cast_fp16")];
            tensor<fp16, [1, 2, 4, 64]> var_1049_cast_fp16 = mul(x = rotated_half_3_cast_fp16, y = sin)[name = string("op_1049_cast_fp16")];
            tensor<fp16, [1, 2, 4, 64]> var_1050_cast_fp16 = add(x = var_1048_cast_fp16, y = var_1049_cast_fp16)[name = string("op_1050_cast_fp16")];
            bool key_13_interleave_0 = const()[name = string("key_13_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 2, 4, 256]> key_13_cast_fp16 = concat(axis = var_972, interleave = key_13_interleave_0, values = (var_1050_cast_fp16, passthrough_3_cast_fp16))[name = string("key_13_cast_fp16")];
            tensor<int32, [4]> var_1053 = const()[name = string("op_1053"), val = tensor<int32, [4]>([2, 4, 4, 256])];
            tensor<fp16, [2, 4, 4, 256]> var_1054_cast_fp16 = reshape(shape = var_1053, x = query_13_cast_fp16)[name = string("op_1054_cast_fp16")];
            tensor<int32, [4]> transpose_0_perm_0 = const()[name = string("transpose_0_perm_0"), val = tensor<int32, [4]>([0, 1, 3, 2])];
            tensor<int32, [3]> var_1057 = const()[name = string("op_1057"), val = tensor<int32, [3]>([2, 4, 256])];
            tensor<fp16, [2, 4, 256]> key_15_cast_fp16 = reshape(shape = var_1057, x = key_13_cast_fp16)[name = string("key_15_cast_fp16")];
            tensor<int32, [3]> var_1059 = const()[name = string("op_1059"), val = tensor<int32, [3]>([1, 0, 2])];
            tensor<int32, [2]> var_1061 = const()[name = string("op_1061"), val = tensor<int32, [2]>([4, 2048])];
            tensor<fp16, [4, 2048]> gate_3_cast_fp16 = reshape(shape = var_1061, x = var_993_cast_fp16_1)[name = string("gate_3_cast_fp16")];
            tensor<int32, [1]> var_1069_axes_0 = const()[name = string("op_1069_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [1, 2, 4, 256]> var_1069_cast_fp16 = expand_dims(axes = var_1069_axes_0, x = key_15_cast_fp16)[name = string("op_1069_cast_fp16")];
            tensor<int32, [1]> var_1071_axes_0 = const()[name = string("op_1071_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [1, 1, 2, 4, 256]> var_1071_cast_fp16 = expand_dims(axes = var_1071_axes_0, x = var_1069_cast_fp16)[name = string("op_1071_cast_fp16")];
            int32 var_1073 = const()[name = string("op_1073"), val = int32(4)];
            tensor<int32, [1]> var_1074 = add(x = current_pos, y = var_1073)[name = string("op_1074")];
            tensor<fp16, [4, 2, 2, 2048, 256]> read_state_0 = read_state(input = kv_cache)[name = string("read_state_0")];
            tensor<int32, [1]> expand_dims_0 = const()[name = string("expand_dims_0"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_1 = const()[name = string("expand_dims_1"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_2 = const()[name = string("expand_dims_2"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_4 = const()[name = string("expand_dims_4"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_5 = const()[name = string("expand_dims_5"), val = tensor<int32, [1]>([1])];
            tensor<int32, [1]> expand_dims_6 = const()[name = string("expand_dims_6"), val = tensor<int32, [1]>([1])];
            int32 concat_2_axis_0 = const()[name = string("concat_2_axis_0"), val = int32(0)];
            bool concat_2_interleave_0 = const()[name = string("concat_2_interleave_0"), val = bool(false)];
            tensor<int32, [5]> concat_2 = concat(axis = concat_2_axis_0, interleave = concat_2_interleave_0, values = (expand_dims_0, expand_dims_1, expand_dims_2, current_pos, expand_dims_4))[name = string("concat_2")];
            tensor<int32, [1]> concat_3_values2_0 = const()[name = string("concat_3_values2_0"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> concat_3_values4_0 = const()[name = string("concat_3_values4_0"), val = tensor<int32, [1]>([0])];
            int32 concat_3_axis_0 = const()[name = string("concat_3_axis_0"), val = int32(0)];
            bool concat_3_interleave_0 = const()[name = string("concat_3_interleave_0"), val = bool(false)];
            tensor<int32, [5]> concat_3 = concat(axis = concat_3_axis_0, interleave = concat_3_interleave_0, values = (expand_dims_5, expand_dims_6, concat_3_values2_0, var_1074, concat_3_values4_0))[name = string("concat_3")];
            tensor<int32, [5]> kv_cache_internal_tensor_assign_1_stride_0 = const()[name = string("kv_cache_internal_tensor_assign_1_stride_0"), val = tensor<int32, [5]>([1, 1, 1, 1, 1])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_1_begin_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_1_begin_mask_0"), val = tensor<bool, [5]>([false, false, false, false, false])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_1_end_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_1_end_mask_0"), val = tensor<bool, [5]>([false, false, true, false, true])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_1_squeeze_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_1_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, false, false, false])];
            tensor<fp16, [4, 2, 2, 2048, 256]> kv_cache_internal_tensor_assign_1_cast_fp16 = slice_update(begin = concat_2, begin_mask = kv_cache_internal_tensor_assign_1_begin_mask_0, end = concat_3, end_mask = kv_cache_internal_tensor_assign_1_end_mask_0, squeeze_mask = kv_cache_internal_tensor_assign_1_squeeze_mask_0, stride = kv_cache_internal_tensor_assign_1_stride_0, update = var_1071_cast_fp16, x = read_state_0)[name = string("kv_cache_internal_tensor_assign_1_cast_fp16")];
            write_state(data = kv_cache_internal_tensor_assign_1_cast_fp16, input = kv_cache)[name = string("coreml_update_state_16_write_state")];
            tensor<fp16, [4, 2, 2, 2048, 256]> coreml_update_state_8 = read_state(input = kv_cache)[name = string("coreml_update_state_16")];
            tensor<int32, [1]> var_1102_axes_0 = const()[name = string("op_1102_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [2, 4, 256]> value_55_cast_fp16 = transpose(perm = var_1059, x = value_53_cast_fp16)[name = string("transpose_118")];
            tensor<fp16, [1, 2, 4, 256]> var_1102_cast_fp16 = expand_dims(axes = var_1102_axes_0, x = value_55_cast_fp16)[name = string("op_1102_cast_fp16")];
            tensor<int32, [1]> var_1104_axes_0 = const()[name = string("op_1104_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [1, 1, 2, 4, 256]> var_1104_cast_fp16 = expand_dims(axes = var_1104_axes_0, x = var_1102_cast_fp16)[name = string("op_1104_cast_fp16")];
            tensor<int32, [1]> expand_dims_8 = const()[name = string("expand_dims_8"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_9 = const()[name = string("expand_dims_9"), val = tensor<int32, [1]>([1])];
            tensor<int32, [1]> expand_dims_10 = const()[name = string("expand_dims_10"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_12 = const()[name = string("expand_dims_12"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_13 = const()[name = string("expand_dims_13"), val = tensor<int32, [1]>([1])];
            tensor<int32, [1]> expand_dims_14 = const()[name = string("expand_dims_14"), val = tensor<int32, [1]>([2])];
            int32 concat_6_axis_0 = const()[name = string("concat_6_axis_0"), val = int32(0)];
            bool concat_6_interleave_0 = const()[name = string("concat_6_interleave_0"), val = bool(false)];
            tensor<int32, [5]> concat_6 = concat(axis = concat_6_axis_0, interleave = concat_6_interleave_0, values = (expand_dims_8, expand_dims_9, expand_dims_10, current_pos, expand_dims_12))[name = string("concat_6")];
            tensor<int32, [1]> concat_7_values2_0 = const()[name = string("concat_7_values2_0"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> concat_7_values4_0 = const()[name = string("concat_7_values4_0"), val = tensor<int32, [1]>([0])];
            int32 concat_7_axis_0 = const()[name = string("concat_7_axis_0"), val = int32(0)];
            bool concat_7_interleave_0 = const()[name = string("concat_7_interleave_0"), val = bool(false)];
            tensor<int32, [5]> concat_7 = concat(axis = concat_7_axis_0, interleave = concat_7_interleave_0, values = (expand_dims_13, expand_dims_14, concat_7_values2_0, var_1074, concat_7_values4_0))[name = string("concat_7")];
            tensor<int32, [5]> kv_cache_internal_tensor_assign_2_stride_0 = const()[name = string("kv_cache_internal_tensor_assign_2_stride_0"), val = tensor<int32, [5]>([1, 1, 1, 1, 1])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_2_begin_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_2_begin_mask_0"), val = tensor<bool, [5]>([false, false, false, false, false])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_2_end_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_2_end_mask_0"), val = tensor<bool, [5]>([false, false, true, false, true])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_2_squeeze_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_2_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, false, false, false])];
            tensor<fp16, [4, 2, 2, 2048, 256]> kv_cache_internal_tensor_assign_2_cast_fp16 = slice_update(begin = concat_6, begin_mask = kv_cache_internal_tensor_assign_2_begin_mask_0, end = concat_7, end_mask = kv_cache_internal_tensor_assign_2_end_mask_0, squeeze_mask = kv_cache_internal_tensor_assign_2_squeeze_mask_0, stride = kv_cache_internal_tensor_assign_2_stride_0, update = var_1104_cast_fp16, x = coreml_update_state_8)[name = string("kv_cache_internal_tensor_assign_2_cast_fp16")];
            write_state(data = kv_cache_internal_tensor_assign_2_cast_fp16, input = kv_cache)[name = string("coreml_update_state_17_write_state")];
            tensor<fp16, [4, 2, 2, 2048, 256]> coreml_update_state_9 = read_state(input = kv_cache)[name = string("coreml_update_state_17")];
            tensor<int32, [5]> var_1136_begin_0 = const()[name = string("op_1136_begin_0"), val = tensor<int32, [5]>([0, 0, 0, 0, 0])];
            tensor<int32, [5]> var_1136_end_0 = const()[name = string("op_1136_end_0"), val = tensor<int32, [5]>([1, 2, 2, 2048, 256])];
            tensor<bool, [5]> var_1136_end_mask_0 = const()[name = string("op_1136_end_mask_0"), val = tensor<bool, [5]>([false, true, true, true, true])];
            tensor<bool, [5]> var_1136_squeeze_mask_0 = const()[name = string("op_1136_squeeze_mask_0"), val = tensor<bool, [5]>([true, false, false, false, false])];
            tensor<fp16, [2, 2, 2048, 256]> var_1136_cast_fp16 = slice_by_index(begin = var_1136_begin_0, end = var_1136_end_0, end_mask = var_1136_end_mask_0, squeeze_mask = var_1136_squeeze_mask_0, x = coreml_update_state_9)[name = string("op_1136_cast_fp16")];
            tensor<int32, [4]> keys_1_begin_0 = const()[name = string("keys_1_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> keys_1_end_0 = const()[name = string("keys_1_end_0"), val = tensor<int32, [4]>([1, 2, 2048, 256])];
            tensor<bool, [4]> keys_1_end_mask_0 = const()[name = string("keys_1_end_mask_0"), val = tensor<bool, [4]>([false, true, true, true])];
            tensor<bool, [4]> keys_1_squeeze_mask_0 = const()[name = string("keys_1_squeeze_mask_0"), val = tensor<bool, [4]>([true, false, false, false])];
            tensor<fp16, [2, 2048, 256]> keys_1_cast_fp16 = slice_by_index(begin = keys_1_begin_0, end = keys_1_end_0, end_mask = keys_1_end_mask_0, squeeze_mask = keys_1_squeeze_mask_0, x = var_1136_cast_fp16)[name = string("keys_1_cast_fp16")];
            tensor<int32, [4]> values_1_begin_0 = const()[name = string("values_1_begin_0"), val = tensor<int32, [4]>([1, 0, 0, 0])];
            tensor<int32, [4]> values_1_end_0 = const()[name = string("values_1_end_0"), val = tensor<int32, [4]>([2, 2, 2048, 256])];
            tensor<bool, [4]> values_1_end_mask_0 = const()[name = string("values_1_end_mask_0"), val = tensor<bool, [4]>([false, true, true, true])];
            tensor<bool, [4]> values_1_squeeze_mask_0 = const()[name = string("values_1_squeeze_mask_0"), val = tensor<bool, [4]>([true, false, false, false])];
            tensor<fp16, [2, 2048, 256]> values_1_cast_fp16 = slice_by_index(begin = values_1_begin_0, end = values_1_end_0, end_mask = values_1_end_mask_0, squeeze_mask = values_1_squeeze_mask_0, x = var_1136_cast_fp16)[name = string("values_1_cast_fp16")];
            int32 var_1153 = const()[name = string("op_1153"), val = int32(1)];
            tensor<int32, [4]> var_1160 = const()[name = string("op_1160"), val = tensor<int32, [4]>([1, 2048, 1, 4])];
            tensor<fp16, [2, 4, 256, 4]> transpose_0_cast_fp16 = transpose(perm = transpose_0_perm_0, x = var_1054_cast_fp16)[name = string("transpose_117")];
            tensor<fp16, [1, 2048, 1, 4]> var_1161_cast_fp16 = reshape(shape = var_1160, x = transpose_0_cast_fp16)[name = string("op_1161_cast_fp16")];
            tensor<int32, [3]> var_1162 = const()[name = string("op_1162"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [4]> var_1165 = const()[name = string("op_1165"), val = tensor<int32, [4]>([1, 512, 1, 2048])];
            tensor<fp16, [2, 256, 2048]> var_1163_cast_fp16 = transpose(perm = var_1162, x = keys_1_cast_fp16)[name = string("transpose_116")];
            tensor<fp16, [1, 512, 1, 2048]> key_native_1_cast_fp16 = reshape(shape = var_1165, x = var_1163_cast_fp16)[name = string("key_native_1_cast_fp16")];
            tensor<int32, [3]> var_1167 = const()[name = string("op_1167"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [4]> var_1170 = const()[name = string("op_1170"), val = tensor<int32, [4]>([1, 512, 1, 2048])];
            tensor<fp16, [2, 256, 2048]> var_1168_cast_fp16 = transpose(perm = var_1167, x = values_1_cast_fp16)[name = string("transpose_115")];
            tensor<fp16, [1, 512, 1, 2048]> var_1171_cast_fp16 = reshape(shape = var_1170, x = var_1168_cast_fp16)[name = string("op_1171_cast_fp16")];
            tensor<int32, [3]> var_1172 = const()[name = string("op_1172"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [1]> mask_native_1_axes_0 = const()[name = string("mask_native_1_axes_0"), val = tensor<int32, [1]>([2])];
            tensor<fp16, [1, 2048, 4]> var_1173_cast_fp16 = transpose(perm = var_1172, x = mask3)[name = string("transpose_114")];
            tensor<fp16, [1, 2048, 1, 4]> mask_native_1_cast_fp16 = expand_dims(axes = mask_native_1_axes_0, x = var_1173_cast_fp16)[name = string("mask_native_1_cast_fp16")];
            tensor<int32, [8]> tile_0 = const()[name = string("tile_0"), val = tensor<int32, [8]>([256, 256, 256, 256, 256, 256, 256, 256])];
            int32 var_1175_axis_0 = const()[name = string("op_1175_axis_0"), val = int32(1)];
            tensor<fp16, [1, 256, 1, 4]> var_1175_cast_fp16_0, tensor<fp16, [1, 256, 1, 4]> var_1175_cast_fp16_1, tensor<fp16, [1, 256, 1, 4]> var_1175_cast_fp16_2, tensor<fp16, [1, 256, 1, 4]> var_1175_cast_fp16_3, tensor<fp16, [1, 256, 1, 4]> var_1175_cast_fp16_4, tensor<fp16, [1, 256, 1, 4]> var_1175_cast_fp16_5, tensor<fp16, [1, 256, 1, 4]> var_1175_cast_fp16_6, tensor<fp16, [1, 256, 1, 4]> var_1175_cast_fp16_7 = split(axis = var_1175_axis_0, split_sizes = tile_0, x = var_1161_cast_fp16)[name = string("op_1175_cast_fp16")];
            tensor<int32, [4]> var_1184_perm_0 = const()[name = string("op_1184_perm_0"), val = tensor<int32, [4]>([0, 3, 2, 1])];
            tensor<int32, [2]> tile_1 = const()[name = string("tile_1"), val = tensor<int32, [2]>([256, 256])];
            int32 var_1185_axis_0 = const()[name = string("op_1185_axis_0"), val = int32(3)];
            tensor<fp16, [1, 2048, 1, 512]> var_1184_cast_fp16 = transpose(perm = var_1184_perm_0, x = key_native_1_cast_fp16)[name = string("transpose_113")];
            tensor<fp16, [1, 2048, 1, 256]> var_1185_cast_fp16_0, tensor<fp16, [1, 2048, 1, 256]> var_1185_cast_fp16_1 = split(axis = var_1185_axis_0, split_sizes = tile_1, x = var_1184_cast_fp16)[name = string("op_1185_cast_fp16")];
            tensor<int32, [2]> tile_2 = const()[name = string("tile_2"), val = tensor<int32, [2]>([256, 256])];
            int32 var_1188_axis_0 = const()[name = string("op_1188_axis_0"), val = int32(1)];
            tensor<fp16, [1, 256, 1, 2048]> var_1188_cast_fp16_0, tensor<fp16, [1, 256, 1, 2048]> var_1188_cast_fp16_1 = split(axis = var_1188_axis_0, split_sizes = tile_2, x = var_1171_cast_fp16)[name = string("op_1188_cast_fp16")];
            string scores_1_equation_0 = const()[name = string("scores_1_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 4]> scores_1_cast_fp16 = einsum(equation = scores_1_equation_0, values = (var_1185_cast_fp16_0, var_1175_cast_fp16_0))[name = string("scores_1_cast_fp16")];
            fp16 var_1193_to_fp16 = const()[name = string("op_1193_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 4]> var_1194_cast_fp16 = mul(x = scores_1_cast_fp16, y = var_1193_to_fp16)[name = string("op_1194_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_1195_cast_fp16 = add(x = var_1194_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_1195_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_1196_cast_fp16 = softmax(axis = var_1153, x = var_1195_cast_fp16)[name = string("op_1196_cast_fp16")];
            tensor<int32, [4]> tile_3 = const()[name = string("tile_3"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_1197_axis_0 = const()[name = string("op_1197_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 4]> var_1197_cast_fp16_0, tensor<fp16, [1, 512, 1, 4]> var_1197_cast_fp16_1, tensor<fp16, [1, 512, 1, 4]> var_1197_cast_fp16_2, tensor<fp16, [1, 512, 1, 4]> var_1197_cast_fp16_3 = split(axis = var_1197_axis_0, split_sizes = tile_3, x = var_1196_cast_fp16)[name = string("op_1197_cast_fp16")];
            tensor<int32, [4]> tile_4 = const()[name = string("tile_4"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_1202_axis_0 = const()[name = string("op_1202_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_1202_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_1202_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_1202_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_1202_cast_fp16_3 = split(axis = var_1202_axis_0, split_sizes = tile_4, x = var_1188_cast_fp16_0)[name = string("op_1202_cast_fp16")];
            fp16 var_1207_to_fp16 = const()[name = string("op_1207_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_1208_cast_fp16 = mul(x = var_1197_cast_fp16_0, y = var_1207_to_fp16)[name = string("op_1208_cast_fp16")];
            string var_1210_equation_0 = const()[name = string("op_1210_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_1210_cast_fp16 = einsum(equation = var_1210_equation_0, values = (var_1202_cast_fp16_0, var_1208_cast_fp16))[name = string("op_1210_cast_fp16")];
            fp16 var_1211_to_fp16 = const()[name = string("op_1211_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_1212_cast_fp16 = mul(x = var_1197_cast_fp16_1, y = var_1211_to_fp16)[name = string("op_1212_cast_fp16")];
            string var_1214_equation_0 = const()[name = string("op_1214_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_1214_cast_fp16 = einsum(equation = var_1214_equation_0, values = (var_1202_cast_fp16_1, var_1212_cast_fp16))[name = string("op_1214_cast_fp16")];
            fp16 var_1215_to_fp16 = const()[name = string("op_1215_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_1216_cast_fp16 = mul(x = var_1197_cast_fp16_2, y = var_1215_to_fp16)[name = string("op_1216_cast_fp16")];
            string var_1218_equation_0 = const()[name = string("op_1218_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_1218_cast_fp16 = einsum(equation = var_1218_equation_0, values = (var_1202_cast_fp16_2, var_1216_cast_fp16))[name = string("op_1218_cast_fp16")];
            fp16 var_1219_to_fp16 = const()[name = string("op_1219_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_1220_cast_fp16 = mul(x = var_1197_cast_fp16_3, y = var_1219_to_fp16)[name = string("op_1220_cast_fp16")];
            string var_1222_equation_0 = const()[name = string("op_1222_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_1222_cast_fp16 = einsum(equation = var_1222_equation_0, values = (var_1202_cast_fp16_3, var_1220_cast_fp16))[name = string("op_1222_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_1223_cast_fp16 = add(x = var_1210_cast_fp16, y = var_1214_cast_fp16)[name = string("op_1223_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_1224_cast_fp16 = add(x = var_1218_cast_fp16, y = var_1222_cast_fp16)[name = string("op_1224_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_1225_cast_fp16 = add(x = var_1223_cast_fp16, y = var_1224_cast_fp16)[name = string("op_1225_cast_fp16")];
            fp16 _inversed_1227_y_0_to_fp16 = const()[name = string("_inversed_1227_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 4]> _inversed_1227_cast_fp16 = mul(x = var_1225_cast_fp16, y = _inversed_1227_y_0_to_fp16)[name = string("_inversed_1227_cast_fp16")];
            string scores_3_equation_0 = const()[name = string("scores_3_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 4]> scores_3_cast_fp16 = einsum(equation = scores_3_equation_0, values = (var_1185_cast_fp16_0, var_1175_cast_fp16_1))[name = string("scores_3_cast_fp16")];
            fp16 var_1230_to_fp16 = const()[name = string("op_1230_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 4]> var_1231_cast_fp16 = mul(x = scores_3_cast_fp16, y = var_1230_to_fp16)[name = string("op_1231_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_1232_cast_fp16 = add(x = var_1231_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_1232_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_1233_cast_fp16 = softmax(axis = var_1153, x = var_1232_cast_fp16)[name = string("op_1233_cast_fp16")];
            tensor<int32, [4]> tile_5 = const()[name = string("tile_5"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_1234_axis_0 = const()[name = string("op_1234_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 4]> var_1234_cast_fp16_0, tensor<fp16, [1, 512, 1, 4]> var_1234_cast_fp16_1, tensor<fp16, [1, 512, 1, 4]> var_1234_cast_fp16_2, tensor<fp16, [1, 512, 1, 4]> var_1234_cast_fp16_3 = split(axis = var_1234_axis_0, split_sizes = tile_5, x = var_1233_cast_fp16)[name = string("op_1234_cast_fp16")];
            tensor<int32, [4]> tile_6 = const()[name = string("tile_6"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_1239_axis_0 = const()[name = string("op_1239_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_1239_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_1239_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_1239_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_1239_cast_fp16_3 = split(axis = var_1239_axis_0, split_sizes = tile_6, x = var_1188_cast_fp16_0)[name = string("op_1239_cast_fp16")];
            fp16 var_1244_to_fp16 = const()[name = string("op_1244_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_1245_cast_fp16 = mul(x = var_1234_cast_fp16_0, y = var_1244_to_fp16)[name = string("op_1245_cast_fp16")];
            string var_1247_equation_0 = const()[name = string("op_1247_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_1247_cast_fp16 = einsum(equation = var_1247_equation_0, values = (var_1239_cast_fp16_0, var_1245_cast_fp16))[name = string("op_1247_cast_fp16")];
            fp16 var_1248_to_fp16 = const()[name = string("op_1248_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_1249_cast_fp16 = mul(x = var_1234_cast_fp16_1, y = var_1248_to_fp16)[name = string("op_1249_cast_fp16")];
            string var_1251_equation_0 = const()[name = string("op_1251_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_1251_cast_fp16 = einsum(equation = var_1251_equation_0, values = (var_1239_cast_fp16_1, var_1249_cast_fp16))[name = string("op_1251_cast_fp16")];
            fp16 var_1252_to_fp16 = const()[name = string("op_1252_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_1253_cast_fp16 = mul(x = var_1234_cast_fp16_2, y = var_1252_to_fp16)[name = string("op_1253_cast_fp16")];
            string var_1255_equation_0 = const()[name = string("op_1255_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_1255_cast_fp16 = einsum(equation = var_1255_equation_0, values = (var_1239_cast_fp16_2, var_1253_cast_fp16))[name = string("op_1255_cast_fp16")];
            fp16 var_1256_to_fp16 = const()[name = string("op_1256_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_1257_cast_fp16 = mul(x = var_1234_cast_fp16_3, y = var_1256_to_fp16)[name = string("op_1257_cast_fp16")];
            string var_1259_equation_0 = const()[name = string("op_1259_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_1259_cast_fp16 = einsum(equation = var_1259_equation_0, values = (var_1239_cast_fp16_3, var_1257_cast_fp16))[name = string("op_1259_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_1260_cast_fp16 = add(x = var_1247_cast_fp16, y = var_1251_cast_fp16)[name = string("op_1260_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_1261_cast_fp16 = add(x = var_1255_cast_fp16, y = var_1259_cast_fp16)[name = string("op_1261_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_1262_cast_fp16 = add(x = var_1260_cast_fp16, y = var_1261_cast_fp16)[name = string("op_1262_cast_fp16")];
            fp16 _inversed_1264_y_0_to_fp16 = const()[name = string("_inversed_1264_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 4]> _inversed_1264_cast_fp16 = mul(x = var_1262_cast_fp16, y = _inversed_1264_y_0_to_fp16)[name = string("_inversed_1264_cast_fp16")];
            string scores_5_equation_0 = const()[name = string("scores_5_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 4]> scores_5_cast_fp16 = einsum(equation = scores_5_equation_0, values = (var_1185_cast_fp16_0, var_1175_cast_fp16_2))[name = string("scores_5_cast_fp16")];
            fp16 var_1267_to_fp16 = const()[name = string("op_1267_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 4]> var_1268_cast_fp16 = mul(x = scores_5_cast_fp16, y = var_1267_to_fp16)[name = string("op_1268_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_1269_cast_fp16 = add(x = var_1268_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_1269_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_1270_cast_fp16 = softmax(axis = var_1153, x = var_1269_cast_fp16)[name = string("op_1270_cast_fp16")];
            tensor<int32, [4]> tile_7 = const()[name = string("tile_7"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_1271_axis_0 = const()[name = string("op_1271_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 4]> var_1271_cast_fp16_0, tensor<fp16, [1, 512, 1, 4]> var_1271_cast_fp16_1, tensor<fp16, [1, 512, 1, 4]> var_1271_cast_fp16_2, tensor<fp16, [1, 512, 1, 4]> var_1271_cast_fp16_3 = split(axis = var_1271_axis_0, split_sizes = tile_7, x = var_1270_cast_fp16)[name = string("op_1271_cast_fp16")];
            tensor<int32, [4]> tile_8 = const()[name = string("tile_8"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_1276_axis_0 = const()[name = string("op_1276_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_1276_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_1276_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_1276_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_1276_cast_fp16_3 = split(axis = var_1276_axis_0, split_sizes = tile_8, x = var_1188_cast_fp16_0)[name = string("op_1276_cast_fp16")];
            fp16 var_1281_to_fp16 = const()[name = string("op_1281_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_1282_cast_fp16 = mul(x = var_1271_cast_fp16_0, y = var_1281_to_fp16)[name = string("op_1282_cast_fp16")];
            string var_1284_equation_0 = const()[name = string("op_1284_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_1284_cast_fp16 = einsum(equation = var_1284_equation_0, values = (var_1276_cast_fp16_0, var_1282_cast_fp16))[name = string("op_1284_cast_fp16")];
            fp16 var_1285_to_fp16 = const()[name = string("op_1285_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_1286_cast_fp16 = mul(x = var_1271_cast_fp16_1, y = var_1285_to_fp16)[name = string("op_1286_cast_fp16")];
            string var_1288_equation_0 = const()[name = string("op_1288_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_1288_cast_fp16 = einsum(equation = var_1288_equation_0, values = (var_1276_cast_fp16_1, var_1286_cast_fp16))[name = string("op_1288_cast_fp16")];
            fp16 var_1289_to_fp16 = const()[name = string("op_1289_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_1290_cast_fp16 = mul(x = var_1271_cast_fp16_2, y = var_1289_to_fp16)[name = string("op_1290_cast_fp16")];
            string var_1292_equation_0 = const()[name = string("op_1292_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_1292_cast_fp16 = einsum(equation = var_1292_equation_0, values = (var_1276_cast_fp16_2, var_1290_cast_fp16))[name = string("op_1292_cast_fp16")];
            fp16 var_1293_to_fp16 = const()[name = string("op_1293_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_1294_cast_fp16 = mul(x = var_1271_cast_fp16_3, y = var_1293_to_fp16)[name = string("op_1294_cast_fp16")];
            string var_1296_equation_0 = const()[name = string("op_1296_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_1296_cast_fp16 = einsum(equation = var_1296_equation_0, values = (var_1276_cast_fp16_3, var_1294_cast_fp16))[name = string("op_1296_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_1297_cast_fp16 = add(x = var_1284_cast_fp16, y = var_1288_cast_fp16)[name = string("op_1297_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_1298_cast_fp16 = add(x = var_1292_cast_fp16, y = var_1296_cast_fp16)[name = string("op_1298_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_1299_cast_fp16 = add(x = var_1297_cast_fp16, y = var_1298_cast_fp16)[name = string("op_1299_cast_fp16")];
            fp16 _inversed_1301_y_0_to_fp16 = const()[name = string("_inversed_1301_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 4]> _inversed_1301_cast_fp16 = mul(x = var_1299_cast_fp16, y = _inversed_1301_y_0_to_fp16)[name = string("_inversed_1301_cast_fp16")];
            string scores_7_equation_0 = const()[name = string("scores_7_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 4]> scores_7_cast_fp16 = einsum(equation = scores_7_equation_0, values = (var_1185_cast_fp16_0, var_1175_cast_fp16_3))[name = string("scores_7_cast_fp16")];
            fp16 var_1304_to_fp16 = const()[name = string("op_1304_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 4]> var_1305_cast_fp16 = mul(x = scores_7_cast_fp16, y = var_1304_to_fp16)[name = string("op_1305_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_1306_cast_fp16 = add(x = var_1305_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_1306_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_1307_cast_fp16 = softmax(axis = var_1153, x = var_1306_cast_fp16)[name = string("op_1307_cast_fp16")];
            tensor<int32, [4]> tile_9 = const()[name = string("tile_9"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_1308_axis_0 = const()[name = string("op_1308_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 4]> var_1308_cast_fp16_0, tensor<fp16, [1, 512, 1, 4]> var_1308_cast_fp16_1, tensor<fp16, [1, 512, 1, 4]> var_1308_cast_fp16_2, tensor<fp16, [1, 512, 1, 4]> var_1308_cast_fp16_3 = split(axis = var_1308_axis_0, split_sizes = tile_9, x = var_1307_cast_fp16)[name = string("op_1308_cast_fp16")];
            tensor<int32, [4]> tile_10 = const()[name = string("tile_10"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_1313_axis_0 = const()[name = string("op_1313_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_1313_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_1313_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_1313_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_1313_cast_fp16_3 = split(axis = var_1313_axis_0, split_sizes = tile_10, x = var_1188_cast_fp16_0)[name = string("op_1313_cast_fp16")];
            fp16 var_1318_to_fp16 = const()[name = string("op_1318_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_1319_cast_fp16 = mul(x = var_1308_cast_fp16_0, y = var_1318_to_fp16)[name = string("op_1319_cast_fp16")];
            string var_1321_equation_0 = const()[name = string("op_1321_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_1321_cast_fp16 = einsum(equation = var_1321_equation_0, values = (var_1313_cast_fp16_0, var_1319_cast_fp16))[name = string("op_1321_cast_fp16")];
            fp16 var_1322_to_fp16 = const()[name = string("op_1322_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_1323_cast_fp16 = mul(x = var_1308_cast_fp16_1, y = var_1322_to_fp16)[name = string("op_1323_cast_fp16")];
            string var_1325_equation_0 = const()[name = string("op_1325_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_1325_cast_fp16 = einsum(equation = var_1325_equation_0, values = (var_1313_cast_fp16_1, var_1323_cast_fp16))[name = string("op_1325_cast_fp16")];
            fp16 var_1326_to_fp16 = const()[name = string("op_1326_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_1327_cast_fp16 = mul(x = var_1308_cast_fp16_2, y = var_1326_to_fp16)[name = string("op_1327_cast_fp16")];
            string var_1329_equation_0 = const()[name = string("op_1329_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_1329_cast_fp16 = einsum(equation = var_1329_equation_0, values = (var_1313_cast_fp16_2, var_1327_cast_fp16))[name = string("op_1329_cast_fp16")];
            fp16 var_1330_to_fp16 = const()[name = string("op_1330_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_1331_cast_fp16 = mul(x = var_1308_cast_fp16_3, y = var_1330_to_fp16)[name = string("op_1331_cast_fp16")];
            string var_1333_equation_0 = const()[name = string("op_1333_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_1333_cast_fp16 = einsum(equation = var_1333_equation_0, values = (var_1313_cast_fp16_3, var_1331_cast_fp16))[name = string("op_1333_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_1334_cast_fp16 = add(x = var_1321_cast_fp16, y = var_1325_cast_fp16)[name = string("op_1334_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_1335_cast_fp16 = add(x = var_1329_cast_fp16, y = var_1333_cast_fp16)[name = string("op_1335_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_1336_cast_fp16 = add(x = var_1334_cast_fp16, y = var_1335_cast_fp16)[name = string("op_1336_cast_fp16")];
            fp16 _inversed_1338_y_0_to_fp16 = const()[name = string("_inversed_1338_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 4]> _inversed_1338_cast_fp16 = mul(x = var_1336_cast_fp16, y = _inversed_1338_y_0_to_fp16)[name = string("_inversed_1338_cast_fp16")];
            string scores_9_equation_0 = const()[name = string("scores_9_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 4]> scores_9_cast_fp16 = einsum(equation = scores_9_equation_0, values = (var_1185_cast_fp16_1, var_1175_cast_fp16_4))[name = string("scores_9_cast_fp16")];
            fp16 var_1341_to_fp16 = const()[name = string("op_1341_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 4]> var_1342_cast_fp16 = mul(x = scores_9_cast_fp16, y = var_1341_to_fp16)[name = string("op_1342_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_1343_cast_fp16 = add(x = var_1342_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_1343_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_1344_cast_fp16 = softmax(axis = var_1153, x = var_1343_cast_fp16)[name = string("op_1344_cast_fp16")];
            tensor<int32, [4]> tile_11 = const()[name = string("tile_11"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_1345_axis_0 = const()[name = string("op_1345_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 4]> var_1345_cast_fp16_0, tensor<fp16, [1, 512, 1, 4]> var_1345_cast_fp16_1, tensor<fp16, [1, 512, 1, 4]> var_1345_cast_fp16_2, tensor<fp16, [1, 512, 1, 4]> var_1345_cast_fp16_3 = split(axis = var_1345_axis_0, split_sizes = tile_11, x = var_1344_cast_fp16)[name = string("op_1345_cast_fp16")];
            tensor<int32, [4]> tile_12 = const()[name = string("tile_12"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_1350_axis_0 = const()[name = string("op_1350_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_1350_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_1350_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_1350_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_1350_cast_fp16_3 = split(axis = var_1350_axis_0, split_sizes = tile_12, x = var_1188_cast_fp16_1)[name = string("op_1350_cast_fp16")];
            fp16 var_1355_to_fp16 = const()[name = string("op_1355_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_1356_cast_fp16 = mul(x = var_1345_cast_fp16_0, y = var_1355_to_fp16)[name = string("op_1356_cast_fp16")];
            string var_1358_equation_0 = const()[name = string("op_1358_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_1358_cast_fp16 = einsum(equation = var_1358_equation_0, values = (var_1350_cast_fp16_0, var_1356_cast_fp16))[name = string("op_1358_cast_fp16")];
            fp16 var_1359_to_fp16 = const()[name = string("op_1359_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_1360_cast_fp16 = mul(x = var_1345_cast_fp16_1, y = var_1359_to_fp16)[name = string("op_1360_cast_fp16")];
            string var_1362_equation_0 = const()[name = string("op_1362_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_1362_cast_fp16 = einsum(equation = var_1362_equation_0, values = (var_1350_cast_fp16_1, var_1360_cast_fp16))[name = string("op_1362_cast_fp16")];
            fp16 var_1363_to_fp16 = const()[name = string("op_1363_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_1364_cast_fp16 = mul(x = var_1345_cast_fp16_2, y = var_1363_to_fp16)[name = string("op_1364_cast_fp16")];
            string var_1366_equation_0 = const()[name = string("op_1366_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_1366_cast_fp16 = einsum(equation = var_1366_equation_0, values = (var_1350_cast_fp16_2, var_1364_cast_fp16))[name = string("op_1366_cast_fp16")];
            fp16 var_1367_to_fp16 = const()[name = string("op_1367_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_1368_cast_fp16 = mul(x = var_1345_cast_fp16_3, y = var_1367_to_fp16)[name = string("op_1368_cast_fp16")];
            string var_1370_equation_0 = const()[name = string("op_1370_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_1370_cast_fp16 = einsum(equation = var_1370_equation_0, values = (var_1350_cast_fp16_3, var_1368_cast_fp16))[name = string("op_1370_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_1371_cast_fp16 = add(x = var_1358_cast_fp16, y = var_1362_cast_fp16)[name = string("op_1371_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_1372_cast_fp16 = add(x = var_1366_cast_fp16, y = var_1370_cast_fp16)[name = string("op_1372_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_1373_cast_fp16 = add(x = var_1371_cast_fp16, y = var_1372_cast_fp16)[name = string("op_1373_cast_fp16")];
            fp16 _inversed_1375_y_0_to_fp16 = const()[name = string("_inversed_1375_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 4]> _inversed_1375_cast_fp16 = mul(x = var_1373_cast_fp16, y = _inversed_1375_y_0_to_fp16)[name = string("_inversed_1375_cast_fp16")];
            string scores_11_equation_0 = const()[name = string("scores_11_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 4]> scores_11_cast_fp16 = einsum(equation = scores_11_equation_0, values = (var_1185_cast_fp16_1, var_1175_cast_fp16_5))[name = string("scores_11_cast_fp16")];
            fp16 var_1378_to_fp16 = const()[name = string("op_1378_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 4]> var_1379_cast_fp16 = mul(x = scores_11_cast_fp16, y = var_1378_to_fp16)[name = string("op_1379_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_1380_cast_fp16 = add(x = var_1379_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_1380_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_1381_cast_fp16 = softmax(axis = var_1153, x = var_1380_cast_fp16)[name = string("op_1381_cast_fp16")];
            tensor<int32, [4]> tile_13 = const()[name = string("tile_13"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_1382_axis_0 = const()[name = string("op_1382_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 4]> var_1382_cast_fp16_0, tensor<fp16, [1, 512, 1, 4]> var_1382_cast_fp16_1, tensor<fp16, [1, 512, 1, 4]> var_1382_cast_fp16_2, tensor<fp16, [1, 512, 1, 4]> var_1382_cast_fp16_3 = split(axis = var_1382_axis_0, split_sizes = tile_13, x = var_1381_cast_fp16)[name = string("op_1382_cast_fp16")];
            tensor<int32, [4]> tile_14 = const()[name = string("tile_14"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_1387_axis_0 = const()[name = string("op_1387_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_1387_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_1387_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_1387_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_1387_cast_fp16_3 = split(axis = var_1387_axis_0, split_sizes = tile_14, x = var_1188_cast_fp16_1)[name = string("op_1387_cast_fp16")];
            fp16 var_1392_to_fp16 = const()[name = string("op_1392_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_1393_cast_fp16 = mul(x = var_1382_cast_fp16_0, y = var_1392_to_fp16)[name = string("op_1393_cast_fp16")];
            string var_1395_equation_0 = const()[name = string("op_1395_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_1395_cast_fp16 = einsum(equation = var_1395_equation_0, values = (var_1387_cast_fp16_0, var_1393_cast_fp16))[name = string("op_1395_cast_fp16")];
            fp16 var_1396_to_fp16 = const()[name = string("op_1396_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_1397_cast_fp16 = mul(x = var_1382_cast_fp16_1, y = var_1396_to_fp16)[name = string("op_1397_cast_fp16")];
            string var_1399_equation_0 = const()[name = string("op_1399_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_1399_cast_fp16 = einsum(equation = var_1399_equation_0, values = (var_1387_cast_fp16_1, var_1397_cast_fp16))[name = string("op_1399_cast_fp16")];
            fp16 var_1400_to_fp16 = const()[name = string("op_1400_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_1401_cast_fp16 = mul(x = var_1382_cast_fp16_2, y = var_1400_to_fp16)[name = string("op_1401_cast_fp16")];
            string var_1403_equation_0 = const()[name = string("op_1403_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_1403_cast_fp16 = einsum(equation = var_1403_equation_0, values = (var_1387_cast_fp16_2, var_1401_cast_fp16))[name = string("op_1403_cast_fp16")];
            fp16 var_1404_to_fp16 = const()[name = string("op_1404_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_1405_cast_fp16 = mul(x = var_1382_cast_fp16_3, y = var_1404_to_fp16)[name = string("op_1405_cast_fp16")];
            string var_1407_equation_0 = const()[name = string("op_1407_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_1407_cast_fp16 = einsum(equation = var_1407_equation_0, values = (var_1387_cast_fp16_3, var_1405_cast_fp16))[name = string("op_1407_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_1408_cast_fp16 = add(x = var_1395_cast_fp16, y = var_1399_cast_fp16)[name = string("op_1408_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_1409_cast_fp16 = add(x = var_1403_cast_fp16, y = var_1407_cast_fp16)[name = string("op_1409_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_1410_cast_fp16 = add(x = var_1408_cast_fp16, y = var_1409_cast_fp16)[name = string("op_1410_cast_fp16")];
            fp16 _inversed_1412_y_0_to_fp16 = const()[name = string("_inversed_1412_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 4]> _inversed_1412_cast_fp16 = mul(x = var_1410_cast_fp16, y = _inversed_1412_y_0_to_fp16)[name = string("_inversed_1412_cast_fp16")];
            string scores_13_equation_0 = const()[name = string("scores_13_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 4]> scores_13_cast_fp16 = einsum(equation = scores_13_equation_0, values = (var_1185_cast_fp16_1, var_1175_cast_fp16_6))[name = string("scores_13_cast_fp16")];
            fp16 var_1415_to_fp16 = const()[name = string("op_1415_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 4]> var_1416_cast_fp16 = mul(x = scores_13_cast_fp16, y = var_1415_to_fp16)[name = string("op_1416_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_1417_cast_fp16 = add(x = var_1416_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_1417_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_1418_cast_fp16 = softmax(axis = var_1153, x = var_1417_cast_fp16)[name = string("op_1418_cast_fp16")];
            tensor<int32, [4]> tile_15 = const()[name = string("tile_15"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_1419_axis_0 = const()[name = string("op_1419_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 4]> var_1419_cast_fp16_0, tensor<fp16, [1, 512, 1, 4]> var_1419_cast_fp16_1, tensor<fp16, [1, 512, 1, 4]> var_1419_cast_fp16_2, tensor<fp16, [1, 512, 1, 4]> var_1419_cast_fp16_3 = split(axis = var_1419_axis_0, split_sizes = tile_15, x = var_1418_cast_fp16)[name = string("op_1419_cast_fp16")];
            tensor<int32, [4]> tile_16 = const()[name = string("tile_16"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_1424_axis_0 = const()[name = string("op_1424_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_1424_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_1424_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_1424_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_1424_cast_fp16_3 = split(axis = var_1424_axis_0, split_sizes = tile_16, x = var_1188_cast_fp16_1)[name = string("op_1424_cast_fp16")];
            fp16 var_1429_to_fp16 = const()[name = string("op_1429_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_1430_cast_fp16 = mul(x = var_1419_cast_fp16_0, y = var_1429_to_fp16)[name = string("op_1430_cast_fp16")];
            string var_1432_equation_0 = const()[name = string("op_1432_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_1432_cast_fp16 = einsum(equation = var_1432_equation_0, values = (var_1424_cast_fp16_0, var_1430_cast_fp16))[name = string("op_1432_cast_fp16")];
            fp16 var_1433_to_fp16 = const()[name = string("op_1433_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_1434_cast_fp16 = mul(x = var_1419_cast_fp16_1, y = var_1433_to_fp16)[name = string("op_1434_cast_fp16")];
            string var_1436_equation_0 = const()[name = string("op_1436_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_1436_cast_fp16 = einsum(equation = var_1436_equation_0, values = (var_1424_cast_fp16_1, var_1434_cast_fp16))[name = string("op_1436_cast_fp16")];
            fp16 var_1437_to_fp16 = const()[name = string("op_1437_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_1438_cast_fp16 = mul(x = var_1419_cast_fp16_2, y = var_1437_to_fp16)[name = string("op_1438_cast_fp16")];
            string var_1440_equation_0 = const()[name = string("op_1440_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_1440_cast_fp16 = einsum(equation = var_1440_equation_0, values = (var_1424_cast_fp16_2, var_1438_cast_fp16))[name = string("op_1440_cast_fp16")];
            fp16 var_1441_to_fp16 = const()[name = string("op_1441_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_1442_cast_fp16 = mul(x = var_1419_cast_fp16_3, y = var_1441_to_fp16)[name = string("op_1442_cast_fp16")];
            string var_1444_equation_0 = const()[name = string("op_1444_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_1444_cast_fp16 = einsum(equation = var_1444_equation_0, values = (var_1424_cast_fp16_3, var_1442_cast_fp16))[name = string("op_1444_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_1445_cast_fp16 = add(x = var_1432_cast_fp16, y = var_1436_cast_fp16)[name = string("op_1445_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_1446_cast_fp16 = add(x = var_1440_cast_fp16, y = var_1444_cast_fp16)[name = string("op_1446_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_1447_cast_fp16 = add(x = var_1445_cast_fp16, y = var_1446_cast_fp16)[name = string("op_1447_cast_fp16")];
            fp16 _inversed_1449_y_0_to_fp16 = const()[name = string("_inversed_1449_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 4]> _inversed_1449_cast_fp16 = mul(x = var_1447_cast_fp16, y = _inversed_1449_y_0_to_fp16)[name = string("_inversed_1449_cast_fp16")];
            string scores_15_equation_0 = const()[name = string("scores_15_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 4]> scores_15_cast_fp16 = einsum(equation = scores_15_equation_0, values = (var_1185_cast_fp16_1, var_1175_cast_fp16_7))[name = string("scores_15_cast_fp16")];
            fp16 var_1452_to_fp16 = const()[name = string("op_1452_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 4]> var_1453_cast_fp16 = mul(x = scores_15_cast_fp16, y = var_1452_to_fp16)[name = string("op_1453_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_1454_cast_fp16 = add(x = var_1453_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_1454_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_1455_cast_fp16 = softmax(axis = var_1153, x = var_1454_cast_fp16)[name = string("op_1455_cast_fp16")];
            tensor<int32, [4]> tile_17 = const()[name = string("tile_17"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_1456_axis_0 = const()[name = string("op_1456_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 4]> var_1456_cast_fp16_0, tensor<fp16, [1, 512, 1, 4]> var_1456_cast_fp16_1, tensor<fp16, [1, 512, 1, 4]> var_1456_cast_fp16_2, tensor<fp16, [1, 512, 1, 4]> var_1456_cast_fp16_3 = split(axis = var_1456_axis_0, split_sizes = tile_17, x = var_1455_cast_fp16)[name = string("op_1456_cast_fp16")];
            tensor<int32, [4]> tile_18 = const()[name = string("tile_18"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_1461_axis_0 = const()[name = string("op_1461_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_1461_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_1461_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_1461_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_1461_cast_fp16_3 = split(axis = var_1461_axis_0, split_sizes = tile_18, x = var_1188_cast_fp16_1)[name = string("op_1461_cast_fp16")];
            fp16 var_1466_to_fp16 = const()[name = string("op_1466_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_1467_cast_fp16 = mul(x = var_1456_cast_fp16_0, y = var_1466_to_fp16)[name = string("op_1467_cast_fp16")];
            string var_1469_equation_0 = const()[name = string("op_1469_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_1469_cast_fp16 = einsum(equation = var_1469_equation_0, values = (var_1461_cast_fp16_0, var_1467_cast_fp16))[name = string("op_1469_cast_fp16")];
            fp16 var_1470_to_fp16 = const()[name = string("op_1470_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_1471_cast_fp16 = mul(x = var_1456_cast_fp16_1, y = var_1470_to_fp16)[name = string("op_1471_cast_fp16")];
            string var_1473_equation_0 = const()[name = string("op_1473_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_1473_cast_fp16 = einsum(equation = var_1473_equation_0, values = (var_1461_cast_fp16_1, var_1471_cast_fp16))[name = string("op_1473_cast_fp16")];
            fp16 var_1474_to_fp16 = const()[name = string("op_1474_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_1475_cast_fp16 = mul(x = var_1456_cast_fp16_2, y = var_1474_to_fp16)[name = string("op_1475_cast_fp16")];
            string var_1477_equation_0 = const()[name = string("op_1477_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_1477_cast_fp16 = einsum(equation = var_1477_equation_0, values = (var_1461_cast_fp16_2, var_1475_cast_fp16))[name = string("op_1477_cast_fp16")];
            fp16 var_1478_to_fp16 = const()[name = string("op_1478_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_1479_cast_fp16 = mul(x = var_1456_cast_fp16_3, y = var_1478_to_fp16)[name = string("op_1479_cast_fp16")];
            string var_1481_equation_0 = const()[name = string("op_1481_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_1481_cast_fp16 = einsum(equation = var_1481_equation_0, values = (var_1461_cast_fp16_3, var_1479_cast_fp16))[name = string("op_1481_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_1482_cast_fp16 = add(x = var_1469_cast_fp16, y = var_1473_cast_fp16)[name = string("op_1482_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_1483_cast_fp16 = add(x = var_1477_cast_fp16, y = var_1481_cast_fp16)[name = string("op_1483_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_1484_cast_fp16 = add(x = var_1482_cast_fp16, y = var_1483_cast_fp16)[name = string("op_1484_cast_fp16")];
            fp16 _inversed_1486_y_0_to_fp16 = const()[name = string("_inversed_1486_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 4]> _inversed_1486_cast_fp16 = mul(x = var_1484_cast_fp16, y = _inversed_1486_y_0_to_fp16)[name = string("_inversed_1486_cast_fp16")];
            bool var_1488_interleave_0 = const()[name = string("op_1488_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 2048, 1, 4]> var_1488_cast_fp16 = concat(axis = var_1153, interleave = var_1488_interleave_0, values = (_inversed_1227_cast_fp16, _inversed_1264_cast_fp16, _inversed_1301_cast_fp16, _inversed_1338_cast_fp16, _inversed_1375_cast_fp16, _inversed_1412_cast_fp16, _inversed_1449_cast_fp16, _inversed_1486_cast_fp16))[name = string("op_1488_cast_fp16")];
            tensor<int32, [4]> var_1489 = const()[name = string("op_1489"), val = tensor<int32, [4]>([2, 4, 256, 4])];
            tensor<fp16, [2, 4, 256, 4]> var_1490_cast_fp16 = reshape(shape = var_1489, x = var_1488_cast_fp16)[name = string("op_1490_cast_fp16")];
            tensor<int32, [4]> transpose_1_perm_0 = const()[name = string("transpose_1_perm_0"), val = tensor<int32, [4]>([3, 0, 1, 2])];
            tensor<int32, [2]> var_1509 = const()[name = string("op_1509"), val = tensor<int32, [2]>([4, 2048])];
            tensor<fp16, [4, 2, 4, 256]> transpose_1_cast_fp16 = transpose(perm = transpose_1_perm_0, x = var_1490_cast_fp16)[name = string("transpose_112")];
            tensor<fp16, [4, 2048]> attention_output_3_cast_fp16 = reshape(shape = var_1509, x = transpose_1_cast_fp16)[name = string("attention_output_3_cast_fp16")];
            tensor<fp16, [4, 2048]> var_1511_cast_fp16 = sigmoid(x = gate_3_cast_fp16)[name = string("op_1511_cast_fp16")];
            tensor<fp16, [4, 2048]> input_45_cast_fp16 = mul(x = attention_output_3_cast_fp16, y = var_1511_cast_fp16)[name = string("input_45_cast_fp16")];
            tensor<fp16, [1024, 2048]> completions_0_o_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 2048]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(53090624))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(54663552))))[name = string("completions_0_o_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 1024]> linear_27_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = completions_0_o_proj_weight_promoted_to_fp16_palettized, x = input_45_cast_fp16)[name = string("linear_27_cast_fp16")];
            tensor<fp16, [4, 1024]> value_57_cast_fp16 = add(x = value_43_cast_fp16, y = linear_27_cast_fp16)[name = string("value_57_cast_fp16")];
            tensor<fp16, [4, 1024]> var_1516_cast_fp16 = mul(x = value_57_cast_fp16, y = value_57_cast_fp16)[name = string("op_1516_cast_fp16")];
            tensor<int32, [1]> variance_25_axes_0 = const()[name = string("variance_25_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_25_keep_dims_0 = const()[name = string("variance_25_keep_dims_0"), val = bool(true)];
            tensor<fp16, [4, 1]> variance_25_cast_fp16 = reduce_mean(axes = variance_25_axes_0, keep_dims = variance_25_keep_dims_0, x = var_1516_cast_fp16)[name = string("variance_25_cast_fp16")];
            fp16 var_1519_to_fp16 = const()[name = string("op_1519_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [4, 1]> var_1520_cast_fp16 = add(x = variance_25_cast_fp16, y = var_1519_to_fp16)[name = string("op_1520_cast_fp16")];
            fp32 var_1521_epsilon_0 = const()[name = string("op_1521_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [4, 1]> var_1521_cast_fp16 = rsqrt(epsilon = var_1521_epsilon_0, x = var_1520_cast_fp16)[name = string("op_1521_cast_fp16")];
            tensor<fp16, [4, 1024]> var_1522_cast_fp16 = mul(x = value_57_cast_fp16, y = var_1521_cast_fp16)[name = string("op_1522_cast_fp16")];
            tensor<fp16, [1024]> var_1524_to_fp16 = const()[name = string("op_1524_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(54671808)))];
            tensor<fp16, [4, 1024]> input_47_cast_fp16 = mul(x = var_1522_cast_fp16, y = var_1524_to_fp16)[name = string("input_47_cast_fp16")];
            tensor<fp16, [3584, 1024]> completions_0_gate_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(54673920))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(57426496))))[name = string("completions_0_gate_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 3584]> linear_28_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = completions_0_gate_proj_weight_promoted_to_fp16_palettized, x = input_47_cast_fp16)[name = string("linear_28_cast_fp16")];
            tensor<fp16, [4, 3584]> var_1528_cast_fp16 = silu(x = linear_28_cast_fp16)[name = string("op_1528_cast_fp16")];
            tensor<fp16, [3584, 1024]> completions_0_up_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(57455232))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(60207808))))[name = string("completions_0_up_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 3584]> linear_29_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = completions_0_up_proj_weight_promoted_to_fp16_palettized, x = input_47_cast_fp16)[name = string("linear_29_cast_fp16")];
            tensor<fp16, [4, 3584]> input_51_cast_fp16 = mul(x = var_1528_cast_fp16, y = linear_29_cast_fp16)[name = string("input_51_cast_fp16")];
            tensor<fp16, [1024, 3584]> completions_0_down_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 3584]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(60236544))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(62989120))))[name = string("completions_0_down_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 1024]> linear_30_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = completions_0_down_proj_weight_promoted_to_fp16_palettized, x = input_51_cast_fp16)[name = string("linear_30_cast_fp16")];
            tensor<fp16, [4, 1024]> value_59_cast_fp16 = add(x = value_57_cast_fp16, y = linear_30_cast_fp16)[name = string("value_59_cast_fp16")];
            int32 var_1551 = const()[name = string("op_1551"), val = int32(-1)];
            tensor<fp16, [4, 1024]> var_1566_cast_fp16 = mul(x = value_59_cast_fp16, y = value_59_cast_fp16)[name = string("op_1566_cast_fp16")];
            tensor<int32, [1]> variance_27_axes_0 = const()[name = string("variance_27_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_27_keep_dims_0 = const()[name = string("variance_27_keep_dims_0"), val = bool(true)];
            tensor<fp16, [4, 1]> variance_27_cast_fp16 = reduce_mean(axes = variance_27_axes_0, keep_dims = variance_27_keep_dims_0, x = var_1566_cast_fp16)[name = string("variance_27_cast_fp16")];
            fp16 var_1569_to_fp16 = const()[name = string("op_1569_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [4, 1]> var_1570_cast_fp16 = add(x = variance_27_cast_fp16, y = var_1569_to_fp16)[name = string("op_1570_cast_fp16")];
            fp32 var_1571_epsilon_0 = const()[name = string("op_1571_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [4, 1]> var_1571_cast_fp16 = rsqrt(epsilon = var_1571_epsilon_0, x = var_1570_cast_fp16)[name = string("op_1571_cast_fp16")];
            tensor<fp16, [4, 1024]> var_1572_cast_fp16 = mul(x = value_59_cast_fp16, y = var_1571_cast_fp16)[name = string("op_1572_cast_fp16")];
            tensor<fp16, [1024]> var_1574_to_fp16 = const()[name = string("op_1574_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(62997376)))];
            tensor<fp16, [4, 1024]> input_53_cast_fp16 = mul(x = var_1572_cast_fp16, y = var_1574_to_fp16)[name = string("input_53_cast_fp16")];
            tensor<fp16, [6144, 1024]> groups_1_0_in_proj_qkv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(62999488))), lut = tensor<fp16, [384, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(67718144))))[name = string("groups_1_0_in_proj_qkv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 6144]> linear_31_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = groups_1_0_in_proj_qkv_weight_promoted_to_fp16_palettized, x = input_53_cast_fp16)[name = string("linear_31_cast_fp16")];
            tensor<int32, [2]> var_1578_perm_0 = const()[name = string("op_1578_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> mixed_7_axes_0 = const()[name = string("mixed_7_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [6144, 4]> var_1578_cast_fp16 = transpose(perm = var_1578_perm_0, x = linear_31_cast_fp16)[name = string("transpose_111")];
            tensor<fp16, [1, 6144, 4]> mixed_7_cast_fp16 = expand_dims(axes = mixed_7_axes_0, x = var_1578_cast_fp16)[name = string("mixed_7_cast_fp16")];
            bool convolution_input_7_interleave_0 = const()[name = string("convolution_input_7_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 6144, 7]> convolution_input_7_cast_fp16 = concat(axis = var_1551, interleave = convolution_input_7_interleave_0, values = (conv4, mixed_7_cast_fp16))[name = string("convolution_input_7_cast_fp16")];
            string input_55_pad_type_0 = const()[name = string("input_55_pad_type_0"), val = string("valid")];
            int32 input_55_groups_0 = const()[name = string("input_55_groups_0"), val = int32(6144)];
            tensor<int32, [1]> input_55_strides_0 = const()[name = string("input_55_strides_0"), val = tensor<int32, [1]>([1])];
            tensor<int32, [2]> input_55_pad_0 = const()[name = string("input_55_pad_0"), val = tensor<int32, [2]>([0, 0])];
            tensor<int32, [1]> input_55_dilations_0 = const()[name = string("input_55_dilations_0"), val = tensor<int32, [1]>([1])];
            tensor<fp16, [6144, 1, 4]> groups_1_0_conv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1, 4]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(67767360))), lut = tensor<fp16, [384, 1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(67785856))))[name = string("groups_1_0_conv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 6144, 4]> input_55_cast_fp16 = conv(dilations = input_55_dilations_0, groups = input_55_groups_0, pad = input_55_pad_0, pad_type = input_55_pad_type_0, strides = input_55_strides_0, weight = groups_1_0_conv_weight_promoted_to_fp16_palettized, x = convolution_input_7_cast_fp16)[name = string("input_55_cast_fp16")];
            tensor<fp16, [1, 6144, 4]> var_1587_cast_fp16 = silu(x = input_55_cast_fp16)[name = string("op_1587_cast_fp16")];
            tensor<int32, [3]> var_1588_perm_0 = const()[name = string("op_1588_perm_0"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [3]> var_1591_begin_0 = const()[name = string("op_1591_begin_0"), val = tensor<int32, [3]>([0, 0, 4])];
            tensor<int32, [3]> var_1591_end_0 = const()[name = string("op_1591_end_0"), val = tensor<int32, [3]>([1, 6144, 7])];
            tensor<bool, [3]> var_1591_end_mask_0 = const()[name = string("op_1591_end_mask_0"), val = tensor<bool, [3]>([true, true, true])];
            tensor<fp16, [1, 6144, 3]> conv4_out = slice_by_index(begin = var_1591_begin_0, end = var_1591_end_0, end_mask = var_1591_end_mask_0, x = convolution_input_7_cast_fp16)[name = string("op_1591_cast_fp16")];
            tensor<int32, [3]> var_1592 = const()[name = string("op_1592"), val = tensor<int32, [3]>([2048, 2048, 2048])];
            int32 var_1593_axis_0 = const()[name = string("op_1593_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 4, 6144]> var_1588_cast_fp16 = transpose(perm = var_1588_perm_0, x = var_1587_cast_fp16)[name = string("transpose_110")];
            tensor<fp16, [1, 4, 2048]> var_1593_cast_fp16_0, tensor<fp16, [1, 4, 2048]> var_1593_cast_fp16_1, tensor<fp16, [1, 4, 2048]> var_1593_cast_fp16_2 = split(axis = var_1593_axis_0, split_sizes = var_1592, x = var_1588_cast_fp16)[name = string("op_1593_cast_fp16")];
            tensor<int32, [4]> var_1597 = const()[name = string("op_1597"), val = tensor<int32, [4]>([1, 4, 16, 128])];
            tensor<fp16, [1, 4, 16, 128]> value_63_cast_fp16 = reshape(shape = var_1597, x = var_1593_cast_fp16_0)[name = string("value_63_cast_fp16")];
            tensor<int32, [4]> var_1599 = const()[name = string("op_1599"), val = tensor<int32, [4]>([1, 4, 16, 128])];
            tensor<fp16, [1, 4, 16, 128]> value_65_cast_fp16 = reshape(shape = var_1599, x = var_1593_cast_fp16_1)[name = string("value_65_cast_fp16")];
            tensor<int32, [4]> var_1601 = const()[name = string("op_1601"), val = tensor<int32, [4]>([1, 4, 16, 128])];
            tensor<fp16, [1, 4, 16, 128]> value_67_cast_fp16 = reshape(shape = var_1601, x = var_1593_cast_fp16_2)[name = string("value_67_cast_fp16")];
            tensor<fp16, [1, 4, 16, 128]> var_1603_cast_fp16 = mul(x = value_63_cast_fp16, y = value_63_cast_fp16)[name = string("op_1603_cast_fp16")];
            tensor<int32, [1]> var_1605_axes_0 = const()[name = string("op_1605_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_1605_keep_dims_0 = const()[name = string("op_1605_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 4, 16, 1]> var_1605_cast_fp16 = reduce_sum(axes = var_1605_axes_0, keep_dims = var_1605_keep_dims_0, x = var_1603_cast_fp16)[name = string("op_1605_cast_fp16")];
            fp16 var_1606_to_fp16 = const()[name = string("op_1606_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 4, 16, 1]> var_1607_cast_fp16 = add(x = var_1605_cast_fp16, y = var_1606_to_fp16)[name = string("op_1607_cast_fp16")];
            fp32 var_1608_epsilon_0 = const()[name = string("op_1608_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 4, 16, 1]> var_1608_cast_fp16 = rsqrt(epsilon = var_1608_epsilon_0, x = var_1607_cast_fp16)[name = string("op_1608_cast_fp16")];
            tensor<fp16, [1, 4, 16, 128]> var_1609_cast_fp16 = mul(x = value_63_cast_fp16, y = var_1608_cast_fp16)[name = string("op_1609_cast_fp16")];
            tensor<int32, [4]> var_1610_perm_0 = const()[name = string("op_1610_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            fp16 _inversed_query_19_y_0_to_fp16 = const()[name = string("_inversed_query_19_y_0_to_fp16"), val = fp16(0x1.6ap-4)];
            tensor<fp16, [1, 16, 4, 128]> var_1610_cast_fp16 = transpose(perm = var_1610_perm_0, x = var_1609_cast_fp16)[name = string("transpose_109")];
            tensor<fp16, [1, 16, 4, 128]> _inversed_query_19_cast_fp16 = mul(x = var_1610_cast_fp16, y = _inversed_query_19_y_0_to_fp16)[name = string("_inversed_query_19_cast_fp16")];
            tensor<fp16, [1, 4, 16, 128]> var_1613_cast_fp16 = mul(x = value_65_cast_fp16, y = value_65_cast_fp16)[name = string("op_1613_cast_fp16")];
            tensor<int32, [1]> var_1615_axes_0 = const()[name = string("op_1615_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_1615_keep_dims_0 = const()[name = string("op_1615_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 4, 16, 1]> var_1615_cast_fp16 = reduce_sum(axes = var_1615_axes_0, keep_dims = var_1615_keep_dims_0, x = var_1613_cast_fp16)[name = string("op_1615_cast_fp16")];
            fp16 var_1616_to_fp16 = const()[name = string("op_1616_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 4, 16, 1]> var_1617_cast_fp16 = add(x = var_1615_cast_fp16, y = var_1616_to_fp16)[name = string("op_1617_cast_fp16")];
            fp32 var_1618_epsilon_0 = const()[name = string("op_1618_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 4, 16, 1]> var_1618_cast_fp16 = rsqrt(epsilon = var_1618_epsilon_0, x = var_1617_cast_fp16)[name = string("op_1618_cast_fp16")];
            tensor<fp16, [1, 4, 16, 128]> var_1619_cast_fp16 = mul(x = value_65_cast_fp16, y = var_1618_cast_fp16)[name = string("op_1619_cast_fp16")];
            tensor<int32, [4]> key_19_perm_0 = const()[name = string("key_19_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<int32, [4]> value_69_perm_0 = const()[name = string("value_69_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<fp16, [16, 1024]> groups_1_0_in_proj_b_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(67835072))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(67847424))))[name = string("groups_1_0_in_proj_b_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16]> linear_32_bias_0_to_fp16 = const()[name = string("linear_32_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(4862656)))];
            tensor<fp16, [4, 16]> linear_32_cast_fp16 = linear(bias = linear_32_bias_0_to_fp16, weight = groups_1_0_in_proj_b_weight_promoted_to_fp16_palettized, x = input_53_cast_fp16)[name = string("linear_32_cast_fp16")];
            tensor<fp16, [4, 16]> var_1624_cast_fp16 = sigmoid(x = linear_32_cast_fp16)[name = string("op_1624_cast_fp16")];
            tensor<int32, [2]> var_1625_perm_0 = const()[name = string("op_1625_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> beta_7_axes_0 = const()[name = string("beta_7_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 4]> var_1625_cast_fp16 = transpose(perm = var_1625_perm_0, x = var_1624_cast_fp16)[name = string("transpose_108")];
            tensor<fp16, [1, 16, 4]> beta_7_cast_fp16 = expand_dims(axes = beta_7_axes_0, x = var_1625_cast_fp16)[name = string("beta_7_cast_fp16")];
            tensor<fp16, [16, 1024]> op_1631_weight_0_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(67847616))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(67859968))))[name = string("op_1631_weight_0_to_fp16_palettized")];
            tensor<fp16, [16]> var_1631_bias_0_to_fp16 = const()[name = string("op_1631_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(67860160)))];
            tensor<fp16, [4, 16]> var_1631_cast_fp16 = linear(bias = var_1631_bias_0_to_fp16, weight = op_1631_weight_0_to_fp16_palettized, x = input_53_cast_fp16)[name = string("op_1631_cast_fp16")];
            tensor<fp16, [4, 16]> var_1632_cast_fp16 = softplus(x = var_1631_cast_fp16)[name = string("op_1632_cast_fp16")];
            tensor<fp16, [16]> var_1628_promoted_to_fp16 = const()[name = string("op_1628_promoted_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(67860288)))];
            tensor<fp16, [4, 16]> var_1633_cast_fp16 = mul(x = var_1628_promoted_to_fp16, y = var_1632_cast_fp16)[name = string("op_1633_cast_fp16")];
            tensor<int32, [2]> var_1634_perm_0 = const()[name = string("op_1634_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> decay_7_axes_0 = const()[name = string("decay_7_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 4]> var_1634_cast_fp16 = transpose(perm = var_1634_perm_0, x = var_1633_cast_fp16)[name = string("transpose_107")];
            tensor<fp16, [1, 16, 4]> decay_7_cast_fp16 = expand_dims(axes = decay_7_axes_0, x = var_1634_cast_fp16)[name = string("decay_7_cast_fp16")];
            tensor<fp16, [2048, 1024]> groups_1_0_in_proj_z_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [2048, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(67860416))), lut = tensor<fp16, [128, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(69433344))))[name = string("groups_1_0_in_proj_z_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 2048]> linear_34_cast_fp16 = linear(bias = linear_3_bias_0_to_fp16, weight = groups_1_0_in_proj_z_weight_promoted_to_fp16_palettized, x = input_53_cast_fp16)[name = string("linear_34_cast_fp16")];
            tensor<int32, [3]> var_1642_begin_0 = const()[name = string("op_1642_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_1642_end_0 = const()[name = string("op_1642_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_1642_end_mask_0 = const()[name = string("op_1642_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1642_squeeze_mask_0 = const()[name = string("op_1642_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1642_cast_fp16 = slice_by_index(begin = var_1642_begin_0, end = var_1642_end_0, end_mask = var_1642_end_mask_0, squeeze_mask = var_1642_squeeze_mask_0, x = decay_7_cast_fp16)[name = string("op_1642_cast_fp16")];
            tensor<fp16, [1, 16]> var_1643_cast_fp16 = exp(x = var_1642_cast_fp16)[name = string("op_1643_cast_fp16")];
            tensor<int32, [1]> var_1644_axes_0 = const()[name = string("op_1644_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1644_cast_fp16 = expand_dims(axes = var_1644_axes_0, x = var_1643_cast_fp16)[name = string("op_1644_cast_fp16")];
            tensor<int32, [1]> var_1645_axes_0 = const()[name = string("op_1645_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_1645_cast_fp16 = expand_dims(axes = var_1645_axes_0, x = var_1644_cast_fp16)[name = string("op_1645_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_49_cast_fp16 = mul(x = rec4, y = var_1645_cast_fp16)[name = string("state_49_cast_fp16")];
            tensor<int32, [4]> key_token_25_begin_0 = const()[name = string("key_token_25_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> key_token_25_end_0 = const()[name = string("key_token_25_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> key_token_25_end_mask_0 = const()[name = string("key_token_25_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_25_squeeze_mask_0 = const()[name = string("key_token_25_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 4, 128]> key_19_cast_fp16 = transpose(perm = key_19_perm_0, x = var_1619_cast_fp16)[name = string("transpose_106")];
            tensor<fp16, [1, 16, 128]> key_token_25_cast_fp16 = slice_by_index(begin = key_token_25_begin_0, end = key_token_25_end_0, end_mask = key_token_25_end_mask_0, squeeze_mask = key_token_25_squeeze_mask_0, x = key_19_cast_fp16)[name = string("key_token_25_cast_fp16")];
            tensor<int32, [4]> value_token_25_begin_0 = const()[name = string("value_token_25_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> value_token_25_end_0 = const()[name = string("value_token_25_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> value_token_25_end_mask_0 = const()[name = string("value_token_25_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_25_squeeze_mask_0 = const()[name = string("value_token_25_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 4, 128]> value_69_cast_fp16 = transpose(perm = value_69_perm_0, x = value_67_cast_fp16)[name = string("transpose_105")];
            tensor<fp16, [1, 16, 128]> value_token_25_cast_fp16 = slice_by_index(begin = value_token_25_begin_0, end = value_token_25_end_0, end_mask = value_token_25_end_mask_0, squeeze_mask = value_token_25_squeeze_mask_0, x = value_69_cast_fp16)[name = string("value_token_25_cast_fp16")];
            tensor<int32, [1]> var_1653_axes_0 = const()[name = string("op_1653_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1653_cast_fp16 = expand_dims(axes = var_1653_axes_0, x = key_token_25_cast_fp16)[name = string("op_1653_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1654_cast_fp16 = mul(x = state_49_cast_fp16, y = var_1653_cast_fp16)[name = string("op_1654_cast_fp16")];
            tensor<int32, [1]> memory_25_axes_0 = const()[name = string("memory_25_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_25_keep_dims_0 = const()[name = string("memory_25_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_25_cast_fp16 = reduce_sum(axes = memory_25_axes_0, keep_dims = memory_25_keep_dims_0, x = var_1654_cast_fp16)[name = string("memory_25_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_1657_cast_fp16 = sub(x = value_token_25_cast_fp16, y = memory_25_cast_fp16)[name = string("op_1657_cast_fp16")];
            tensor<int32, [3]> var_1660_begin_0 = const()[name = string("op_1660_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_1660_end_0 = const()[name = string("op_1660_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_1660_end_mask_0 = const()[name = string("op_1660_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1660_squeeze_mask_0 = const()[name = string("op_1660_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1660_cast_fp16 = slice_by_index(begin = var_1660_begin_0, end = var_1660_end_0, end_mask = var_1660_end_mask_0, squeeze_mask = var_1660_squeeze_mask_0, x = beta_7_cast_fp16)[name = string("op_1660_cast_fp16")];
            tensor<int32, [1]> var_1661_axes_0 = const()[name = string("op_1661_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1661_cast_fp16 = expand_dims(axes = var_1661_axes_0, x = var_1660_cast_fp16)[name = string("op_1661_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_25_cast_fp16 = mul(x = var_1657_cast_fp16, y = var_1661_cast_fp16)[name = string("delta_25_cast_fp16")];
            tensor<int32, [1]> var_1664_axes_0 = const()[name = string("op_1664_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_1664_cast_fp16 = expand_dims(axes = var_1664_axes_0, x = delta_25_cast_fp16)[name = string("op_1664_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1665_cast_fp16 = mul(x = var_1653_cast_fp16, y = var_1664_cast_fp16)[name = string("op_1665_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_51_cast_fp16 = add(x = state_49_cast_fp16, y = var_1665_cast_fp16)[name = string("state_51_cast_fp16")];
            tensor<int32, [4]> var_1669_begin_0 = const()[name = string("op_1669_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> var_1669_end_0 = const()[name = string("op_1669_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> var_1669_end_mask_0 = const()[name = string("op_1669_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_1669_squeeze_mask_0 = const()[name = string("op_1669_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_1669_cast_fp16 = slice_by_index(begin = var_1669_begin_0, end = var_1669_end_0, end_mask = var_1669_end_mask_0, squeeze_mask = var_1669_squeeze_mask_0, x = _inversed_query_19_cast_fp16)[name = string("op_1669_cast_fp16")];
            tensor<int32, [1]> var_1670_axes_0 = const()[name = string("op_1670_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1670_cast_fp16 = expand_dims(axes = var_1670_axes_0, x = var_1669_cast_fp16)[name = string("op_1670_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1671_cast_fp16 = mul(x = state_51_cast_fp16, y = var_1670_cast_fp16)[name = string("op_1671_cast_fp16")];
            tensor<int32, [1]> var_1673_axes_0 = const()[name = string("op_1673_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_1673_keep_dims_0 = const()[name = string("op_1673_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_1673_cast_fp16 = reduce_sum(axes = var_1673_axes_0, keep_dims = var_1673_keep_dims_0, x = var_1671_cast_fp16)[name = string("op_1673_cast_fp16")];
            tensor<int32, [3]> var_1676_begin_0 = const()[name = string("op_1676_begin_0"), val = tensor<int32, [3]>([0, 0, 1])];
            tensor<int32, [3]> var_1676_end_0 = const()[name = string("op_1676_end_0"), val = tensor<int32, [3]>([1, 16, 2])];
            tensor<bool, [3]> var_1676_end_mask_0 = const()[name = string("op_1676_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1676_squeeze_mask_0 = const()[name = string("op_1676_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1676_cast_fp16 = slice_by_index(begin = var_1676_begin_0, end = var_1676_end_0, end_mask = var_1676_end_mask_0, squeeze_mask = var_1676_squeeze_mask_0, x = decay_7_cast_fp16)[name = string("op_1676_cast_fp16")];
            tensor<fp16, [1, 16]> var_1677_cast_fp16 = exp(x = var_1676_cast_fp16)[name = string("op_1677_cast_fp16")];
            tensor<int32, [1]> var_1678_axes_0 = const()[name = string("op_1678_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1678_cast_fp16 = expand_dims(axes = var_1678_axes_0, x = var_1677_cast_fp16)[name = string("op_1678_cast_fp16")];
            tensor<int32, [1]> var_1679_axes_0 = const()[name = string("op_1679_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_1679_cast_fp16 = expand_dims(axes = var_1679_axes_0, x = var_1678_cast_fp16)[name = string("op_1679_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_53_cast_fp16 = mul(x = state_51_cast_fp16, y = var_1679_cast_fp16)[name = string("state_53_cast_fp16")];
            tensor<int32, [4]> key_token_27_begin_0 = const()[name = string("key_token_27_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> key_token_27_end_0 = const()[name = string("key_token_27_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> key_token_27_end_mask_0 = const()[name = string("key_token_27_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_27_squeeze_mask_0 = const()[name = string("key_token_27_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_27_cast_fp16 = slice_by_index(begin = key_token_27_begin_0, end = key_token_27_end_0, end_mask = key_token_27_end_mask_0, squeeze_mask = key_token_27_squeeze_mask_0, x = key_19_cast_fp16)[name = string("key_token_27_cast_fp16")];
            tensor<int32, [4]> value_token_27_begin_0 = const()[name = string("value_token_27_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> value_token_27_end_0 = const()[name = string("value_token_27_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> value_token_27_end_mask_0 = const()[name = string("value_token_27_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_27_squeeze_mask_0 = const()[name = string("value_token_27_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_27_cast_fp16 = slice_by_index(begin = value_token_27_begin_0, end = value_token_27_end_0, end_mask = value_token_27_end_mask_0, squeeze_mask = value_token_27_squeeze_mask_0, x = value_69_cast_fp16)[name = string("value_token_27_cast_fp16")];
            tensor<int32, [1]> var_1687_axes_0 = const()[name = string("op_1687_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1687_cast_fp16 = expand_dims(axes = var_1687_axes_0, x = key_token_27_cast_fp16)[name = string("op_1687_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1688_cast_fp16 = mul(x = state_53_cast_fp16, y = var_1687_cast_fp16)[name = string("op_1688_cast_fp16")];
            tensor<int32, [1]> memory_27_axes_0 = const()[name = string("memory_27_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_27_keep_dims_0 = const()[name = string("memory_27_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_27_cast_fp16 = reduce_sum(axes = memory_27_axes_0, keep_dims = memory_27_keep_dims_0, x = var_1688_cast_fp16)[name = string("memory_27_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_1691_cast_fp16 = sub(x = value_token_27_cast_fp16, y = memory_27_cast_fp16)[name = string("op_1691_cast_fp16")];
            tensor<int32, [3]> var_1694_begin_0 = const()[name = string("op_1694_begin_0"), val = tensor<int32, [3]>([0, 0, 1])];
            tensor<int32, [3]> var_1694_end_0 = const()[name = string("op_1694_end_0"), val = tensor<int32, [3]>([1, 16, 2])];
            tensor<bool, [3]> var_1694_end_mask_0 = const()[name = string("op_1694_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1694_squeeze_mask_0 = const()[name = string("op_1694_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1694_cast_fp16 = slice_by_index(begin = var_1694_begin_0, end = var_1694_end_0, end_mask = var_1694_end_mask_0, squeeze_mask = var_1694_squeeze_mask_0, x = beta_7_cast_fp16)[name = string("op_1694_cast_fp16")];
            tensor<int32, [1]> var_1695_axes_0 = const()[name = string("op_1695_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1695_cast_fp16 = expand_dims(axes = var_1695_axes_0, x = var_1694_cast_fp16)[name = string("op_1695_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_27_cast_fp16 = mul(x = var_1691_cast_fp16, y = var_1695_cast_fp16)[name = string("delta_27_cast_fp16")];
            tensor<int32, [1]> var_1698_axes_0 = const()[name = string("op_1698_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_1698_cast_fp16 = expand_dims(axes = var_1698_axes_0, x = delta_27_cast_fp16)[name = string("op_1698_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1699_cast_fp16 = mul(x = var_1687_cast_fp16, y = var_1698_cast_fp16)[name = string("op_1699_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_55_cast_fp16 = add(x = state_53_cast_fp16, y = var_1699_cast_fp16)[name = string("state_55_cast_fp16")];
            tensor<int32, [4]> var_1703_begin_0 = const()[name = string("op_1703_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> var_1703_end_0 = const()[name = string("op_1703_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> var_1703_end_mask_0 = const()[name = string("op_1703_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_1703_squeeze_mask_0 = const()[name = string("op_1703_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_1703_cast_fp16 = slice_by_index(begin = var_1703_begin_0, end = var_1703_end_0, end_mask = var_1703_end_mask_0, squeeze_mask = var_1703_squeeze_mask_0, x = _inversed_query_19_cast_fp16)[name = string("op_1703_cast_fp16")];
            tensor<int32, [1]> var_1704_axes_0 = const()[name = string("op_1704_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1704_cast_fp16 = expand_dims(axes = var_1704_axes_0, x = var_1703_cast_fp16)[name = string("op_1704_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1705_cast_fp16 = mul(x = state_55_cast_fp16, y = var_1704_cast_fp16)[name = string("op_1705_cast_fp16")];
            tensor<int32, [1]> var_1707_axes_0 = const()[name = string("op_1707_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_1707_keep_dims_0 = const()[name = string("op_1707_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_1707_cast_fp16 = reduce_sum(axes = var_1707_axes_0, keep_dims = var_1707_keep_dims_0, x = var_1705_cast_fp16)[name = string("op_1707_cast_fp16")];
            tensor<int32, [3]> var_1710_begin_0 = const()[name = string("op_1710_begin_0"), val = tensor<int32, [3]>([0, 0, 2])];
            tensor<int32, [3]> var_1710_end_0 = const()[name = string("op_1710_end_0"), val = tensor<int32, [3]>([1, 16, 3])];
            tensor<bool, [3]> var_1710_end_mask_0 = const()[name = string("op_1710_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1710_squeeze_mask_0 = const()[name = string("op_1710_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1710_cast_fp16 = slice_by_index(begin = var_1710_begin_0, end = var_1710_end_0, end_mask = var_1710_end_mask_0, squeeze_mask = var_1710_squeeze_mask_0, x = decay_7_cast_fp16)[name = string("op_1710_cast_fp16")];
            tensor<fp16, [1, 16]> var_1711_cast_fp16 = exp(x = var_1710_cast_fp16)[name = string("op_1711_cast_fp16")];
            tensor<int32, [1]> var_1712_axes_0 = const()[name = string("op_1712_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1712_cast_fp16 = expand_dims(axes = var_1712_axes_0, x = var_1711_cast_fp16)[name = string("op_1712_cast_fp16")];
            tensor<int32, [1]> var_1713_axes_0 = const()[name = string("op_1713_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_1713_cast_fp16 = expand_dims(axes = var_1713_axes_0, x = var_1712_cast_fp16)[name = string("op_1713_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_57_cast_fp16 = mul(x = state_55_cast_fp16, y = var_1713_cast_fp16)[name = string("state_57_cast_fp16")];
            tensor<int32, [4]> key_token_29_begin_0 = const()[name = string("key_token_29_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> key_token_29_end_0 = const()[name = string("key_token_29_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> key_token_29_end_mask_0 = const()[name = string("key_token_29_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_29_squeeze_mask_0 = const()[name = string("key_token_29_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_29_cast_fp16 = slice_by_index(begin = key_token_29_begin_0, end = key_token_29_end_0, end_mask = key_token_29_end_mask_0, squeeze_mask = key_token_29_squeeze_mask_0, x = key_19_cast_fp16)[name = string("key_token_29_cast_fp16")];
            tensor<int32, [4]> value_token_29_begin_0 = const()[name = string("value_token_29_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> value_token_29_end_0 = const()[name = string("value_token_29_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> value_token_29_end_mask_0 = const()[name = string("value_token_29_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_29_squeeze_mask_0 = const()[name = string("value_token_29_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_29_cast_fp16 = slice_by_index(begin = value_token_29_begin_0, end = value_token_29_end_0, end_mask = value_token_29_end_mask_0, squeeze_mask = value_token_29_squeeze_mask_0, x = value_69_cast_fp16)[name = string("value_token_29_cast_fp16")];
            tensor<int32, [1]> var_1721_axes_0 = const()[name = string("op_1721_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1721_cast_fp16 = expand_dims(axes = var_1721_axes_0, x = key_token_29_cast_fp16)[name = string("op_1721_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1722_cast_fp16 = mul(x = state_57_cast_fp16, y = var_1721_cast_fp16)[name = string("op_1722_cast_fp16")];
            tensor<int32, [1]> memory_29_axes_0 = const()[name = string("memory_29_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_29_keep_dims_0 = const()[name = string("memory_29_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_29_cast_fp16 = reduce_sum(axes = memory_29_axes_0, keep_dims = memory_29_keep_dims_0, x = var_1722_cast_fp16)[name = string("memory_29_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_1725_cast_fp16 = sub(x = value_token_29_cast_fp16, y = memory_29_cast_fp16)[name = string("op_1725_cast_fp16")];
            tensor<int32, [3]> var_1728_begin_0 = const()[name = string("op_1728_begin_0"), val = tensor<int32, [3]>([0, 0, 2])];
            tensor<int32, [3]> var_1728_end_0 = const()[name = string("op_1728_end_0"), val = tensor<int32, [3]>([1, 16, 3])];
            tensor<bool, [3]> var_1728_end_mask_0 = const()[name = string("op_1728_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1728_squeeze_mask_0 = const()[name = string("op_1728_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1728_cast_fp16 = slice_by_index(begin = var_1728_begin_0, end = var_1728_end_0, end_mask = var_1728_end_mask_0, squeeze_mask = var_1728_squeeze_mask_0, x = beta_7_cast_fp16)[name = string("op_1728_cast_fp16")];
            tensor<int32, [1]> var_1729_axes_0 = const()[name = string("op_1729_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1729_cast_fp16 = expand_dims(axes = var_1729_axes_0, x = var_1728_cast_fp16)[name = string("op_1729_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_29_cast_fp16 = mul(x = var_1725_cast_fp16, y = var_1729_cast_fp16)[name = string("delta_29_cast_fp16")];
            tensor<int32, [1]> var_1732_axes_0 = const()[name = string("op_1732_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_1732_cast_fp16 = expand_dims(axes = var_1732_axes_0, x = delta_29_cast_fp16)[name = string("op_1732_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1733_cast_fp16 = mul(x = var_1721_cast_fp16, y = var_1732_cast_fp16)[name = string("op_1733_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_59_cast_fp16 = add(x = state_57_cast_fp16, y = var_1733_cast_fp16)[name = string("state_59_cast_fp16")];
            tensor<int32, [4]> var_1737_begin_0 = const()[name = string("op_1737_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> var_1737_end_0 = const()[name = string("op_1737_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> var_1737_end_mask_0 = const()[name = string("op_1737_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_1737_squeeze_mask_0 = const()[name = string("op_1737_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_1737_cast_fp16 = slice_by_index(begin = var_1737_begin_0, end = var_1737_end_0, end_mask = var_1737_end_mask_0, squeeze_mask = var_1737_squeeze_mask_0, x = _inversed_query_19_cast_fp16)[name = string("op_1737_cast_fp16")];
            tensor<int32, [1]> var_1738_axes_0 = const()[name = string("op_1738_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1738_cast_fp16 = expand_dims(axes = var_1738_axes_0, x = var_1737_cast_fp16)[name = string("op_1738_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1739_cast_fp16 = mul(x = state_59_cast_fp16, y = var_1738_cast_fp16)[name = string("op_1739_cast_fp16")];
            tensor<int32, [1]> var_1741_axes_0 = const()[name = string("op_1741_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_1741_keep_dims_0 = const()[name = string("op_1741_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_1741_cast_fp16 = reduce_sum(axes = var_1741_axes_0, keep_dims = var_1741_keep_dims_0, x = var_1739_cast_fp16)[name = string("op_1741_cast_fp16")];
            tensor<int32, [3]> var_1744_begin_0 = const()[name = string("op_1744_begin_0"), val = tensor<int32, [3]>([0, 0, 3])];
            tensor<int32, [3]> var_1744_end_0 = const()[name = string("op_1744_end_0"), val = tensor<int32, [3]>([1, 16, 4])];
            tensor<bool, [3]> var_1744_end_mask_0 = const()[name = string("op_1744_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1744_squeeze_mask_0 = const()[name = string("op_1744_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1744_cast_fp16 = slice_by_index(begin = var_1744_begin_0, end = var_1744_end_0, end_mask = var_1744_end_mask_0, squeeze_mask = var_1744_squeeze_mask_0, x = decay_7_cast_fp16)[name = string("op_1744_cast_fp16")];
            tensor<fp16, [1, 16]> var_1745_cast_fp16 = exp(x = var_1744_cast_fp16)[name = string("op_1745_cast_fp16")];
            tensor<int32, [1]> var_1746_axes_0 = const()[name = string("op_1746_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1746_cast_fp16 = expand_dims(axes = var_1746_axes_0, x = var_1745_cast_fp16)[name = string("op_1746_cast_fp16")];
            tensor<int32, [1]> var_1747_axes_0 = const()[name = string("op_1747_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_1747_cast_fp16 = expand_dims(axes = var_1747_axes_0, x = var_1746_cast_fp16)[name = string("op_1747_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_61_cast_fp16 = mul(x = state_59_cast_fp16, y = var_1747_cast_fp16)[name = string("state_61_cast_fp16")];
            tensor<int32, [4]> key_token_31_begin_0 = const()[name = string("key_token_31_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> key_token_31_end_0 = const()[name = string("key_token_31_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> key_token_31_end_mask_0 = const()[name = string("key_token_31_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_31_squeeze_mask_0 = const()[name = string("key_token_31_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_31_cast_fp16 = slice_by_index(begin = key_token_31_begin_0, end = key_token_31_end_0, end_mask = key_token_31_end_mask_0, squeeze_mask = key_token_31_squeeze_mask_0, x = key_19_cast_fp16)[name = string("key_token_31_cast_fp16")];
            tensor<int32, [4]> value_token_31_begin_0 = const()[name = string("value_token_31_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> value_token_31_end_0 = const()[name = string("value_token_31_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> value_token_31_end_mask_0 = const()[name = string("value_token_31_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_31_squeeze_mask_0 = const()[name = string("value_token_31_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_31_cast_fp16 = slice_by_index(begin = value_token_31_begin_0, end = value_token_31_end_0, end_mask = value_token_31_end_mask_0, squeeze_mask = value_token_31_squeeze_mask_0, x = value_69_cast_fp16)[name = string("value_token_31_cast_fp16")];
            tensor<int32, [1]> var_1755_axes_0 = const()[name = string("op_1755_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1755_cast_fp16 = expand_dims(axes = var_1755_axes_0, x = key_token_31_cast_fp16)[name = string("op_1755_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1756_cast_fp16 = mul(x = state_61_cast_fp16, y = var_1755_cast_fp16)[name = string("op_1756_cast_fp16")];
            tensor<int32, [1]> memory_31_axes_0 = const()[name = string("memory_31_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_31_keep_dims_0 = const()[name = string("memory_31_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_31_cast_fp16 = reduce_sum(axes = memory_31_axes_0, keep_dims = memory_31_keep_dims_0, x = var_1756_cast_fp16)[name = string("memory_31_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_1759_cast_fp16 = sub(x = value_token_31_cast_fp16, y = memory_31_cast_fp16)[name = string("op_1759_cast_fp16")];
            tensor<int32, [3]> var_1762_begin_0 = const()[name = string("op_1762_begin_0"), val = tensor<int32, [3]>([0, 0, 3])];
            tensor<int32, [3]> var_1762_end_0 = const()[name = string("op_1762_end_0"), val = tensor<int32, [3]>([1, 16, 4])];
            tensor<bool, [3]> var_1762_end_mask_0 = const()[name = string("op_1762_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1762_squeeze_mask_0 = const()[name = string("op_1762_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1762_cast_fp16 = slice_by_index(begin = var_1762_begin_0, end = var_1762_end_0, end_mask = var_1762_end_mask_0, squeeze_mask = var_1762_squeeze_mask_0, x = beta_7_cast_fp16)[name = string("op_1762_cast_fp16")];
            tensor<int32, [1]> var_1763_axes_0 = const()[name = string("op_1763_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1763_cast_fp16 = expand_dims(axes = var_1763_axes_0, x = var_1762_cast_fp16)[name = string("op_1763_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_31_cast_fp16 = mul(x = var_1759_cast_fp16, y = var_1763_cast_fp16)[name = string("delta_31_cast_fp16")];
            tensor<int32, [1]> var_1766_axes_0 = const()[name = string("op_1766_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_1766_cast_fp16 = expand_dims(axes = var_1766_axes_0, x = delta_31_cast_fp16)[name = string("op_1766_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1767_cast_fp16 = mul(x = var_1755_cast_fp16, y = var_1766_cast_fp16)[name = string("op_1767_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> rec4_out = add(x = state_61_cast_fp16, y = var_1767_cast_fp16)[name = string("state_63_cast_fp16")];
            tensor<int32, [4]> var_1771_begin_0 = const()[name = string("op_1771_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> var_1771_end_0 = const()[name = string("op_1771_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> var_1771_end_mask_0 = const()[name = string("op_1771_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_1771_squeeze_mask_0 = const()[name = string("op_1771_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_1771_cast_fp16 = slice_by_index(begin = var_1771_begin_0, end = var_1771_end_0, end_mask = var_1771_end_mask_0, squeeze_mask = var_1771_squeeze_mask_0, x = _inversed_query_19_cast_fp16)[name = string("op_1771_cast_fp16")];
            tensor<int32, [1]> var_1772_axes_0 = const()[name = string("op_1772_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1772_cast_fp16 = expand_dims(axes = var_1772_axes_0, x = var_1771_cast_fp16)[name = string("op_1772_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1773_cast_fp16 = mul(x = rec4_out, y = var_1772_cast_fp16)[name = string("op_1773_cast_fp16")];
            tensor<int32, [1]> var_1775_axes_0 = const()[name = string("op_1775_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_1775_keep_dims_0 = const()[name = string("op_1775_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_1775_cast_fp16 = reduce_sum(axes = var_1775_axes_0, keep_dims = var_1775_keep_dims_0, x = var_1773_cast_fp16)[name = string("op_1775_cast_fp16")];
            int32 attention_31_axis_0 = const()[name = string("attention_31_axis_0"), val = int32(1)];
            tensor<fp16, [1, 4, 16, 128]> attention_31_cast_fp16 = stack(axis = attention_31_axis_0, values = (var_1673_cast_fp16, var_1707_cast_fp16, var_1741_cast_fp16, var_1775_cast_fp16))[name = string("attention_31_cast_fp16")];
            tensor<int32, [2]> var_1778 = const()[name = string("op_1778"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [64, 128]> attention_33_cast_fp16 = reshape(shape = var_1778, x = attention_31_cast_fp16)[name = string("attention_33_cast_fp16")];
            tensor<int32, [2]> var_1780 = const()[name = string("op_1780"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [64, 128]> input_57_cast_fp16 = reshape(shape = var_1780, x = linear_34_cast_fp16)[name = string("input_57_cast_fp16")];
            tensor<fp16, [64, 128]> var_1782_cast_fp16 = mul(x = attention_33_cast_fp16, y = attention_33_cast_fp16)[name = string("op_1782_cast_fp16")];
            tensor<int32, [1]> variance_29_axes_0 = const()[name = string("variance_29_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_29_keep_dims_0 = const()[name = string("variance_29_keep_dims_0"), val = bool(true)];
            tensor<fp16, [64, 1]> variance_29_cast_fp16 = reduce_mean(axes = variance_29_axes_0, keep_dims = variance_29_keep_dims_0, x = var_1782_cast_fp16)[name = string("variance_29_cast_fp16")];
            fp16 var_1785_to_fp16 = const()[name = string("op_1785_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [64, 1]> var_1786_cast_fp16 = add(x = variance_29_cast_fp16, y = var_1785_to_fp16)[name = string("op_1786_cast_fp16")];
            fp32 var_1787_epsilon_0 = const()[name = string("op_1787_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [64, 1]> var_1787_cast_fp16 = rsqrt(epsilon = var_1787_epsilon_0, x = var_1786_cast_fp16)[name = string("op_1787_cast_fp16")];
            tensor<fp16, [64, 128]> attention_35_cast_fp16 = mul(x = attention_33_cast_fp16, y = var_1787_cast_fp16)[name = string("attention_35_cast_fp16")];
            tensor<fp16, [128]> groups_1_0_gated_norm_promoted_to_fp16 = const()[name = string("groups_1_0_gated_norm_promoted_to_fp16"), val = tensor<fp16, [128]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(69449792)))];
            tensor<fp16, [64, 128]> attention_37_cast_fp16 = mul(x = attention_35_cast_fp16, y = groups_1_0_gated_norm_promoted_to_fp16)[name = string("attention_37_cast_fp16")];
            tensor<fp16, [64, 128]> var_1790_cast_fp16 = silu(x = input_57_cast_fp16)[name = string("op_1790_cast_fp16")];
            tensor<fp16, [64, 128]> attention_39_cast_fp16 = mul(x = attention_37_cast_fp16, y = var_1790_cast_fp16)[name = string("attention_39_cast_fp16")];
            tensor<int32, [2]> var_1792 = const()[name = string("op_1792"), val = tensor<int32, [2]>([4, 2048])];
            tensor<fp16, [4, 2048]> input_59_cast_fp16 = reshape(shape = var_1792, x = attention_39_cast_fp16)[name = string("input_59_cast_fp16")];
            tensor<fp16, [1024, 2048]> groups_1_0_out_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 2048]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(69450112))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(71023040))))[name = string("groups_1_0_out_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 1024]> linear_35_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = groups_1_0_out_proj_weight_promoted_to_fp16_palettized, x = input_59_cast_fp16)[name = string("linear_35_cast_fp16")];
            tensor<fp16, [4, 1024]> value_71_cast_fp16 = add(x = value_59_cast_fp16, y = linear_35_cast_fp16)[name = string("value_71_cast_fp16")];
            tensor<fp16, [4, 1024]> var_1797_cast_fp16 = mul(x = value_71_cast_fp16, y = value_71_cast_fp16)[name = string("op_1797_cast_fp16")];
            tensor<int32, [1]> variance_31_axes_0 = const()[name = string("variance_31_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_31_keep_dims_0 = const()[name = string("variance_31_keep_dims_0"), val = bool(true)];
            tensor<fp16, [4, 1]> variance_31_cast_fp16 = reduce_mean(axes = variance_31_axes_0, keep_dims = variance_31_keep_dims_0, x = var_1797_cast_fp16)[name = string("variance_31_cast_fp16")];
            fp16 var_1800_to_fp16 = const()[name = string("op_1800_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [4, 1]> var_1801_cast_fp16 = add(x = variance_31_cast_fp16, y = var_1800_to_fp16)[name = string("op_1801_cast_fp16")];
            fp32 var_1802_epsilon_0 = const()[name = string("op_1802_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [4, 1]> var_1802_cast_fp16 = rsqrt(epsilon = var_1802_epsilon_0, x = var_1801_cast_fp16)[name = string("op_1802_cast_fp16")];
            tensor<fp16, [4, 1024]> var_1803_cast_fp16 = mul(x = value_71_cast_fp16, y = var_1802_cast_fp16)[name = string("op_1803_cast_fp16")];
            tensor<fp16, [1024]> var_1805_to_fp16 = const()[name = string("op_1805_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(71031296)))];
            tensor<fp16, [4, 1024]> input_61_cast_fp16 = mul(x = var_1803_cast_fp16, y = var_1805_to_fp16)[name = string("input_61_cast_fp16")];
            tensor<fp16, [3584, 1024]> groups_1_0_gate_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(71033408))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(73785984))))[name = string("groups_1_0_gate_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 3584]> linear_36_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = groups_1_0_gate_proj_weight_promoted_to_fp16_palettized, x = input_61_cast_fp16)[name = string("linear_36_cast_fp16")];
            tensor<fp16, [4, 3584]> var_1809_cast_fp16 = silu(x = linear_36_cast_fp16)[name = string("op_1809_cast_fp16")];
            tensor<fp16, [3584, 1024]> groups_1_0_up_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(73814720))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(76567296))))[name = string("groups_1_0_up_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 3584]> linear_37_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = groups_1_0_up_proj_weight_promoted_to_fp16_palettized, x = input_61_cast_fp16)[name = string("linear_37_cast_fp16")];
            tensor<fp16, [4, 3584]> input_65_cast_fp16 = mul(x = var_1809_cast_fp16, y = linear_37_cast_fp16)[name = string("input_65_cast_fp16")];
            tensor<fp16, [1024, 3584]> groups_1_0_down_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 3584]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(76596032))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(79348608))))[name = string("groups_1_0_down_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 1024]> linear_38_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = groups_1_0_down_proj_weight_promoted_to_fp16_palettized, x = input_65_cast_fp16)[name = string("linear_38_cast_fp16")];
            tensor<fp16, [4, 1024]> value_73_cast_fp16 = add(x = value_71_cast_fp16, y = linear_38_cast_fp16)[name = string("value_73_cast_fp16")];
            int32 var_1836 = const()[name = string("op_1836"), val = int32(-1)];
            tensor<fp16, [4, 1024]> var_1851_cast_fp16 = mul(x = value_73_cast_fp16, y = value_73_cast_fp16)[name = string("op_1851_cast_fp16")];
            tensor<int32, [1]> variance_33_axes_0 = const()[name = string("variance_33_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_33_keep_dims_0 = const()[name = string("variance_33_keep_dims_0"), val = bool(true)];
            tensor<fp16, [4, 1]> variance_33_cast_fp16 = reduce_mean(axes = variance_33_axes_0, keep_dims = variance_33_keep_dims_0, x = var_1851_cast_fp16)[name = string("variance_33_cast_fp16")];
            fp16 var_1854_to_fp16 = const()[name = string("op_1854_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [4, 1]> var_1855_cast_fp16 = add(x = variance_33_cast_fp16, y = var_1854_to_fp16)[name = string("op_1855_cast_fp16")];
            fp32 var_1856_epsilon_0 = const()[name = string("op_1856_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [4, 1]> var_1856_cast_fp16 = rsqrt(epsilon = var_1856_epsilon_0, x = var_1855_cast_fp16)[name = string("op_1856_cast_fp16")];
            tensor<fp16, [4, 1024]> var_1857_cast_fp16 = mul(x = value_73_cast_fp16, y = var_1856_cast_fp16)[name = string("op_1857_cast_fp16")];
            tensor<fp16, [1024]> var_1859_to_fp16 = const()[name = string("op_1859_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(79356864)))];
            tensor<fp16, [4, 1024]> input_67_cast_fp16 = mul(x = var_1857_cast_fp16, y = var_1859_to_fp16)[name = string("input_67_cast_fp16")];
            tensor<fp16, [6144, 1024]> groups_1_1_in_proj_qkv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(79358976))), lut = tensor<fp16, [384, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(84077632))))[name = string("groups_1_1_in_proj_qkv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 6144]> linear_39_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = groups_1_1_in_proj_qkv_weight_promoted_to_fp16_palettized, x = input_67_cast_fp16)[name = string("linear_39_cast_fp16")];
            tensor<int32, [2]> var_1863_perm_0 = const()[name = string("op_1863_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> mixed_9_axes_0 = const()[name = string("mixed_9_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [6144, 4]> var_1863_cast_fp16 = transpose(perm = var_1863_perm_0, x = linear_39_cast_fp16)[name = string("transpose_104")];
            tensor<fp16, [1, 6144, 4]> mixed_9_cast_fp16 = expand_dims(axes = mixed_9_axes_0, x = var_1863_cast_fp16)[name = string("mixed_9_cast_fp16")];
            bool convolution_input_9_interleave_0 = const()[name = string("convolution_input_9_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 6144, 7]> convolution_input_9_cast_fp16 = concat(axis = var_1836, interleave = convolution_input_9_interleave_0, values = (conv5, mixed_9_cast_fp16))[name = string("convolution_input_9_cast_fp16")];
            string input_69_pad_type_0 = const()[name = string("input_69_pad_type_0"), val = string("valid")];
            int32 input_69_groups_0 = const()[name = string("input_69_groups_0"), val = int32(6144)];
            tensor<int32, [1]> input_69_strides_0 = const()[name = string("input_69_strides_0"), val = tensor<int32, [1]>([1])];
            tensor<int32, [2]> input_69_pad_0 = const()[name = string("input_69_pad_0"), val = tensor<int32, [2]>([0, 0])];
            tensor<int32, [1]> input_69_dilations_0 = const()[name = string("input_69_dilations_0"), val = tensor<int32, [1]>([1])];
            tensor<fp16, [6144, 1, 4]> groups_1_1_conv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1, 4]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(84126848))), lut = tensor<fp16, [384, 1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(84145344))))[name = string("groups_1_1_conv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 6144, 4]> input_69_cast_fp16 = conv(dilations = input_69_dilations_0, groups = input_69_groups_0, pad = input_69_pad_0, pad_type = input_69_pad_type_0, strides = input_69_strides_0, weight = groups_1_1_conv_weight_promoted_to_fp16_palettized, x = convolution_input_9_cast_fp16)[name = string("input_69_cast_fp16")];
            tensor<fp16, [1, 6144, 4]> var_1872_cast_fp16 = silu(x = input_69_cast_fp16)[name = string("op_1872_cast_fp16")];
            tensor<int32, [3]> var_1873_perm_0 = const()[name = string("op_1873_perm_0"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [3]> var_1876_begin_0 = const()[name = string("op_1876_begin_0"), val = tensor<int32, [3]>([0, 0, 4])];
            tensor<int32, [3]> var_1876_end_0 = const()[name = string("op_1876_end_0"), val = tensor<int32, [3]>([1, 6144, 7])];
            tensor<bool, [3]> var_1876_end_mask_0 = const()[name = string("op_1876_end_mask_0"), val = tensor<bool, [3]>([true, true, true])];
            tensor<fp16, [1, 6144, 3]> conv5_out = slice_by_index(begin = var_1876_begin_0, end = var_1876_end_0, end_mask = var_1876_end_mask_0, x = convolution_input_9_cast_fp16)[name = string("op_1876_cast_fp16")];
            tensor<int32, [3]> var_1877 = const()[name = string("op_1877"), val = tensor<int32, [3]>([2048, 2048, 2048])];
            int32 var_1878_axis_0 = const()[name = string("op_1878_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 4, 6144]> var_1873_cast_fp16 = transpose(perm = var_1873_perm_0, x = var_1872_cast_fp16)[name = string("transpose_103")];
            tensor<fp16, [1, 4, 2048]> var_1878_cast_fp16_0, tensor<fp16, [1, 4, 2048]> var_1878_cast_fp16_1, tensor<fp16, [1, 4, 2048]> var_1878_cast_fp16_2 = split(axis = var_1878_axis_0, split_sizes = var_1877, x = var_1873_cast_fp16)[name = string("op_1878_cast_fp16")];
            tensor<int32, [4]> var_1882 = const()[name = string("op_1882"), val = tensor<int32, [4]>([1, 4, 16, 128])];
            tensor<fp16, [1, 4, 16, 128]> value_77_cast_fp16 = reshape(shape = var_1882, x = var_1878_cast_fp16_0)[name = string("value_77_cast_fp16")];
            tensor<int32, [4]> var_1884 = const()[name = string("op_1884"), val = tensor<int32, [4]>([1, 4, 16, 128])];
            tensor<fp16, [1, 4, 16, 128]> value_79_cast_fp16 = reshape(shape = var_1884, x = var_1878_cast_fp16_1)[name = string("value_79_cast_fp16")];
            tensor<int32, [4]> var_1886 = const()[name = string("op_1886"), val = tensor<int32, [4]>([1, 4, 16, 128])];
            tensor<fp16, [1, 4, 16, 128]> value_81_cast_fp16 = reshape(shape = var_1886, x = var_1878_cast_fp16_2)[name = string("value_81_cast_fp16")];
            tensor<fp16, [1, 4, 16, 128]> var_1888_cast_fp16 = mul(x = value_77_cast_fp16, y = value_77_cast_fp16)[name = string("op_1888_cast_fp16")];
            tensor<int32, [1]> var_1890_axes_0 = const()[name = string("op_1890_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_1890_keep_dims_0 = const()[name = string("op_1890_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 4, 16, 1]> var_1890_cast_fp16 = reduce_sum(axes = var_1890_axes_0, keep_dims = var_1890_keep_dims_0, x = var_1888_cast_fp16)[name = string("op_1890_cast_fp16")];
            fp16 var_1891_to_fp16 = const()[name = string("op_1891_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 4, 16, 1]> var_1892_cast_fp16 = add(x = var_1890_cast_fp16, y = var_1891_to_fp16)[name = string("op_1892_cast_fp16")];
            fp32 var_1893_epsilon_0 = const()[name = string("op_1893_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 4, 16, 1]> var_1893_cast_fp16 = rsqrt(epsilon = var_1893_epsilon_0, x = var_1892_cast_fp16)[name = string("op_1893_cast_fp16")];
            tensor<fp16, [1, 4, 16, 128]> var_1894_cast_fp16 = mul(x = value_77_cast_fp16, y = var_1893_cast_fp16)[name = string("op_1894_cast_fp16")];
            tensor<int32, [4]> var_1895_perm_0 = const()[name = string("op_1895_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            fp16 _inversed_query_23_y_0_to_fp16 = const()[name = string("_inversed_query_23_y_0_to_fp16"), val = fp16(0x1.6ap-4)];
            tensor<fp16, [1, 16, 4, 128]> var_1895_cast_fp16 = transpose(perm = var_1895_perm_0, x = var_1894_cast_fp16)[name = string("transpose_102")];
            tensor<fp16, [1, 16, 4, 128]> _inversed_query_23_cast_fp16 = mul(x = var_1895_cast_fp16, y = _inversed_query_23_y_0_to_fp16)[name = string("_inversed_query_23_cast_fp16")];
            tensor<fp16, [1, 4, 16, 128]> var_1898_cast_fp16 = mul(x = value_79_cast_fp16, y = value_79_cast_fp16)[name = string("op_1898_cast_fp16")];
            tensor<int32, [1]> var_1900_axes_0 = const()[name = string("op_1900_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_1900_keep_dims_0 = const()[name = string("op_1900_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 4, 16, 1]> var_1900_cast_fp16 = reduce_sum(axes = var_1900_axes_0, keep_dims = var_1900_keep_dims_0, x = var_1898_cast_fp16)[name = string("op_1900_cast_fp16")];
            fp16 var_1901_to_fp16 = const()[name = string("op_1901_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 4, 16, 1]> var_1902_cast_fp16 = add(x = var_1900_cast_fp16, y = var_1901_to_fp16)[name = string("op_1902_cast_fp16")];
            fp32 var_1903_epsilon_0 = const()[name = string("op_1903_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 4, 16, 1]> var_1903_cast_fp16 = rsqrt(epsilon = var_1903_epsilon_0, x = var_1902_cast_fp16)[name = string("op_1903_cast_fp16")];
            tensor<fp16, [1, 4, 16, 128]> var_1904_cast_fp16 = mul(x = value_79_cast_fp16, y = var_1903_cast_fp16)[name = string("op_1904_cast_fp16")];
            tensor<int32, [4]> key_23_perm_0 = const()[name = string("key_23_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<int32, [4]> value_83_perm_0 = const()[name = string("value_83_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<fp16, [16, 1024]> groups_1_1_in_proj_b_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(84194560))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(84206912))))[name = string("groups_1_1_in_proj_b_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16]> linear_40_bias_0_to_fp16 = const()[name = string("linear_40_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(4862656)))];
            tensor<fp16, [4, 16]> linear_40_cast_fp16 = linear(bias = linear_40_bias_0_to_fp16, weight = groups_1_1_in_proj_b_weight_promoted_to_fp16_palettized, x = input_67_cast_fp16)[name = string("linear_40_cast_fp16")];
            tensor<fp16, [4, 16]> var_1909_cast_fp16 = sigmoid(x = linear_40_cast_fp16)[name = string("op_1909_cast_fp16")];
            tensor<int32, [2]> var_1910_perm_0 = const()[name = string("op_1910_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> beta_9_axes_0 = const()[name = string("beta_9_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 4]> var_1910_cast_fp16 = transpose(perm = var_1910_perm_0, x = var_1909_cast_fp16)[name = string("transpose_101")];
            tensor<fp16, [1, 16, 4]> beta_9_cast_fp16 = expand_dims(axes = beta_9_axes_0, x = var_1910_cast_fp16)[name = string("beta_9_cast_fp16")];
            tensor<fp16, [16, 1024]> op_1916_weight_0_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(84207104))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(84219456))))[name = string("op_1916_weight_0_to_fp16_palettized")];
            tensor<fp16, [16]> var_1916_bias_0_to_fp16 = const()[name = string("op_1916_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(84219648)))];
            tensor<fp16, [4, 16]> var_1916_cast_fp16 = linear(bias = var_1916_bias_0_to_fp16, weight = op_1916_weight_0_to_fp16_palettized, x = input_67_cast_fp16)[name = string("op_1916_cast_fp16")];
            tensor<fp16, [4, 16]> var_1917_cast_fp16 = softplus(x = var_1916_cast_fp16)[name = string("op_1917_cast_fp16")];
            tensor<fp16, [16]> var_1913_promoted_to_fp16 = const()[name = string("op_1913_promoted_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(84219776)))];
            tensor<fp16, [4, 16]> var_1918_cast_fp16 = mul(x = var_1913_promoted_to_fp16, y = var_1917_cast_fp16)[name = string("op_1918_cast_fp16")];
            tensor<int32, [2]> var_1919_perm_0 = const()[name = string("op_1919_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> decay_9_axes_0 = const()[name = string("decay_9_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 4]> var_1919_cast_fp16 = transpose(perm = var_1919_perm_0, x = var_1918_cast_fp16)[name = string("transpose_100")];
            tensor<fp16, [1, 16, 4]> decay_9_cast_fp16 = expand_dims(axes = decay_9_axes_0, x = var_1919_cast_fp16)[name = string("decay_9_cast_fp16")];
            tensor<fp16, [2048, 1024]> groups_1_1_in_proj_z_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [2048, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(84219904))), lut = tensor<fp16, [128, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(85792832))))[name = string("groups_1_1_in_proj_z_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 2048]> linear_42_cast_fp16 = linear(bias = linear_3_bias_0_to_fp16, weight = groups_1_1_in_proj_z_weight_promoted_to_fp16_palettized, x = input_67_cast_fp16)[name = string("linear_42_cast_fp16")];
            tensor<int32, [3]> var_1927_begin_0 = const()[name = string("op_1927_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_1927_end_0 = const()[name = string("op_1927_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_1927_end_mask_0 = const()[name = string("op_1927_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1927_squeeze_mask_0 = const()[name = string("op_1927_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1927_cast_fp16 = slice_by_index(begin = var_1927_begin_0, end = var_1927_end_0, end_mask = var_1927_end_mask_0, squeeze_mask = var_1927_squeeze_mask_0, x = decay_9_cast_fp16)[name = string("op_1927_cast_fp16")];
            tensor<fp16, [1, 16]> var_1928_cast_fp16 = exp(x = var_1927_cast_fp16)[name = string("op_1928_cast_fp16")];
            tensor<int32, [1]> var_1929_axes_0 = const()[name = string("op_1929_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1929_cast_fp16 = expand_dims(axes = var_1929_axes_0, x = var_1928_cast_fp16)[name = string("op_1929_cast_fp16")];
            tensor<int32, [1]> var_1930_axes_0 = const()[name = string("op_1930_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_1930_cast_fp16 = expand_dims(axes = var_1930_axes_0, x = var_1929_cast_fp16)[name = string("op_1930_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_65_cast_fp16 = mul(x = rec5, y = var_1930_cast_fp16)[name = string("state_65_cast_fp16")];
            tensor<int32, [4]> key_token_33_begin_0 = const()[name = string("key_token_33_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> key_token_33_end_0 = const()[name = string("key_token_33_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> key_token_33_end_mask_0 = const()[name = string("key_token_33_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_33_squeeze_mask_0 = const()[name = string("key_token_33_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 4, 128]> key_23_cast_fp16 = transpose(perm = key_23_perm_0, x = var_1904_cast_fp16)[name = string("transpose_99")];
            tensor<fp16, [1, 16, 128]> key_token_33_cast_fp16 = slice_by_index(begin = key_token_33_begin_0, end = key_token_33_end_0, end_mask = key_token_33_end_mask_0, squeeze_mask = key_token_33_squeeze_mask_0, x = key_23_cast_fp16)[name = string("key_token_33_cast_fp16")];
            tensor<int32, [4]> value_token_33_begin_0 = const()[name = string("value_token_33_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> value_token_33_end_0 = const()[name = string("value_token_33_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> value_token_33_end_mask_0 = const()[name = string("value_token_33_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_33_squeeze_mask_0 = const()[name = string("value_token_33_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 4, 128]> value_83_cast_fp16 = transpose(perm = value_83_perm_0, x = value_81_cast_fp16)[name = string("transpose_98")];
            tensor<fp16, [1, 16, 128]> value_token_33_cast_fp16 = slice_by_index(begin = value_token_33_begin_0, end = value_token_33_end_0, end_mask = value_token_33_end_mask_0, squeeze_mask = value_token_33_squeeze_mask_0, x = value_83_cast_fp16)[name = string("value_token_33_cast_fp16")];
            tensor<int32, [1]> var_1938_axes_0 = const()[name = string("op_1938_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1938_cast_fp16 = expand_dims(axes = var_1938_axes_0, x = key_token_33_cast_fp16)[name = string("op_1938_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1939_cast_fp16 = mul(x = state_65_cast_fp16, y = var_1938_cast_fp16)[name = string("op_1939_cast_fp16")];
            tensor<int32, [1]> memory_33_axes_0 = const()[name = string("memory_33_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_33_keep_dims_0 = const()[name = string("memory_33_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_33_cast_fp16 = reduce_sum(axes = memory_33_axes_0, keep_dims = memory_33_keep_dims_0, x = var_1939_cast_fp16)[name = string("memory_33_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_1942_cast_fp16 = sub(x = value_token_33_cast_fp16, y = memory_33_cast_fp16)[name = string("op_1942_cast_fp16")];
            tensor<int32, [3]> var_1945_begin_0 = const()[name = string("op_1945_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_1945_end_0 = const()[name = string("op_1945_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_1945_end_mask_0 = const()[name = string("op_1945_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1945_squeeze_mask_0 = const()[name = string("op_1945_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1945_cast_fp16 = slice_by_index(begin = var_1945_begin_0, end = var_1945_end_0, end_mask = var_1945_end_mask_0, squeeze_mask = var_1945_squeeze_mask_0, x = beta_9_cast_fp16)[name = string("op_1945_cast_fp16")];
            tensor<int32, [1]> var_1946_axes_0 = const()[name = string("op_1946_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1946_cast_fp16 = expand_dims(axes = var_1946_axes_0, x = var_1945_cast_fp16)[name = string("op_1946_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_33_cast_fp16 = mul(x = var_1942_cast_fp16, y = var_1946_cast_fp16)[name = string("delta_33_cast_fp16")];
            tensor<int32, [1]> var_1949_axes_0 = const()[name = string("op_1949_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_1949_cast_fp16 = expand_dims(axes = var_1949_axes_0, x = delta_33_cast_fp16)[name = string("op_1949_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1950_cast_fp16 = mul(x = var_1938_cast_fp16, y = var_1949_cast_fp16)[name = string("op_1950_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_67_cast_fp16 = add(x = state_65_cast_fp16, y = var_1950_cast_fp16)[name = string("state_67_cast_fp16")];
            tensor<int32, [4]> var_1954_begin_0 = const()[name = string("op_1954_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> var_1954_end_0 = const()[name = string("op_1954_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> var_1954_end_mask_0 = const()[name = string("op_1954_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_1954_squeeze_mask_0 = const()[name = string("op_1954_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_1954_cast_fp16 = slice_by_index(begin = var_1954_begin_0, end = var_1954_end_0, end_mask = var_1954_end_mask_0, squeeze_mask = var_1954_squeeze_mask_0, x = _inversed_query_23_cast_fp16)[name = string("op_1954_cast_fp16")];
            tensor<int32, [1]> var_1955_axes_0 = const()[name = string("op_1955_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1955_cast_fp16 = expand_dims(axes = var_1955_axes_0, x = var_1954_cast_fp16)[name = string("op_1955_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1956_cast_fp16 = mul(x = state_67_cast_fp16, y = var_1955_cast_fp16)[name = string("op_1956_cast_fp16")];
            tensor<int32, [1]> var_1958_axes_0 = const()[name = string("op_1958_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_1958_keep_dims_0 = const()[name = string("op_1958_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_1958_cast_fp16 = reduce_sum(axes = var_1958_axes_0, keep_dims = var_1958_keep_dims_0, x = var_1956_cast_fp16)[name = string("op_1958_cast_fp16")];
            tensor<int32, [3]> var_1961_begin_0 = const()[name = string("op_1961_begin_0"), val = tensor<int32, [3]>([0, 0, 1])];
            tensor<int32, [3]> var_1961_end_0 = const()[name = string("op_1961_end_0"), val = tensor<int32, [3]>([1, 16, 2])];
            tensor<bool, [3]> var_1961_end_mask_0 = const()[name = string("op_1961_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1961_squeeze_mask_0 = const()[name = string("op_1961_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1961_cast_fp16 = slice_by_index(begin = var_1961_begin_0, end = var_1961_end_0, end_mask = var_1961_end_mask_0, squeeze_mask = var_1961_squeeze_mask_0, x = decay_9_cast_fp16)[name = string("op_1961_cast_fp16")];
            tensor<fp16, [1, 16]> var_1962_cast_fp16 = exp(x = var_1961_cast_fp16)[name = string("op_1962_cast_fp16")];
            tensor<int32, [1]> var_1963_axes_0 = const()[name = string("op_1963_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1963_cast_fp16 = expand_dims(axes = var_1963_axes_0, x = var_1962_cast_fp16)[name = string("op_1963_cast_fp16")];
            tensor<int32, [1]> var_1964_axes_0 = const()[name = string("op_1964_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_1964_cast_fp16 = expand_dims(axes = var_1964_axes_0, x = var_1963_cast_fp16)[name = string("op_1964_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_69_cast_fp16 = mul(x = state_67_cast_fp16, y = var_1964_cast_fp16)[name = string("state_69_cast_fp16")];
            tensor<int32, [4]> key_token_35_begin_0 = const()[name = string("key_token_35_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> key_token_35_end_0 = const()[name = string("key_token_35_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> key_token_35_end_mask_0 = const()[name = string("key_token_35_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_35_squeeze_mask_0 = const()[name = string("key_token_35_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_35_cast_fp16 = slice_by_index(begin = key_token_35_begin_0, end = key_token_35_end_0, end_mask = key_token_35_end_mask_0, squeeze_mask = key_token_35_squeeze_mask_0, x = key_23_cast_fp16)[name = string("key_token_35_cast_fp16")];
            tensor<int32, [4]> value_token_35_begin_0 = const()[name = string("value_token_35_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> value_token_35_end_0 = const()[name = string("value_token_35_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> value_token_35_end_mask_0 = const()[name = string("value_token_35_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_35_squeeze_mask_0 = const()[name = string("value_token_35_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_35_cast_fp16 = slice_by_index(begin = value_token_35_begin_0, end = value_token_35_end_0, end_mask = value_token_35_end_mask_0, squeeze_mask = value_token_35_squeeze_mask_0, x = value_83_cast_fp16)[name = string("value_token_35_cast_fp16")];
            tensor<int32, [1]> var_1972_axes_0 = const()[name = string("op_1972_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1972_cast_fp16 = expand_dims(axes = var_1972_axes_0, x = key_token_35_cast_fp16)[name = string("op_1972_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1973_cast_fp16 = mul(x = state_69_cast_fp16, y = var_1972_cast_fp16)[name = string("op_1973_cast_fp16")];
            tensor<int32, [1]> memory_35_axes_0 = const()[name = string("memory_35_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_35_keep_dims_0 = const()[name = string("memory_35_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_35_cast_fp16 = reduce_sum(axes = memory_35_axes_0, keep_dims = memory_35_keep_dims_0, x = var_1973_cast_fp16)[name = string("memory_35_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_1976_cast_fp16 = sub(x = value_token_35_cast_fp16, y = memory_35_cast_fp16)[name = string("op_1976_cast_fp16")];
            tensor<int32, [3]> var_1979_begin_0 = const()[name = string("op_1979_begin_0"), val = tensor<int32, [3]>([0, 0, 1])];
            tensor<int32, [3]> var_1979_end_0 = const()[name = string("op_1979_end_0"), val = tensor<int32, [3]>([1, 16, 2])];
            tensor<bool, [3]> var_1979_end_mask_0 = const()[name = string("op_1979_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1979_squeeze_mask_0 = const()[name = string("op_1979_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1979_cast_fp16 = slice_by_index(begin = var_1979_begin_0, end = var_1979_end_0, end_mask = var_1979_end_mask_0, squeeze_mask = var_1979_squeeze_mask_0, x = beta_9_cast_fp16)[name = string("op_1979_cast_fp16")];
            tensor<int32, [1]> var_1980_axes_0 = const()[name = string("op_1980_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1980_cast_fp16 = expand_dims(axes = var_1980_axes_0, x = var_1979_cast_fp16)[name = string("op_1980_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_35_cast_fp16 = mul(x = var_1976_cast_fp16, y = var_1980_cast_fp16)[name = string("delta_35_cast_fp16")];
            tensor<int32, [1]> var_1983_axes_0 = const()[name = string("op_1983_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_1983_cast_fp16 = expand_dims(axes = var_1983_axes_0, x = delta_35_cast_fp16)[name = string("op_1983_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1984_cast_fp16 = mul(x = var_1972_cast_fp16, y = var_1983_cast_fp16)[name = string("op_1984_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_71_cast_fp16 = add(x = state_69_cast_fp16, y = var_1984_cast_fp16)[name = string("state_71_cast_fp16")];
            tensor<int32, [4]> var_1988_begin_0 = const()[name = string("op_1988_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> var_1988_end_0 = const()[name = string("op_1988_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> var_1988_end_mask_0 = const()[name = string("op_1988_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_1988_squeeze_mask_0 = const()[name = string("op_1988_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_1988_cast_fp16 = slice_by_index(begin = var_1988_begin_0, end = var_1988_end_0, end_mask = var_1988_end_mask_0, squeeze_mask = var_1988_squeeze_mask_0, x = _inversed_query_23_cast_fp16)[name = string("op_1988_cast_fp16")];
            tensor<int32, [1]> var_1989_axes_0 = const()[name = string("op_1989_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1989_cast_fp16 = expand_dims(axes = var_1989_axes_0, x = var_1988_cast_fp16)[name = string("op_1989_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1990_cast_fp16 = mul(x = state_71_cast_fp16, y = var_1989_cast_fp16)[name = string("op_1990_cast_fp16")];
            tensor<int32, [1]> var_1992_axes_0 = const()[name = string("op_1992_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_1992_keep_dims_0 = const()[name = string("op_1992_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_1992_cast_fp16 = reduce_sum(axes = var_1992_axes_0, keep_dims = var_1992_keep_dims_0, x = var_1990_cast_fp16)[name = string("op_1992_cast_fp16")];
            tensor<int32, [3]> var_1995_begin_0 = const()[name = string("op_1995_begin_0"), val = tensor<int32, [3]>([0, 0, 2])];
            tensor<int32, [3]> var_1995_end_0 = const()[name = string("op_1995_end_0"), val = tensor<int32, [3]>([1, 16, 3])];
            tensor<bool, [3]> var_1995_end_mask_0 = const()[name = string("op_1995_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1995_squeeze_mask_0 = const()[name = string("op_1995_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1995_cast_fp16 = slice_by_index(begin = var_1995_begin_0, end = var_1995_end_0, end_mask = var_1995_end_mask_0, squeeze_mask = var_1995_squeeze_mask_0, x = decay_9_cast_fp16)[name = string("op_1995_cast_fp16")];
            tensor<fp16, [1, 16]> var_1996_cast_fp16 = exp(x = var_1995_cast_fp16)[name = string("op_1996_cast_fp16")];
            tensor<int32, [1]> var_1997_axes_0 = const()[name = string("op_1997_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1997_cast_fp16 = expand_dims(axes = var_1997_axes_0, x = var_1996_cast_fp16)[name = string("op_1997_cast_fp16")];
            tensor<int32, [1]> var_1998_axes_0 = const()[name = string("op_1998_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_1998_cast_fp16 = expand_dims(axes = var_1998_axes_0, x = var_1997_cast_fp16)[name = string("op_1998_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_73_cast_fp16 = mul(x = state_71_cast_fp16, y = var_1998_cast_fp16)[name = string("state_73_cast_fp16")];
            tensor<int32, [4]> key_token_37_begin_0 = const()[name = string("key_token_37_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> key_token_37_end_0 = const()[name = string("key_token_37_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> key_token_37_end_mask_0 = const()[name = string("key_token_37_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_37_squeeze_mask_0 = const()[name = string("key_token_37_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_37_cast_fp16 = slice_by_index(begin = key_token_37_begin_0, end = key_token_37_end_0, end_mask = key_token_37_end_mask_0, squeeze_mask = key_token_37_squeeze_mask_0, x = key_23_cast_fp16)[name = string("key_token_37_cast_fp16")];
            tensor<int32, [4]> value_token_37_begin_0 = const()[name = string("value_token_37_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> value_token_37_end_0 = const()[name = string("value_token_37_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> value_token_37_end_mask_0 = const()[name = string("value_token_37_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_37_squeeze_mask_0 = const()[name = string("value_token_37_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_37_cast_fp16 = slice_by_index(begin = value_token_37_begin_0, end = value_token_37_end_0, end_mask = value_token_37_end_mask_0, squeeze_mask = value_token_37_squeeze_mask_0, x = value_83_cast_fp16)[name = string("value_token_37_cast_fp16")];
            tensor<int32, [1]> var_2006_axes_0 = const()[name = string("op_2006_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_2006_cast_fp16 = expand_dims(axes = var_2006_axes_0, x = key_token_37_cast_fp16)[name = string("op_2006_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2007_cast_fp16 = mul(x = state_73_cast_fp16, y = var_2006_cast_fp16)[name = string("op_2007_cast_fp16")];
            tensor<int32, [1]> memory_37_axes_0 = const()[name = string("memory_37_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_37_keep_dims_0 = const()[name = string("memory_37_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_37_cast_fp16 = reduce_sum(axes = memory_37_axes_0, keep_dims = memory_37_keep_dims_0, x = var_2007_cast_fp16)[name = string("memory_37_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_2010_cast_fp16 = sub(x = value_token_37_cast_fp16, y = memory_37_cast_fp16)[name = string("op_2010_cast_fp16")];
            tensor<int32, [3]> var_2013_begin_0 = const()[name = string("op_2013_begin_0"), val = tensor<int32, [3]>([0, 0, 2])];
            tensor<int32, [3]> var_2013_end_0 = const()[name = string("op_2013_end_0"), val = tensor<int32, [3]>([1, 16, 3])];
            tensor<bool, [3]> var_2013_end_mask_0 = const()[name = string("op_2013_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_2013_squeeze_mask_0 = const()[name = string("op_2013_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_2013_cast_fp16 = slice_by_index(begin = var_2013_begin_0, end = var_2013_end_0, end_mask = var_2013_end_mask_0, squeeze_mask = var_2013_squeeze_mask_0, x = beta_9_cast_fp16)[name = string("op_2013_cast_fp16")];
            tensor<int32, [1]> var_2014_axes_0 = const()[name = string("op_2014_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_2014_cast_fp16 = expand_dims(axes = var_2014_axes_0, x = var_2013_cast_fp16)[name = string("op_2014_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_37_cast_fp16 = mul(x = var_2010_cast_fp16, y = var_2014_cast_fp16)[name = string("delta_37_cast_fp16")];
            tensor<int32, [1]> var_2017_axes_0 = const()[name = string("op_2017_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_2017_cast_fp16 = expand_dims(axes = var_2017_axes_0, x = delta_37_cast_fp16)[name = string("op_2017_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2018_cast_fp16 = mul(x = var_2006_cast_fp16, y = var_2017_cast_fp16)[name = string("op_2018_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_75_cast_fp16 = add(x = state_73_cast_fp16, y = var_2018_cast_fp16)[name = string("state_75_cast_fp16")];
            tensor<int32, [4]> var_2022_begin_0 = const()[name = string("op_2022_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> var_2022_end_0 = const()[name = string("op_2022_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> var_2022_end_mask_0 = const()[name = string("op_2022_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_2022_squeeze_mask_0 = const()[name = string("op_2022_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_2022_cast_fp16 = slice_by_index(begin = var_2022_begin_0, end = var_2022_end_0, end_mask = var_2022_end_mask_0, squeeze_mask = var_2022_squeeze_mask_0, x = _inversed_query_23_cast_fp16)[name = string("op_2022_cast_fp16")];
            tensor<int32, [1]> var_2023_axes_0 = const()[name = string("op_2023_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_2023_cast_fp16 = expand_dims(axes = var_2023_axes_0, x = var_2022_cast_fp16)[name = string("op_2023_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2024_cast_fp16 = mul(x = state_75_cast_fp16, y = var_2023_cast_fp16)[name = string("op_2024_cast_fp16")];
            tensor<int32, [1]> var_2026_axes_0 = const()[name = string("op_2026_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_2026_keep_dims_0 = const()[name = string("op_2026_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_2026_cast_fp16 = reduce_sum(axes = var_2026_axes_0, keep_dims = var_2026_keep_dims_0, x = var_2024_cast_fp16)[name = string("op_2026_cast_fp16")];
            tensor<int32, [3]> var_2029_begin_0 = const()[name = string("op_2029_begin_0"), val = tensor<int32, [3]>([0, 0, 3])];
            tensor<int32, [3]> var_2029_end_0 = const()[name = string("op_2029_end_0"), val = tensor<int32, [3]>([1, 16, 4])];
            tensor<bool, [3]> var_2029_end_mask_0 = const()[name = string("op_2029_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_2029_squeeze_mask_0 = const()[name = string("op_2029_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_2029_cast_fp16 = slice_by_index(begin = var_2029_begin_0, end = var_2029_end_0, end_mask = var_2029_end_mask_0, squeeze_mask = var_2029_squeeze_mask_0, x = decay_9_cast_fp16)[name = string("op_2029_cast_fp16")];
            tensor<fp16, [1, 16]> var_2030_cast_fp16 = exp(x = var_2029_cast_fp16)[name = string("op_2030_cast_fp16")];
            tensor<int32, [1]> var_2031_axes_0 = const()[name = string("op_2031_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_2031_cast_fp16 = expand_dims(axes = var_2031_axes_0, x = var_2030_cast_fp16)[name = string("op_2031_cast_fp16")];
            tensor<int32, [1]> var_2032_axes_0 = const()[name = string("op_2032_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_2032_cast_fp16 = expand_dims(axes = var_2032_axes_0, x = var_2031_cast_fp16)[name = string("op_2032_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_77_cast_fp16 = mul(x = state_75_cast_fp16, y = var_2032_cast_fp16)[name = string("state_77_cast_fp16")];
            tensor<int32, [4]> key_token_39_begin_0 = const()[name = string("key_token_39_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> key_token_39_end_0 = const()[name = string("key_token_39_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> key_token_39_end_mask_0 = const()[name = string("key_token_39_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_39_squeeze_mask_0 = const()[name = string("key_token_39_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_39_cast_fp16 = slice_by_index(begin = key_token_39_begin_0, end = key_token_39_end_0, end_mask = key_token_39_end_mask_0, squeeze_mask = key_token_39_squeeze_mask_0, x = key_23_cast_fp16)[name = string("key_token_39_cast_fp16")];
            tensor<int32, [4]> value_token_39_begin_0 = const()[name = string("value_token_39_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> value_token_39_end_0 = const()[name = string("value_token_39_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> value_token_39_end_mask_0 = const()[name = string("value_token_39_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_39_squeeze_mask_0 = const()[name = string("value_token_39_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_39_cast_fp16 = slice_by_index(begin = value_token_39_begin_0, end = value_token_39_end_0, end_mask = value_token_39_end_mask_0, squeeze_mask = value_token_39_squeeze_mask_0, x = value_83_cast_fp16)[name = string("value_token_39_cast_fp16")];
            tensor<int32, [1]> var_2040_axes_0 = const()[name = string("op_2040_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_2040_cast_fp16 = expand_dims(axes = var_2040_axes_0, x = key_token_39_cast_fp16)[name = string("op_2040_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2041_cast_fp16 = mul(x = state_77_cast_fp16, y = var_2040_cast_fp16)[name = string("op_2041_cast_fp16")];
            tensor<int32, [1]> memory_39_axes_0 = const()[name = string("memory_39_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_39_keep_dims_0 = const()[name = string("memory_39_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_39_cast_fp16 = reduce_sum(axes = memory_39_axes_0, keep_dims = memory_39_keep_dims_0, x = var_2041_cast_fp16)[name = string("memory_39_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_2044_cast_fp16 = sub(x = value_token_39_cast_fp16, y = memory_39_cast_fp16)[name = string("op_2044_cast_fp16")];
            tensor<int32, [3]> var_2047_begin_0 = const()[name = string("op_2047_begin_0"), val = tensor<int32, [3]>([0, 0, 3])];
            tensor<int32, [3]> var_2047_end_0 = const()[name = string("op_2047_end_0"), val = tensor<int32, [3]>([1, 16, 4])];
            tensor<bool, [3]> var_2047_end_mask_0 = const()[name = string("op_2047_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_2047_squeeze_mask_0 = const()[name = string("op_2047_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_2047_cast_fp16 = slice_by_index(begin = var_2047_begin_0, end = var_2047_end_0, end_mask = var_2047_end_mask_0, squeeze_mask = var_2047_squeeze_mask_0, x = beta_9_cast_fp16)[name = string("op_2047_cast_fp16")];
            tensor<int32, [1]> var_2048_axes_0 = const()[name = string("op_2048_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_2048_cast_fp16 = expand_dims(axes = var_2048_axes_0, x = var_2047_cast_fp16)[name = string("op_2048_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_39_cast_fp16 = mul(x = var_2044_cast_fp16, y = var_2048_cast_fp16)[name = string("delta_39_cast_fp16")];
            tensor<int32, [1]> var_2051_axes_0 = const()[name = string("op_2051_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_2051_cast_fp16 = expand_dims(axes = var_2051_axes_0, x = delta_39_cast_fp16)[name = string("op_2051_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2052_cast_fp16 = mul(x = var_2040_cast_fp16, y = var_2051_cast_fp16)[name = string("op_2052_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> rec5_out = add(x = state_77_cast_fp16, y = var_2052_cast_fp16)[name = string("state_79_cast_fp16")];
            tensor<int32, [4]> var_2056_begin_0 = const()[name = string("op_2056_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> var_2056_end_0 = const()[name = string("op_2056_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> var_2056_end_mask_0 = const()[name = string("op_2056_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_2056_squeeze_mask_0 = const()[name = string("op_2056_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_2056_cast_fp16 = slice_by_index(begin = var_2056_begin_0, end = var_2056_end_0, end_mask = var_2056_end_mask_0, squeeze_mask = var_2056_squeeze_mask_0, x = _inversed_query_23_cast_fp16)[name = string("op_2056_cast_fp16")];
            tensor<int32, [1]> var_2057_axes_0 = const()[name = string("op_2057_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_2057_cast_fp16 = expand_dims(axes = var_2057_axes_0, x = var_2056_cast_fp16)[name = string("op_2057_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2058_cast_fp16 = mul(x = rec5_out, y = var_2057_cast_fp16)[name = string("op_2058_cast_fp16")];
            tensor<int32, [1]> var_2060_axes_0 = const()[name = string("op_2060_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_2060_keep_dims_0 = const()[name = string("op_2060_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_2060_cast_fp16 = reduce_sum(axes = var_2060_axes_0, keep_dims = var_2060_keep_dims_0, x = var_2058_cast_fp16)[name = string("op_2060_cast_fp16")];
            int32 attention_41_axis_0 = const()[name = string("attention_41_axis_0"), val = int32(1)];
            tensor<fp16, [1, 4, 16, 128]> attention_41_cast_fp16 = stack(axis = attention_41_axis_0, values = (var_1958_cast_fp16, var_1992_cast_fp16, var_2026_cast_fp16, var_2060_cast_fp16))[name = string("attention_41_cast_fp16")];
            tensor<int32, [2]> var_2063 = const()[name = string("op_2063"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [64, 128]> attention_43_cast_fp16 = reshape(shape = var_2063, x = attention_41_cast_fp16)[name = string("attention_43_cast_fp16")];
            tensor<int32, [2]> var_2065 = const()[name = string("op_2065"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [64, 128]> input_71_cast_fp16 = reshape(shape = var_2065, x = linear_42_cast_fp16)[name = string("input_71_cast_fp16")];
            tensor<fp16, [64, 128]> var_2067_cast_fp16 = mul(x = attention_43_cast_fp16, y = attention_43_cast_fp16)[name = string("op_2067_cast_fp16")];
            tensor<int32, [1]> variance_35_axes_0 = const()[name = string("variance_35_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_35_keep_dims_0 = const()[name = string("variance_35_keep_dims_0"), val = bool(true)];
            tensor<fp16, [64, 1]> variance_35_cast_fp16 = reduce_mean(axes = variance_35_axes_0, keep_dims = variance_35_keep_dims_0, x = var_2067_cast_fp16)[name = string("variance_35_cast_fp16")];
            fp16 var_2070_to_fp16 = const()[name = string("op_2070_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [64, 1]> var_2071_cast_fp16 = add(x = variance_35_cast_fp16, y = var_2070_to_fp16)[name = string("op_2071_cast_fp16")];
            fp32 var_2072_epsilon_0 = const()[name = string("op_2072_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [64, 1]> var_2072_cast_fp16 = rsqrt(epsilon = var_2072_epsilon_0, x = var_2071_cast_fp16)[name = string("op_2072_cast_fp16")];
            tensor<fp16, [64, 128]> attention_45_cast_fp16 = mul(x = attention_43_cast_fp16, y = var_2072_cast_fp16)[name = string("attention_45_cast_fp16")];
            tensor<fp16, [128]> groups_1_1_gated_norm_promoted_to_fp16 = const()[name = string("groups_1_1_gated_norm_promoted_to_fp16"), val = tensor<fp16, [128]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(85809280)))];
            tensor<fp16, [64, 128]> attention_47_cast_fp16 = mul(x = attention_45_cast_fp16, y = groups_1_1_gated_norm_promoted_to_fp16)[name = string("attention_47_cast_fp16")];
            tensor<fp16, [64, 128]> var_2075_cast_fp16 = silu(x = input_71_cast_fp16)[name = string("op_2075_cast_fp16")];
            tensor<fp16, [64, 128]> attention_49_cast_fp16 = mul(x = attention_47_cast_fp16, y = var_2075_cast_fp16)[name = string("attention_49_cast_fp16")];
            tensor<int32, [2]> var_2077 = const()[name = string("op_2077"), val = tensor<int32, [2]>([4, 2048])];
            tensor<fp16, [4, 2048]> input_73_cast_fp16 = reshape(shape = var_2077, x = attention_49_cast_fp16)[name = string("input_73_cast_fp16")];
            tensor<fp16, [1024, 2048]> groups_1_1_out_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 2048]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(85809600))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(87382528))))[name = string("groups_1_1_out_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 1024]> linear_43_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = groups_1_1_out_proj_weight_promoted_to_fp16_palettized, x = input_73_cast_fp16)[name = string("linear_43_cast_fp16")];
            tensor<fp16, [4, 1024]> value_85_cast_fp16 = add(x = value_73_cast_fp16, y = linear_43_cast_fp16)[name = string("value_85_cast_fp16")];
            tensor<fp16, [4, 1024]> var_2082_cast_fp16 = mul(x = value_85_cast_fp16, y = value_85_cast_fp16)[name = string("op_2082_cast_fp16")];
            tensor<int32, [1]> variance_37_axes_0 = const()[name = string("variance_37_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_37_keep_dims_0 = const()[name = string("variance_37_keep_dims_0"), val = bool(true)];
            tensor<fp16, [4, 1]> variance_37_cast_fp16 = reduce_mean(axes = variance_37_axes_0, keep_dims = variance_37_keep_dims_0, x = var_2082_cast_fp16)[name = string("variance_37_cast_fp16")];
            fp16 var_2085_to_fp16 = const()[name = string("op_2085_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [4, 1]> var_2086_cast_fp16 = add(x = variance_37_cast_fp16, y = var_2085_to_fp16)[name = string("op_2086_cast_fp16")];
            fp32 var_2087_epsilon_0 = const()[name = string("op_2087_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [4, 1]> var_2087_cast_fp16 = rsqrt(epsilon = var_2087_epsilon_0, x = var_2086_cast_fp16)[name = string("op_2087_cast_fp16")];
            tensor<fp16, [4, 1024]> var_2088_cast_fp16 = mul(x = value_85_cast_fp16, y = var_2087_cast_fp16)[name = string("op_2088_cast_fp16")];
            tensor<fp16, [1024]> var_2090_to_fp16 = const()[name = string("op_2090_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(87390784)))];
            tensor<fp16, [4, 1024]> input_75_cast_fp16 = mul(x = var_2088_cast_fp16, y = var_2090_to_fp16)[name = string("input_75_cast_fp16")];
            tensor<fp16, [3584, 1024]> groups_1_1_gate_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(87392896))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(90145472))))[name = string("groups_1_1_gate_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 3584]> linear_44_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = groups_1_1_gate_proj_weight_promoted_to_fp16_palettized, x = input_75_cast_fp16)[name = string("linear_44_cast_fp16")];
            tensor<fp16, [4, 3584]> var_2094_cast_fp16 = silu(x = linear_44_cast_fp16)[name = string("op_2094_cast_fp16")];
            tensor<fp16, [3584, 1024]> groups_1_1_up_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(90174208))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(92926784))))[name = string("groups_1_1_up_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 3584]> linear_45_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = groups_1_1_up_proj_weight_promoted_to_fp16_palettized, x = input_75_cast_fp16)[name = string("linear_45_cast_fp16")];
            tensor<fp16, [4, 3584]> input_79_cast_fp16 = mul(x = var_2094_cast_fp16, y = linear_45_cast_fp16)[name = string("input_79_cast_fp16")];
            tensor<fp16, [1024, 3584]> groups_1_1_down_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 3584]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(92955520))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(95708096))))[name = string("groups_1_1_down_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 1024]> linear_46_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = groups_1_1_down_proj_weight_promoted_to_fp16_palettized, x = input_79_cast_fp16)[name = string("linear_46_cast_fp16")];
            tensor<fp16, [4, 1024]> value_87_cast_fp16 = add(x = value_85_cast_fp16, y = linear_46_cast_fp16)[name = string("value_87_cast_fp16")];
            int32 var_2121 = const()[name = string("op_2121"), val = int32(-1)];
            tensor<fp16, [4, 1024]> var_2136_cast_fp16 = mul(x = value_87_cast_fp16, y = value_87_cast_fp16)[name = string("op_2136_cast_fp16")];
            tensor<int32, [1]> variance_39_axes_0 = const()[name = string("variance_39_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_39_keep_dims_0 = const()[name = string("variance_39_keep_dims_0"), val = bool(true)];
            tensor<fp16, [4, 1]> variance_39_cast_fp16 = reduce_mean(axes = variance_39_axes_0, keep_dims = variance_39_keep_dims_0, x = var_2136_cast_fp16)[name = string("variance_39_cast_fp16")];
            fp16 var_2139_to_fp16 = const()[name = string("op_2139_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [4, 1]> var_2140_cast_fp16 = add(x = variance_39_cast_fp16, y = var_2139_to_fp16)[name = string("op_2140_cast_fp16")];
            fp32 var_2141_epsilon_0 = const()[name = string("op_2141_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [4, 1]> var_2141_cast_fp16 = rsqrt(epsilon = var_2141_epsilon_0, x = var_2140_cast_fp16)[name = string("op_2141_cast_fp16")];
            tensor<fp16, [4, 1024]> var_2142_cast_fp16 = mul(x = value_87_cast_fp16, y = var_2141_cast_fp16)[name = string("op_2142_cast_fp16")];
            tensor<fp16, [1024]> var_2144_to_fp16 = const()[name = string("op_2144_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(95716352)))];
            tensor<fp16, [4, 1024]> input_81_cast_fp16 = mul(x = var_2142_cast_fp16, y = var_2144_to_fp16)[name = string("input_81_cast_fp16")];
            tensor<fp16, [6144, 1024]> groups_1_2_in_proj_qkv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(95718464))), lut = tensor<fp16, [384, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(100437120))))[name = string("groups_1_2_in_proj_qkv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 6144]> linear_47_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = groups_1_2_in_proj_qkv_weight_promoted_to_fp16_palettized, x = input_81_cast_fp16)[name = string("linear_47_cast_fp16")];
            tensor<int32, [2]> var_2148_perm_0 = const()[name = string("op_2148_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> mixed_11_axes_0 = const()[name = string("mixed_11_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [6144, 4]> var_2148_cast_fp16 = transpose(perm = var_2148_perm_0, x = linear_47_cast_fp16)[name = string("transpose_97")];
            tensor<fp16, [1, 6144, 4]> mixed_11_cast_fp16 = expand_dims(axes = mixed_11_axes_0, x = var_2148_cast_fp16)[name = string("mixed_11_cast_fp16")];
            bool convolution_input_11_interleave_0 = const()[name = string("convolution_input_11_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 6144, 7]> convolution_input_11_cast_fp16 = concat(axis = var_2121, interleave = convolution_input_11_interleave_0, values = (conv6, mixed_11_cast_fp16))[name = string("convolution_input_11_cast_fp16")];
            string input_83_pad_type_0 = const()[name = string("input_83_pad_type_0"), val = string("valid")];
            int32 input_83_groups_0 = const()[name = string("input_83_groups_0"), val = int32(6144)];
            tensor<int32, [1]> input_83_strides_0 = const()[name = string("input_83_strides_0"), val = tensor<int32, [1]>([1])];
            tensor<int32, [2]> input_83_pad_0 = const()[name = string("input_83_pad_0"), val = tensor<int32, [2]>([0, 0])];
            tensor<int32, [1]> input_83_dilations_0 = const()[name = string("input_83_dilations_0"), val = tensor<int32, [1]>([1])];
            tensor<fp16, [6144, 1, 4]> groups_1_2_conv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1, 4]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(100486336))), lut = tensor<fp16, [384, 1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(100504832))))[name = string("groups_1_2_conv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 6144, 4]> input_83_cast_fp16 = conv(dilations = input_83_dilations_0, groups = input_83_groups_0, pad = input_83_pad_0, pad_type = input_83_pad_type_0, strides = input_83_strides_0, weight = groups_1_2_conv_weight_promoted_to_fp16_palettized, x = convolution_input_11_cast_fp16)[name = string("input_83_cast_fp16")];
            tensor<fp16, [1, 6144, 4]> var_2157_cast_fp16 = silu(x = input_83_cast_fp16)[name = string("op_2157_cast_fp16")];
            tensor<int32, [3]> var_2158_perm_0 = const()[name = string("op_2158_perm_0"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [3]> var_2161_begin_0 = const()[name = string("op_2161_begin_0"), val = tensor<int32, [3]>([0, 0, 4])];
            tensor<int32, [3]> var_2161_end_0 = const()[name = string("op_2161_end_0"), val = tensor<int32, [3]>([1, 6144, 7])];
            tensor<bool, [3]> var_2161_end_mask_0 = const()[name = string("op_2161_end_mask_0"), val = tensor<bool, [3]>([true, true, true])];
            tensor<fp16, [1, 6144, 3]> conv6_out = slice_by_index(begin = var_2161_begin_0, end = var_2161_end_0, end_mask = var_2161_end_mask_0, x = convolution_input_11_cast_fp16)[name = string("op_2161_cast_fp16")];
            tensor<int32, [3]> var_2162 = const()[name = string("op_2162"), val = tensor<int32, [3]>([2048, 2048, 2048])];
            int32 var_2163_axis_0 = const()[name = string("op_2163_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 4, 6144]> var_2158_cast_fp16 = transpose(perm = var_2158_perm_0, x = var_2157_cast_fp16)[name = string("transpose_96")];
            tensor<fp16, [1, 4, 2048]> var_2163_cast_fp16_0, tensor<fp16, [1, 4, 2048]> var_2163_cast_fp16_1, tensor<fp16, [1, 4, 2048]> var_2163_cast_fp16_2 = split(axis = var_2163_axis_0, split_sizes = var_2162, x = var_2158_cast_fp16)[name = string("op_2163_cast_fp16")];
            tensor<int32, [4]> var_2167 = const()[name = string("op_2167"), val = tensor<int32, [4]>([1, 4, 16, 128])];
            tensor<fp16, [1, 4, 16, 128]> value_91_cast_fp16 = reshape(shape = var_2167, x = var_2163_cast_fp16_0)[name = string("value_91_cast_fp16")];
            tensor<int32, [4]> var_2169 = const()[name = string("op_2169"), val = tensor<int32, [4]>([1, 4, 16, 128])];
            tensor<fp16, [1, 4, 16, 128]> value_93_cast_fp16 = reshape(shape = var_2169, x = var_2163_cast_fp16_1)[name = string("value_93_cast_fp16")];
            tensor<int32, [4]> var_2171 = const()[name = string("op_2171"), val = tensor<int32, [4]>([1, 4, 16, 128])];
            tensor<fp16, [1, 4, 16, 128]> value_95_cast_fp16 = reshape(shape = var_2171, x = var_2163_cast_fp16_2)[name = string("value_95_cast_fp16")];
            tensor<fp16, [1, 4, 16, 128]> var_2173_cast_fp16 = mul(x = value_91_cast_fp16, y = value_91_cast_fp16)[name = string("op_2173_cast_fp16")];
            tensor<int32, [1]> var_2175_axes_0 = const()[name = string("op_2175_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_2175_keep_dims_0 = const()[name = string("op_2175_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 4, 16, 1]> var_2175_cast_fp16 = reduce_sum(axes = var_2175_axes_0, keep_dims = var_2175_keep_dims_0, x = var_2173_cast_fp16)[name = string("op_2175_cast_fp16")];
            fp16 var_2176_to_fp16 = const()[name = string("op_2176_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 4, 16, 1]> var_2177_cast_fp16 = add(x = var_2175_cast_fp16, y = var_2176_to_fp16)[name = string("op_2177_cast_fp16")];
            fp32 var_2178_epsilon_0 = const()[name = string("op_2178_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 4, 16, 1]> var_2178_cast_fp16 = rsqrt(epsilon = var_2178_epsilon_0, x = var_2177_cast_fp16)[name = string("op_2178_cast_fp16")];
            tensor<fp16, [1, 4, 16, 128]> var_2179_cast_fp16 = mul(x = value_91_cast_fp16, y = var_2178_cast_fp16)[name = string("op_2179_cast_fp16")];
            tensor<int32, [4]> var_2180_perm_0 = const()[name = string("op_2180_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            fp16 _inversed_query_27_y_0_to_fp16 = const()[name = string("_inversed_query_27_y_0_to_fp16"), val = fp16(0x1.6ap-4)];
            tensor<fp16, [1, 16, 4, 128]> var_2180_cast_fp16 = transpose(perm = var_2180_perm_0, x = var_2179_cast_fp16)[name = string("transpose_95")];
            tensor<fp16, [1, 16, 4, 128]> _inversed_query_27_cast_fp16 = mul(x = var_2180_cast_fp16, y = _inversed_query_27_y_0_to_fp16)[name = string("_inversed_query_27_cast_fp16")];
            tensor<fp16, [1, 4, 16, 128]> var_2183_cast_fp16 = mul(x = value_93_cast_fp16, y = value_93_cast_fp16)[name = string("op_2183_cast_fp16")];
            tensor<int32, [1]> var_2185_axes_0 = const()[name = string("op_2185_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_2185_keep_dims_0 = const()[name = string("op_2185_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 4, 16, 1]> var_2185_cast_fp16 = reduce_sum(axes = var_2185_axes_0, keep_dims = var_2185_keep_dims_0, x = var_2183_cast_fp16)[name = string("op_2185_cast_fp16")];
            fp16 var_2186_to_fp16 = const()[name = string("op_2186_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 4, 16, 1]> var_2187_cast_fp16 = add(x = var_2185_cast_fp16, y = var_2186_to_fp16)[name = string("op_2187_cast_fp16")];
            fp32 var_2188_epsilon_0 = const()[name = string("op_2188_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 4, 16, 1]> var_2188_cast_fp16 = rsqrt(epsilon = var_2188_epsilon_0, x = var_2187_cast_fp16)[name = string("op_2188_cast_fp16")];
            tensor<fp16, [1, 4, 16, 128]> var_2189_cast_fp16 = mul(x = value_93_cast_fp16, y = var_2188_cast_fp16)[name = string("op_2189_cast_fp16")];
            tensor<int32, [4]> key_27_perm_0 = const()[name = string("key_27_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<int32, [4]> value_97_perm_0 = const()[name = string("value_97_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<fp16, [16, 1024]> groups_1_2_in_proj_b_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(100554048))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(100566400))))[name = string("groups_1_2_in_proj_b_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16]> linear_48_bias_0_to_fp16 = const()[name = string("linear_48_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(4862656)))];
            tensor<fp16, [4, 16]> linear_48_cast_fp16 = linear(bias = linear_48_bias_0_to_fp16, weight = groups_1_2_in_proj_b_weight_promoted_to_fp16_palettized, x = input_81_cast_fp16)[name = string("linear_48_cast_fp16")];
            tensor<fp16, [4, 16]> var_2194_cast_fp16 = sigmoid(x = linear_48_cast_fp16)[name = string("op_2194_cast_fp16")];
            tensor<int32, [2]> var_2195_perm_0 = const()[name = string("op_2195_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> beta_11_axes_0 = const()[name = string("beta_11_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 4]> var_2195_cast_fp16 = transpose(perm = var_2195_perm_0, x = var_2194_cast_fp16)[name = string("transpose_94")];
            tensor<fp16, [1, 16, 4]> beta_11_cast_fp16 = expand_dims(axes = beta_11_axes_0, x = var_2195_cast_fp16)[name = string("beta_11_cast_fp16")];
            tensor<fp16, [16, 1024]> op_2201_weight_0_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(100566592))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(100578944))))[name = string("op_2201_weight_0_to_fp16_palettized")];
            tensor<fp16, [16]> var_2201_bias_0_to_fp16 = const()[name = string("op_2201_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(100579136)))];
            tensor<fp16, [4, 16]> var_2201_cast_fp16 = linear(bias = var_2201_bias_0_to_fp16, weight = op_2201_weight_0_to_fp16_palettized, x = input_81_cast_fp16)[name = string("op_2201_cast_fp16")];
            tensor<fp16, [4, 16]> var_2202_cast_fp16 = softplus(x = var_2201_cast_fp16)[name = string("op_2202_cast_fp16")];
            tensor<fp16, [16]> var_2198_promoted_to_fp16 = const()[name = string("op_2198_promoted_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(100579264)))];
            tensor<fp16, [4, 16]> var_2203_cast_fp16 = mul(x = var_2198_promoted_to_fp16, y = var_2202_cast_fp16)[name = string("op_2203_cast_fp16")];
            tensor<int32, [2]> var_2204_perm_0 = const()[name = string("op_2204_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> decay_11_axes_0 = const()[name = string("decay_11_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 4]> var_2204_cast_fp16 = transpose(perm = var_2204_perm_0, x = var_2203_cast_fp16)[name = string("transpose_93")];
            tensor<fp16, [1, 16, 4]> decay_11_cast_fp16 = expand_dims(axes = decay_11_axes_0, x = var_2204_cast_fp16)[name = string("decay_11_cast_fp16")];
            tensor<fp16, [2048, 1024]> groups_1_2_in_proj_z_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [2048, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(100579392))), lut = tensor<fp16, [128, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(102152320))))[name = string("groups_1_2_in_proj_z_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 2048]> linear_50_cast_fp16 = linear(bias = linear_3_bias_0_to_fp16, weight = groups_1_2_in_proj_z_weight_promoted_to_fp16_palettized, x = input_81_cast_fp16)[name = string("linear_50_cast_fp16")];
            tensor<int32, [3]> var_2212_begin_0 = const()[name = string("op_2212_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_2212_end_0 = const()[name = string("op_2212_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_2212_end_mask_0 = const()[name = string("op_2212_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_2212_squeeze_mask_0 = const()[name = string("op_2212_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_2212_cast_fp16 = slice_by_index(begin = var_2212_begin_0, end = var_2212_end_0, end_mask = var_2212_end_mask_0, squeeze_mask = var_2212_squeeze_mask_0, x = decay_11_cast_fp16)[name = string("op_2212_cast_fp16")];
            tensor<fp16, [1, 16]> var_2213_cast_fp16 = exp(x = var_2212_cast_fp16)[name = string("op_2213_cast_fp16")];
            tensor<int32, [1]> var_2214_axes_0 = const()[name = string("op_2214_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_2214_cast_fp16 = expand_dims(axes = var_2214_axes_0, x = var_2213_cast_fp16)[name = string("op_2214_cast_fp16")];
            tensor<int32, [1]> var_2215_axes_0 = const()[name = string("op_2215_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_2215_cast_fp16 = expand_dims(axes = var_2215_axes_0, x = var_2214_cast_fp16)[name = string("op_2215_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_81_cast_fp16 = mul(x = rec6, y = var_2215_cast_fp16)[name = string("state_81_cast_fp16")];
            tensor<int32, [4]> key_token_41_begin_0 = const()[name = string("key_token_41_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> key_token_41_end_0 = const()[name = string("key_token_41_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> key_token_41_end_mask_0 = const()[name = string("key_token_41_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_41_squeeze_mask_0 = const()[name = string("key_token_41_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 4, 128]> key_27_cast_fp16 = transpose(perm = key_27_perm_0, x = var_2189_cast_fp16)[name = string("transpose_92")];
            tensor<fp16, [1, 16, 128]> key_token_41_cast_fp16 = slice_by_index(begin = key_token_41_begin_0, end = key_token_41_end_0, end_mask = key_token_41_end_mask_0, squeeze_mask = key_token_41_squeeze_mask_0, x = key_27_cast_fp16)[name = string("key_token_41_cast_fp16")];
            tensor<int32, [4]> value_token_41_begin_0 = const()[name = string("value_token_41_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> value_token_41_end_0 = const()[name = string("value_token_41_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> value_token_41_end_mask_0 = const()[name = string("value_token_41_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_41_squeeze_mask_0 = const()[name = string("value_token_41_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 4, 128]> value_97_cast_fp16 = transpose(perm = value_97_perm_0, x = value_95_cast_fp16)[name = string("transpose_91")];
            tensor<fp16, [1, 16, 128]> value_token_41_cast_fp16 = slice_by_index(begin = value_token_41_begin_0, end = value_token_41_end_0, end_mask = value_token_41_end_mask_0, squeeze_mask = value_token_41_squeeze_mask_0, x = value_97_cast_fp16)[name = string("value_token_41_cast_fp16")];
            tensor<int32, [1]> var_2223_axes_0 = const()[name = string("op_2223_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_2223_cast_fp16 = expand_dims(axes = var_2223_axes_0, x = key_token_41_cast_fp16)[name = string("op_2223_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2224_cast_fp16 = mul(x = state_81_cast_fp16, y = var_2223_cast_fp16)[name = string("op_2224_cast_fp16")];
            tensor<int32, [1]> memory_41_axes_0 = const()[name = string("memory_41_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_41_keep_dims_0 = const()[name = string("memory_41_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_41_cast_fp16 = reduce_sum(axes = memory_41_axes_0, keep_dims = memory_41_keep_dims_0, x = var_2224_cast_fp16)[name = string("memory_41_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_2227_cast_fp16 = sub(x = value_token_41_cast_fp16, y = memory_41_cast_fp16)[name = string("op_2227_cast_fp16")];
            tensor<int32, [3]> var_2230_begin_0 = const()[name = string("op_2230_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_2230_end_0 = const()[name = string("op_2230_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_2230_end_mask_0 = const()[name = string("op_2230_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_2230_squeeze_mask_0 = const()[name = string("op_2230_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_2230_cast_fp16 = slice_by_index(begin = var_2230_begin_0, end = var_2230_end_0, end_mask = var_2230_end_mask_0, squeeze_mask = var_2230_squeeze_mask_0, x = beta_11_cast_fp16)[name = string("op_2230_cast_fp16")];
            tensor<int32, [1]> var_2231_axes_0 = const()[name = string("op_2231_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_2231_cast_fp16 = expand_dims(axes = var_2231_axes_0, x = var_2230_cast_fp16)[name = string("op_2231_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_41_cast_fp16 = mul(x = var_2227_cast_fp16, y = var_2231_cast_fp16)[name = string("delta_41_cast_fp16")];
            tensor<int32, [1]> var_2234_axes_0 = const()[name = string("op_2234_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_2234_cast_fp16 = expand_dims(axes = var_2234_axes_0, x = delta_41_cast_fp16)[name = string("op_2234_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2235_cast_fp16 = mul(x = var_2223_cast_fp16, y = var_2234_cast_fp16)[name = string("op_2235_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_83_cast_fp16 = add(x = state_81_cast_fp16, y = var_2235_cast_fp16)[name = string("state_83_cast_fp16")];
            tensor<int32, [4]> var_2239_begin_0 = const()[name = string("op_2239_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> var_2239_end_0 = const()[name = string("op_2239_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> var_2239_end_mask_0 = const()[name = string("op_2239_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_2239_squeeze_mask_0 = const()[name = string("op_2239_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_2239_cast_fp16 = slice_by_index(begin = var_2239_begin_0, end = var_2239_end_0, end_mask = var_2239_end_mask_0, squeeze_mask = var_2239_squeeze_mask_0, x = _inversed_query_27_cast_fp16)[name = string("op_2239_cast_fp16")];
            tensor<int32, [1]> var_2240_axes_0 = const()[name = string("op_2240_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_2240_cast_fp16 = expand_dims(axes = var_2240_axes_0, x = var_2239_cast_fp16)[name = string("op_2240_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2241_cast_fp16 = mul(x = state_83_cast_fp16, y = var_2240_cast_fp16)[name = string("op_2241_cast_fp16")];
            tensor<int32, [1]> var_2243_axes_0 = const()[name = string("op_2243_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_2243_keep_dims_0 = const()[name = string("op_2243_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_2243_cast_fp16 = reduce_sum(axes = var_2243_axes_0, keep_dims = var_2243_keep_dims_0, x = var_2241_cast_fp16)[name = string("op_2243_cast_fp16")];
            tensor<int32, [3]> var_2246_begin_0 = const()[name = string("op_2246_begin_0"), val = tensor<int32, [3]>([0, 0, 1])];
            tensor<int32, [3]> var_2246_end_0 = const()[name = string("op_2246_end_0"), val = tensor<int32, [3]>([1, 16, 2])];
            tensor<bool, [3]> var_2246_end_mask_0 = const()[name = string("op_2246_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_2246_squeeze_mask_0 = const()[name = string("op_2246_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_2246_cast_fp16 = slice_by_index(begin = var_2246_begin_0, end = var_2246_end_0, end_mask = var_2246_end_mask_0, squeeze_mask = var_2246_squeeze_mask_0, x = decay_11_cast_fp16)[name = string("op_2246_cast_fp16")];
            tensor<fp16, [1, 16]> var_2247_cast_fp16 = exp(x = var_2246_cast_fp16)[name = string("op_2247_cast_fp16")];
            tensor<int32, [1]> var_2248_axes_0 = const()[name = string("op_2248_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_2248_cast_fp16 = expand_dims(axes = var_2248_axes_0, x = var_2247_cast_fp16)[name = string("op_2248_cast_fp16")];
            tensor<int32, [1]> var_2249_axes_0 = const()[name = string("op_2249_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_2249_cast_fp16 = expand_dims(axes = var_2249_axes_0, x = var_2248_cast_fp16)[name = string("op_2249_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_85_cast_fp16 = mul(x = state_83_cast_fp16, y = var_2249_cast_fp16)[name = string("state_85_cast_fp16")];
            tensor<int32, [4]> key_token_43_begin_0 = const()[name = string("key_token_43_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> key_token_43_end_0 = const()[name = string("key_token_43_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> key_token_43_end_mask_0 = const()[name = string("key_token_43_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_43_squeeze_mask_0 = const()[name = string("key_token_43_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_43_cast_fp16 = slice_by_index(begin = key_token_43_begin_0, end = key_token_43_end_0, end_mask = key_token_43_end_mask_0, squeeze_mask = key_token_43_squeeze_mask_0, x = key_27_cast_fp16)[name = string("key_token_43_cast_fp16")];
            tensor<int32, [4]> value_token_43_begin_0 = const()[name = string("value_token_43_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> value_token_43_end_0 = const()[name = string("value_token_43_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> value_token_43_end_mask_0 = const()[name = string("value_token_43_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_43_squeeze_mask_0 = const()[name = string("value_token_43_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_43_cast_fp16 = slice_by_index(begin = value_token_43_begin_0, end = value_token_43_end_0, end_mask = value_token_43_end_mask_0, squeeze_mask = value_token_43_squeeze_mask_0, x = value_97_cast_fp16)[name = string("value_token_43_cast_fp16")];
            tensor<int32, [1]> var_2257_axes_0 = const()[name = string("op_2257_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_2257_cast_fp16 = expand_dims(axes = var_2257_axes_0, x = key_token_43_cast_fp16)[name = string("op_2257_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2258_cast_fp16 = mul(x = state_85_cast_fp16, y = var_2257_cast_fp16)[name = string("op_2258_cast_fp16")];
            tensor<int32, [1]> memory_43_axes_0 = const()[name = string("memory_43_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_43_keep_dims_0 = const()[name = string("memory_43_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_43_cast_fp16 = reduce_sum(axes = memory_43_axes_0, keep_dims = memory_43_keep_dims_0, x = var_2258_cast_fp16)[name = string("memory_43_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_2261_cast_fp16 = sub(x = value_token_43_cast_fp16, y = memory_43_cast_fp16)[name = string("op_2261_cast_fp16")];
            tensor<int32, [3]> var_2264_begin_0 = const()[name = string("op_2264_begin_0"), val = tensor<int32, [3]>([0, 0, 1])];
            tensor<int32, [3]> var_2264_end_0 = const()[name = string("op_2264_end_0"), val = tensor<int32, [3]>([1, 16, 2])];
            tensor<bool, [3]> var_2264_end_mask_0 = const()[name = string("op_2264_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_2264_squeeze_mask_0 = const()[name = string("op_2264_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_2264_cast_fp16 = slice_by_index(begin = var_2264_begin_0, end = var_2264_end_0, end_mask = var_2264_end_mask_0, squeeze_mask = var_2264_squeeze_mask_0, x = beta_11_cast_fp16)[name = string("op_2264_cast_fp16")];
            tensor<int32, [1]> var_2265_axes_0 = const()[name = string("op_2265_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_2265_cast_fp16 = expand_dims(axes = var_2265_axes_0, x = var_2264_cast_fp16)[name = string("op_2265_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_43_cast_fp16 = mul(x = var_2261_cast_fp16, y = var_2265_cast_fp16)[name = string("delta_43_cast_fp16")];
            tensor<int32, [1]> var_2268_axes_0 = const()[name = string("op_2268_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_2268_cast_fp16 = expand_dims(axes = var_2268_axes_0, x = delta_43_cast_fp16)[name = string("op_2268_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2269_cast_fp16 = mul(x = var_2257_cast_fp16, y = var_2268_cast_fp16)[name = string("op_2269_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_87_cast_fp16 = add(x = state_85_cast_fp16, y = var_2269_cast_fp16)[name = string("state_87_cast_fp16")];
            tensor<int32, [4]> var_2273_begin_0 = const()[name = string("op_2273_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> var_2273_end_0 = const()[name = string("op_2273_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> var_2273_end_mask_0 = const()[name = string("op_2273_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_2273_squeeze_mask_0 = const()[name = string("op_2273_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_2273_cast_fp16 = slice_by_index(begin = var_2273_begin_0, end = var_2273_end_0, end_mask = var_2273_end_mask_0, squeeze_mask = var_2273_squeeze_mask_0, x = _inversed_query_27_cast_fp16)[name = string("op_2273_cast_fp16")];
            tensor<int32, [1]> var_2274_axes_0 = const()[name = string("op_2274_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_2274_cast_fp16 = expand_dims(axes = var_2274_axes_0, x = var_2273_cast_fp16)[name = string("op_2274_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2275_cast_fp16 = mul(x = state_87_cast_fp16, y = var_2274_cast_fp16)[name = string("op_2275_cast_fp16")];
            tensor<int32, [1]> var_2277_axes_0 = const()[name = string("op_2277_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_2277_keep_dims_0 = const()[name = string("op_2277_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_2277_cast_fp16 = reduce_sum(axes = var_2277_axes_0, keep_dims = var_2277_keep_dims_0, x = var_2275_cast_fp16)[name = string("op_2277_cast_fp16")];
            tensor<int32, [3]> var_2280_begin_0 = const()[name = string("op_2280_begin_0"), val = tensor<int32, [3]>([0, 0, 2])];
            tensor<int32, [3]> var_2280_end_0 = const()[name = string("op_2280_end_0"), val = tensor<int32, [3]>([1, 16, 3])];
            tensor<bool, [3]> var_2280_end_mask_0 = const()[name = string("op_2280_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_2280_squeeze_mask_0 = const()[name = string("op_2280_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_2280_cast_fp16 = slice_by_index(begin = var_2280_begin_0, end = var_2280_end_0, end_mask = var_2280_end_mask_0, squeeze_mask = var_2280_squeeze_mask_0, x = decay_11_cast_fp16)[name = string("op_2280_cast_fp16")];
            tensor<fp16, [1, 16]> var_2281_cast_fp16 = exp(x = var_2280_cast_fp16)[name = string("op_2281_cast_fp16")];
            tensor<int32, [1]> var_2282_axes_0 = const()[name = string("op_2282_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_2282_cast_fp16 = expand_dims(axes = var_2282_axes_0, x = var_2281_cast_fp16)[name = string("op_2282_cast_fp16")];
            tensor<int32, [1]> var_2283_axes_0 = const()[name = string("op_2283_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_2283_cast_fp16 = expand_dims(axes = var_2283_axes_0, x = var_2282_cast_fp16)[name = string("op_2283_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_89_cast_fp16 = mul(x = state_87_cast_fp16, y = var_2283_cast_fp16)[name = string("state_89_cast_fp16")];
            tensor<int32, [4]> key_token_45_begin_0 = const()[name = string("key_token_45_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> key_token_45_end_0 = const()[name = string("key_token_45_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> key_token_45_end_mask_0 = const()[name = string("key_token_45_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_45_squeeze_mask_0 = const()[name = string("key_token_45_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_45_cast_fp16 = slice_by_index(begin = key_token_45_begin_0, end = key_token_45_end_0, end_mask = key_token_45_end_mask_0, squeeze_mask = key_token_45_squeeze_mask_0, x = key_27_cast_fp16)[name = string("key_token_45_cast_fp16")];
            tensor<int32, [4]> value_token_45_begin_0 = const()[name = string("value_token_45_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> value_token_45_end_0 = const()[name = string("value_token_45_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> value_token_45_end_mask_0 = const()[name = string("value_token_45_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_45_squeeze_mask_0 = const()[name = string("value_token_45_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_45_cast_fp16 = slice_by_index(begin = value_token_45_begin_0, end = value_token_45_end_0, end_mask = value_token_45_end_mask_0, squeeze_mask = value_token_45_squeeze_mask_0, x = value_97_cast_fp16)[name = string("value_token_45_cast_fp16")];
            tensor<int32, [1]> var_2291_axes_0 = const()[name = string("op_2291_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_2291_cast_fp16 = expand_dims(axes = var_2291_axes_0, x = key_token_45_cast_fp16)[name = string("op_2291_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2292_cast_fp16 = mul(x = state_89_cast_fp16, y = var_2291_cast_fp16)[name = string("op_2292_cast_fp16")];
            tensor<int32, [1]> memory_45_axes_0 = const()[name = string("memory_45_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_45_keep_dims_0 = const()[name = string("memory_45_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_45_cast_fp16 = reduce_sum(axes = memory_45_axes_0, keep_dims = memory_45_keep_dims_0, x = var_2292_cast_fp16)[name = string("memory_45_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_2295_cast_fp16 = sub(x = value_token_45_cast_fp16, y = memory_45_cast_fp16)[name = string("op_2295_cast_fp16")];
            tensor<int32, [3]> var_2298_begin_0 = const()[name = string("op_2298_begin_0"), val = tensor<int32, [3]>([0, 0, 2])];
            tensor<int32, [3]> var_2298_end_0 = const()[name = string("op_2298_end_0"), val = tensor<int32, [3]>([1, 16, 3])];
            tensor<bool, [3]> var_2298_end_mask_0 = const()[name = string("op_2298_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_2298_squeeze_mask_0 = const()[name = string("op_2298_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_2298_cast_fp16 = slice_by_index(begin = var_2298_begin_0, end = var_2298_end_0, end_mask = var_2298_end_mask_0, squeeze_mask = var_2298_squeeze_mask_0, x = beta_11_cast_fp16)[name = string("op_2298_cast_fp16")];
            tensor<int32, [1]> var_2299_axes_0 = const()[name = string("op_2299_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_2299_cast_fp16 = expand_dims(axes = var_2299_axes_0, x = var_2298_cast_fp16)[name = string("op_2299_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_45_cast_fp16 = mul(x = var_2295_cast_fp16, y = var_2299_cast_fp16)[name = string("delta_45_cast_fp16")];
            tensor<int32, [1]> var_2302_axes_0 = const()[name = string("op_2302_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_2302_cast_fp16 = expand_dims(axes = var_2302_axes_0, x = delta_45_cast_fp16)[name = string("op_2302_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2303_cast_fp16 = mul(x = var_2291_cast_fp16, y = var_2302_cast_fp16)[name = string("op_2303_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_91_cast_fp16 = add(x = state_89_cast_fp16, y = var_2303_cast_fp16)[name = string("state_91_cast_fp16")];
            tensor<int32, [4]> var_2307_begin_0 = const()[name = string("op_2307_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> var_2307_end_0 = const()[name = string("op_2307_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> var_2307_end_mask_0 = const()[name = string("op_2307_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_2307_squeeze_mask_0 = const()[name = string("op_2307_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_2307_cast_fp16 = slice_by_index(begin = var_2307_begin_0, end = var_2307_end_0, end_mask = var_2307_end_mask_0, squeeze_mask = var_2307_squeeze_mask_0, x = _inversed_query_27_cast_fp16)[name = string("op_2307_cast_fp16")];
            tensor<int32, [1]> var_2308_axes_0 = const()[name = string("op_2308_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_2308_cast_fp16 = expand_dims(axes = var_2308_axes_0, x = var_2307_cast_fp16)[name = string("op_2308_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2309_cast_fp16 = mul(x = state_91_cast_fp16, y = var_2308_cast_fp16)[name = string("op_2309_cast_fp16")];
            tensor<int32, [1]> var_2311_axes_0 = const()[name = string("op_2311_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_2311_keep_dims_0 = const()[name = string("op_2311_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_2311_cast_fp16 = reduce_sum(axes = var_2311_axes_0, keep_dims = var_2311_keep_dims_0, x = var_2309_cast_fp16)[name = string("op_2311_cast_fp16")];
            tensor<int32, [3]> var_2314_begin_0 = const()[name = string("op_2314_begin_0"), val = tensor<int32, [3]>([0, 0, 3])];
            tensor<int32, [3]> var_2314_end_0 = const()[name = string("op_2314_end_0"), val = tensor<int32, [3]>([1, 16, 4])];
            tensor<bool, [3]> var_2314_end_mask_0 = const()[name = string("op_2314_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_2314_squeeze_mask_0 = const()[name = string("op_2314_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_2314_cast_fp16 = slice_by_index(begin = var_2314_begin_0, end = var_2314_end_0, end_mask = var_2314_end_mask_0, squeeze_mask = var_2314_squeeze_mask_0, x = decay_11_cast_fp16)[name = string("op_2314_cast_fp16")];
            tensor<fp16, [1, 16]> var_2315_cast_fp16 = exp(x = var_2314_cast_fp16)[name = string("op_2315_cast_fp16")];
            tensor<int32, [1]> var_2316_axes_0 = const()[name = string("op_2316_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_2316_cast_fp16 = expand_dims(axes = var_2316_axes_0, x = var_2315_cast_fp16)[name = string("op_2316_cast_fp16")];
            tensor<int32, [1]> var_2317_axes_0 = const()[name = string("op_2317_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_2317_cast_fp16 = expand_dims(axes = var_2317_axes_0, x = var_2316_cast_fp16)[name = string("op_2317_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_93_cast_fp16 = mul(x = state_91_cast_fp16, y = var_2317_cast_fp16)[name = string("state_93_cast_fp16")];
            tensor<int32, [4]> key_token_47_begin_0 = const()[name = string("key_token_47_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> key_token_47_end_0 = const()[name = string("key_token_47_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> key_token_47_end_mask_0 = const()[name = string("key_token_47_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_47_squeeze_mask_0 = const()[name = string("key_token_47_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_47_cast_fp16 = slice_by_index(begin = key_token_47_begin_0, end = key_token_47_end_0, end_mask = key_token_47_end_mask_0, squeeze_mask = key_token_47_squeeze_mask_0, x = key_27_cast_fp16)[name = string("key_token_47_cast_fp16")];
            tensor<int32, [4]> value_token_47_begin_0 = const()[name = string("value_token_47_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> value_token_47_end_0 = const()[name = string("value_token_47_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> value_token_47_end_mask_0 = const()[name = string("value_token_47_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_47_squeeze_mask_0 = const()[name = string("value_token_47_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_47_cast_fp16 = slice_by_index(begin = value_token_47_begin_0, end = value_token_47_end_0, end_mask = value_token_47_end_mask_0, squeeze_mask = value_token_47_squeeze_mask_0, x = value_97_cast_fp16)[name = string("value_token_47_cast_fp16")];
            tensor<int32, [1]> var_2325_axes_0 = const()[name = string("op_2325_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_2325_cast_fp16 = expand_dims(axes = var_2325_axes_0, x = key_token_47_cast_fp16)[name = string("op_2325_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2326_cast_fp16 = mul(x = state_93_cast_fp16, y = var_2325_cast_fp16)[name = string("op_2326_cast_fp16")];
            tensor<int32, [1]> memory_47_axes_0 = const()[name = string("memory_47_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_47_keep_dims_0 = const()[name = string("memory_47_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_47_cast_fp16 = reduce_sum(axes = memory_47_axes_0, keep_dims = memory_47_keep_dims_0, x = var_2326_cast_fp16)[name = string("memory_47_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_2329_cast_fp16 = sub(x = value_token_47_cast_fp16, y = memory_47_cast_fp16)[name = string("op_2329_cast_fp16")];
            tensor<int32, [3]> var_2332_begin_0 = const()[name = string("op_2332_begin_0"), val = tensor<int32, [3]>([0, 0, 3])];
            tensor<int32, [3]> var_2332_end_0 = const()[name = string("op_2332_end_0"), val = tensor<int32, [3]>([1, 16, 4])];
            tensor<bool, [3]> var_2332_end_mask_0 = const()[name = string("op_2332_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_2332_squeeze_mask_0 = const()[name = string("op_2332_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_2332_cast_fp16 = slice_by_index(begin = var_2332_begin_0, end = var_2332_end_0, end_mask = var_2332_end_mask_0, squeeze_mask = var_2332_squeeze_mask_0, x = beta_11_cast_fp16)[name = string("op_2332_cast_fp16")];
            tensor<int32, [1]> var_2333_axes_0 = const()[name = string("op_2333_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_2333_cast_fp16 = expand_dims(axes = var_2333_axes_0, x = var_2332_cast_fp16)[name = string("op_2333_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_47_cast_fp16 = mul(x = var_2329_cast_fp16, y = var_2333_cast_fp16)[name = string("delta_47_cast_fp16")];
            tensor<int32, [1]> var_2336_axes_0 = const()[name = string("op_2336_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_2336_cast_fp16 = expand_dims(axes = var_2336_axes_0, x = delta_47_cast_fp16)[name = string("op_2336_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2337_cast_fp16 = mul(x = var_2325_cast_fp16, y = var_2336_cast_fp16)[name = string("op_2337_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> rec6_out = add(x = state_93_cast_fp16, y = var_2337_cast_fp16)[name = string("state_95_cast_fp16")];
            tensor<int32, [4]> var_2341_begin_0 = const()[name = string("op_2341_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> var_2341_end_0 = const()[name = string("op_2341_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> var_2341_end_mask_0 = const()[name = string("op_2341_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_2341_squeeze_mask_0 = const()[name = string("op_2341_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_2341_cast_fp16 = slice_by_index(begin = var_2341_begin_0, end = var_2341_end_0, end_mask = var_2341_end_mask_0, squeeze_mask = var_2341_squeeze_mask_0, x = _inversed_query_27_cast_fp16)[name = string("op_2341_cast_fp16")];
            tensor<int32, [1]> var_2342_axes_0 = const()[name = string("op_2342_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_2342_cast_fp16 = expand_dims(axes = var_2342_axes_0, x = var_2341_cast_fp16)[name = string("op_2342_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2343_cast_fp16 = mul(x = rec6_out, y = var_2342_cast_fp16)[name = string("op_2343_cast_fp16")];
            tensor<int32, [1]> var_2345_axes_0 = const()[name = string("op_2345_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_2345_keep_dims_0 = const()[name = string("op_2345_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_2345_cast_fp16 = reduce_sum(axes = var_2345_axes_0, keep_dims = var_2345_keep_dims_0, x = var_2343_cast_fp16)[name = string("op_2345_cast_fp16")];
            int32 attention_51_axis_0 = const()[name = string("attention_51_axis_0"), val = int32(1)];
            tensor<fp16, [1, 4, 16, 128]> attention_51_cast_fp16 = stack(axis = attention_51_axis_0, values = (var_2243_cast_fp16, var_2277_cast_fp16, var_2311_cast_fp16, var_2345_cast_fp16))[name = string("attention_51_cast_fp16")];
            tensor<int32, [2]> var_2348 = const()[name = string("op_2348"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [64, 128]> attention_53_cast_fp16 = reshape(shape = var_2348, x = attention_51_cast_fp16)[name = string("attention_53_cast_fp16")];
            tensor<int32, [2]> var_2350 = const()[name = string("op_2350"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [64, 128]> input_85_cast_fp16 = reshape(shape = var_2350, x = linear_50_cast_fp16)[name = string("input_85_cast_fp16")];
            tensor<fp16, [64, 128]> var_2352_cast_fp16 = mul(x = attention_53_cast_fp16, y = attention_53_cast_fp16)[name = string("op_2352_cast_fp16")];
            tensor<int32, [1]> variance_41_axes_0 = const()[name = string("variance_41_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_41_keep_dims_0 = const()[name = string("variance_41_keep_dims_0"), val = bool(true)];
            tensor<fp16, [64, 1]> variance_41_cast_fp16 = reduce_mean(axes = variance_41_axes_0, keep_dims = variance_41_keep_dims_0, x = var_2352_cast_fp16)[name = string("variance_41_cast_fp16")];
            fp16 var_2355_to_fp16 = const()[name = string("op_2355_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [64, 1]> var_2356_cast_fp16 = add(x = variance_41_cast_fp16, y = var_2355_to_fp16)[name = string("op_2356_cast_fp16")];
            fp32 var_2357_epsilon_0 = const()[name = string("op_2357_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [64, 1]> var_2357_cast_fp16 = rsqrt(epsilon = var_2357_epsilon_0, x = var_2356_cast_fp16)[name = string("op_2357_cast_fp16")];
            tensor<fp16, [64, 128]> attention_55_cast_fp16 = mul(x = attention_53_cast_fp16, y = var_2357_cast_fp16)[name = string("attention_55_cast_fp16")];
            tensor<fp16, [128]> groups_1_2_gated_norm_promoted_to_fp16 = const()[name = string("groups_1_2_gated_norm_promoted_to_fp16"), val = tensor<fp16, [128]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(102168768)))];
            tensor<fp16, [64, 128]> attention_57_cast_fp16 = mul(x = attention_55_cast_fp16, y = groups_1_2_gated_norm_promoted_to_fp16)[name = string("attention_57_cast_fp16")];
            tensor<fp16, [64, 128]> var_2360_cast_fp16 = silu(x = input_85_cast_fp16)[name = string("op_2360_cast_fp16")];
            tensor<fp16, [64, 128]> attention_59_cast_fp16 = mul(x = attention_57_cast_fp16, y = var_2360_cast_fp16)[name = string("attention_59_cast_fp16")];
            tensor<int32, [2]> var_2362 = const()[name = string("op_2362"), val = tensor<int32, [2]>([4, 2048])];
            tensor<fp16, [4, 2048]> input_87_cast_fp16 = reshape(shape = var_2362, x = attention_59_cast_fp16)[name = string("input_87_cast_fp16")];
            tensor<fp16, [1024, 2048]> groups_1_2_out_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 2048]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(102169088))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(103742016))))[name = string("groups_1_2_out_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 1024]> linear_51_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = groups_1_2_out_proj_weight_promoted_to_fp16_palettized, x = input_87_cast_fp16)[name = string("linear_51_cast_fp16")];
            tensor<fp16, [4, 1024]> value_99_cast_fp16 = add(x = value_87_cast_fp16, y = linear_51_cast_fp16)[name = string("value_99_cast_fp16")];
            tensor<fp16, [4, 1024]> var_2367_cast_fp16 = mul(x = value_99_cast_fp16, y = value_99_cast_fp16)[name = string("op_2367_cast_fp16")];
            tensor<int32, [1]> variance_43_axes_0 = const()[name = string("variance_43_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_43_keep_dims_0 = const()[name = string("variance_43_keep_dims_0"), val = bool(true)];
            tensor<fp16, [4, 1]> variance_43_cast_fp16 = reduce_mean(axes = variance_43_axes_0, keep_dims = variance_43_keep_dims_0, x = var_2367_cast_fp16)[name = string("variance_43_cast_fp16")];
            fp16 var_2370_to_fp16 = const()[name = string("op_2370_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [4, 1]> var_2371_cast_fp16 = add(x = variance_43_cast_fp16, y = var_2370_to_fp16)[name = string("op_2371_cast_fp16")];
            fp32 var_2372_epsilon_0 = const()[name = string("op_2372_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [4, 1]> var_2372_cast_fp16 = rsqrt(epsilon = var_2372_epsilon_0, x = var_2371_cast_fp16)[name = string("op_2372_cast_fp16")];
            tensor<fp16, [4, 1024]> var_2373_cast_fp16 = mul(x = value_99_cast_fp16, y = var_2372_cast_fp16)[name = string("op_2373_cast_fp16")];
            tensor<fp16, [1024]> var_2375_to_fp16 = const()[name = string("op_2375_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(103750272)))];
            tensor<fp16, [4, 1024]> input_89_cast_fp16 = mul(x = var_2373_cast_fp16, y = var_2375_to_fp16)[name = string("input_89_cast_fp16")];
            tensor<fp16, [3584, 1024]> groups_1_2_gate_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(103752384))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(106504960))))[name = string("groups_1_2_gate_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 3584]> linear_52_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = groups_1_2_gate_proj_weight_promoted_to_fp16_palettized, x = input_89_cast_fp16)[name = string("linear_52_cast_fp16")];
            tensor<fp16, [4, 3584]> var_2379_cast_fp16 = silu(x = linear_52_cast_fp16)[name = string("op_2379_cast_fp16")];
            tensor<fp16, [3584, 1024]> groups_1_2_up_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(106533696))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(109286272))))[name = string("groups_1_2_up_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 3584]> linear_53_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = groups_1_2_up_proj_weight_promoted_to_fp16_palettized, x = input_89_cast_fp16)[name = string("linear_53_cast_fp16")];
            tensor<fp16, [4, 3584]> input_93_cast_fp16 = mul(x = var_2379_cast_fp16, y = linear_53_cast_fp16)[name = string("input_93_cast_fp16")];
            tensor<fp16, [1024, 3584]> groups_1_2_down_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 3584]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(109315008))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(112067584))))[name = string("groups_1_2_down_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 1024]> linear_54_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = groups_1_2_down_proj_weight_promoted_to_fp16_palettized, x = input_93_cast_fp16)[name = string("linear_54_cast_fp16")];
            tensor<fp16, [4, 1024]> value_101_cast_fp16 = add(x = value_99_cast_fp16, y = linear_54_cast_fp16)[name = string("value_101_cast_fp16")];
            int32 var_2403 = const()[name = string("op_2403"), val = int32(-1)];
            tensor<fp16, [4, 1024]> var_2410_cast_fp16 = mul(x = value_101_cast_fp16, y = value_101_cast_fp16)[name = string("op_2410_cast_fp16")];
            tensor<int32, [1]> variance_45_axes_0 = const()[name = string("variance_45_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_45_keep_dims_0 = const()[name = string("variance_45_keep_dims_0"), val = bool(true)];
            tensor<fp16, [4, 1]> variance_45_cast_fp16 = reduce_mean(axes = variance_45_axes_0, keep_dims = variance_45_keep_dims_0, x = var_2410_cast_fp16)[name = string("variance_45_cast_fp16")];
            fp16 var_2413_to_fp16 = const()[name = string("op_2413_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [4, 1]> var_2414_cast_fp16 = add(x = variance_45_cast_fp16, y = var_2413_to_fp16)[name = string("op_2414_cast_fp16")];
            fp32 var_2415_epsilon_0 = const()[name = string("op_2415_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [4, 1]> var_2415_cast_fp16 = rsqrt(epsilon = var_2415_epsilon_0, x = var_2414_cast_fp16)[name = string("op_2415_cast_fp16")];
            tensor<fp16, [4, 1024]> var_2416_cast_fp16 = mul(x = value_101_cast_fp16, y = var_2415_cast_fp16)[name = string("op_2416_cast_fp16")];
            tensor<fp16, [1024]> var_2418_to_fp16 = const()[name = string("op_2418_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(112075840)))];
            tensor<fp16, [4, 1024]> input_95_cast_fp16 = mul(x = var_2416_cast_fp16, y = var_2418_to_fp16)[name = string("input_95_cast_fp16")];
            tensor<fp16, [4096, 1024]> projections_1_q_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [4096, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(112077952))), lut = tensor<fp16, [256, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(115223744))))[name = string("projections_1_q_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 4096]> linear_55_cast_fp16 = linear(bias = linear_24_bias_0_to_fp16, weight = projections_1_q_proj_weight_promoted_to_fp16_palettized, x = input_95_cast_fp16)[name = string("linear_55_cast_fp16")];
            tensor<int32, [3]> var_2422 = const()[name = string("op_2422"), val = tensor<int32, [3]>([4, 8, 512])];
            tensor<fp16, [4, 8, 512]> query_and_gate_3_cast_fp16 = reshape(shape = var_2422, x = linear_55_cast_fp16)[name = string("query_and_gate_3_cast_fp16")];
            tensor<int32, [2]> var_2424_split_sizes_0 = const()[name = string("op_2424_split_sizes_0"), val = tensor<int32, [2]>([256, 256])];
            int32 var_2424_axis_0 = const()[name = string("op_2424_axis_0"), val = int32(-1)];
            tensor<fp16, [4, 8, 256]> var_2424_cast_fp16_0, tensor<fp16, [4, 8, 256]> var_2424_cast_fp16_1 = split(axis = var_2424_axis_0, split_sizes = var_2424_split_sizes_0, x = query_and_gate_3_cast_fp16)[name = string("op_2424_cast_fp16")];
            tensor<fp16, [512, 1024]> projections_1_k_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [512, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(115256576))), lut = tensor<fp16, [32, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(115649856))))[name = string("projections_1_k_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 512]> linear_56_cast_fp16 = linear(bias = linear_25_bias_0_to_fp16, weight = projections_1_k_proj_weight_promoted_to_fp16_palettized, x = input_95_cast_fp16)[name = string("linear_56_cast_fp16")];
            tensor<int32, [3]> var_2428 = const()[name = string("op_2428"), val = tensor<int32, [3]>([4, 2, 256])];
            tensor<fp16, [4, 2, 256]> value_105_cast_fp16 = reshape(shape = var_2428, x = linear_56_cast_fp16)[name = string("value_105_cast_fp16")];
            tensor<fp16, [512, 1024]> projections_1_v_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [512, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(115654016))), lut = tensor<fp16, [32, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(116047296))))[name = string("projections_1_v_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 512]> linear_57_cast_fp16 = linear(bias = linear_25_bias_0_to_fp16, weight = projections_1_v_proj_weight_promoted_to_fp16_palettized, x = input_95_cast_fp16)[name = string("linear_57_cast_fp16")];
            tensor<int32, [3]> var_2432 = const()[name = string("op_2432"), val = tensor<int32, [3]>([4, 2, 256])];
            tensor<fp16, [4, 2, 256]> value_111_cast_fp16 = reshape(shape = var_2432, x = linear_57_cast_fp16)[name = string("value_111_cast_fp16")];
            tensor<fp16, [4, 8, 256]> var_2434_cast_fp16 = mul(x = var_2424_cast_fp16_0, y = var_2424_cast_fp16_0)[name = string("op_2434_cast_fp16")];
            tensor<int32, [1]> variance_47_axes_0 = const()[name = string("variance_47_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_47_keep_dims_0 = const()[name = string("variance_47_keep_dims_0"), val = bool(true)];
            tensor<fp16, [4, 8, 1]> variance_47_cast_fp16 = reduce_mean(axes = variance_47_axes_0, keep_dims = variance_47_keep_dims_0, x = var_2434_cast_fp16)[name = string("variance_47_cast_fp16")];
            fp16 var_2437_to_fp16 = const()[name = string("op_2437_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [4, 8, 1]> var_2438_cast_fp16 = add(x = variance_47_cast_fp16, y = var_2437_to_fp16)[name = string("op_2438_cast_fp16")];
            fp32 var_2439_epsilon_0 = const()[name = string("op_2439_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [4, 8, 1]> var_2439_cast_fp16 = rsqrt(epsilon = var_2439_epsilon_0, x = var_2438_cast_fp16)[name = string("op_2439_cast_fp16")];
            tensor<fp16, [4, 8, 256]> var_2440_cast_fp16 = mul(x = var_2424_cast_fp16_0, y = var_2439_cast_fp16)[name = string("op_2440_cast_fp16")];
            tensor<fp16, [256]> var_2442_to_fp16 = const()[name = string("op_2442_to_fp16"), val = tensor<fp16, [256]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(116051456)))];
            tensor<fp16, [4, 8, 256]> var_2443_cast_fp16 = mul(x = var_2440_cast_fp16, y = var_2442_to_fp16)[name = string("op_2443_cast_fp16")];
            tensor<int32, [3]> var_2444 = const()[name = string("op_2444"), val = tensor<int32, [3]>([1, 0, 2])];
            tensor<int32, [1]> value_107_axes_0 = const()[name = string("value_107_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [8, 4, 256]> var_2445_cast_fp16 = transpose(perm = var_2444, x = var_2443_cast_fp16)[name = string("transpose_90")];
            tensor<fp16, [1, 8, 4, 256]> value_107_cast_fp16 = expand_dims(axes = value_107_axes_0, x = var_2445_cast_fp16)[name = string("value_107_cast_fp16")];
            tensor<fp16, [4, 2, 256]> var_2447_cast_fp16 = mul(x = value_105_cast_fp16, y = value_105_cast_fp16)[name = string("op_2447_cast_fp16")];
            tensor<int32, [1]> variance_49_axes_0 = const()[name = string("variance_49_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_49_keep_dims_0 = const()[name = string("variance_49_keep_dims_0"), val = bool(true)];
            tensor<fp16, [4, 2, 1]> variance_49_cast_fp16 = reduce_mean(axes = variance_49_axes_0, keep_dims = variance_49_keep_dims_0, x = var_2447_cast_fp16)[name = string("variance_49_cast_fp16")];
            fp16 var_2450_to_fp16 = const()[name = string("op_2450_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [4, 2, 1]> var_2451_cast_fp16 = add(x = variance_49_cast_fp16, y = var_2450_to_fp16)[name = string("op_2451_cast_fp16")];
            fp32 var_2452_epsilon_0 = const()[name = string("op_2452_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [4, 2, 1]> var_2452_cast_fp16 = rsqrt(epsilon = var_2452_epsilon_0, x = var_2451_cast_fp16)[name = string("op_2452_cast_fp16")];
            tensor<fp16, [4, 2, 256]> var_2453_cast_fp16 = mul(x = value_105_cast_fp16, y = var_2452_cast_fp16)[name = string("op_2453_cast_fp16")];
            tensor<fp16, [256]> var_2455_to_fp16 = const()[name = string("op_2455_to_fp16"), val = tensor<fp16, [256]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(116052032)))];
            tensor<fp16, [4, 2, 256]> var_2456_cast_fp16 = mul(x = var_2453_cast_fp16, y = var_2455_to_fp16)[name = string("op_2456_cast_fp16")];
            tensor<int32, [3]> var_2457 = const()[name = string("op_2457"), val = tensor<int32, [3]>([1, 0, 2])];
            tensor<int32, [1]> value_109_axes_0 = const()[name = string("value_109_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [2, 4, 256]> var_2458_cast_fp16 = transpose(perm = var_2457, x = var_2456_cast_fp16)[name = string("transpose_89")];
            tensor<fp16, [1, 2, 4, 256]> value_109_cast_fp16 = expand_dims(axes = value_109_axes_0, x = var_2458_cast_fp16)[name = string("value_109_cast_fp16")];
            tensor<int32, [4]> rotated_5_begin_0 = const()[name = string("rotated_5_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> rotated_5_end_0 = const()[name = string("rotated_5_end_0"), val = tensor<int32, [4]>([1, 8, 4, 64])];
            tensor<bool, [4]> rotated_5_end_mask_0 = const()[name = string("rotated_5_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
            tensor<fp16, [1, 8, 4, 64]> rotated_5_cast_fp16 = slice_by_index(begin = rotated_5_begin_0, end = rotated_5_end_0, end_mask = rotated_5_end_mask_0, x = value_107_cast_fp16)[name = string("rotated_5_cast_fp16")];
            tensor<int32, [4]> passthrough_5_begin_0 = const()[name = string("passthrough_5_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 64])];
            tensor<int32, [4]> passthrough_5_end_0 = const()[name = string("passthrough_5_end_0"), val = tensor<int32, [4]>([1, 8, 4, 256])];
            tensor<bool, [4]> passthrough_5_end_mask_0 = const()[name = string("passthrough_5_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
            tensor<fp16, [1, 8, 4, 192]> passthrough_5_cast_fp16 = slice_by_index(begin = passthrough_5_begin_0, end = passthrough_5_end_0, end_mask = passthrough_5_end_mask_0, x = value_107_cast_fp16)[name = string("passthrough_5_cast_fp16")];
            tensor<int32, [2]> var_2462_split_sizes_0 = const()[name = string("op_2462_split_sizes_0"), val = tensor<int32, [2]>([32, 32])];
            int32 var_2462_axis_0 = const()[name = string("op_2462_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 8, 4, 32]> var_2462_cast_fp16_0, tensor<fp16, [1, 8, 4, 32]> var_2462_cast_fp16_1 = split(axis = var_2462_axis_0, split_sizes = var_2462_split_sizes_0, x = rotated_5_cast_fp16)[name = string("op_2462_cast_fp16")];
            fp16 const_46_promoted_to_fp16 = const()[name = string("const_46_promoted_to_fp16"), val = fp16(-0x1p+0)];
            tensor<fp16, [1, 8, 4, 32]> var_2464_cast_fp16 = mul(x = var_2462_cast_fp16_1, y = const_46_promoted_to_fp16)[name = string("op_2464_cast_fp16")];
            bool rotated_half_5_interleave_0 = const()[name = string("rotated_half_5_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 8, 4, 64]> rotated_half_5_cast_fp16 = concat(axis = var_2403, interleave = rotated_half_5_interleave_0, values = (var_2464_cast_fp16, var_2462_cast_fp16_0))[name = string("rotated_half_5_cast_fp16")];
            tensor<fp16, [1, 8, 4, 64]> var_2467_cast_fp16 = mul(x = rotated_5_cast_fp16, y = cos)[name = string("op_2467_cast_fp16")];
            tensor<fp16, [1, 8, 4, 64]> var_2468_cast_fp16 = mul(x = rotated_half_5_cast_fp16, y = sin)[name = string("op_2468_cast_fp16")];
            tensor<fp16, [1, 8, 4, 64]> var_2469_cast_fp16 = add(x = var_2467_cast_fp16, y = var_2468_cast_fp16)[name = string("op_2469_cast_fp16")];
            bool query_29_interleave_0 = const()[name = string("query_29_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 8, 4, 256]> query_29_cast_fp16 = concat(axis = var_2403, interleave = query_29_interleave_0, values = (var_2469_cast_fp16, passthrough_5_cast_fp16))[name = string("query_29_cast_fp16")];
            tensor<int32, [4]> rotated_7_begin_0 = const()[name = string("rotated_7_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> rotated_7_end_0 = const()[name = string("rotated_7_end_0"), val = tensor<int32, [4]>([1, 2, 4, 64])];
            tensor<bool, [4]> rotated_7_end_mask_0 = const()[name = string("rotated_7_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
            tensor<fp16, [1, 2, 4, 64]> rotated_7_cast_fp16 = slice_by_index(begin = rotated_7_begin_0, end = rotated_7_end_0, end_mask = rotated_7_end_mask_0, x = value_109_cast_fp16)[name = string("rotated_7_cast_fp16")];
            tensor<int32, [4]> passthrough_7_begin_0 = const()[name = string("passthrough_7_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 64])];
            tensor<int32, [4]> passthrough_7_end_0 = const()[name = string("passthrough_7_end_0"), val = tensor<int32, [4]>([1, 2, 4, 256])];
            tensor<bool, [4]> passthrough_7_end_mask_0 = const()[name = string("passthrough_7_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
            tensor<fp16, [1, 2, 4, 192]> passthrough_7_cast_fp16 = slice_by_index(begin = passthrough_7_begin_0, end = passthrough_7_end_0, end_mask = passthrough_7_end_mask_0, x = value_109_cast_fp16)[name = string("passthrough_7_cast_fp16")];
            tensor<int32, [2]> var_2474_split_sizes_0 = const()[name = string("op_2474_split_sizes_0"), val = tensor<int32, [2]>([32, 32])];
            int32 var_2474_axis_0 = const()[name = string("op_2474_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 2, 4, 32]> var_2474_cast_fp16_0, tensor<fp16, [1, 2, 4, 32]> var_2474_cast_fp16_1 = split(axis = var_2474_axis_0, split_sizes = var_2474_split_sizes_0, x = rotated_7_cast_fp16)[name = string("op_2474_cast_fp16")];
            fp16 const_47_promoted_to_fp16 = const()[name = string("const_47_promoted_to_fp16"), val = fp16(-0x1p+0)];
            tensor<fp16, [1, 2, 4, 32]> var_2476_cast_fp16 = mul(x = var_2474_cast_fp16_1, y = const_47_promoted_to_fp16)[name = string("op_2476_cast_fp16")];
            bool rotated_half_7_interleave_0 = const()[name = string("rotated_half_7_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 2, 4, 64]> rotated_half_7_cast_fp16 = concat(axis = var_2403, interleave = rotated_half_7_interleave_0, values = (var_2476_cast_fp16, var_2474_cast_fp16_0))[name = string("rotated_half_7_cast_fp16")];
            tensor<fp16, [1, 2, 4, 64]> var_2479_cast_fp16 = mul(x = rotated_7_cast_fp16, y = cos)[name = string("op_2479_cast_fp16")];
            tensor<fp16, [1, 2, 4, 64]> var_2480_cast_fp16 = mul(x = rotated_half_7_cast_fp16, y = sin)[name = string("op_2480_cast_fp16")];
            tensor<fp16, [1, 2, 4, 64]> var_2481_cast_fp16 = add(x = var_2479_cast_fp16, y = var_2480_cast_fp16)[name = string("op_2481_cast_fp16")];
            bool key_29_interleave_0 = const()[name = string("key_29_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 2, 4, 256]> key_29_cast_fp16 = concat(axis = var_2403, interleave = key_29_interleave_0, values = (var_2481_cast_fp16, passthrough_7_cast_fp16))[name = string("key_29_cast_fp16")];
            tensor<int32, [4]> var_2484 = const()[name = string("op_2484"), val = tensor<int32, [4]>([2, 4, 4, 256])];
            tensor<fp16, [2, 4, 4, 256]> var_2485_cast_fp16 = reshape(shape = var_2484, x = query_29_cast_fp16)[name = string("op_2485_cast_fp16")];
            tensor<int32, [4]> transpose_2_perm_0 = const()[name = string("transpose_2_perm_0"), val = tensor<int32, [4]>([0, 1, 3, 2])];
            tensor<int32, [3]> var_2488 = const()[name = string("op_2488"), val = tensor<int32, [3]>([2, 4, 256])];
            tensor<fp16, [2, 4, 256]> key_31_cast_fp16 = reshape(shape = var_2488, x = key_29_cast_fp16)[name = string("key_31_cast_fp16")];
            tensor<int32, [3]> var_2490 = const()[name = string("op_2490"), val = tensor<int32, [3]>([1, 0, 2])];
            tensor<int32, [2]> var_2492 = const()[name = string("op_2492"), val = tensor<int32, [2]>([4, 2048])];
            tensor<fp16, [4, 2048]> gate_7_cast_fp16 = reshape(shape = var_2492, x = var_2424_cast_fp16_1)[name = string("gate_7_cast_fp16")];
            tensor<int32, [1]> var_2500_axes_0 = const()[name = string("op_2500_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [1, 2, 4, 256]> var_2500_cast_fp16 = expand_dims(axes = var_2500_axes_0, x = key_31_cast_fp16)[name = string("op_2500_cast_fp16")];
            tensor<int32, [1]> var_2502_axes_0 = const()[name = string("op_2502_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [1, 1, 2, 4, 256]> var_2502_cast_fp16 = expand_dims(axes = var_2502_axes_0, x = var_2500_cast_fp16)[name = string("op_2502_cast_fp16")];
            tensor<int32, [1]> expand_dims_16 = const()[name = string("expand_dims_16"), val = tensor<int32, [1]>([1])];
            tensor<int32, [1]> expand_dims_17 = const()[name = string("expand_dims_17"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_18 = const()[name = string("expand_dims_18"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_20 = const()[name = string("expand_dims_20"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_21 = const()[name = string("expand_dims_21"), val = tensor<int32, [1]>([2])];
            tensor<int32, [1]> expand_dims_22 = const()[name = string("expand_dims_22"), val = tensor<int32, [1]>([1])];
            int32 concat_10_axis_0 = const()[name = string("concat_10_axis_0"), val = int32(0)];
            bool concat_10_interleave_0 = const()[name = string("concat_10_interleave_0"), val = bool(false)];
            tensor<int32, [5]> concat_10 = concat(axis = concat_10_axis_0, interleave = concat_10_interleave_0, values = (expand_dims_16, expand_dims_17, expand_dims_18, current_pos, expand_dims_20))[name = string("concat_10")];
            tensor<int32, [1]> concat_11_values2_0 = const()[name = string("concat_11_values2_0"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> concat_11_values4_0 = const()[name = string("concat_11_values4_0"), val = tensor<int32, [1]>([0])];
            int32 concat_11_axis_0 = const()[name = string("concat_11_axis_0"), val = int32(0)];
            bool concat_11_interleave_0 = const()[name = string("concat_11_interleave_0"), val = bool(false)];
            tensor<int32, [5]> concat_11 = concat(axis = concat_11_axis_0, interleave = concat_11_interleave_0, values = (expand_dims_21, expand_dims_22, concat_11_values2_0, var_1074, concat_11_values4_0))[name = string("concat_11")];
            tensor<int32, [5]> kv_cache_internal_tensor_assign_3_stride_0 = const()[name = string("kv_cache_internal_tensor_assign_3_stride_0"), val = tensor<int32, [5]>([1, 1, 1, 1, 1])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_3_begin_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_3_begin_mask_0"), val = tensor<bool, [5]>([false, false, false, false, false])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_3_end_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_3_end_mask_0"), val = tensor<bool, [5]>([false, false, true, false, true])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_3_squeeze_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_3_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, false, false, false])];
            tensor<fp16, [4, 2, 2, 2048, 256]> kv_cache_internal_tensor_assign_3_cast_fp16 = slice_update(begin = concat_10, begin_mask = kv_cache_internal_tensor_assign_3_begin_mask_0, end = concat_11, end_mask = kv_cache_internal_tensor_assign_3_end_mask_0, squeeze_mask = kv_cache_internal_tensor_assign_3_squeeze_mask_0, stride = kv_cache_internal_tensor_assign_3_stride_0, update = var_2502_cast_fp16, x = coreml_update_state_9)[name = string("kv_cache_internal_tensor_assign_3_cast_fp16")];
            write_state(data = kv_cache_internal_tensor_assign_3_cast_fp16, input = kv_cache)[name = string("coreml_update_state_18_write_state")];
            tensor<fp16, [4, 2, 2, 2048, 256]> coreml_update_state_10 = read_state(input = kv_cache)[name = string("coreml_update_state_18")];
            tensor<int32, [1]> var_2533_axes_0 = const()[name = string("op_2533_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [2, 4, 256]> value_113_cast_fp16 = transpose(perm = var_2490, x = value_111_cast_fp16)[name = string("transpose_88")];
            tensor<fp16, [1, 2, 4, 256]> var_2533_cast_fp16 = expand_dims(axes = var_2533_axes_0, x = value_113_cast_fp16)[name = string("op_2533_cast_fp16")];
            tensor<int32, [1]> var_2535_axes_0 = const()[name = string("op_2535_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [1, 1, 2, 4, 256]> var_2535_cast_fp16 = expand_dims(axes = var_2535_axes_0, x = var_2533_cast_fp16)[name = string("op_2535_cast_fp16")];
            tensor<int32, [1]> expand_dims_24 = const()[name = string("expand_dims_24"), val = tensor<int32, [1]>([1])];
            tensor<int32, [1]> expand_dims_25 = const()[name = string("expand_dims_25"), val = tensor<int32, [1]>([1])];
            tensor<int32, [1]> expand_dims_26 = const()[name = string("expand_dims_26"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_28 = const()[name = string("expand_dims_28"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_29 = const()[name = string("expand_dims_29"), val = tensor<int32, [1]>([2])];
            tensor<int32, [1]> expand_dims_30 = const()[name = string("expand_dims_30"), val = tensor<int32, [1]>([2])];
            int32 concat_14_axis_0 = const()[name = string("concat_14_axis_0"), val = int32(0)];
            bool concat_14_interleave_0 = const()[name = string("concat_14_interleave_0"), val = bool(false)];
            tensor<int32, [5]> concat_14 = concat(axis = concat_14_axis_0, interleave = concat_14_interleave_0, values = (expand_dims_24, expand_dims_25, expand_dims_26, current_pos, expand_dims_28))[name = string("concat_14")];
            tensor<int32, [1]> concat_15_values2_0 = const()[name = string("concat_15_values2_0"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> concat_15_values4_0 = const()[name = string("concat_15_values4_0"), val = tensor<int32, [1]>([0])];
            int32 concat_15_axis_0 = const()[name = string("concat_15_axis_0"), val = int32(0)];
            bool concat_15_interleave_0 = const()[name = string("concat_15_interleave_0"), val = bool(false)];
            tensor<int32, [5]> concat_15 = concat(axis = concat_15_axis_0, interleave = concat_15_interleave_0, values = (expand_dims_29, expand_dims_30, concat_15_values2_0, var_1074, concat_15_values4_0))[name = string("concat_15")];
            tensor<int32, [5]> kv_cache_internal_tensor_assign_4_stride_0 = const()[name = string("kv_cache_internal_tensor_assign_4_stride_0"), val = tensor<int32, [5]>([1, 1, 1, 1, 1])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_4_begin_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_4_begin_mask_0"), val = tensor<bool, [5]>([false, false, false, false, false])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_4_end_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_4_end_mask_0"), val = tensor<bool, [5]>([false, false, true, false, true])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_4_squeeze_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_4_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, false, false, false])];
            tensor<fp16, [4, 2, 2, 2048, 256]> kv_cache_internal_tensor_assign_4_cast_fp16 = slice_update(begin = concat_14, begin_mask = kv_cache_internal_tensor_assign_4_begin_mask_0, end = concat_15, end_mask = kv_cache_internal_tensor_assign_4_end_mask_0, squeeze_mask = kv_cache_internal_tensor_assign_4_squeeze_mask_0, stride = kv_cache_internal_tensor_assign_4_stride_0, update = var_2535_cast_fp16, x = coreml_update_state_10)[name = string("kv_cache_internal_tensor_assign_4_cast_fp16")];
            write_state(data = kv_cache_internal_tensor_assign_4_cast_fp16, input = kv_cache)[name = string("coreml_update_state_19_write_state")];
            tensor<fp16, [4, 2, 2, 2048, 256]> coreml_update_state_11 = read_state(input = kv_cache)[name = string("coreml_update_state_19")];
            tensor<int32, [5]> var_2567_begin_0 = const()[name = string("op_2567_begin_0"), val = tensor<int32, [5]>([1, 0, 0, 0, 0])];
            tensor<int32, [5]> var_2567_end_0 = const()[name = string("op_2567_end_0"), val = tensor<int32, [5]>([2, 2, 2, 2048, 256])];
            tensor<bool, [5]> var_2567_end_mask_0 = const()[name = string("op_2567_end_mask_0"), val = tensor<bool, [5]>([false, true, true, true, true])];
            tensor<bool, [5]> var_2567_squeeze_mask_0 = const()[name = string("op_2567_squeeze_mask_0"), val = tensor<bool, [5]>([true, false, false, false, false])];
            tensor<fp16, [2, 2, 2048, 256]> var_2567_cast_fp16 = slice_by_index(begin = var_2567_begin_0, end = var_2567_end_0, end_mask = var_2567_end_mask_0, squeeze_mask = var_2567_squeeze_mask_0, x = coreml_update_state_11)[name = string("op_2567_cast_fp16")];
            tensor<int32, [4]> keys_3_begin_0 = const()[name = string("keys_3_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> keys_3_end_0 = const()[name = string("keys_3_end_0"), val = tensor<int32, [4]>([1, 2, 2048, 256])];
            tensor<bool, [4]> keys_3_end_mask_0 = const()[name = string("keys_3_end_mask_0"), val = tensor<bool, [4]>([false, true, true, true])];
            tensor<bool, [4]> keys_3_squeeze_mask_0 = const()[name = string("keys_3_squeeze_mask_0"), val = tensor<bool, [4]>([true, false, false, false])];
            tensor<fp16, [2, 2048, 256]> keys_3_cast_fp16 = slice_by_index(begin = keys_3_begin_0, end = keys_3_end_0, end_mask = keys_3_end_mask_0, squeeze_mask = keys_3_squeeze_mask_0, x = var_2567_cast_fp16)[name = string("keys_3_cast_fp16")];
            tensor<int32, [4]> values_3_begin_0 = const()[name = string("values_3_begin_0"), val = tensor<int32, [4]>([1, 0, 0, 0])];
            tensor<int32, [4]> values_3_end_0 = const()[name = string("values_3_end_0"), val = tensor<int32, [4]>([2, 2, 2048, 256])];
            tensor<bool, [4]> values_3_end_mask_0 = const()[name = string("values_3_end_mask_0"), val = tensor<bool, [4]>([false, true, true, true])];
            tensor<bool, [4]> values_3_squeeze_mask_0 = const()[name = string("values_3_squeeze_mask_0"), val = tensor<bool, [4]>([true, false, false, false])];
            tensor<fp16, [2, 2048, 256]> values_3_cast_fp16 = slice_by_index(begin = values_3_begin_0, end = values_3_end_0, end_mask = values_3_end_mask_0, squeeze_mask = values_3_squeeze_mask_0, x = var_2567_cast_fp16)[name = string("values_3_cast_fp16")];
            int32 var_2584 = const()[name = string("op_2584"), val = int32(1)];
            tensor<int32, [4]> var_2591 = const()[name = string("op_2591"), val = tensor<int32, [4]>([1, 2048, 1, 4])];
            tensor<fp16, [2, 4, 256, 4]> transpose_2_cast_fp16 = transpose(perm = transpose_2_perm_0, x = var_2485_cast_fp16)[name = string("transpose_87")];
            tensor<fp16, [1, 2048, 1, 4]> var_2592_cast_fp16 = reshape(shape = var_2591, x = transpose_2_cast_fp16)[name = string("op_2592_cast_fp16")];
            tensor<int32, [3]> var_2593 = const()[name = string("op_2593"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [4]> var_2596 = const()[name = string("op_2596"), val = tensor<int32, [4]>([1, 512, 1, 2048])];
            tensor<fp16, [2, 256, 2048]> var_2594_cast_fp16 = transpose(perm = var_2593, x = keys_3_cast_fp16)[name = string("transpose_86")];
            tensor<fp16, [1, 512, 1, 2048]> key_native_3_cast_fp16 = reshape(shape = var_2596, x = var_2594_cast_fp16)[name = string("key_native_3_cast_fp16")];
            tensor<int32, [3]> var_2598 = const()[name = string("op_2598"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [4]> var_2601 = const()[name = string("op_2601"), val = tensor<int32, [4]>([1, 512, 1, 2048])];
            tensor<fp16, [2, 256, 2048]> var_2599_cast_fp16 = transpose(perm = var_2598, x = values_3_cast_fp16)[name = string("transpose_85")];
            tensor<fp16, [1, 512, 1, 2048]> var_2602_cast_fp16 = reshape(shape = var_2601, x = var_2599_cast_fp16)[name = string("op_2602_cast_fp16")];
            tensor<int32, [8]> tile_19 = const()[name = string("tile_19"), val = tensor<int32, [8]>([256, 256, 256, 256, 256, 256, 256, 256])];
            int32 var_2606_axis_0 = const()[name = string("op_2606_axis_0"), val = int32(1)];
            tensor<fp16, [1, 256, 1, 4]> var_2606_cast_fp16_0, tensor<fp16, [1, 256, 1, 4]> var_2606_cast_fp16_1, tensor<fp16, [1, 256, 1, 4]> var_2606_cast_fp16_2, tensor<fp16, [1, 256, 1, 4]> var_2606_cast_fp16_3, tensor<fp16, [1, 256, 1, 4]> var_2606_cast_fp16_4, tensor<fp16, [1, 256, 1, 4]> var_2606_cast_fp16_5, tensor<fp16, [1, 256, 1, 4]> var_2606_cast_fp16_6, tensor<fp16, [1, 256, 1, 4]> var_2606_cast_fp16_7 = split(axis = var_2606_axis_0, split_sizes = tile_19, x = var_2592_cast_fp16)[name = string("op_2606_cast_fp16")];
            tensor<int32, [4]> var_2615_perm_0 = const()[name = string("op_2615_perm_0"), val = tensor<int32, [4]>([0, 3, 2, 1])];
            tensor<int32, [2]> tile_20 = const()[name = string("tile_20"), val = tensor<int32, [2]>([256, 256])];
            int32 var_2616_axis_0 = const()[name = string("op_2616_axis_0"), val = int32(3)];
            tensor<fp16, [1, 2048, 1, 512]> var_2615_cast_fp16 = transpose(perm = var_2615_perm_0, x = key_native_3_cast_fp16)[name = string("transpose_84")];
            tensor<fp16, [1, 2048, 1, 256]> var_2616_cast_fp16_0, tensor<fp16, [1, 2048, 1, 256]> var_2616_cast_fp16_1 = split(axis = var_2616_axis_0, split_sizes = tile_20, x = var_2615_cast_fp16)[name = string("op_2616_cast_fp16")];
            tensor<int32, [2]> tile_21 = const()[name = string("tile_21"), val = tensor<int32, [2]>([256, 256])];
            int32 var_2619_axis_0 = const()[name = string("op_2619_axis_0"), val = int32(1)];
            tensor<fp16, [1, 256, 1, 2048]> var_2619_cast_fp16_0, tensor<fp16, [1, 256, 1, 2048]> var_2619_cast_fp16_1 = split(axis = var_2619_axis_0, split_sizes = tile_21, x = var_2602_cast_fp16)[name = string("op_2619_cast_fp16")];
            string scores_17_equation_0 = const()[name = string("scores_17_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 4]> scores_17_cast_fp16 = einsum(equation = scores_17_equation_0, values = (var_2616_cast_fp16_0, var_2606_cast_fp16_0))[name = string("scores_17_cast_fp16")];
            fp16 var_2624_to_fp16 = const()[name = string("op_2624_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 4]> var_2625_cast_fp16 = mul(x = scores_17_cast_fp16, y = var_2624_to_fp16)[name = string("op_2625_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_2626_cast_fp16 = add(x = var_2625_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_2626_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_2627_cast_fp16 = softmax(axis = var_2584, x = var_2626_cast_fp16)[name = string("op_2627_cast_fp16")];
            tensor<int32, [4]> tile_22 = const()[name = string("tile_22"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_2628_axis_0 = const()[name = string("op_2628_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 4]> var_2628_cast_fp16_0, tensor<fp16, [1, 512, 1, 4]> var_2628_cast_fp16_1, tensor<fp16, [1, 512, 1, 4]> var_2628_cast_fp16_2, tensor<fp16, [1, 512, 1, 4]> var_2628_cast_fp16_3 = split(axis = var_2628_axis_0, split_sizes = tile_22, x = var_2627_cast_fp16)[name = string("op_2628_cast_fp16")];
            tensor<int32, [4]> tile_23 = const()[name = string("tile_23"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_2633_axis_0 = const()[name = string("op_2633_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_2633_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_2633_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_2633_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_2633_cast_fp16_3 = split(axis = var_2633_axis_0, split_sizes = tile_23, x = var_2619_cast_fp16_0)[name = string("op_2633_cast_fp16")];
            fp16 var_2638_to_fp16 = const()[name = string("op_2638_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_2639_cast_fp16 = mul(x = var_2628_cast_fp16_0, y = var_2638_to_fp16)[name = string("op_2639_cast_fp16")];
            string var_2641_equation_0 = const()[name = string("op_2641_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_2641_cast_fp16 = einsum(equation = var_2641_equation_0, values = (var_2633_cast_fp16_0, var_2639_cast_fp16))[name = string("op_2641_cast_fp16")];
            fp16 var_2642_to_fp16 = const()[name = string("op_2642_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_2643_cast_fp16 = mul(x = var_2628_cast_fp16_1, y = var_2642_to_fp16)[name = string("op_2643_cast_fp16")];
            string var_2645_equation_0 = const()[name = string("op_2645_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_2645_cast_fp16 = einsum(equation = var_2645_equation_0, values = (var_2633_cast_fp16_1, var_2643_cast_fp16))[name = string("op_2645_cast_fp16")];
            fp16 var_2646_to_fp16 = const()[name = string("op_2646_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_2647_cast_fp16 = mul(x = var_2628_cast_fp16_2, y = var_2646_to_fp16)[name = string("op_2647_cast_fp16")];
            string var_2649_equation_0 = const()[name = string("op_2649_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_2649_cast_fp16 = einsum(equation = var_2649_equation_0, values = (var_2633_cast_fp16_2, var_2647_cast_fp16))[name = string("op_2649_cast_fp16")];
            fp16 var_2650_to_fp16 = const()[name = string("op_2650_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_2651_cast_fp16 = mul(x = var_2628_cast_fp16_3, y = var_2650_to_fp16)[name = string("op_2651_cast_fp16")];
            string var_2653_equation_0 = const()[name = string("op_2653_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_2653_cast_fp16 = einsum(equation = var_2653_equation_0, values = (var_2633_cast_fp16_3, var_2651_cast_fp16))[name = string("op_2653_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_2654_cast_fp16 = add(x = var_2641_cast_fp16, y = var_2645_cast_fp16)[name = string("op_2654_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_2655_cast_fp16 = add(x = var_2649_cast_fp16, y = var_2653_cast_fp16)[name = string("op_2655_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_2656_cast_fp16 = add(x = var_2654_cast_fp16, y = var_2655_cast_fp16)[name = string("op_2656_cast_fp16")];
            fp16 _inversed_2658_y_0_to_fp16 = const()[name = string("_inversed_2658_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 4]> _inversed_2658_cast_fp16 = mul(x = var_2656_cast_fp16, y = _inversed_2658_y_0_to_fp16)[name = string("_inversed_2658_cast_fp16")];
            string scores_19_equation_0 = const()[name = string("scores_19_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 4]> scores_19_cast_fp16 = einsum(equation = scores_19_equation_0, values = (var_2616_cast_fp16_0, var_2606_cast_fp16_1))[name = string("scores_19_cast_fp16")];
            fp16 var_2661_to_fp16 = const()[name = string("op_2661_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 4]> var_2662_cast_fp16 = mul(x = scores_19_cast_fp16, y = var_2661_to_fp16)[name = string("op_2662_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_2663_cast_fp16 = add(x = var_2662_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_2663_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_2664_cast_fp16 = softmax(axis = var_2584, x = var_2663_cast_fp16)[name = string("op_2664_cast_fp16")];
            tensor<int32, [4]> tile_24 = const()[name = string("tile_24"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_2665_axis_0 = const()[name = string("op_2665_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 4]> var_2665_cast_fp16_0, tensor<fp16, [1, 512, 1, 4]> var_2665_cast_fp16_1, tensor<fp16, [1, 512, 1, 4]> var_2665_cast_fp16_2, tensor<fp16, [1, 512, 1, 4]> var_2665_cast_fp16_3 = split(axis = var_2665_axis_0, split_sizes = tile_24, x = var_2664_cast_fp16)[name = string("op_2665_cast_fp16")];
            tensor<int32, [4]> tile_25 = const()[name = string("tile_25"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_2670_axis_0 = const()[name = string("op_2670_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_2670_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_2670_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_2670_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_2670_cast_fp16_3 = split(axis = var_2670_axis_0, split_sizes = tile_25, x = var_2619_cast_fp16_0)[name = string("op_2670_cast_fp16")];
            fp16 var_2675_to_fp16 = const()[name = string("op_2675_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_2676_cast_fp16 = mul(x = var_2665_cast_fp16_0, y = var_2675_to_fp16)[name = string("op_2676_cast_fp16")];
            string var_2678_equation_0 = const()[name = string("op_2678_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_2678_cast_fp16 = einsum(equation = var_2678_equation_0, values = (var_2670_cast_fp16_0, var_2676_cast_fp16))[name = string("op_2678_cast_fp16")];
            fp16 var_2679_to_fp16 = const()[name = string("op_2679_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_2680_cast_fp16 = mul(x = var_2665_cast_fp16_1, y = var_2679_to_fp16)[name = string("op_2680_cast_fp16")];
            string var_2682_equation_0 = const()[name = string("op_2682_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_2682_cast_fp16 = einsum(equation = var_2682_equation_0, values = (var_2670_cast_fp16_1, var_2680_cast_fp16))[name = string("op_2682_cast_fp16")];
            fp16 var_2683_to_fp16 = const()[name = string("op_2683_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_2684_cast_fp16 = mul(x = var_2665_cast_fp16_2, y = var_2683_to_fp16)[name = string("op_2684_cast_fp16")];
            string var_2686_equation_0 = const()[name = string("op_2686_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_2686_cast_fp16 = einsum(equation = var_2686_equation_0, values = (var_2670_cast_fp16_2, var_2684_cast_fp16))[name = string("op_2686_cast_fp16")];
            fp16 var_2687_to_fp16 = const()[name = string("op_2687_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_2688_cast_fp16 = mul(x = var_2665_cast_fp16_3, y = var_2687_to_fp16)[name = string("op_2688_cast_fp16")];
            string var_2690_equation_0 = const()[name = string("op_2690_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_2690_cast_fp16 = einsum(equation = var_2690_equation_0, values = (var_2670_cast_fp16_3, var_2688_cast_fp16))[name = string("op_2690_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_2691_cast_fp16 = add(x = var_2678_cast_fp16, y = var_2682_cast_fp16)[name = string("op_2691_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_2692_cast_fp16 = add(x = var_2686_cast_fp16, y = var_2690_cast_fp16)[name = string("op_2692_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_2693_cast_fp16 = add(x = var_2691_cast_fp16, y = var_2692_cast_fp16)[name = string("op_2693_cast_fp16")];
            fp16 _inversed_2695_y_0_to_fp16 = const()[name = string("_inversed_2695_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 4]> _inversed_2695_cast_fp16 = mul(x = var_2693_cast_fp16, y = _inversed_2695_y_0_to_fp16)[name = string("_inversed_2695_cast_fp16")];
            string scores_21_equation_0 = const()[name = string("scores_21_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 4]> scores_21_cast_fp16 = einsum(equation = scores_21_equation_0, values = (var_2616_cast_fp16_0, var_2606_cast_fp16_2))[name = string("scores_21_cast_fp16")];
            fp16 var_2698_to_fp16 = const()[name = string("op_2698_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 4]> var_2699_cast_fp16 = mul(x = scores_21_cast_fp16, y = var_2698_to_fp16)[name = string("op_2699_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_2700_cast_fp16 = add(x = var_2699_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_2700_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_2701_cast_fp16 = softmax(axis = var_2584, x = var_2700_cast_fp16)[name = string("op_2701_cast_fp16")];
            tensor<int32, [4]> tile_26 = const()[name = string("tile_26"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_2702_axis_0 = const()[name = string("op_2702_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 4]> var_2702_cast_fp16_0, tensor<fp16, [1, 512, 1, 4]> var_2702_cast_fp16_1, tensor<fp16, [1, 512, 1, 4]> var_2702_cast_fp16_2, tensor<fp16, [1, 512, 1, 4]> var_2702_cast_fp16_3 = split(axis = var_2702_axis_0, split_sizes = tile_26, x = var_2701_cast_fp16)[name = string("op_2702_cast_fp16")];
            tensor<int32, [4]> tile_27 = const()[name = string("tile_27"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_2707_axis_0 = const()[name = string("op_2707_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_2707_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_2707_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_2707_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_2707_cast_fp16_3 = split(axis = var_2707_axis_0, split_sizes = tile_27, x = var_2619_cast_fp16_0)[name = string("op_2707_cast_fp16")];
            fp16 var_2712_to_fp16 = const()[name = string("op_2712_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_2713_cast_fp16 = mul(x = var_2702_cast_fp16_0, y = var_2712_to_fp16)[name = string("op_2713_cast_fp16")];
            string var_2715_equation_0 = const()[name = string("op_2715_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_2715_cast_fp16 = einsum(equation = var_2715_equation_0, values = (var_2707_cast_fp16_0, var_2713_cast_fp16))[name = string("op_2715_cast_fp16")];
            fp16 var_2716_to_fp16 = const()[name = string("op_2716_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_2717_cast_fp16 = mul(x = var_2702_cast_fp16_1, y = var_2716_to_fp16)[name = string("op_2717_cast_fp16")];
            string var_2719_equation_0 = const()[name = string("op_2719_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_2719_cast_fp16 = einsum(equation = var_2719_equation_0, values = (var_2707_cast_fp16_1, var_2717_cast_fp16))[name = string("op_2719_cast_fp16")];
            fp16 var_2720_to_fp16 = const()[name = string("op_2720_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_2721_cast_fp16 = mul(x = var_2702_cast_fp16_2, y = var_2720_to_fp16)[name = string("op_2721_cast_fp16")];
            string var_2723_equation_0 = const()[name = string("op_2723_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_2723_cast_fp16 = einsum(equation = var_2723_equation_0, values = (var_2707_cast_fp16_2, var_2721_cast_fp16))[name = string("op_2723_cast_fp16")];
            fp16 var_2724_to_fp16 = const()[name = string("op_2724_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_2725_cast_fp16 = mul(x = var_2702_cast_fp16_3, y = var_2724_to_fp16)[name = string("op_2725_cast_fp16")];
            string var_2727_equation_0 = const()[name = string("op_2727_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_2727_cast_fp16 = einsum(equation = var_2727_equation_0, values = (var_2707_cast_fp16_3, var_2725_cast_fp16))[name = string("op_2727_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_2728_cast_fp16 = add(x = var_2715_cast_fp16, y = var_2719_cast_fp16)[name = string("op_2728_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_2729_cast_fp16 = add(x = var_2723_cast_fp16, y = var_2727_cast_fp16)[name = string("op_2729_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_2730_cast_fp16 = add(x = var_2728_cast_fp16, y = var_2729_cast_fp16)[name = string("op_2730_cast_fp16")];
            fp16 _inversed_2732_y_0_to_fp16 = const()[name = string("_inversed_2732_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 4]> _inversed_2732_cast_fp16 = mul(x = var_2730_cast_fp16, y = _inversed_2732_y_0_to_fp16)[name = string("_inversed_2732_cast_fp16")];
            string scores_23_equation_0 = const()[name = string("scores_23_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 4]> scores_23_cast_fp16 = einsum(equation = scores_23_equation_0, values = (var_2616_cast_fp16_0, var_2606_cast_fp16_3))[name = string("scores_23_cast_fp16")];
            fp16 var_2735_to_fp16 = const()[name = string("op_2735_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 4]> var_2736_cast_fp16 = mul(x = scores_23_cast_fp16, y = var_2735_to_fp16)[name = string("op_2736_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_2737_cast_fp16 = add(x = var_2736_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_2737_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_2738_cast_fp16 = softmax(axis = var_2584, x = var_2737_cast_fp16)[name = string("op_2738_cast_fp16")];
            tensor<int32, [4]> tile_28 = const()[name = string("tile_28"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_2739_axis_0 = const()[name = string("op_2739_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 4]> var_2739_cast_fp16_0, tensor<fp16, [1, 512, 1, 4]> var_2739_cast_fp16_1, tensor<fp16, [1, 512, 1, 4]> var_2739_cast_fp16_2, tensor<fp16, [1, 512, 1, 4]> var_2739_cast_fp16_3 = split(axis = var_2739_axis_0, split_sizes = tile_28, x = var_2738_cast_fp16)[name = string("op_2739_cast_fp16")];
            tensor<int32, [4]> tile_29 = const()[name = string("tile_29"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_2744_axis_0 = const()[name = string("op_2744_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_2744_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_2744_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_2744_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_2744_cast_fp16_3 = split(axis = var_2744_axis_0, split_sizes = tile_29, x = var_2619_cast_fp16_0)[name = string("op_2744_cast_fp16")];
            fp16 var_2749_to_fp16 = const()[name = string("op_2749_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_2750_cast_fp16 = mul(x = var_2739_cast_fp16_0, y = var_2749_to_fp16)[name = string("op_2750_cast_fp16")];
            string var_2752_equation_0 = const()[name = string("op_2752_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_2752_cast_fp16 = einsum(equation = var_2752_equation_0, values = (var_2744_cast_fp16_0, var_2750_cast_fp16))[name = string("op_2752_cast_fp16")];
            fp16 var_2753_to_fp16 = const()[name = string("op_2753_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_2754_cast_fp16 = mul(x = var_2739_cast_fp16_1, y = var_2753_to_fp16)[name = string("op_2754_cast_fp16")];
            string var_2756_equation_0 = const()[name = string("op_2756_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_2756_cast_fp16 = einsum(equation = var_2756_equation_0, values = (var_2744_cast_fp16_1, var_2754_cast_fp16))[name = string("op_2756_cast_fp16")];
            fp16 var_2757_to_fp16 = const()[name = string("op_2757_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_2758_cast_fp16 = mul(x = var_2739_cast_fp16_2, y = var_2757_to_fp16)[name = string("op_2758_cast_fp16")];
            string var_2760_equation_0 = const()[name = string("op_2760_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_2760_cast_fp16 = einsum(equation = var_2760_equation_0, values = (var_2744_cast_fp16_2, var_2758_cast_fp16))[name = string("op_2760_cast_fp16")];
            fp16 var_2761_to_fp16 = const()[name = string("op_2761_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_2762_cast_fp16 = mul(x = var_2739_cast_fp16_3, y = var_2761_to_fp16)[name = string("op_2762_cast_fp16")];
            string var_2764_equation_0 = const()[name = string("op_2764_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_2764_cast_fp16 = einsum(equation = var_2764_equation_0, values = (var_2744_cast_fp16_3, var_2762_cast_fp16))[name = string("op_2764_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_2765_cast_fp16 = add(x = var_2752_cast_fp16, y = var_2756_cast_fp16)[name = string("op_2765_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_2766_cast_fp16 = add(x = var_2760_cast_fp16, y = var_2764_cast_fp16)[name = string("op_2766_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_2767_cast_fp16 = add(x = var_2765_cast_fp16, y = var_2766_cast_fp16)[name = string("op_2767_cast_fp16")];
            fp16 _inversed_2769_y_0_to_fp16 = const()[name = string("_inversed_2769_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 4]> _inversed_2769_cast_fp16 = mul(x = var_2767_cast_fp16, y = _inversed_2769_y_0_to_fp16)[name = string("_inversed_2769_cast_fp16")];
            string scores_25_equation_0 = const()[name = string("scores_25_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 4]> scores_25_cast_fp16 = einsum(equation = scores_25_equation_0, values = (var_2616_cast_fp16_1, var_2606_cast_fp16_4))[name = string("scores_25_cast_fp16")];
            fp16 var_2772_to_fp16 = const()[name = string("op_2772_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 4]> var_2773_cast_fp16 = mul(x = scores_25_cast_fp16, y = var_2772_to_fp16)[name = string("op_2773_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_2774_cast_fp16 = add(x = var_2773_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_2774_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_2775_cast_fp16 = softmax(axis = var_2584, x = var_2774_cast_fp16)[name = string("op_2775_cast_fp16")];
            tensor<int32, [4]> tile_30 = const()[name = string("tile_30"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_2776_axis_0 = const()[name = string("op_2776_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 4]> var_2776_cast_fp16_0, tensor<fp16, [1, 512, 1, 4]> var_2776_cast_fp16_1, tensor<fp16, [1, 512, 1, 4]> var_2776_cast_fp16_2, tensor<fp16, [1, 512, 1, 4]> var_2776_cast_fp16_3 = split(axis = var_2776_axis_0, split_sizes = tile_30, x = var_2775_cast_fp16)[name = string("op_2776_cast_fp16")];
            tensor<int32, [4]> tile_31 = const()[name = string("tile_31"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_2781_axis_0 = const()[name = string("op_2781_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_2781_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_2781_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_2781_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_2781_cast_fp16_3 = split(axis = var_2781_axis_0, split_sizes = tile_31, x = var_2619_cast_fp16_1)[name = string("op_2781_cast_fp16")];
            fp16 var_2786_to_fp16 = const()[name = string("op_2786_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_2787_cast_fp16 = mul(x = var_2776_cast_fp16_0, y = var_2786_to_fp16)[name = string("op_2787_cast_fp16")];
            string var_2789_equation_0 = const()[name = string("op_2789_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_2789_cast_fp16 = einsum(equation = var_2789_equation_0, values = (var_2781_cast_fp16_0, var_2787_cast_fp16))[name = string("op_2789_cast_fp16")];
            fp16 var_2790_to_fp16 = const()[name = string("op_2790_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_2791_cast_fp16 = mul(x = var_2776_cast_fp16_1, y = var_2790_to_fp16)[name = string("op_2791_cast_fp16")];
            string var_2793_equation_0 = const()[name = string("op_2793_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_2793_cast_fp16 = einsum(equation = var_2793_equation_0, values = (var_2781_cast_fp16_1, var_2791_cast_fp16))[name = string("op_2793_cast_fp16")];
            fp16 var_2794_to_fp16 = const()[name = string("op_2794_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_2795_cast_fp16 = mul(x = var_2776_cast_fp16_2, y = var_2794_to_fp16)[name = string("op_2795_cast_fp16")];
            string var_2797_equation_0 = const()[name = string("op_2797_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_2797_cast_fp16 = einsum(equation = var_2797_equation_0, values = (var_2781_cast_fp16_2, var_2795_cast_fp16))[name = string("op_2797_cast_fp16")];
            fp16 var_2798_to_fp16 = const()[name = string("op_2798_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_2799_cast_fp16 = mul(x = var_2776_cast_fp16_3, y = var_2798_to_fp16)[name = string("op_2799_cast_fp16")];
            string var_2801_equation_0 = const()[name = string("op_2801_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_2801_cast_fp16 = einsum(equation = var_2801_equation_0, values = (var_2781_cast_fp16_3, var_2799_cast_fp16))[name = string("op_2801_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_2802_cast_fp16 = add(x = var_2789_cast_fp16, y = var_2793_cast_fp16)[name = string("op_2802_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_2803_cast_fp16 = add(x = var_2797_cast_fp16, y = var_2801_cast_fp16)[name = string("op_2803_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_2804_cast_fp16 = add(x = var_2802_cast_fp16, y = var_2803_cast_fp16)[name = string("op_2804_cast_fp16")];
            fp16 _inversed_2806_y_0_to_fp16 = const()[name = string("_inversed_2806_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 4]> _inversed_2806_cast_fp16 = mul(x = var_2804_cast_fp16, y = _inversed_2806_y_0_to_fp16)[name = string("_inversed_2806_cast_fp16")];
            string scores_27_equation_0 = const()[name = string("scores_27_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 4]> scores_27_cast_fp16 = einsum(equation = scores_27_equation_0, values = (var_2616_cast_fp16_1, var_2606_cast_fp16_5))[name = string("scores_27_cast_fp16")];
            fp16 var_2809_to_fp16 = const()[name = string("op_2809_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 4]> var_2810_cast_fp16 = mul(x = scores_27_cast_fp16, y = var_2809_to_fp16)[name = string("op_2810_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_2811_cast_fp16 = add(x = var_2810_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_2811_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_2812_cast_fp16 = softmax(axis = var_2584, x = var_2811_cast_fp16)[name = string("op_2812_cast_fp16")];
            tensor<int32, [4]> tile_32 = const()[name = string("tile_32"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_2813_axis_0 = const()[name = string("op_2813_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 4]> var_2813_cast_fp16_0, tensor<fp16, [1, 512, 1, 4]> var_2813_cast_fp16_1, tensor<fp16, [1, 512, 1, 4]> var_2813_cast_fp16_2, tensor<fp16, [1, 512, 1, 4]> var_2813_cast_fp16_3 = split(axis = var_2813_axis_0, split_sizes = tile_32, x = var_2812_cast_fp16)[name = string("op_2813_cast_fp16")];
            tensor<int32, [4]> tile_33 = const()[name = string("tile_33"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_2818_axis_0 = const()[name = string("op_2818_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_2818_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_2818_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_2818_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_2818_cast_fp16_3 = split(axis = var_2818_axis_0, split_sizes = tile_33, x = var_2619_cast_fp16_1)[name = string("op_2818_cast_fp16")];
            fp16 var_2823_to_fp16 = const()[name = string("op_2823_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_2824_cast_fp16 = mul(x = var_2813_cast_fp16_0, y = var_2823_to_fp16)[name = string("op_2824_cast_fp16")];
            string var_2826_equation_0 = const()[name = string("op_2826_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_2826_cast_fp16 = einsum(equation = var_2826_equation_0, values = (var_2818_cast_fp16_0, var_2824_cast_fp16))[name = string("op_2826_cast_fp16")];
            fp16 var_2827_to_fp16 = const()[name = string("op_2827_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_2828_cast_fp16 = mul(x = var_2813_cast_fp16_1, y = var_2827_to_fp16)[name = string("op_2828_cast_fp16")];
            string var_2830_equation_0 = const()[name = string("op_2830_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_2830_cast_fp16 = einsum(equation = var_2830_equation_0, values = (var_2818_cast_fp16_1, var_2828_cast_fp16))[name = string("op_2830_cast_fp16")];
            fp16 var_2831_to_fp16 = const()[name = string("op_2831_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_2832_cast_fp16 = mul(x = var_2813_cast_fp16_2, y = var_2831_to_fp16)[name = string("op_2832_cast_fp16")];
            string var_2834_equation_0 = const()[name = string("op_2834_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_2834_cast_fp16 = einsum(equation = var_2834_equation_0, values = (var_2818_cast_fp16_2, var_2832_cast_fp16))[name = string("op_2834_cast_fp16")];
            fp16 var_2835_to_fp16 = const()[name = string("op_2835_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_2836_cast_fp16 = mul(x = var_2813_cast_fp16_3, y = var_2835_to_fp16)[name = string("op_2836_cast_fp16")];
            string var_2838_equation_0 = const()[name = string("op_2838_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_2838_cast_fp16 = einsum(equation = var_2838_equation_0, values = (var_2818_cast_fp16_3, var_2836_cast_fp16))[name = string("op_2838_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_2839_cast_fp16 = add(x = var_2826_cast_fp16, y = var_2830_cast_fp16)[name = string("op_2839_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_2840_cast_fp16 = add(x = var_2834_cast_fp16, y = var_2838_cast_fp16)[name = string("op_2840_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_2841_cast_fp16 = add(x = var_2839_cast_fp16, y = var_2840_cast_fp16)[name = string("op_2841_cast_fp16")];
            fp16 _inversed_2843_y_0_to_fp16 = const()[name = string("_inversed_2843_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 4]> _inversed_2843_cast_fp16 = mul(x = var_2841_cast_fp16, y = _inversed_2843_y_0_to_fp16)[name = string("_inversed_2843_cast_fp16")];
            string scores_29_equation_0 = const()[name = string("scores_29_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 4]> scores_29_cast_fp16 = einsum(equation = scores_29_equation_0, values = (var_2616_cast_fp16_1, var_2606_cast_fp16_6))[name = string("scores_29_cast_fp16")];
            fp16 var_2846_to_fp16 = const()[name = string("op_2846_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 4]> var_2847_cast_fp16 = mul(x = scores_29_cast_fp16, y = var_2846_to_fp16)[name = string("op_2847_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_2848_cast_fp16 = add(x = var_2847_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_2848_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_2849_cast_fp16 = softmax(axis = var_2584, x = var_2848_cast_fp16)[name = string("op_2849_cast_fp16")];
            tensor<int32, [4]> tile_34 = const()[name = string("tile_34"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_2850_axis_0 = const()[name = string("op_2850_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 4]> var_2850_cast_fp16_0, tensor<fp16, [1, 512, 1, 4]> var_2850_cast_fp16_1, tensor<fp16, [1, 512, 1, 4]> var_2850_cast_fp16_2, tensor<fp16, [1, 512, 1, 4]> var_2850_cast_fp16_3 = split(axis = var_2850_axis_0, split_sizes = tile_34, x = var_2849_cast_fp16)[name = string("op_2850_cast_fp16")];
            tensor<int32, [4]> tile_35 = const()[name = string("tile_35"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_2855_axis_0 = const()[name = string("op_2855_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_2855_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_2855_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_2855_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_2855_cast_fp16_3 = split(axis = var_2855_axis_0, split_sizes = tile_35, x = var_2619_cast_fp16_1)[name = string("op_2855_cast_fp16")];
            fp16 var_2860_to_fp16 = const()[name = string("op_2860_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_2861_cast_fp16 = mul(x = var_2850_cast_fp16_0, y = var_2860_to_fp16)[name = string("op_2861_cast_fp16")];
            string var_2863_equation_0 = const()[name = string("op_2863_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_2863_cast_fp16 = einsum(equation = var_2863_equation_0, values = (var_2855_cast_fp16_0, var_2861_cast_fp16))[name = string("op_2863_cast_fp16")];
            fp16 var_2864_to_fp16 = const()[name = string("op_2864_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_2865_cast_fp16 = mul(x = var_2850_cast_fp16_1, y = var_2864_to_fp16)[name = string("op_2865_cast_fp16")];
            string var_2867_equation_0 = const()[name = string("op_2867_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_2867_cast_fp16 = einsum(equation = var_2867_equation_0, values = (var_2855_cast_fp16_1, var_2865_cast_fp16))[name = string("op_2867_cast_fp16")];
            fp16 var_2868_to_fp16 = const()[name = string("op_2868_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_2869_cast_fp16 = mul(x = var_2850_cast_fp16_2, y = var_2868_to_fp16)[name = string("op_2869_cast_fp16")];
            string var_2871_equation_0 = const()[name = string("op_2871_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_2871_cast_fp16 = einsum(equation = var_2871_equation_0, values = (var_2855_cast_fp16_2, var_2869_cast_fp16))[name = string("op_2871_cast_fp16")];
            fp16 var_2872_to_fp16 = const()[name = string("op_2872_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_2873_cast_fp16 = mul(x = var_2850_cast_fp16_3, y = var_2872_to_fp16)[name = string("op_2873_cast_fp16")];
            string var_2875_equation_0 = const()[name = string("op_2875_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_2875_cast_fp16 = einsum(equation = var_2875_equation_0, values = (var_2855_cast_fp16_3, var_2873_cast_fp16))[name = string("op_2875_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_2876_cast_fp16 = add(x = var_2863_cast_fp16, y = var_2867_cast_fp16)[name = string("op_2876_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_2877_cast_fp16 = add(x = var_2871_cast_fp16, y = var_2875_cast_fp16)[name = string("op_2877_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_2878_cast_fp16 = add(x = var_2876_cast_fp16, y = var_2877_cast_fp16)[name = string("op_2878_cast_fp16")];
            fp16 _inversed_2880_y_0_to_fp16 = const()[name = string("_inversed_2880_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 4]> _inversed_2880_cast_fp16 = mul(x = var_2878_cast_fp16, y = _inversed_2880_y_0_to_fp16)[name = string("_inversed_2880_cast_fp16")];
            string scores_31_equation_0 = const()[name = string("scores_31_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 4]> scores_31_cast_fp16 = einsum(equation = scores_31_equation_0, values = (var_2616_cast_fp16_1, var_2606_cast_fp16_7))[name = string("scores_31_cast_fp16")];
            fp16 var_2883_to_fp16 = const()[name = string("op_2883_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 4]> var_2884_cast_fp16 = mul(x = scores_31_cast_fp16, y = var_2883_to_fp16)[name = string("op_2884_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_2885_cast_fp16 = add(x = var_2884_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_2885_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_2886_cast_fp16 = softmax(axis = var_2584, x = var_2885_cast_fp16)[name = string("op_2886_cast_fp16")];
            tensor<int32, [4]> tile_36 = const()[name = string("tile_36"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_2887_axis_0 = const()[name = string("op_2887_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 4]> var_2887_cast_fp16_0, tensor<fp16, [1, 512, 1, 4]> var_2887_cast_fp16_1, tensor<fp16, [1, 512, 1, 4]> var_2887_cast_fp16_2, tensor<fp16, [1, 512, 1, 4]> var_2887_cast_fp16_3 = split(axis = var_2887_axis_0, split_sizes = tile_36, x = var_2886_cast_fp16)[name = string("op_2887_cast_fp16")];
            tensor<int32, [4]> tile_37 = const()[name = string("tile_37"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_2892_axis_0 = const()[name = string("op_2892_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_2892_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_2892_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_2892_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_2892_cast_fp16_3 = split(axis = var_2892_axis_0, split_sizes = tile_37, x = var_2619_cast_fp16_1)[name = string("op_2892_cast_fp16")];
            fp16 var_2897_to_fp16 = const()[name = string("op_2897_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_2898_cast_fp16 = mul(x = var_2887_cast_fp16_0, y = var_2897_to_fp16)[name = string("op_2898_cast_fp16")];
            string var_2900_equation_0 = const()[name = string("op_2900_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_2900_cast_fp16 = einsum(equation = var_2900_equation_0, values = (var_2892_cast_fp16_0, var_2898_cast_fp16))[name = string("op_2900_cast_fp16")];
            fp16 var_2901_to_fp16 = const()[name = string("op_2901_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_2902_cast_fp16 = mul(x = var_2887_cast_fp16_1, y = var_2901_to_fp16)[name = string("op_2902_cast_fp16")];
            string var_2904_equation_0 = const()[name = string("op_2904_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_2904_cast_fp16 = einsum(equation = var_2904_equation_0, values = (var_2892_cast_fp16_1, var_2902_cast_fp16))[name = string("op_2904_cast_fp16")];
            fp16 var_2905_to_fp16 = const()[name = string("op_2905_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_2906_cast_fp16 = mul(x = var_2887_cast_fp16_2, y = var_2905_to_fp16)[name = string("op_2906_cast_fp16")];
            string var_2908_equation_0 = const()[name = string("op_2908_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_2908_cast_fp16 = einsum(equation = var_2908_equation_0, values = (var_2892_cast_fp16_2, var_2906_cast_fp16))[name = string("op_2908_cast_fp16")];
            fp16 var_2909_to_fp16 = const()[name = string("op_2909_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_2910_cast_fp16 = mul(x = var_2887_cast_fp16_3, y = var_2909_to_fp16)[name = string("op_2910_cast_fp16")];
            string var_2912_equation_0 = const()[name = string("op_2912_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_2912_cast_fp16 = einsum(equation = var_2912_equation_0, values = (var_2892_cast_fp16_3, var_2910_cast_fp16))[name = string("op_2912_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_2913_cast_fp16 = add(x = var_2900_cast_fp16, y = var_2904_cast_fp16)[name = string("op_2913_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_2914_cast_fp16 = add(x = var_2908_cast_fp16, y = var_2912_cast_fp16)[name = string("op_2914_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_2915_cast_fp16 = add(x = var_2913_cast_fp16, y = var_2914_cast_fp16)[name = string("op_2915_cast_fp16")];
            fp16 _inversed_2917_y_0_to_fp16 = const()[name = string("_inversed_2917_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 4]> _inversed_2917_cast_fp16 = mul(x = var_2915_cast_fp16, y = _inversed_2917_y_0_to_fp16)[name = string("_inversed_2917_cast_fp16")];
            bool var_2919_interleave_0 = const()[name = string("op_2919_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 2048, 1, 4]> var_2919_cast_fp16 = concat(axis = var_2584, interleave = var_2919_interleave_0, values = (_inversed_2658_cast_fp16, _inversed_2695_cast_fp16, _inversed_2732_cast_fp16, _inversed_2769_cast_fp16, _inversed_2806_cast_fp16, _inversed_2843_cast_fp16, _inversed_2880_cast_fp16, _inversed_2917_cast_fp16))[name = string("op_2919_cast_fp16")];
            tensor<int32, [4]> var_2920 = const()[name = string("op_2920"), val = tensor<int32, [4]>([2, 4, 256, 4])];
            tensor<fp16, [2, 4, 256, 4]> var_2921_cast_fp16 = reshape(shape = var_2920, x = var_2919_cast_fp16)[name = string("op_2921_cast_fp16")];
            tensor<int32, [4]> transpose_3_perm_0 = const()[name = string("transpose_3_perm_0"), val = tensor<int32, [4]>([3, 0, 1, 2])];
            tensor<int32, [2]> var_2940 = const()[name = string("op_2940"), val = tensor<int32, [2]>([4, 2048])];
            tensor<fp16, [4, 2, 4, 256]> transpose_3_cast_fp16 = transpose(perm = transpose_3_perm_0, x = var_2921_cast_fp16)[name = string("transpose_83")];
            tensor<fp16, [4, 2048]> attention_output_7_cast_fp16 = reshape(shape = var_2940, x = transpose_3_cast_fp16)[name = string("attention_output_7_cast_fp16")];
            tensor<fp16, [4, 2048]> var_2942_cast_fp16 = sigmoid(x = gate_7_cast_fp16)[name = string("op_2942_cast_fp16")];
            tensor<fp16, [4, 2048]> input_97_cast_fp16 = mul(x = attention_output_7_cast_fp16, y = var_2942_cast_fp16)[name = string("input_97_cast_fp16")];
            tensor<fp16, [1024, 2048]> completions_1_o_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 2048]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(116052608))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(117625536))))[name = string("completions_1_o_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 1024]> linear_58_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = completions_1_o_proj_weight_promoted_to_fp16_palettized, x = input_97_cast_fp16)[name = string("linear_58_cast_fp16")];
            tensor<fp16, [4, 1024]> value_115_cast_fp16 = add(x = value_101_cast_fp16, y = linear_58_cast_fp16)[name = string("value_115_cast_fp16")];
            tensor<fp16, [4, 1024]> var_2947_cast_fp16 = mul(x = value_115_cast_fp16, y = value_115_cast_fp16)[name = string("op_2947_cast_fp16")];
            tensor<int32, [1]> variance_51_axes_0 = const()[name = string("variance_51_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_51_keep_dims_0 = const()[name = string("variance_51_keep_dims_0"), val = bool(true)];
            tensor<fp16, [4, 1]> variance_51_cast_fp16 = reduce_mean(axes = variance_51_axes_0, keep_dims = variance_51_keep_dims_0, x = var_2947_cast_fp16)[name = string("variance_51_cast_fp16")];
            fp16 var_2950_to_fp16 = const()[name = string("op_2950_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [4, 1]> var_2951_cast_fp16 = add(x = variance_51_cast_fp16, y = var_2950_to_fp16)[name = string("op_2951_cast_fp16")];
            fp32 var_2952_epsilon_0 = const()[name = string("op_2952_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [4, 1]> var_2952_cast_fp16 = rsqrt(epsilon = var_2952_epsilon_0, x = var_2951_cast_fp16)[name = string("op_2952_cast_fp16")];
            tensor<fp16, [4, 1024]> var_2953_cast_fp16 = mul(x = value_115_cast_fp16, y = var_2952_cast_fp16)[name = string("op_2953_cast_fp16")];
            tensor<fp16, [1024]> var_2955_to_fp16 = const()[name = string("op_2955_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(117633792)))];
            tensor<fp16, [4, 1024]> input_99_cast_fp16 = mul(x = var_2953_cast_fp16, y = var_2955_to_fp16)[name = string("input_99_cast_fp16")];
            tensor<fp16, [3584, 1024]> completions_1_gate_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(117635904))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(120388480))))[name = string("completions_1_gate_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 3584]> linear_59_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = completions_1_gate_proj_weight_promoted_to_fp16_palettized, x = input_99_cast_fp16)[name = string("linear_59_cast_fp16")];
            tensor<fp16, [4, 3584]> var_2959_cast_fp16 = silu(x = linear_59_cast_fp16)[name = string("op_2959_cast_fp16")];
            tensor<fp16, [3584, 1024]> completions_1_up_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(120417216))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(123169792))))[name = string("completions_1_up_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 3584]> linear_60_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = completions_1_up_proj_weight_promoted_to_fp16_palettized, x = input_99_cast_fp16)[name = string("linear_60_cast_fp16")];
            tensor<fp16, [4, 3584]> input_103_cast_fp16 = mul(x = var_2959_cast_fp16, y = linear_60_cast_fp16)[name = string("input_103_cast_fp16")];
            tensor<fp16, [1024, 3584]> completions_1_down_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 3584]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(123198528))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(125951104))))[name = string("completions_1_down_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 1024]> linear_61_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = completions_1_down_proj_weight_promoted_to_fp16_palettized, x = input_103_cast_fp16)[name = string("linear_61_cast_fp16")];
            tensor<fp16, [4, 1024]> value_117_cast_fp16 = add(x = value_115_cast_fp16, y = linear_61_cast_fp16)[name = string("value_117_cast_fp16")];
            int32 var_2982 = const()[name = string("op_2982"), val = int32(-1)];
            tensor<fp16, [4, 1024]> var_2997_cast_fp16 = mul(x = value_117_cast_fp16, y = value_117_cast_fp16)[name = string("op_2997_cast_fp16")];
            tensor<int32, [1]> variance_53_axes_0 = const()[name = string("variance_53_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_53_keep_dims_0 = const()[name = string("variance_53_keep_dims_0"), val = bool(true)];
            tensor<fp16, [4, 1]> variance_53_cast_fp16 = reduce_mean(axes = variance_53_axes_0, keep_dims = variance_53_keep_dims_0, x = var_2997_cast_fp16)[name = string("variance_53_cast_fp16")];
            fp16 var_3000_to_fp16 = const()[name = string("op_3000_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [4, 1]> var_3001_cast_fp16 = add(x = variance_53_cast_fp16, y = var_3000_to_fp16)[name = string("op_3001_cast_fp16")];
            fp32 var_3002_epsilon_0 = const()[name = string("op_3002_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [4, 1]> var_3002_cast_fp16 = rsqrt(epsilon = var_3002_epsilon_0, x = var_3001_cast_fp16)[name = string("op_3002_cast_fp16")];
            tensor<fp16, [4, 1024]> var_3003_cast_fp16 = mul(x = value_117_cast_fp16, y = var_3002_cast_fp16)[name = string("op_3003_cast_fp16")];
            tensor<fp16, [1024]> var_3005_to_fp16 = const()[name = string("op_3005_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(125959360)))];
            tensor<fp16, [4, 1024]> input_105_cast_fp16 = mul(x = var_3003_cast_fp16, y = var_3005_to_fp16)[name = string("input_105_cast_fp16")];
            tensor<fp16, [6144, 1024]> groups_2_0_in_proj_qkv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(125961472))), lut = tensor<fp16, [384, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(130680128))))[name = string("groups_2_0_in_proj_qkv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 6144]> linear_62_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = groups_2_0_in_proj_qkv_weight_promoted_to_fp16_palettized, x = input_105_cast_fp16)[name = string("linear_62_cast_fp16")];
            tensor<int32, [2]> var_3009_perm_0 = const()[name = string("op_3009_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> mixed_13_axes_0 = const()[name = string("mixed_13_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [6144, 4]> var_3009_cast_fp16 = transpose(perm = var_3009_perm_0, x = linear_62_cast_fp16)[name = string("transpose_82")];
            tensor<fp16, [1, 6144, 4]> mixed_13_cast_fp16 = expand_dims(axes = mixed_13_axes_0, x = var_3009_cast_fp16)[name = string("mixed_13_cast_fp16")];
            bool convolution_input_13_interleave_0 = const()[name = string("convolution_input_13_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 6144, 7]> convolution_input_13_cast_fp16 = concat(axis = var_2982, interleave = convolution_input_13_interleave_0, values = (conv8, mixed_13_cast_fp16))[name = string("convolution_input_13_cast_fp16")];
            string input_107_pad_type_0 = const()[name = string("input_107_pad_type_0"), val = string("valid")];
            int32 input_107_groups_0 = const()[name = string("input_107_groups_0"), val = int32(6144)];
            tensor<int32, [1]> input_107_strides_0 = const()[name = string("input_107_strides_0"), val = tensor<int32, [1]>([1])];
            tensor<int32, [2]> input_107_pad_0 = const()[name = string("input_107_pad_0"), val = tensor<int32, [2]>([0, 0])];
            tensor<int32, [1]> input_107_dilations_0 = const()[name = string("input_107_dilations_0"), val = tensor<int32, [1]>([1])];
            tensor<fp16, [6144, 1, 4]> groups_2_0_conv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1, 4]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(130729344))), lut = tensor<fp16, [384, 1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(130747840))))[name = string("groups_2_0_conv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 6144, 4]> input_107_cast_fp16 = conv(dilations = input_107_dilations_0, groups = input_107_groups_0, pad = input_107_pad_0, pad_type = input_107_pad_type_0, strides = input_107_strides_0, weight = groups_2_0_conv_weight_promoted_to_fp16_palettized, x = convolution_input_13_cast_fp16)[name = string("input_107_cast_fp16")];
            tensor<fp16, [1, 6144, 4]> var_3018_cast_fp16 = silu(x = input_107_cast_fp16)[name = string("op_3018_cast_fp16")];
            tensor<int32, [3]> var_3019_perm_0 = const()[name = string("op_3019_perm_0"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [3]> var_3022_begin_0 = const()[name = string("op_3022_begin_0"), val = tensor<int32, [3]>([0, 0, 4])];
            tensor<int32, [3]> var_3022_end_0 = const()[name = string("op_3022_end_0"), val = tensor<int32, [3]>([1, 6144, 7])];
            tensor<bool, [3]> var_3022_end_mask_0 = const()[name = string("op_3022_end_mask_0"), val = tensor<bool, [3]>([true, true, true])];
            tensor<fp16, [1, 6144, 3]> conv8_out = slice_by_index(begin = var_3022_begin_0, end = var_3022_end_0, end_mask = var_3022_end_mask_0, x = convolution_input_13_cast_fp16)[name = string("op_3022_cast_fp16")];
            tensor<int32, [3]> var_3023 = const()[name = string("op_3023"), val = tensor<int32, [3]>([2048, 2048, 2048])];
            int32 var_3024_axis_0 = const()[name = string("op_3024_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 4, 6144]> var_3019_cast_fp16 = transpose(perm = var_3019_perm_0, x = var_3018_cast_fp16)[name = string("transpose_81")];
            tensor<fp16, [1, 4, 2048]> var_3024_cast_fp16_0, tensor<fp16, [1, 4, 2048]> var_3024_cast_fp16_1, tensor<fp16, [1, 4, 2048]> var_3024_cast_fp16_2 = split(axis = var_3024_axis_0, split_sizes = var_3023, x = var_3019_cast_fp16)[name = string("op_3024_cast_fp16")];
            tensor<int32, [4]> var_3028 = const()[name = string("op_3028"), val = tensor<int32, [4]>([1, 4, 16, 128])];
            tensor<fp16, [1, 4, 16, 128]> value_121_cast_fp16 = reshape(shape = var_3028, x = var_3024_cast_fp16_0)[name = string("value_121_cast_fp16")];
            tensor<int32, [4]> var_3030 = const()[name = string("op_3030"), val = tensor<int32, [4]>([1, 4, 16, 128])];
            tensor<fp16, [1, 4, 16, 128]> value_123_cast_fp16 = reshape(shape = var_3030, x = var_3024_cast_fp16_1)[name = string("value_123_cast_fp16")];
            tensor<int32, [4]> var_3032 = const()[name = string("op_3032"), val = tensor<int32, [4]>([1, 4, 16, 128])];
            tensor<fp16, [1, 4, 16, 128]> value_125_cast_fp16 = reshape(shape = var_3032, x = var_3024_cast_fp16_2)[name = string("value_125_cast_fp16")];
            tensor<fp16, [1, 4, 16, 128]> var_3034_cast_fp16 = mul(x = value_121_cast_fp16, y = value_121_cast_fp16)[name = string("op_3034_cast_fp16")];
            tensor<int32, [1]> var_3036_axes_0 = const()[name = string("op_3036_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_3036_keep_dims_0 = const()[name = string("op_3036_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 4, 16, 1]> var_3036_cast_fp16 = reduce_sum(axes = var_3036_axes_0, keep_dims = var_3036_keep_dims_0, x = var_3034_cast_fp16)[name = string("op_3036_cast_fp16")];
            fp16 var_3037_to_fp16 = const()[name = string("op_3037_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 4, 16, 1]> var_3038_cast_fp16 = add(x = var_3036_cast_fp16, y = var_3037_to_fp16)[name = string("op_3038_cast_fp16")];
            fp32 var_3039_epsilon_0 = const()[name = string("op_3039_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 4, 16, 1]> var_3039_cast_fp16 = rsqrt(epsilon = var_3039_epsilon_0, x = var_3038_cast_fp16)[name = string("op_3039_cast_fp16")];
            tensor<fp16, [1, 4, 16, 128]> var_3040_cast_fp16 = mul(x = value_121_cast_fp16, y = var_3039_cast_fp16)[name = string("op_3040_cast_fp16")];
            tensor<int32, [4]> var_3041_perm_0 = const()[name = string("op_3041_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            fp16 _inversed_query_35_y_0_to_fp16 = const()[name = string("_inversed_query_35_y_0_to_fp16"), val = fp16(0x1.6ap-4)];
            tensor<fp16, [1, 16, 4, 128]> var_3041_cast_fp16 = transpose(perm = var_3041_perm_0, x = var_3040_cast_fp16)[name = string("transpose_80")];
            tensor<fp16, [1, 16, 4, 128]> _inversed_query_35_cast_fp16 = mul(x = var_3041_cast_fp16, y = _inversed_query_35_y_0_to_fp16)[name = string("_inversed_query_35_cast_fp16")];
            tensor<fp16, [1, 4, 16, 128]> var_3044_cast_fp16 = mul(x = value_123_cast_fp16, y = value_123_cast_fp16)[name = string("op_3044_cast_fp16")];
            tensor<int32, [1]> var_3046_axes_0 = const()[name = string("op_3046_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_3046_keep_dims_0 = const()[name = string("op_3046_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 4, 16, 1]> var_3046_cast_fp16 = reduce_sum(axes = var_3046_axes_0, keep_dims = var_3046_keep_dims_0, x = var_3044_cast_fp16)[name = string("op_3046_cast_fp16")];
            fp16 var_3047_to_fp16 = const()[name = string("op_3047_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 4, 16, 1]> var_3048_cast_fp16 = add(x = var_3046_cast_fp16, y = var_3047_to_fp16)[name = string("op_3048_cast_fp16")];
            fp32 var_3049_epsilon_0 = const()[name = string("op_3049_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 4, 16, 1]> var_3049_cast_fp16 = rsqrt(epsilon = var_3049_epsilon_0, x = var_3048_cast_fp16)[name = string("op_3049_cast_fp16")];
            tensor<fp16, [1, 4, 16, 128]> var_3050_cast_fp16 = mul(x = value_123_cast_fp16, y = var_3049_cast_fp16)[name = string("op_3050_cast_fp16")];
            tensor<int32, [4]> key_35_perm_0 = const()[name = string("key_35_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<int32, [4]> value_127_perm_0 = const()[name = string("value_127_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<fp16, [16, 1024]> groups_2_0_in_proj_b_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(130797056))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(130809408))))[name = string("groups_2_0_in_proj_b_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16]> linear_63_bias_0_to_fp16 = const()[name = string("linear_63_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(4862656)))];
            tensor<fp16, [4, 16]> linear_63_cast_fp16 = linear(bias = linear_63_bias_0_to_fp16, weight = groups_2_0_in_proj_b_weight_promoted_to_fp16_palettized, x = input_105_cast_fp16)[name = string("linear_63_cast_fp16")];
            tensor<fp16, [4, 16]> var_3055_cast_fp16 = sigmoid(x = linear_63_cast_fp16)[name = string("op_3055_cast_fp16")];
            tensor<int32, [2]> var_3056_perm_0 = const()[name = string("op_3056_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> beta_13_axes_0 = const()[name = string("beta_13_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 4]> var_3056_cast_fp16 = transpose(perm = var_3056_perm_0, x = var_3055_cast_fp16)[name = string("transpose_79")];
            tensor<fp16, [1, 16, 4]> beta_13_cast_fp16 = expand_dims(axes = beta_13_axes_0, x = var_3056_cast_fp16)[name = string("beta_13_cast_fp16")];
            tensor<fp16, [16, 1024]> op_3062_weight_0_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(130809600))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(130821952))))[name = string("op_3062_weight_0_to_fp16_palettized")];
            tensor<fp16, [16]> var_3062_bias_0_to_fp16 = const()[name = string("op_3062_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(130822144)))];
            tensor<fp16, [4, 16]> var_3062_cast_fp16 = linear(bias = var_3062_bias_0_to_fp16, weight = op_3062_weight_0_to_fp16_palettized, x = input_105_cast_fp16)[name = string("op_3062_cast_fp16")];
            tensor<fp16, [4, 16]> var_3063_cast_fp16 = softplus(x = var_3062_cast_fp16)[name = string("op_3063_cast_fp16")];
            tensor<fp16, [16]> var_3059_promoted_to_fp16 = const()[name = string("op_3059_promoted_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(130822272)))];
            tensor<fp16, [4, 16]> var_3064_cast_fp16 = mul(x = var_3059_promoted_to_fp16, y = var_3063_cast_fp16)[name = string("op_3064_cast_fp16")];
            tensor<int32, [2]> var_3065_perm_0 = const()[name = string("op_3065_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> decay_13_axes_0 = const()[name = string("decay_13_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 4]> var_3065_cast_fp16 = transpose(perm = var_3065_perm_0, x = var_3064_cast_fp16)[name = string("transpose_78")];
            tensor<fp16, [1, 16, 4]> decay_13_cast_fp16 = expand_dims(axes = decay_13_axes_0, x = var_3065_cast_fp16)[name = string("decay_13_cast_fp16")];
            tensor<fp16, [2048, 1024]> groups_2_0_in_proj_z_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [2048, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(130822400))), lut = tensor<fp16, [128, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(132395328))))[name = string("groups_2_0_in_proj_z_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 2048]> linear_65_cast_fp16 = linear(bias = linear_3_bias_0_to_fp16, weight = groups_2_0_in_proj_z_weight_promoted_to_fp16_palettized, x = input_105_cast_fp16)[name = string("linear_65_cast_fp16")];
            tensor<int32, [3]> var_3073_begin_0 = const()[name = string("op_3073_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_3073_end_0 = const()[name = string("op_3073_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_3073_end_mask_0 = const()[name = string("op_3073_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_3073_squeeze_mask_0 = const()[name = string("op_3073_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_3073_cast_fp16 = slice_by_index(begin = var_3073_begin_0, end = var_3073_end_0, end_mask = var_3073_end_mask_0, squeeze_mask = var_3073_squeeze_mask_0, x = decay_13_cast_fp16)[name = string("op_3073_cast_fp16")];
            tensor<fp16, [1, 16]> var_3074_cast_fp16 = exp(x = var_3073_cast_fp16)[name = string("op_3074_cast_fp16")];
            tensor<int32, [1]> var_3075_axes_0 = const()[name = string("op_3075_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_3075_cast_fp16 = expand_dims(axes = var_3075_axes_0, x = var_3074_cast_fp16)[name = string("op_3075_cast_fp16")];
            tensor<int32, [1]> var_3076_axes_0 = const()[name = string("op_3076_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_3076_cast_fp16 = expand_dims(axes = var_3076_axes_0, x = var_3075_cast_fp16)[name = string("op_3076_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_97_cast_fp16 = mul(x = rec8, y = var_3076_cast_fp16)[name = string("state_97_cast_fp16")];
            tensor<int32, [4]> key_token_49_begin_0 = const()[name = string("key_token_49_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> key_token_49_end_0 = const()[name = string("key_token_49_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> key_token_49_end_mask_0 = const()[name = string("key_token_49_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_49_squeeze_mask_0 = const()[name = string("key_token_49_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 4, 128]> key_35_cast_fp16 = transpose(perm = key_35_perm_0, x = var_3050_cast_fp16)[name = string("transpose_77")];
            tensor<fp16, [1, 16, 128]> key_token_49_cast_fp16 = slice_by_index(begin = key_token_49_begin_0, end = key_token_49_end_0, end_mask = key_token_49_end_mask_0, squeeze_mask = key_token_49_squeeze_mask_0, x = key_35_cast_fp16)[name = string("key_token_49_cast_fp16")];
            tensor<int32, [4]> value_token_49_begin_0 = const()[name = string("value_token_49_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> value_token_49_end_0 = const()[name = string("value_token_49_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> value_token_49_end_mask_0 = const()[name = string("value_token_49_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_49_squeeze_mask_0 = const()[name = string("value_token_49_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 4, 128]> value_127_cast_fp16 = transpose(perm = value_127_perm_0, x = value_125_cast_fp16)[name = string("transpose_76")];
            tensor<fp16, [1, 16, 128]> value_token_49_cast_fp16 = slice_by_index(begin = value_token_49_begin_0, end = value_token_49_end_0, end_mask = value_token_49_end_mask_0, squeeze_mask = value_token_49_squeeze_mask_0, x = value_127_cast_fp16)[name = string("value_token_49_cast_fp16")];
            tensor<int32, [1]> var_3084_axes_0 = const()[name = string("op_3084_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_3084_cast_fp16 = expand_dims(axes = var_3084_axes_0, x = key_token_49_cast_fp16)[name = string("op_3084_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3085_cast_fp16 = mul(x = state_97_cast_fp16, y = var_3084_cast_fp16)[name = string("op_3085_cast_fp16")];
            tensor<int32, [1]> memory_49_axes_0 = const()[name = string("memory_49_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_49_keep_dims_0 = const()[name = string("memory_49_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_49_cast_fp16 = reduce_sum(axes = memory_49_axes_0, keep_dims = memory_49_keep_dims_0, x = var_3085_cast_fp16)[name = string("memory_49_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_3088_cast_fp16 = sub(x = value_token_49_cast_fp16, y = memory_49_cast_fp16)[name = string("op_3088_cast_fp16")];
            tensor<int32, [3]> var_3091_begin_0 = const()[name = string("op_3091_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_3091_end_0 = const()[name = string("op_3091_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_3091_end_mask_0 = const()[name = string("op_3091_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_3091_squeeze_mask_0 = const()[name = string("op_3091_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_3091_cast_fp16 = slice_by_index(begin = var_3091_begin_0, end = var_3091_end_0, end_mask = var_3091_end_mask_0, squeeze_mask = var_3091_squeeze_mask_0, x = beta_13_cast_fp16)[name = string("op_3091_cast_fp16")];
            tensor<int32, [1]> var_3092_axes_0 = const()[name = string("op_3092_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_3092_cast_fp16 = expand_dims(axes = var_3092_axes_0, x = var_3091_cast_fp16)[name = string("op_3092_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_49_cast_fp16 = mul(x = var_3088_cast_fp16, y = var_3092_cast_fp16)[name = string("delta_49_cast_fp16")];
            tensor<int32, [1]> var_3095_axes_0 = const()[name = string("op_3095_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_3095_cast_fp16 = expand_dims(axes = var_3095_axes_0, x = delta_49_cast_fp16)[name = string("op_3095_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3096_cast_fp16 = mul(x = var_3084_cast_fp16, y = var_3095_cast_fp16)[name = string("op_3096_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_99_cast_fp16 = add(x = state_97_cast_fp16, y = var_3096_cast_fp16)[name = string("state_99_cast_fp16")];
            tensor<int32, [4]> var_3100_begin_0 = const()[name = string("op_3100_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> var_3100_end_0 = const()[name = string("op_3100_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> var_3100_end_mask_0 = const()[name = string("op_3100_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_3100_squeeze_mask_0 = const()[name = string("op_3100_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_3100_cast_fp16 = slice_by_index(begin = var_3100_begin_0, end = var_3100_end_0, end_mask = var_3100_end_mask_0, squeeze_mask = var_3100_squeeze_mask_0, x = _inversed_query_35_cast_fp16)[name = string("op_3100_cast_fp16")];
            tensor<int32, [1]> var_3101_axes_0 = const()[name = string("op_3101_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_3101_cast_fp16 = expand_dims(axes = var_3101_axes_0, x = var_3100_cast_fp16)[name = string("op_3101_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3102_cast_fp16 = mul(x = state_99_cast_fp16, y = var_3101_cast_fp16)[name = string("op_3102_cast_fp16")];
            tensor<int32, [1]> var_3104_axes_0 = const()[name = string("op_3104_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_3104_keep_dims_0 = const()[name = string("op_3104_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_3104_cast_fp16 = reduce_sum(axes = var_3104_axes_0, keep_dims = var_3104_keep_dims_0, x = var_3102_cast_fp16)[name = string("op_3104_cast_fp16")];
            tensor<int32, [3]> var_3107_begin_0 = const()[name = string("op_3107_begin_0"), val = tensor<int32, [3]>([0, 0, 1])];
            tensor<int32, [3]> var_3107_end_0 = const()[name = string("op_3107_end_0"), val = tensor<int32, [3]>([1, 16, 2])];
            tensor<bool, [3]> var_3107_end_mask_0 = const()[name = string("op_3107_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_3107_squeeze_mask_0 = const()[name = string("op_3107_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_3107_cast_fp16 = slice_by_index(begin = var_3107_begin_0, end = var_3107_end_0, end_mask = var_3107_end_mask_0, squeeze_mask = var_3107_squeeze_mask_0, x = decay_13_cast_fp16)[name = string("op_3107_cast_fp16")];
            tensor<fp16, [1, 16]> var_3108_cast_fp16 = exp(x = var_3107_cast_fp16)[name = string("op_3108_cast_fp16")];
            tensor<int32, [1]> var_3109_axes_0 = const()[name = string("op_3109_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_3109_cast_fp16 = expand_dims(axes = var_3109_axes_0, x = var_3108_cast_fp16)[name = string("op_3109_cast_fp16")];
            tensor<int32, [1]> var_3110_axes_0 = const()[name = string("op_3110_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_3110_cast_fp16 = expand_dims(axes = var_3110_axes_0, x = var_3109_cast_fp16)[name = string("op_3110_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_101_cast_fp16 = mul(x = state_99_cast_fp16, y = var_3110_cast_fp16)[name = string("state_101_cast_fp16")];
            tensor<int32, [4]> key_token_51_begin_0 = const()[name = string("key_token_51_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> key_token_51_end_0 = const()[name = string("key_token_51_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> key_token_51_end_mask_0 = const()[name = string("key_token_51_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_51_squeeze_mask_0 = const()[name = string("key_token_51_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_51_cast_fp16 = slice_by_index(begin = key_token_51_begin_0, end = key_token_51_end_0, end_mask = key_token_51_end_mask_0, squeeze_mask = key_token_51_squeeze_mask_0, x = key_35_cast_fp16)[name = string("key_token_51_cast_fp16")];
            tensor<int32, [4]> value_token_51_begin_0 = const()[name = string("value_token_51_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> value_token_51_end_0 = const()[name = string("value_token_51_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> value_token_51_end_mask_0 = const()[name = string("value_token_51_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_51_squeeze_mask_0 = const()[name = string("value_token_51_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_51_cast_fp16 = slice_by_index(begin = value_token_51_begin_0, end = value_token_51_end_0, end_mask = value_token_51_end_mask_0, squeeze_mask = value_token_51_squeeze_mask_0, x = value_127_cast_fp16)[name = string("value_token_51_cast_fp16")];
            tensor<int32, [1]> var_3118_axes_0 = const()[name = string("op_3118_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_3118_cast_fp16 = expand_dims(axes = var_3118_axes_0, x = key_token_51_cast_fp16)[name = string("op_3118_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3119_cast_fp16 = mul(x = state_101_cast_fp16, y = var_3118_cast_fp16)[name = string("op_3119_cast_fp16")];
            tensor<int32, [1]> memory_51_axes_0 = const()[name = string("memory_51_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_51_keep_dims_0 = const()[name = string("memory_51_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_51_cast_fp16 = reduce_sum(axes = memory_51_axes_0, keep_dims = memory_51_keep_dims_0, x = var_3119_cast_fp16)[name = string("memory_51_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_3122_cast_fp16 = sub(x = value_token_51_cast_fp16, y = memory_51_cast_fp16)[name = string("op_3122_cast_fp16")];
            tensor<int32, [3]> var_3125_begin_0 = const()[name = string("op_3125_begin_0"), val = tensor<int32, [3]>([0, 0, 1])];
            tensor<int32, [3]> var_3125_end_0 = const()[name = string("op_3125_end_0"), val = tensor<int32, [3]>([1, 16, 2])];
            tensor<bool, [3]> var_3125_end_mask_0 = const()[name = string("op_3125_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_3125_squeeze_mask_0 = const()[name = string("op_3125_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_3125_cast_fp16 = slice_by_index(begin = var_3125_begin_0, end = var_3125_end_0, end_mask = var_3125_end_mask_0, squeeze_mask = var_3125_squeeze_mask_0, x = beta_13_cast_fp16)[name = string("op_3125_cast_fp16")];
            tensor<int32, [1]> var_3126_axes_0 = const()[name = string("op_3126_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_3126_cast_fp16 = expand_dims(axes = var_3126_axes_0, x = var_3125_cast_fp16)[name = string("op_3126_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_51_cast_fp16 = mul(x = var_3122_cast_fp16, y = var_3126_cast_fp16)[name = string("delta_51_cast_fp16")];
            tensor<int32, [1]> var_3129_axes_0 = const()[name = string("op_3129_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_3129_cast_fp16 = expand_dims(axes = var_3129_axes_0, x = delta_51_cast_fp16)[name = string("op_3129_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3130_cast_fp16 = mul(x = var_3118_cast_fp16, y = var_3129_cast_fp16)[name = string("op_3130_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_103_cast_fp16 = add(x = state_101_cast_fp16, y = var_3130_cast_fp16)[name = string("state_103_cast_fp16")];
            tensor<int32, [4]> var_3134_begin_0 = const()[name = string("op_3134_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> var_3134_end_0 = const()[name = string("op_3134_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> var_3134_end_mask_0 = const()[name = string("op_3134_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_3134_squeeze_mask_0 = const()[name = string("op_3134_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_3134_cast_fp16 = slice_by_index(begin = var_3134_begin_0, end = var_3134_end_0, end_mask = var_3134_end_mask_0, squeeze_mask = var_3134_squeeze_mask_0, x = _inversed_query_35_cast_fp16)[name = string("op_3134_cast_fp16")];
            tensor<int32, [1]> var_3135_axes_0 = const()[name = string("op_3135_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_3135_cast_fp16 = expand_dims(axes = var_3135_axes_0, x = var_3134_cast_fp16)[name = string("op_3135_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3136_cast_fp16 = mul(x = state_103_cast_fp16, y = var_3135_cast_fp16)[name = string("op_3136_cast_fp16")];
            tensor<int32, [1]> var_3138_axes_0 = const()[name = string("op_3138_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_3138_keep_dims_0 = const()[name = string("op_3138_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_3138_cast_fp16 = reduce_sum(axes = var_3138_axes_0, keep_dims = var_3138_keep_dims_0, x = var_3136_cast_fp16)[name = string("op_3138_cast_fp16")];
            tensor<int32, [3]> var_3141_begin_0 = const()[name = string("op_3141_begin_0"), val = tensor<int32, [3]>([0, 0, 2])];
            tensor<int32, [3]> var_3141_end_0 = const()[name = string("op_3141_end_0"), val = tensor<int32, [3]>([1, 16, 3])];
            tensor<bool, [3]> var_3141_end_mask_0 = const()[name = string("op_3141_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_3141_squeeze_mask_0 = const()[name = string("op_3141_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_3141_cast_fp16 = slice_by_index(begin = var_3141_begin_0, end = var_3141_end_0, end_mask = var_3141_end_mask_0, squeeze_mask = var_3141_squeeze_mask_0, x = decay_13_cast_fp16)[name = string("op_3141_cast_fp16")];
            tensor<fp16, [1, 16]> var_3142_cast_fp16 = exp(x = var_3141_cast_fp16)[name = string("op_3142_cast_fp16")];
            tensor<int32, [1]> var_3143_axes_0 = const()[name = string("op_3143_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_3143_cast_fp16 = expand_dims(axes = var_3143_axes_0, x = var_3142_cast_fp16)[name = string("op_3143_cast_fp16")];
            tensor<int32, [1]> var_3144_axes_0 = const()[name = string("op_3144_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_3144_cast_fp16 = expand_dims(axes = var_3144_axes_0, x = var_3143_cast_fp16)[name = string("op_3144_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_105_cast_fp16 = mul(x = state_103_cast_fp16, y = var_3144_cast_fp16)[name = string("state_105_cast_fp16")];
            tensor<int32, [4]> key_token_53_begin_0 = const()[name = string("key_token_53_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> key_token_53_end_0 = const()[name = string("key_token_53_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> key_token_53_end_mask_0 = const()[name = string("key_token_53_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_53_squeeze_mask_0 = const()[name = string("key_token_53_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_53_cast_fp16 = slice_by_index(begin = key_token_53_begin_0, end = key_token_53_end_0, end_mask = key_token_53_end_mask_0, squeeze_mask = key_token_53_squeeze_mask_0, x = key_35_cast_fp16)[name = string("key_token_53_cast_fp16")];
            tensor<int32, [4]> value_token_53_begin_0 = const()[name = string("value_token_53_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> value_token_53_end_0 = const()[name = string("value_token_53_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> value_token_53_end_mask_0 = const()[name = string("value_token_53_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_53_squeeze_mask_0 = const()[name = string("value_token_53_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_53_cast_fp16 = slice_by_index(begin = value_token_53_begin_0, end = value_token_53_end_0, end_mask = value_token_53_end_mask_0, squeeze_mask = value_token_53_squeeze_mask_0, x = value_127_cast_fp16)[name = string("value_token_53_cast_fp16")];
            tensor<int32, [1]> var_3152_axes_0 = const()[name = string("op_3152_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_3152_cast_fp16 = expand_dims(axes = var_3152_axes_0, x = key_token_53_cast_fp16)[name = string("op_3152_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3153_cast_fp16 = mul(x = state_105_cast_fp16, y = var_3152_cast_fp16)[name = string("op_3153_cast_fp16")];
            tensor<int32, [1]> memory_53_axes_0 = const()[name = string("memory_53_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_53_keep_dims_0 = const()[name = string("memory_53_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_53_cast_fp16 = reduce_sum(axes = memory_53_axes_0, keep_dims = memory_53_keep_dims_0, x = var_3153_cast_fp16)[name = string("memory_53_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_3156_cast_fp16 = sub(x = value_token_53_cast_fp16, y = memory_53_cast_fp16)[name = string("op_3156_cast_fp16")];
            tensor<int32, [3]> var_3159_begin_0 = const()[name = string("op_3159_begin_0"), val = tensor<int32, [3]>([0, 0, 2])];
            tensor<int32, [3]> var_3159_end_0 = const()[name = string("op_3159_end_0"), val = tensor<int32, [3]>([1, 16, 3])];
            tensor<bool, [3]> var_3159_end_mask_0 = const()[name = string("op_3159_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_3159_squeeze_mask_0 = const()[name = string("op_3159_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_3159_cast_fp16 = slice_by_index(begin = var_3159_begin_0, end = var_3159_end_0, end_mask = var_3159_end_mask_0, squeeze_mask = var_3159_squeeze_mask_0, x = beta_13_cast_fp16)[name = string("op_3159_cast_fp16")];
            tensor<int32, [1]> var_3160_axes_0 = const()[name = string("op_3160_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_3160_cast_fp16 = expand_dims(axes = var_3160_axes_0, x = var_3159_cast_fp16)[name = string("op_3160_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_53_cast_fp16 = mul(x = var_3156_cast_fp16, y = var_3160_cast_fp16)[name = string("delta_53_cast_fp16")];
            tensor<int32, [1]> var_3163_axes_0 = const()[name = string("op_3163_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_3163_cast_fp16 = expand_dims(axes = var_3163_axes_0, x = delta_53_cast_fp16)[name = string("op_3163_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3164_cast_fp16 = mul(x = var_3152_cast_fp16, y = var_3163_cast_fp16)[name = string("op_3164_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_107_cast_fp16 = add(x = state_105_cast_fp16, y = var_3164_cast_fp16)[name = string("state_107_cast_fp16")];
            tensor<int32, [4]> var_3168_begin_0 = const()[name = string("op_3168_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> var_3168_end_0 = const()[name = string("op_3168_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> var_3168_end_mask_0 = const()[name = string("op_3168_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_3168_squeeze_mask_0 = const()[name = string("op_3168_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_3168_cast_fp16 = slice_by_index(begin = var_3168_begin_0, end = var_3168_end_0, end_mask = var_3168_end_mask_0, squeeze_mask = var_3168_squeeze_mask_0, x = _inversed_query_35_cast_fp16)[name = string("op_3168_cast_fp16")];
            tensor<int32, [1]> var_3169_axes_0 = const()[name = string("op_3169_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_3169_cast_fp16 = expand_dims(axes = var_3169_axes_0, x = var_3168_cast_fp16)[name = string("op_3169_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3170_cast_fp16 = mul(x = state_107_cast_fp16, y = var_3169_cast_fp16)[name = string("op_3170_cast_fp16")];
            tensor<int32, [1]> var_3172_axes_0 = const()[name = string("op_3172_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_3172_keep_dims_0 = const()[name = string("op_3172_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_3172_cast_fp16 = reduce_sum(axes = var_3172_axes_0, keep_dims = var_3172_keep_dims_0, x = var_3170_cast_fp16)[name = string("op_3172_cast_fp16")];
            tensor<int32, [3]> var_3175_begin_0 = const()[name = string("op_3175_begin_0"), val = tensor<int32, [3]>([0, 0, 3])];
            tensor<int32, [3]> var_3175_end_0 = const()[name = string("op_3175_end_0"), val = tensor<int32, [3]>([1, 16, 4])];
            tensor<bool, [3]> var_3175_end_mask_0 = const()[name = string("op_3175_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_3175_squeeze_mask_0 = const()[name = string("op_3175_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_3175_cast_fp16 = slice_by_index(begin = var_3175_begin_0, end = var_3175_end_0, end_mask = var_3175_end_mask_0, squeeze_mask = var_3175_squeeze_mask_0, x = decay_13_cast_fp16)[name = string("op_3175_cast_fp16")];
            tensor<fp16, [1, 16]> var_3176_cast_fp16 = exp(x = var_3175_cast_fp16)[name = string("op_3176_cast_fp16")];
            tensor<int32, [1]> var_3177_axes_0 = const()[name = string("op_3177_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_3177_cast_fp16 = expand_dims(axes = var_3177_axes_0, x = var_3176_cast_fp16)[name = string("op_3177_cast_fp16")];
            tensor<int32, [1]> var_3178_axes_0 = const()[name = string("op_3178_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_3178_cast_fp16 = expand_dims(axes = var_3178_axes_0, x = var_3177_cast_fp16)[name = string("op_3178_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_109_cast_fp16 = mul(x = state_107_cast_fp16, y = var_3178_cast_fp16)[name = string("state_109_cast_fp16")];
            tensor<int32, [4]> key_token_55_begin_0 = const()[name = string("key_token_55_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> key_token_55_end_0 = const()[name = string("key_token_55_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> key_token_55_end_mask_0 = const()[name = string("key_token_55_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_55_squeeze_mask_0 = const()[name = string("key_token_55_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_55_cast_fp16 = slice_by_index(begin = key_token_55_begin_0, end = key_token_55_end_0, end_mask = key_token_55_end_mask_0, squeeze_mask = key_token_55_squeeze_mask_0, x = key_35_cast_fp16)[name = string("key_token_55_cast_fp16")];
            tensor<int32, [4]> value_token_55_begin_0 = const()[name = string("value_token_55_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> value_token_55_end_0 = const()[name = string("value_token_55_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> value_token_55_end_mask_0 = const()[name = string("value_token_55_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_55_squeeze_mask_0 = const()[name = string("value_token_55_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_55_cast_fp16 = slice_by_index(begin = value_token_55_begin_0, end = value_token_55_end_0, end_mask = value_token_55_end_mask_0, squeeze_mask = value_token_55_squeeze_mask_0, x = value_127_cast_fp16)[name = string("value_token_55_cast_fp16")];
            tensor<int32, [1]> var_3186_axes_0 = const()[name = string("op_3186_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_3186_cast_fp16 = expand_dims(axes = var_3186_axes_0, x = key_token_55_cast_fp16)[name = string("op_3186_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3187_cast_fp16 = mul(x = state_109_cast_fp16, y = var_3186_cast_fp16)[name = string("op_3187_cast_fp16")];
            tensor<int32, [1]> memory_55_axes_0 = const()[name = string("memory_55_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_55_keep_dims_0 = const()[name = string("memory_55_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_55_cast_fp16 = reduce_sum(axes = memory_55_axes_0, keep_dims = memory_55_keep_dims_0, x = var_3187_cast_fp16)[name = string("memory_55_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_3190_cast_fp16 = sub(x = value_token_55_cast_fp16, y = memory_55_cast_fp16)[name = string("op_3190_cast_fp16")];
            tensor<int32, [3]> var_3193_begin_0 = const()[name = string("op_3193_begin_0"), val = tensor<int32, [3]>([0, 0, 3])];
            tensor<int32, [3]> var_3193_end_0 = const()[name = string("op_3193_end_0"), val = tensor<int32, [3]>([1, 16, 4])];
            tensor<bool, [3]> var_3193_end_mask_0 = const()[name = string("op_3193_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_3193_squeeze_mask_0 = const()[name = string("op_3193_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_3193_cast_fp16 = slice_by_index(begin = var_3193_begin_0, end = var_3193_end_0, end_mask = var_3193_end_mask_0, squeeze_mask = var_3193_squeeze_mask_0, x = beta_13_cast_fp16)[name = string("op_3193_cast_fp16")];
            tensor<int32, [1]> var_3194_axes_0 = const()[name = string("op_3194_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_3194_cast_fp16 = expand_dims(axes = var_3194_axes_0, x = var_3193_cast_fp16)[name = string("op_3194_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_55_cast_fp16 = mul(x = var_3190_cast_fp16, y = var_3194_cast_fp16)[name = string("delta_55_cast_fp16")];
            tensor<int32, [1]> var_3197_axes_0 = const()[name = string("op_3197_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_3197_cast_fp16 = expand_dims(axes = var_3197_axes_0, x = delta_55_cast_fp16)[name = string("op_3197_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3198_cast_fp16 = mul(x = var_3186_cast_fp16, y = var_3197_cast_fp16)[name = string("op_3198_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> rec8_out = add(x = state_109_cast_fp16, y = var_3198_cast_fp16)[name = string("state_111_cast_fp16")];
            tensor<int32, [4]> var_3202_begin_0 = const()[name = string("op_3202_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> var_3202_end_0 = const()[name = string("op_3202_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> var_3202_end_mask_0 = const()[name = string("op_3202_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_3202_squeeze_mask_0 = const()[name = string("op_3202_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_3202_cast_fp16 = slice_by_index(begin = var_3202_begin_0, end = var_3202_end_0, end_mask = var_3202_end_mask_0, squeeze_mask = var_3202_squeeze_mask_0, x = _inversed_query_35_cast_fp16)[name = string("op_3202_cast_fp16")];
            tensor<int32, [1]> var_3203_axes_0 = const()[name = string("op_3203_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_3203_cast_fp16 = expand_dims(axes = var_3203_axes_0, x = var_3202_cast_fp16)[name = string("op_3203_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3204_cast_fp16 = mul(x = rec8_out, y = var_3203_cast_fp16)[name = string("op_3204_cast_fp16")];
            tensor<int32, [1]> var_3206_axes_0 = const()[name = string("op_3206_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_3206_keep_dims_0 = const()[name = string("op_3206_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_3206_cast_fp16 = reduce_sum(axes = var_3206_axes_0, keep_dims = var_3206_keep_dims_0, x = var_3204_cast_fp16)[name = string("op_3206_cast_fp16")];
            int32 attention_61_axis_0 = const()[name = string("attention_61_axis_0"), val = int32(1)];
            tensor<fp16, [1, 4, 16, 128]> attention_61_cast_fp16 = stack(axis = attention_61_axis_0, values = (var_3104_cast_fp16, var_3138_cast_fp16, var_3172_cast_fp16, var_3206_cast_fp16))[name = string("attention_61_cast_fp16")];
            tensor<int32, [2]> var_3209 = const()[name = string("op_3209"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [64, 128]> attention_63_cast_fp16 = reshape(shape = var_3209, x = attention_61_cast_fp16)[name = string("attention_63_cast_fp16")];
            tensor<int32, [2]> var_3211 = const()[name = string("op_3211"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [64, 128]> input_109_cast_fp16 = reshape(shape = var_3211, x = linear_65_cast_fp16)[name = string("input_109_cast_fp16")];
            tensor<fp16, [64, 128]> var_3213_cast_fp16 = mul(x = attention_63_cast_fp16, y = attention_63_cast_fp16)[name = string("op_3213_cast_fp16")];
            tensor<int32, [1]> variance_55_axes_0 = const()[name = string("variance_55_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_55_keep_dims_0 = const()[name = string("variance_55_keep_dims_0"), val = bool(true)];
            tensor<fp16, [64, 1]> variance_55_cast_fp16 = reduce_mean(axes = variance_55_axes_0, keep_dims = variance_55_keep_dims_0, x = var_3213_cast_fp16)[name = string("variance_55_cast_fp16")];
            fp16 var_3216_to_fp16 = const()[name = string("op_3216_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [64, 1]> var_3217_cast_fp16 = add(x = variance_55_cast_fp16, y = var_3216_to_fp16)[name = string("op_3217_cast_fp16")];
            fp32 var_3218_epsilon_0 = const()[name = string("op_3218_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [64, 1]> var_3218_cast_fp16 = rsqrt(epsilon = var_3218_epsilon_0, x = var_3217_cast_fp16)[name = string("op_3218_cast_fp16")];
            tensor<fp16, [64, 128]> attention_65_cast_fp16 = mul(x = attention_63_cast_fp16, y = var_3218_cast_fp16)[name = string("attention_65_cast_fp16")];
            tensor<fp16, [128]> groups_2_0_gated_norm_promoted_to_fp16 = const()[name = string("groups_2_0_gated_norm_promoted_to_fp16"), val = tensor<fp16, [128]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(132411776)))];
            tensor<fp16, [64, 128]> attention_67_cast_fp16 = mul(x = attention_65_cast_fp16, y = groups_2_0_gated_norm_promoted_to_fp16)[name = string("attention_67_cast_fp16")];
            tensor<fp16, [64, 128]> var_3221_cast_fp16 = silu(x = input_109_cast_fp16)[name = string("op_3221_cast_fp16")];
            tensor<fp16, [64, 128]> attention_69_cast_fp16 = mul(x = attention_67_cast_fp16, y = var_3221_cast_fp16)[name = string("attention_69_cast_fp16")];
            tensor<int32, [2]> var_3223 = const()[name = string("op_3223"), val = tensor<int32, [2]>([4, 2048])];
            tensor<fp16, [4, 2048]> input_111_cast_fp16 = reshape(shape = var_3223, x = attention_69_cast_fp16)[name = string("input_111_cast_fp16")];
            tensor<fp16, [1024, 2048]> groups_2_0_out_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 2048]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(132412096))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(133985024))))[name = string("groups_2_0_out_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 1024]> linear_66_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = groups_2_0_out_proj_weight_promoted_to_fp16_palettized, x = input_111_cast_fp16)[name = string("linear_66_cast_fp16")];
            tensor<fp16, [4, 1024]> value_129_cast_fp16 = add(x = value_117_cast_fp16, y = linear_66_cast_fp16)[name = string("value_129_cast_fp16")];
            tensor<fp16, [4, 1024]> var_3228_cast_fp16 = mul(x = value_129_cast_fp16, y = value_129_cast_fp16)[name = string("op_3228_cast_fp16")];
            tensor<int32, [1]> variance_57_axes_0 = const()[name = string("variance_57_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_57_keep_dims_0 = const()[name = string("variance_57_keep_dims_0"), val = bool(true)];
            tensor<fp16, [4, 1]> variance_57_cast_fp16 = reduce_mean(axes = variance_57_axes_0, keep_dims = variance_57_keep_dims_0, x = var_3228_cast_fp16)[name = string("variance_57_cast_fp16")];
            fp16 var_3231_to_fp16 = const()[name = string("op_3231_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [4, 1]> var_3232_cast_fp16 = add(x = variance_57_cast_fp16, y = var_3231_to_fp16)[name = string("op_3232_cast_fp16")];
            fp32 var_3233_epsilon_0 = const()[name = string("op_3233_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [4, 1]> var_3233_cast_fp16 = rsqrt(epsilon = var_3233_epsilon_0, x = var_3232_cast_fp16)[name = string("op_3233_cast_fp16")];
            tensor<fp16, [4, 1024]> var_3234_cast_fp16 = mul(x = value_129_cast_fp16, y = var_3233_cast_fp16)[name = string("op_3234_cast_fp16")];
            tensor<fp16, [1024]> var_3236_to_fp16 = const()[name = string("op_3236_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(133993280)))];
            tensor<fp16, [4, 1024]> input_113_cast_fp16 = mul(x = var_3234_cast_fp16, y = var_3236_to_fp16)[name = string("input_113_cast_fp16")];
            tensor<fp16, [3584, 1024]> groups_2_0_gate_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(133995392))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(136747968))))[name = string("groups_2_0_gate_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 3584]> linear_67_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = groups_2_0_gate_proj_weight_promoted_to_fp16_palettized, x = input_113_cast_fp16)[name = string("linear_67_cast_fp16")];
            tensor<fp16, [4, 3584]> var_3240_cast_fp16 = silu(x = linear_67_cast_fp16)[name = string("op_3240_cast_fp16")];
            tensor<fp16, [3584, 1024]> groups_2_0_up_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(136776704))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(139529280))))[name = string("groups_2_0_up_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 3584]> linear_68_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = groups_2_0_up_proj_weight_promoted_to_fp16_palettized, x = input_113_cast_fp16)[name = string("linear_68_cast_fp16")];
            tensor<fp16, [4, 3584]> input_117_cast_fp16 = mul(x = var_3240_cast_fp16, y = linear_68_cast_fp16)[name = string("input_117_cast_fp16")];
            tensor<fp16, [1024, 3584]> groups_2_0_down_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 3584]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(139558016))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(142310592))))[name = string("groups_2_0_down_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 1024]> linear_69_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = groups_2_0_down_proj_weight_promoted_to_fp16_palettized, x = input_117_cast_fp16)[name = string("linear_69_cast_fp16")];
            tensor<fp16, [4, 1024]> value_131_cast_fp16 = add(x = value_129_cast_fp16, y = linear_69_cast_fp16)[name = string("value_131_cast_fp16")];
            int32 var_3267 = const()[name = string("op_3267"), val = int32(-1)];
            tensor<fp16, [4, 1024]> var_3282_cast_fp16 = mul(x = value_131_cast_fp16, y = value_131_cast_fp16)[name = string("op_3282_cast_fp16")];
            tensor<int32, [1]> variance_59_axes_0 = const()[name = string("variance_59_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_59_keep_dims_0 = const()[name = string("variance_59_keep_dims_0"), val = bool(true)];
            tensor<fp16, [4, 1]> variance_59_cast_fp16 = reduce_mean(axes = variance_59_axes_0, keep_dims = variance_59_keep_dims_0, x = var_3282_cast_fp16)[name = string("variance_59_cast_fp16")];
            fp16 var_3285_to_fp16 = const()[name = string("op_3285_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [4, 1]> var_3286_cast_fp16 = add(x = variance_59_cast_fp16, y = var_3285_to_fp16)[name = string("op_3286_cast_fp16")];
            fp32 var_3287_epsilon_0 = const()[name = string("op_3287_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [4, 1]> var_3287_cast_fp16 = rsqrt(epsilon = var_3287_epsilon_0, x = var_3286_cast_fp16)[name = string("op_3287_cast_fp16")];
            tensor<fp16, [4, 1024]> var_3288_cast_fp16 = mul(x = value_131_cast_fp16, y = var_3287_cast_fp16)[name = string("op_3288_cast_fp16")];
            tensor<fp16, [1024]> var_3290_to_fp16 = const()[name = string("op_3290_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(142318848)))];
            tensor<fp16, [4, 1024]> input_119_cast_fp16 = mul(x = var_3288_cast_fp16, y = var_3290_to_fp16)[name = string("input_119_cast_fp16")];
            tensor<fp16, [6144, 1024]> groups_2_1_in_proj_qkv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(142320960))), lut = tensor<fp16, [384, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(147039616))))[name = string("groups_2_1_in_proj_qkv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 6144]> linear_70_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = groups_2_1_in_proj_qkv_weight_promoted_to_fp16_palettized, x = input_119_cast_fp16)[name = string("linear_70_cast_fp16")];
            tensor<int32, [2]> var_3294_perm_0 = const()[name = string("op_3294_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> mixed_15_axes_0 = const()[name = string("mixed_15_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [6144, 4]> var_3294_cast_fp16 = transpose(perm = var_3294_perm_0, x = linear_70_cast_fp16)[name = string("transpose_75")];
            tensor<fp16, [1, 6144, 4]> mixed_15_cast_fp16 = expand_dims(axes = mixed_15_axes_0, x = var_3294_cast_fp16)[name = string("mixed_15_cast_fp16")];
            bool convolution_input_15_interleave_0 = const()[name = string("convolution_input_15_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 6144, 7]> convolution_input_15_cast_fp16 = concat(axis = var_3267, interleave = convolution_input_15_interleave_0, values = (conv9, mixed_15_cast_fp16))[name = string("convolution_input_15_cast_fp16")];
            string input_121_pad_type_0 = const()[name = string("input_121_pad_type_0"), val = string("valid")];
            int32 input_121_groups_0 = const()[name = string("input_121_groups_0"), val = int32(6144)];
            tensor<int32, [1]> input_121_strides_0 = const()[name = string("input_121_strides_0"), val = tensor<int32, [1]>([1])];
            tensor<int32, [2]> input_121_pad_0 = const()[name = string("input_121_pad_0"), val = tensor<int32, [2]>([0, 0])];
            tensor<int32, [1]> input_121_dilations_0 = const()[name = string("input_121_dilations_0"), val = tensor<int32, [1]>([1])];
            tensor<fp16, [6144, 1, 4]> groups_2_1_conv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1, 4]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(147088832))), lut = tensor<fp16, [384, 1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(147107328))))[name = string("groups_2_1_conv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 6144, 4]> input_121_cast_fp16 = conv(dilations = input_121_dilations_0, groups = input_121_groups_0, pad = input_121_pad_0, pad_type = input_121_pad_type_0, strides = input_121_strides_0, weight = groups_2_1_conv_weight_promoted_to_fp16_palettized, x = convolution_input_15_cast_fp16)[name = string("input_121_cast_fp16")];
            tensor<fp16, [1, 6144, 4]> var_3303_cast_fp16 = silu(x = input_121_cast_fp16)[name = string("op_3303_cast_fp16")];
            tensor<int32, [3]> var_3304_perm_0 = const()[name = string("op_3304_perm_0"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [3]> var_3307_begin_0 = const()[name = string("op_3307_begin_0"), val = tensor<int32, [3]>([0, 0, 4])];
            tensor<int32, [3]> var_3307_end_0 = const()[name = string("op_3307_end_0"), val = tensor<int32, [3]>([1, 6144, 7])];
            tensor<bool, [3]> var_3307_end_mask_0 = const()[name = string("op_3307_end_mask_0"), val = tensor<bool, [3]>([true, true, true])];
            tensor<fp16, [1, 6144, 3]> conv9_out = slice_by_index(begin = var_3307_begin_0, end = var_3307_end_0, end_mask = var_3307_end_mask_0, x = convolution_input_15_cast_fp16)[name = string("op_3307_cast_fp16")];
            tensor<int32, [3]> var_3308 = const()[name = string("op_3308"), val = tensor<int32, [3]>([2048, 2048, 2048])];
            int32 var_3309_axis_0 = const()[name = string("op_3309_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 4, 6144]> var_3304_cast_fp16 = transpose(perm = var_3304_perm_0, x = var_3303_cast_fp16)[name = string("transpose_74")];
            tensor<fp16, [1, 4, 2048]> var_3309_cast_fp16_0, tensor<fp16, [1, 4, 2048]> var_3309_cast_fp16_1, tensor<fp16, [1, 4, 2048]> var_3309_cast_fp16_2 = split(axis = var_3309_axis_0, split_sizes = var_3308, x = var_3304_cast_fp16)[name = string("op_3309_cast_fp16")];
            tensor<int32, [4]> var_3313 = const()[name = string("op_3313"), val = tensor<int32, [4]>([1, 4, 16, 128])];
            tensor<fp16, [1, 4, 16, 128]> value_135_cast_fp16 = reshape(shape = var_3313, x = var_3309_cast_fp16_0)[name = string("value_135_cast_fp16")];
            tensor<int32, [4]> var_3315 = const()[name = string("op_3315"), val = tensor<int32, [4]>([1, 4, 16, 128])];
            tensor<fp16, [1, 4, 16, 128]> value_137_cast_fp16 = reshape(shape = var_3315, x = var_3309_cast_fp16_1)[name = string("value_137_cast_fp16")];
            tensor<int32, [4]> var_3317 = const()[name = string("op_3317"), val = tensor<int32, [4]>([1, 4, 16, 128])];
            tensor<fp16, [1, 4, 16, 128]> value_139_cast_fp16 = reshape(shape = var_3317, x = var_3309_cast_fp16_2)[name = string("value_139_cast_fp16")];
            tensor<fp16, [1, 4, 16, 128]> var_3319_cast_fp16 = mul(x = value_135_cast_fp16, y = value_135_cast_fp16)[name = string("op_3319_cast_fp16")];
            tensor<int32, [1]> var_3321_axes_0 = const()[name = string("op_3321_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_3321_keep_dims_0 = const()[name = string("op_3321_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 4, 16, 1]> var_3321_cast_fp16 = reduce_sum(axes = var_3321_axes_0, keep_dims = var_3321_keep_dims_0, x = var_3319_cast_fp16)[name = string("op_3321_cast_fp16")];
            fp16 var_3322_to_fp16 = const()[name = string("op_3322_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 4, 16, 1]> var_3323_cast_fp16 = add(x = var_3321_cast_fp16, y = var_3322_to_fp16)[name = string("op_3323_cast_fp16")];
            fp32 var_3324_epsilon_0 = const()[name = string("op_3324_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 4, 16, 1]> var_3324_cast_fp16 = rsqrt(epsilon = var_3324_epsilon_0, x = var_3323_cast_fp16)[name = string("op_3324_cast_fp16")];
            tensor<fp16, [1, 4, 16, 128]> var_3325_cast_fp16 = mul(x = value_135_cast_fp16, y = var_3324_cast_fp16)[name = string("op_3325_cast_fp16")];
            tensor<int32, [4]> var_3326_perm_0 = const()[name = string("op_3326_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            fp16 _inversed_query_39_y_0_to_fp16 = const()[name = string("_inversed_query_39_y_0_to_fp16"), val = fp16(0x1.6ap-4)];
            tensor<fp16, [1, 16, 4, 128]> var_3326_cast_fp16 = transpose(perm = var_3326_perm_0, x = var_3325_cast_fp16)[name = string("transpose_73")];
            tensor<fp16, [1, 16, 4, 128]> _inversed_query_39_cast_fp16 = mul(x = var_3326_cast_fp16, y = _inversed_query_39_y_0_to_fp16)[name = string("_inversed_query_39_cast_fp16")];
            tensor<fp16, [1, 4, 16, 128]> var_3329_cast_fp16 = mul(x = value_137_cast_fp16, y = value_137_cast_fp16)[name = string("op_3329_cast_fp16")];
            tensor<int32, [1]> var_3331_axes_0 = const()[name = string("op_3331_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_3331_keep_dims_0 = const()[name = string("op_3331_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 4, 16, 1]> var_3331_cast_fp16 = reduce_sum(axes = var_3331_axes_0, keep_dims = var_3331_keep_dims_0, x = var_3329_cast_fp16)[name = string("op_3331_cast_fp16")];
            fp16 var_3332_to_fp16 = const()[name = string("op_3332_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 4, 16, 1]> var_3333_cast_fp16 = add(x = var_3331_cast_fp16, y = var_3332_to_fp16)[name = string("op_3333_cast_fp16")];
            fp32 var_3334_epsilon_0 = const()[name = string("op_3334_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 4, 16, 1]> var_3334_cast_fp16 = rsqrt(epsilon = var_3334_epsilon_0, x = var_3333_cast_fp16)[name = string("op_3334_cast_fp16")];
            tensor<fp16, [1, 4, 16, 128]> var_3335_cast_fp16 = mul(x = value_137_cast_fp16, y = var_3334_cast_fp16)[name = string("op_3335_cast_fp16")];
            tensor<int32, [4]> key_39_perm_0 = const()[name = string("key_39_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<int32, [4]> value_141_perm_0 = const()[name = string("value_141_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<fp16, [16, 1024]> groups_2_1_in_proj_b_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(147156544))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(147168896))))[name = string("groups_2_1_in_proj_b_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16]> linear_71_bias_0_to_fp16 = const()[name = string("linear_71_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(4862656)))];
            tensor<fp16, [4, 16]> linear_71_cast_fp16 = linear(bias = linear_71_bias_0_to_fp16, weight = groups_2_1_in_proj_b_weight_promoted_to_fp16_palettized, x = input_119_cast_fp16)[name = string("linear_71_cast_fp16")];
            tensor<fp16, [4, 16]> var_3340_cast_fp16 = sigmoid(x = linear_71_cast_fp16)[name = string("op_3340_cast_fp16")];
            tensor<int32, [2]> var_3341_perm_0 = const()[name = string("op_3341_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> beta_15_axes_0 = const()[name = string("beta_15_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 4]> var_3341_cast_fp16 = transpose(perm = var_3341_perm_0, x = var_3340_cast_fp16)[name = string("transpose_72")];
            tensor<fp16, [1, 16, 4]> beta_15_cast_fp16 = expand_dims(axes = beta_15_axes_0, x = var_3341_cast_fp16)[name = string("beta_15_cast_fp16")];
            tensor<fp16, [16, 1024]> op_3347_weight_0_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(147169088))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(147181440))))[name = string("op_3347_weight_0_to_fp16_palettized")];
            tensor<fp16, [16]> var_3347_bias_0_to_fp16 = const()[name = string("op_3347_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(147181632)))];
            tensor<fp16, [4, 16]> var_3347_cast_fp16 = linear(bias = var_3347_bias_0_to_fp16, weight = op_3347_weight_0_to_fp16_palettized, x = input_119_cast_fp16)[name = string("op_3347_cast_fp16")];
            tensor<fp16, [4, 16]> var_3348_cast_fp16 = softplus(x = var_3347_cast_fp16)[name = string("op_3348_cast_fp16")];
            tensor<fp16, [16]> var_3344_promoted_to_fp16 = const()[name = string("op_3344_promoted_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(147181760)))];
            tensor<fp16, [4, 16]> var_3349_cast_fp16 = mul(x = var_3344_promoted_to_fp16, y = var_3348_cast_fp16)[name = string("op_3349_cast_fp16")];
            tensor<int32, [2]> var_3350_perm_0 = const()[name = string("op_3350_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> decay_15_axes_0 = const()[name = string("decay_15_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 4]> var_3350_cast_fp16 = transpose(perm = var_3350_perm_0, x = var_3349_cast_fp16)[name = string("transpose_71")];
            tensor<fp16, [1, 16, 4]> decay_15_cast_fp16 = expand_dims(axes = decay_15_axes_0, x = var_3350_cast_fp16)[name = string("decay_15_cast_fp16")];
            tensor<fp16, [2048, 1024]> groups_2_1_in_proj_z_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [2048, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(147181888))), lut = tensor<fp16, [128, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(148754816))))[name = string("groups_2_1_in_proj_z_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 2048]> linear_73_cast_fp16 = linear(bias = linear_3_bias_0_to_fp16, weight = groups_2_1_in_proj_z_weight_promoted_to_fp16_palettized, x = input_119_cast_fp16)[name = string("linear_73_cast_fp16")];
            tensor<int32, [3]> var_3358_begin_0 = const()[name = string("op_3358_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_3358_end_0 = const()[name = string("op_3358_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_3358_end_mask_0 = const()[name = string("op_3358_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_3358_squeeze_mask_0 = const()[name = string("op_3358_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_3358_cast_fp16 = slice_by_index(begin = var_3358_begin_0, end = var_3358_end_0, end_mask = var_3358_end_mask_0, squeeze_mask = var_3358_squeeze_mask_0, x = decay_15_cast_fp16)[name = string("op_3358_cast_fp16")];
            tensor<fp16, [1, 16]> var_3359_cast_fp16 = exp(x = var_3358_cast_fp16)[name = string("op_3359_cast_fp16")];
            tensor<int32, [1]> var_3360_axes_0 = const()[name = string("op_3360_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_3360_cast_fp16 = expand_dims(axes = var_3360_axes_0, x = var_3359_cast_fp16)[name = string("op_3360_cast_fp16")];
            tensor<int32, [1]> var_3361_axes_0 = const()[name = string("op_3361_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_3361_cast_fp16 = expand_dims(axes = var_3361_axes_0, x = var_3360_cast_fp16)[name = string("op_3361_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_113_cast_fp16 = mul(x = rec9, y = var_3361_cast_fp16)[name = string("state_113_cast_fp16")];
            tensor<int32, [4]> key_token_57_begin_0 = const()[name = string("key_token_57_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> key_token_57_end_0 = const()[name = string("key_token_57_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> key_token_57_end_mask_0 = const()[name = string("key_token_57_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_57_squeeze_mask_0 = const()[name = string("key_token_57_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 4, 128]> key_39_cast_fp16 = transpose(perm = key_39_perm_0, x = var_3335_cast_fp16)[name = string("transpose_70")];
            tensor<fp16, [1, 16, 128]> key_token_57_cast_fp16 = slice_by_index(begin = key_token_57_begin_0, end = key_token_57_end_0, end_mask = key_token_57_end_mask_0, squeeze_mask = key_token_57_squeeze_mask_0, x = key_39_cast_fp16)[name = string("key_token_57_cast_fp16")];
            tensor<int32, [4]> value_token_57_begin_0 = const()[name = string("value_token_57_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> value_token_57_end_0 = const()[name = string("value_token_57_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> value_token_57_end_mask_0 = const()[name = string("value_token_57_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_57_squeeze_mask_0 = const()[name = string("value_token_57_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 4, 128]> value_141_cast_fp16 = transpose(perm = value_141_perm_0, x = value_139_cast_fp16)[name = string("transpose_69")];
            tensor<fp16, [1, 16, 128]> value_token_57_cast_fp16 = slice_by_index(begin = value_token_57_begin_0, end = value_token_57_end_0, end_mask = value_token_57_end_mask_0, squeeze_mask = value_token_57_squeeze_mask_0, x = value_141_cast_fp16)[name = string("value_token_57_cast_fp16")];
            tensor<int32, [1]> var_3369_axes_0 = const()[name = string("op_3369_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_3369_cast_fp16 = expand_dims(axes = var_3369_axes_0, x = key_token_57_cast_fp16)[name = string("op_3369_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3370_cast_fp16 = mul(x = state_113_cast_fp16, y = var_3369_cast_fp16)[name = string("op_3370_cast_fp16")];
            tensor<int32, [1]> memory_57_axes_0 = const()[name = string("memory_57_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_57_keep_dims_0 = const()[name = string("memory_57_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_57_cast_fp16 = reduce_sum(axes = memory_57_axes_0, keep_dims = memory_57_keep_dims_0, x = var_3370_cast_fp16)[name = string("memory_57_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_3373_cast_fp16 = sub(x = value_token_57_cast_fp16, y = memory_57_cast_fp16)[name = string("op_3373_cast_fp16")];
            tensor<int32, [3]> var_3376_begin_0 = const()[name = string("op_3376_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_3376_end_0 = const()[name = string("op_3376_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_3376_end_mask_0 = const()[name = string("op_3376_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_3376_squeeze_mask_0 = const()[name = string("op_3376_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_3376_cast_fp16 = slice_by_index(begin = var_3376_begin_0, end = var_3376_end_0, end_mask = var_3376_end_mask_0, squeeze_mask = var_3376_squeeze_mask_0, x = beta_15_cast_fp16)[name = string("op_3376_cast_fp16")];
            tensor<int32, [1]> var_3377_axes_0 = const()[name = string("op_3377_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_3377_cast_fp16 = expand_dims(axes = var_3377_axes_0, x = var_3376_cast_fp16)[name = string("op_3377_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_57_cast_fp16 = mul(x = var_3373_cast_fp16, y = var_3377_cast_fp16)[name = string("delta_57_cast_fp16")];
            tensor<int32, [1]> var_3380_axes_0 = const()[name = string("op_3380_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_3380_cast_fp16 = expand_dims(axes = var_3380_axes_0, x = delta_57_cast_fp16)[name = string("op_3380_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3381_cast_fp16 = mul(x = var_3369_cast_fp16, y = var_3380_cast_fp16)[name = string("op_3381_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_115_cast_fp16 = add(x = state_113_cast_fp16, y = var_3381_cast_fp16)[name = string("state_115_cast_fp16")];
            tensor<int32, [4]> var_3385_begin_0 = const()[name = string("op_3385_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> var_3385_end_0 = const()[name = string("op_3385_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> var_3385_end_mask_0 = const()[name = string("op_3385_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_3385_squeeze_mask_0 = const()[name = string("op_3385_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_3385_cast_fp16 = slice_by_index(begin = var_3385_begin_0, end = var_3385_end_0, end_mask = var_3385_end_mask_0, squeeze_mask = var_3385_squeeze_mask_0, x = _inversed_query_39_cast_fp16)[name = string("op_3385_cast_fp16")];
            tensor<int32, [1]> var_3386_axes_0 = const()[name = string("op_3386_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_3386_cast_fp16 = expand_dims(axes = var_3386_axes_0, x = var_3385_cast_fp16)[name = string("op_3386_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3387_cast_fp16 = mul(x = state_115_cast_fp16, y = var_3386_cast_fp16)[name = string("op_3387_cast_fp16")];
            tensor<int32, [1]> var_3389_axes_0 = const()[name = string("op_3389_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_3389_keep_dims_0 = const()[name = string("op_3389_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_3389_cast_fp16 = reduce_sum(axes = var_3389_axes_0, keep_dims = var_3389_keep_dims_0, x = var_3387_cast_fp16)[name = string("op_3389_cast_fp16")];
            tensor<int32, [3]> var_3392_begin_0 = const()[name = string("op_3392_begin_0"), val = tensor<int32, [3]>([0, 0, 1])];
            tensor<int32, [3]> var_3392_end_0 = const()[name = string("op_3392_end_0"), val = tensor<int32, [3]>([1, 16, 2])];
            tensor<bool, [3]> var_3392_end_mask_0 = const()[name = string("op_3392_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_3392_squeeze_mask_0 = const()[name = string("op_3392_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_3392_cast_fp16 = slice_by_index(begin = var_3392_begin_0, end = var_3392_end_0, end_mask = var_3392_end_mask_0, squeeze_mask = var_3392_squeeze_mask_0, x = decay_15_cast_fp16)[name = string("op_3392_cast_fp16")];
            tensor<fp16, [1, 16]> var_3393_cast_fp16 = exp(x = var_3392_cast_fp16)[name = string("op_3393_cast_fp16")];
            tensor<int32, [1]> var_3394_axes_0 = const()[name = string("op_3394_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_3394_cast_fp16 = expand_dims(axes = var_3394_axes_0, x = var_3393_cast_fp16)[name = string("op_3394_cast_fp16")];
            tensor<int32, [1]> var_3395_axes_0 = const()[name = string("op_3395_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_3395_cast_fp16 = expand_dims(axes = var_3395_axes_0, x = var_3394_cast_fp16)[name = string("op_3395_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_117_cast_fp16 = mul(x = state_115_cast_fp16, y = var_3395_cast_fp16)[name = string("state_117_cast_fp16")];
            tensor<int32, [4]> key_token_59_begin_0 = const()[name = string("key_token_59_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> key_token_59_end_0 = const()[name = string("key_token_59_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> key_token_59_end_mask_0 = const()[name = string("key_token_59_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_59_squeeze_mask_0 = const()[name = string("key_token_59_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_59_cast_fp16 = slice_by_index(begin = key_token_59_begin_0, end = key_token_59_end_0, end_mask = key_token_59_end_mask_0, squeeze_mask = key_token_59_squeeze_mask_0, x = key_39_cast_fp16)[name = string("key_token_59_cast_fp16")];
            tensor<int32, [4]> value_token_59_begin_0 = const()[name = string("value_token_59_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> value_token_59_end_0 = const()[name = string("value_token_59_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> value_token_59_end_mask_0 = const()[name = string("value_token_59_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_59_squeeze_mask_0 = const()[name = string("value_token_59_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_59_cast_fp16 = slice_by_index(begin = value_token_59_begin_0, end = value_token_59_end_0, end_mask = value_token_59_end_mask_0, squeeze_mask = value_token_59_squeeze_mask_0, x = value_141_cast_fp16)[name = string("value_token_59_cast_fp16")];
            tensor<int32, [1]> var_3403_axes_0 = const()[name = string("op_3403_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_3403_cast_fp16 = expand_dims(axes = var_3403_axes_0, x = key_token_59_cast_fp16)[name = string("op_3403_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3404_cast_fp16 = mul(x = state_117_cast_fp16, y = var_3403_cast_fp16)[name = string("op_3404_cast_fp16")];
            tensor<int32, [1]> memory_59_axes_0 = const()[name = string("memory_59_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_59_keep_dims_0 = const()[name = string("memory_59_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_59_cast_fp16 = reduce_sum(axes = memory_59_axes_0, keep_dims = memory_59_keep_dims_0, x = var_3404_cast_fp16)[name = string("memory_59_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_3407_cast_fp16 = sub(x = value_token_59_cast_fp16, y = memory_59_cast_fp16)[name = string("op_3407_cast_fp16")];
            tensor<int32, [3]> var_3410_begin_0 = const()[name = string("op_3410_begin_0"), val = tensor<int32, [3]>([0, 0, 1])];
            tensor<int32, [3]> var_3410_end_0 = const()[name = string("op_3410_end_0"), val = tensor<int32, [3]>([1, 16, 2])];
            tensor<bool, [3]> var_3410_end_mask_0 = const()[name = string("op_3410_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_3410_squeeze_mask_0 = const()[name = string("op_3410_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_3410_cast_fp16 = slice_by_index(begin = var_3410_begin_0, end = var_3410_end_0, end_mask = var_3410_end_mask_0, squeeze_mask = var_3410_squeeze_mask_0, x = beta_15_cast_fp16)[name = string("op_3410_cast_fp16")];
            tensor<int32, [1]> var_3411_axes_0 = const()[name = string("op_3411_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_3411_cast_fp16 = expand_dims(axes = var_3411_axes_0, x = var_3410_cast_fp16)[name = string("op_3411_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_59_cast_fp16 = mul(x = var_3407_cast_fp16, y = var_3411_cast_fp16)[name = string("delta_59_cast_fp16")];
            tensor<int32, [1]> var_3414_axes_0 = const()[name = string("op_3414_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_3414_cast_fp16 = expand_dims(axes = var_3414_axes_0, x = delta_59_cast_fp16)[name = string("op_3414_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3415_cast_fp16 = mul(x = var_3403_cast_fp16, y = var_3414_cast_fp16)[name = string("op_3415_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_119_cast_fp16 = add(x = state_117_cast_fp16, y = var_3415_cast_fp16)[name = string("state_119_cast_fp16")];
            tensor<int32, [4]> var_3419_begin_0 = const()[name = string("op_3419_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> var_3419_end_0 = const()[name = string("op_3419_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> var_3419_end_mask_0 = const()[name = string("op_3419_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_3419_squeeze_mask_0 = const()[name = string("op_3419_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_3419_cast_fp16 = slice_by_index(begin = var_3419_begin_0, end = var_3419_end_0, end_mask = var_3419_end_mask_0, squeeze_mask = var_3419_squeeze_mask_0, x = _inversed_query_39_cast_fp16)[name = string("op_3419_cast_fp16")];
            tensor<int32, [1]> var_3420_axes_0 = const()[name = string("op_3420_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_3420_cast_fp16 = expand_dims(axes = var_3420_axes_0, x = var_3419_cast_fp16)[name = string("op_3420_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3421_cast_fp16 = mul(x = state_119_cast_fp16, y = var_3420_cast_fp16)[name = string("op_3421_cast_fp16")];
            tensor<int32, [1]> var_3423_axes_0 = const()[name = string("op_3423_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_3423_keep_dims_0 = const()[name = string("op_3423_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_3423_cast_fp16 = reduce_sum(axes = var_3423_axes_0, keep_dims = var_3423_keep_dims_0, x = var_3421_cast_fp16)[name = string("op_3423_cast_fp16")];
            tensor<int32, [3]> var_3426_begin_0 = const()[name = string("op_3426_begin_0"), val = tensor<int32, [3]>([0, 0, 2])];
            tensor<int32, [3]> var_3426_end_0 = const()[name = string("op_3426_end_0"), val = tensor<int32, [3]>([1, 16, 3])];
            tensor<bool, [3]> var_3426_end_mask_0 = const()[name = string("op_3426_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_3426_squeeze_mask_0 = const()[name = string("op_3426_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_3426_cast_fp16 = slice_by_index(begin = var_3426_begin_0, end = var_3426_end_0, end_mask = var_3426_end_mask_0, squeeze_mask = var_3426_squeeze_mask_0, x = decay_15_cast_fp16)[name = string("op_3426_cast_fp16")];
            tensor<fp16, [1, 16]> var_3427_cast_fp16 = exp(x = var_3426_cast_fp16)[name = string("op_3427_cast_fp16")];
            tensor<int32, [1]> var_3428_axes_0 = const()[name = string("op_3428_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_3428_cast_fp16 = expand_dims(axes = var_3428_axes_0, x = var_3427_cast_fp16)[name = string("op_3428_cast_fp16")];
            tensor<int32, [1]> var_3429_axes_0 = const()[name = string("op_3429_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_3429_cast_fp16 = expand_dims(axes = var_3429_axes_0, x = var_3428_cast_fp16)[name = string("op_3429_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_121_cast_fp16 = mul(x = state_119_cast_fp16, y = var_3429_cast_fp16)[name = string("state_121_cast_fp16")];
            tensor<int32, [4]> key_token_61_begin_0 = const()[name = string("key_token_61_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> key_token_61_end_0 = const()[name = string("key_token_61_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> key_token_61_end_mask_0 = const()[name = string("key_token_61_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_61_squeeze_mask_0 = const()[name = string("key_token_61_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_61_cast_fp16 = slice_by_index(begin = key_token_61_begin_0, end = key_token_61_end_0, end_mask = key_token_61_end_mask_0, squeeze_mask = key_token_61_squeeze_mask_0, x = key_39_cast_fp16)[name = string("key_token_61_cast_fp16")];
            tensor<int32, [4]> value_token_61_begin_0 = const()[name = string("value_token_61_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> value_token_61_end_0 = const()[name = string("value_token_61_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> value_token_61_end_mask_0 = const()[name = string("value_token_61_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_61_squeeze_mask_0 = const()[name = string("value_token_61_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_61_cast_fp16 = slice_by_index(begin = value_token_61_begin_0, end = value_token_61_end_0, end_mask = value_token_61_end_mask_0, squeeze_mask = value_token_61_squeeze_mask_0, x = value_141_cast_fp16)[name = string("value_token_61_cast_fp16")];
            tensor<int32, [1]> var_3437_axes_0 = const()[name = string("op_3437_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_3437_cast_fp16 = expand_dims(axes = var_3437_axes_0, x = key_token_61_cast_fp16)[name = string("op_3437_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3438_cast_fp16 = mul(x = state_121_cast_fp16, y = var_3437_cast_fp16)[name = string("op_3438_cast_fp16")];
            tensor<int32, [1]> memory_61_axes_0 = const()[name = string("memory_61_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_61_keep_dims_0 = const()[name = string("memory_61_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_61_cast_fp16 = reduce_sum(axes = memory_61_axes_0, keep_dims = memory_61_keep_dims_0, x = var_3438_cast_fp16)[name = string("memory_61_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_3441_cast_fp16 = sub(x = value_token_61_cast_fp16, y = memory_61_cast_fp16)[name = string("op_3441_cast_fp16")];
            tensor<int32, [3]> var_3444_begin_0 = const()[name = string("op_3444_begin_0"), val = tensor<int32, [3]>([0, 0, 2])];
            tensor<int32, [3]> var_3444_end_0 = const()[name = string("op_3444_end_0"), val = tensor<int32, [3]>([1, 16, 3])];
            tensor<bool, [3]> var_3444_end_mask_0 = const()[name = string("op_3444_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_3444_squeeze_mask_0 = const()[name = string("op_3444_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_3444_cast_fp16 = slice_by_index(begin = var_3444_begin_0, end = var_3444_end_0, end_mask = var_3444_end_mask_0, squeeze_mask = var_3444_squeeze_mask_0, x = beta_15_cast_fp16)[name = string("op_3444_cast_fp16")];
            tensor<int32, [1]> var_3445_axes_0 = const()[name = string("op_3445_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_3445_cast_fp16 = expand_dims(axes = var_3445_axes_0, x = var_3444_cast_fp16)[name = string("op_3445_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_61_cast_fp16 = mul(x = var_3441_cast_fp16, y = var_3445_cast_fp16)[name = string("delta_61_cast_fp16")];
            tensor<int32, [1]> var_3448_axes_0 = const()[name = string("op_3448_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_3448_cast_fp16 = expand_dims(axes = var_3448_axes_0, x = delta_61_cast_fp16)[name = string("op_3448_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3449_cast_fp16 = mul(x = var_3437_cast_fp16, y = var_3448_cast_fp16)[name = string("op_3449_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_123_cast_fp16 = add(x = state_121_cast_fp16, y = var_3449_cast_fp16)[name = string("state_123_cast_fp16")];
            tensor<int32, [4]> var_3453_begin_0 = const()[name = string("op_3453_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> var_3453_end_0 = const()[name = string("op_3453_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> var_3453_end_mask_0 = const()[name = string("op_3453_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_3453_squeeze_mask_0 = const()[name = string("op_3453_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_3453_cast_fp16 = slice_by_index(begin = var_3453_begin_0, end = var_3453_end_0, end_mask = var_3453_end_mask_0, squeeze_mask = var_3453_squeeze_mask_0, x = _inversed_query_39_cast_fp16)[name = string("op_3453_cast_fp16")];
            tensor<int32, [1]> var_3454_axes_0 = const()[name = string("op_3454_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_3454_cast_fp16 = expand_dims(axes = var_3454_axes_0, x = var_3453_cast_fp16)[name = string("op_3454_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3455_cast_fp16 = mul(x = state_123_cast_fp16, y = var_3454_cast_fp16)[name = string("op_3455_cast_fp16")];
            tensor<int32, [1]> var_3457_axes_0 = const()[name = string("op_3457_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_3457_keep_dims_0 = const()[name = string("op_3457_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_3457_cast_fp16 = reduce_sum(axes = var_3457_axes_0, keep_dims = var_3457_keep_dims_0, x = var_3455_cast_fp16)[name = string("op_3457_cast_fp16")];
            tensor<int32, [3]> var_3460_begin_0 = const()[name = string("op_3460_begin_0"), val = tensor<int32, [3]>([0, 0, 3])];
            tensor<int32, [3]> var_3460_end_0 = const()[name = string("op_3460_end_0"), val = tensor<int32, [3]>([1, 16, 4])];
            tensor<bool, [3]> var_3460_end_mask_0 = const()[name = string("op_3460_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_3460_squeeze_mask_0 = const()[name = string("op_3460_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_3460_cast_fp16 = slice_by_index(begin = var_3460_begin_0, end = var_3460_end_0, end_mask = var_3460_end_mask_0, squeeze_mask = var_3460_squeeze_mask_0, x = decay_15_cast_fp16)[name = string("op_3460_cast_fp16")];
            tensor<fp16, [1, 16]> var_3461_cast_fp16 = exp(x = var_3460_cast_fp16)[name = string("op_3461_cast_fp16")];
            tensor<int32, [1]> var_3462_axes_0 = const()[name = string("op_3462_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_3462_cast_fp16 = expand_dims(axes = var_3462_axes_0, x = var_3461_cast_fp16)[name = string("op_3462_cast_fp16")];
            tensor<int32, [1]> var_3463_axes_0 = const()[name = string("op_3463_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_3463_cast_fp16 = expand_dims(axes = var_3463_axes_0, x = var_3462_cast_fp16)[name = string("op_3463_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_125_cast_fp16 = mul(x = state_123_cast_fp16, y = var_3463_cast_fp16)[name = string("state_125_cast_fp16")];
            tensor<int32, [4]> key_token_63_begin_0 = const()[name = string("key_token_63_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> key_token_63_end_0 = const()[name = string("key_token_63_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> key_token_63_end_mask_0 = const()[name = string("key_token_63_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_63_squeeze_mask_0 = const()[name = string("key_token_63_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_63_cast_fp16 = slice_by_index(begin = key_token_63_begin_0, end = key_token_63_end_0, end_mask = key_token_63_end_mask_0, squeeze_mask = key_token_63_squeeze_mask_0, x = key_39_cast_fp16)[name = string("key_token_63_cast_fp16")];
            tensor<int32, [4]> value_token_63_begin_0 = const()[name = string("value_token_63_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> value_token_63_end_0 = const()[name = string("value_token_63_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> value_token_63_end_mask_0 = const()[name = string("value_token_63_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_63_squeeze_mask_0 = const()[name = string("value_token_63_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_63_cast_fp16 = slice_by_index(begin = value_token_63_begin_0, end = value_token_63_end_0, end_mask = value_token_63_end_mask_0, squeeze_mask = value_token_63_squeeze_mask_0, x = value_141_cast_fp16)[name = string("value_token_63_cast_fp16")];
            tensor<int32, [1]> var_3471_axes_0 = const()[name = string("op_3471_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_3471_cast_fp16 = expand_dims(axes = var_3471_axes_0, x = key_token_63_cast_fp16)[name = string("op_3471_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3472_cast_fp16 = mul(x = state_125_cast_fp16, y = var_3471_cast_fp16)[name = string("op_3472_cast_fp16")];
            tensor<int32, [1]> memory_63_axes_0 = const()[name = string("memory_63_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_63_keep_dims_0 = const()[name = string("memory_63_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_63_cast_fp16 = reduce_sum(axes = memory_63_axes_0, keep_dims = memory_63_keep_dims_0, x = var_3472_cast_fp16)[name = string("memory_63_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_3475_cast_fp16 = sub(x = value_token_63_cast_fp16, y = memory_63_cast_fp16)[name = string("op_3475_cast_fp16")];
            tensor<int32, [3]> var_3478_begin_0 = const()[name = string("op_3478_begin_0"), val = tensor<int32, [3]>([0, 0, 3])];
            tensor<int32, [3]> var_3478_end_0 = const()[name = string("op_3478_end_0"), val = tensor<int32, [3]>([1, 16, 4])];
            tensor<bool, [3]> var_3478_end_mask_0 = const()[name = string("op_3478_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_3478_squeeze_mask_0 = const()[name = string("op_3478_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_3478_cast_fp16 = slice_by_index(begin = var_3478_begin_0, end = var_3478_end_0, end_mask = var_3478_end_mask_0, squeeze_mask = var_3478_squeeze_mask_0, x = beta_15_cast_fp16)[name = string("op_3478_cast_fp16")];
            tensor<int32, [1]> var_3479_axes_0 = const()[name = string("op_3479_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_3479_cast_fp16 = expand_dims(axes = var_3479_axes_0, x = var_3478_cast_fp16)[name = string("op_3479_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_63_cast_fp16 = mul(x = var_3475_cast_fp16, y = var_3479_cast_fp16)[name = string("delta_63_cast_fp16")];
            tensor<int32, [1]> var_3482_axes_0 = const()[name = string("op_3482_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_3482_cast_fp16 = expand_dims(axes = var_3482_axes_0, x = delta_63_cast_fp16)[name = string("op_3482_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3483_cast_fp16 = mul(x = var_3471_cast_fp16, y = var_3482_cast_fp16)[name = string("op_3483_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> rec9_out = add(x = state_125_cast_fp16, y = var_3483_cast_fp16)[name = string("state_127_cast_fp16")];
            tensor<int32, [4]> var_3487_begin_0 = const()[name = string("op_3487_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> var_3487_end_0 = const()[name = string("op_3487_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> var_3487_end_mask_0 = const()[name = string("op_3487_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_3487_squeeze_mask_0 = const()[name = string("op_3487_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_3487_cast_fp16 = slice_by_index(begin = var_3487_begin_0, end = var_3487_end_0, end_mask = var_3487_end_mask_0, squeeze_mask = var_3487_squeeze_mask_0, x = _inversed_query_39_cast_fp16)[name = string("op_3487_cast_fp16")];
            tensor<int32, [1]> var_3488_axes_0 = const()[name = string("op_3488_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_3488_cast_fp16 = expand_dims(axes = var_3488_axes_0, x = var_3487_cast_fp16)[name = string("op_3488_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3489_cast_fp16 = mul(x = rec9_out, y = var_3488_cast_fp16)[name = string("op_3489_cast_fp16")];
            tensor<int32, [1]> var_3491_axes_0 = const()[name = string("op_3491_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_3491_keep_dims_0 = const()[name = string("op_3491_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_3491_cast_fp16 = reduce_sum(axes = var_3491_axes_0, keep_dims = var_3491_keep_dims_0, x = var_3489_cast_fp16)[name = string("op_3491_cast_fp16")];
            int32 attention_71_axis_0 = const()[name = string("attention_71_axis_0"), val = int32(1)];
            tensor<fp16, [1, 4, 16, 128]> attention_71_cast_fp16 = stack(axis = attention_71_axis_0, values = (var_3389_cast_fp16, var_3423_cast_fp16, var_3457_cast_fp16, var_3491_cast_fp16))[name = string("attention_71_cast_fp16")];
            tensor<int32, [2]> var_3494 = const()[name = string("op_3494"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [64, 128]> attention_73_cast_fp16 = reshape(shape = var_3494, x = attention_71_cast_fp16)[name = string("attention_73_cast_fp16")];
            tensor<int32, [2]> var_3496 = const()[name = string("op_3496"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [64, 128]> input_123_cast_fp16 = reshape(shape = var_3496, x = linear_73_cast_fp16)[name = string("input_123_cast_fp16")];
            tensor<fp16, [64, 128]> var_3498_cast_fp16 = mul(x = attention_73_cast_fp16, y = attention_73_cast_fp16)[name = string("op_3498_cast_fp16")];
            tensor<int32, [1]> variance_61_axes_0 = const()[name = string("variance_61_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_61_keep_dims_0 = const()[name = string("variance_61_keep_dims_0"), val = bool(true)];
            tensor<fp16, [64, 1]> variance_61_cast_fp16 = reduce_mean(axes = variance_61_axes_0, keep_dims = variance_61_keep_dims_0, x = var_3498_cast_fp16)[name = string("variance_61_cast_fp16")];
            fp16 var_3501_to_fp16 = const()[name = string("op_3501_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [64, 1]> var_3502_cast_fp16 = add(x = variance_61_cast_fp16, y = var_3501_to_fp16)[name = string("op_3502_cast_fp16")];
            fp32 var_3503_epsilon_0 = const()[name = string("op_3503_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [64, 1]> var_3503_cast_fp16 = rsqrt(epsilon = var_3503_epsilon_0, x = var_3502_cast_fp16)[name = string("op_3503_cast_fp16")];
            tensor<fp16, [64, 128]> attention_75_cast_fp16 = mul(x = attention_73_cast_fp16, y = var_3503_cast_fp16)[name = string("attention_75_cast_fp16")];
            tensor<fp16, [128]> groups_2_1_gated_norm_promoted_to_fp16 = const()[name = string("groups_2_1_gated_norm_promoted_to_fp16"), val = tensor<fp16, [128]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(148771264)))];
            tensor<fp16, [64, 128]> attention_77_cast_fp16 = mul(x = attention_75_cast_fp16, y = groups_2_1_gated_norm_promoted_to_fp16)[name = string("attention_77_cast_fp16")];
            tensor<fp16, [64, 128]> var_3506_cast_fp16 = silu(x = input_123_cast_fp16)[name = string("op_3506_cast_fp16")];
            tensor<fp16, [64, 128]> attention_79_cast_fp16 = mul(x = attention_77_cast_fp16, y = var_3506_cast_fp16)[name = string("attention_79_cast_fp16")];
            tensor<int32, [2]> var_3508 = const()[name = string("op_3508"), val = tensor<int32, [2]>([4, 2048])];
            tensor<fp16, [4, 2048]> input_125_cast_fp16 = reshape(shape = var_3508, x = attention_79_cast_fp16)[name = string("input_125_cast_fp16")];
            tensor<fp16, [1024, 2048]> groups_2_1_out_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 2048]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(148771584))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(150344512))))[name = string("groups_2_1_out_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 1024]> linear_74_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = groups_2_1_out_proj_weight_promoted_to_fp16_palettized, x = input_125_cast_fp16)[name = string("linear_74_cast_fp16")];
            tensor<fp16, [4, 1024]> value_143_cast_fp16 = add(x = value_131_cast_fp16, y = linear_74_cast_fp16)[name = string("value_143_cast_fp16")];
            tensor<fp16, [4, 1024]> var_3513_cast_fp16 = mul(x = value_143_cast_fp16, y = value_143_cast_fp16)[name = string("op_3513_cast_fp16")];
            tensor<int32, [1]> variance_63_axes_0 = const()[name = string("variance_63_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_63_keep_dims_0 = const()[name = string("variance_63_keep_dims_0"), val = bool(true)];
            tensor<fp16, [4, 1]> variance_63_cast_fp16 = reduce_mean(axes = variance_63_axes_0, keep_dims = variance_63_keep_dims_0, x = var_3513_cast_fp16)[name = string("variance_63_cast_fp16")];
            fp16 var_3516_to_fp16 = const()[name = string("op_3516_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [4, 1]> var_3517_cast_fp16 = add(x = variance_63_cast_fp16, y = var_3516_to_fp16)[name = string("op_3517_cast_fp16")];
            fp32 var_3518_epsilon_0 = const()[name = string("op_3518_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [4, 1]> var_3518_cast_fp16 = rsqrt(epsilon = var_3518_epsilon_0, x = var_3517_cast_fp16)[name = string("op_3518_cast_fp16")];
            tensor<fp16, [4, 1024]> var_3519_cast_fp16 = mul(x = value_143_cast_fp16, y = var_3518_cast_fp16)[name = string("op_3519_cast_fp16")];
            tensor<fp16, [1024]> var_3521_to_fp16 = const()[name = string("op_3521_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(150352768)))];
            tensor<fp16, [4, 1024]> input_127_cast_fp16 = mul(x = var_3519_cast_fp16, y = var_3521_to_fp16)[name = string("input_127_cast_fp16")];
            tensor<fp16, [3584, 1024]> groups_2_1_gate_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(150354880))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(153107456))))[name = string("groups_2_1_gate_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 3584]> linear_75_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = groups_2_1_gate_proj_weight_promoted_to_fp16_palettized, x = input_127_cast_fp16)[name = string("linear_75_cast_fp16")];
            tensor<fp16, [4, 3584]> var_3525_cast_fp16 = silu(x = linear_75_cast_fp16)[name = string("op_3525_cast_fp16")];
            tensor<fp16, [3584, 1024]> groups_2_1_up_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(153136192))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(155888768))))[name = string("groups_2_1_up_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 3584]> linear_76_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = groups_2_1_up_proj_weight_promoted_to_fp16_palettized, x = input_127_cast_fp16)[name = string("linear_76_cast_fp16")];
            tensor<fp16, [4, 3584]> input_131_cast_fp16 = mul(x = var_3525_cast_fp16, y = linear_76_cast_fp16)[name = string("input_131_cast_fp16")];
            tensor<fp16, [1024, 3584]> groups_2_1_down_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 3584]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(155917504))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(158670080))))[name = string("groups_2_1_down_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 1024]> linear_77_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = groups_2_1_down_proj_weight_promoted_to_fp16_palettized, x = input_131_cast_fp16)[name = string("linear_77_cast_fp16")];
            tensor<fp16, [4, 1024]> value_145_cast_fp16 = add(x = value_143_cast_fp16, y = linear_77_cast_fp16)[name = string("value_145_cast_fp16")];
            int32 var_3552 = const()[name = string("op_3552"), val = int32(-1)];
            tensor<fp16, [4, 1024]> var_3567_cast_fp16 = mul(x = value_145_cast_fp16, y = value_145_cast_fp16)[name = string("op_3567_cast_fp16")];
            tensor<int32, [1]> variance_65_axes_0 = const()[name = string("variance_65_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_65_keep_dims_0 = const()[name = string("variance_65_keep_dims_0"), val = bool(true)];
            tensor<fp16, [4, 1]> variance_65_cast_fp16 = reduce_mean(axes = variance_65_axes_0, keep_dims = variance_65_keep_dims_0, x = var_3567_cast_fp16)[name = string("variance_65_cast_fp16")];
            fp16 var_3570_to_fp16 = const()[name = string("op_3570_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [4, 1]> var_3571_cast_fp16 = add(x = variance_65_cast_fp16, y = var_3570_to_fp16)[name = string("op_3571_cast_fp16")];
            fp32 var_3572_epsilon_0 = const()[name = string("op_3572_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [4, 1]> var_3572_cast_fp16 = rsqrt(epsilon = var_3572_epsilon_0, x = var_3571_cast_fp16)[name = string("op_3572_cast_fp16")];
            tensor<fp16, [4, 1024]> var_3573_cast_fp16 = mul(x = value_145_cast_fp16, y = var_3572_cast_fp16)[name = string("op_3573_cast_fp16")];
            tensor<fp16, [1024]> var_3575_to_fp16 = const()[name = string("op_3575_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(158678336)))];
            tensor<fp16, [4, 1024]> input_133_cast_fp16 = mul(x = var_3573_cast_fp16, y = var_3575_to_fp16)[name = string("input_133_cast_fp16")];
            tensor<fp16, [6144, 1024]> groups_2_2_in_proj_qkv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(158680448))), lut = tensor<fp16, [384, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(163399104))))[name = string("groups_2_2_in_proj_qkv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 6144]> linear_78_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = groups_2_2_in_proj_qkv_weight_promoted_to_fp16_palettized, x = input_133_cast_fp16)[name = string("linear_78_cast_fp16")];
            tensor<int32, [2]> var_3579_perm_0 = const()[name = string("op_3579_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> mixed_17_axes_0 = const()[name = string("mixed_17_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [6144, 4]> var_3579_cast_fp16 = transpose(perm = var_3579_perm_0, x = linear_78_cast_fp16)[name = string("transpose_68")];
            tensor<fp16, [1, 6144, 4]> mixed_17_cast_fp16 = expand_dims(axes = mixed_17_axes_0, x = var_3579_cast_fp16)[name = string("mixed_17_cast_fp16")];
            bool convolution_input_17_interleave_0 = const()[name = string("convolution_input_17_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 6144, 7]> convolution_input_17_cast_fp16 = concat(axis = var_3552, interleave = convolution_input_17_interleave_0, values = (conv10, mixed_17_cast_fp16))[name = string("convolution_input_17_cast_fp16")];
            string input_135_pad_type_0 = const()[name = string("input_135_pad_type_0"), val = string("valid")];
            int32 input_135_groups_0 = const()[name = string("input_135_groups_0"), val = int32(6144)];
            tensor<int32, [1]> input_135_strides_0 = const()[name = string("input_135_strides_0"), val = tensor<int32, [1]>([1])];
            tensor<int32, [2]> input_135_pad_0 = const()[name = string("input_135_pad_0"), val = tensor<int32, [2]>([0, 0])];
            tensor<int32, [1]> input_135_dilations_0 = const()[name = string("input_135_dilations_0"), val = tensor<int32, [1]>([1])];
            tensor<fp16, [6144, 1, 4]> groups_2_2_conv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1, 4]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(163448320))), lut = tensor<fp16, [384, 1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(163466816))))[name = string("groups_2_2_conv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 6144, 4]> input_135_cast_fp16 = conv(dilations = input_135_dilations_0, groups = input_135_groups_0, pad = input_135_pad_0, pad_type = input_135_pad_type_0, strides = input_135_strides_0, weight = groups_2_2_conv_weight_promoted_to_fp16_palettized, x = convolution_input_17_cast_fp16)[name = string("input_135_cast_fp16")];
            tensor<fp16, [1, 6144, 4]> var_3588_cast_fp16 = silu(x = input_135_cast_fp16)[name = string("op_3588_cast_fp16")];
            tensor<int32, [3]> var_3589_perm_0 = const()[name = string("op_3589_perm_0"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [3]> var_3592_begin_0 = const()[name = string("op_3592_begin_0"), val = tensor<int32, [3]>([0, 0, 4])];
            tensor<int32, [3]> var_3592_end_0 = const()[name = string("op_3592_end_0"), val = tensor<int32, [3]>([1, 6144, 7])];
            tensor<bool, [3]> var_3592_end_mask_0 = const()[name = string("op_3592_end_mask_0"), val = tensor<bool, [3]>([true, true, true])];
            tensor<fp16, [1, 6144, 3]> conv10_out = slice_by_index(begin = var_3592_begin_0, end = var_3592_end_0, end_mask = var_3592_end_mask_0, x = convolution_input_17_cast_fp16)[name = string("op_3592_cast_fp16")];
            tensor<int32, [3]> var_3593 = const()[name = string("op_3593"), val = tensor<int32, [3]>([2048, 2048, 2048])];
            int32 var_3594_axis_0 = const()[name = string("op_3594_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 4, 6144]> var_3589_cast_fp16 = transpose(perm = var_3589_perm_0, x = var_3588_cast_fp16)[name = string("transpose_67")];
            tensor<fp16, [1, 4, 2048]> var_3594_cast_fp16_0, tensor<fp16, [1, 4, 2048]> var_3594_cast_fp16_1, tensor<fp16, [1, 4, 2048]> var_3594_cast_fp16_2 = split(axis = var_3594_axis_0, split_sizes = var_3593, x = var_3589_cast_fp16)[name = string("op_3594_cast_fp16")];
            tensor<int32, [4]> var_3598 = const()[name = string("op_3598"), val = tensor<int32, [4]>([1, 4, 16, 128])];
            tensor<fp16, [1, 4, 16, 128]> value_149_cast_fp16 = reshape(shape = var_3598, x = var_3594_cast_fp16_0)[name = string("value_149_cast_fp16")];
            tensor<int32, [4]> var_3600 = const()[name = string("op_3600"), val = tensor<int32, [4]>([1, 4, 16, 128])];
            tensor<fp16, [1, 4, 16, 128]> value_151_cast_fp16 = reshape(shape = var_3600, x = var_3594_cast_fp16_1)[name = string("value_151_cast_fp16")];
            tensor<int32, [4]> var_3602 = const()[name = string("op_3602"), val = tensor<int32, [4]>([1, 4, 16, 128])];
            tensor<fp16, [1, 4, 16, 128]> value_153_cast_fp16 = reshape(shape = var_3602, x = var_3594_cast_fp16_2)[name = string("value_153_cast_fp16")];
            tensor<fp16, [1, 4, 16, 128]> var_3604_cast_fp16 = mul(x = value_149_cast_fp16, y = value_149_cast_fp16)[name = string("op_3604_cast_fp16")];
            tensor<int32, [1]> var_3606_axes_0 = const()[name = string("op_3606_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_3606_keep_dims_0 = const()[name = string("op_3606_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 4, 16, 1]> var_3606_cast_fp16 = reduce_sum(axes = var_3606_axes_0, keep_dims = var_3606_keep_dims_0, x = var_3604_cast_fp16)[name = string("op_3606_cast_fp16")];
            fp16 var_3607_to_fp16 = const()[name = string("op_3607_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 4, 16, 1]> var_3608_cast_fp16 = add(x = var_3606_cast_fp16, y = var_3607_to_fp16)[name = string("op_3608_cast_fp16")];
            fp32 var_3609_epsilon_0 = const()[name = string("op_3609_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 4, 16, 1]> var_3609_cast_fp16 = rsqrt(epsilon = var_3609_epsilon_0, x = var_3608_cast_fp16)[name = string("op_3609_cast_fp16")];
            tensor<fp16, [1, 4, 16, 128]> var_3610_cast_fp16 = mul(x = value_149_cast_fp16, y = var_3609_cast_fp16)[name = string("op_3610_cast_fp16")];
            tensor<int32, [4]> var_3611_perm_0 = const()[name = string("op_3611_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            fp16 _inversed_query_43_y_0_to_fp16 = const()[name = string("_inversed_query_43_y_0_to_fp16"), val = fp16(0x1.6ap-4)];
            tensor<fp16, [1, 16, 4, 128]> var_3611_cast_fp16 = transpose(perm = var_3611_perm_0, x = var_3610_cast_fp16)[name = string("transpose_66")];
            tensor<fp16, [1, 16, 4, 128]> _inversed_query_43_cast_fp16 = mul(x = var_3611_cast_fp16, y = _inversed_query_43_y_0_to_fp16)[name = string("_inversed_query_43_cast_fp16")];
            tensor<fp16, [1, 4, 16, 128]> var_3614_cast_fp16 = mul(x = value_151_cast_fp16, y = value_151_cast_fp16)[name = string("op_3614_cast_fp16")];
            tensor<int32, [1]> var_3616_axes_0 = const()[name = string("op_3616_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_3616_keep_dims_0 = const()[name = string("op_3616_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 4, 16, 1]> var_3616_cast_fp16 = reduce_sum(axes = var_3616_axes_0, keep_dims = var_3616_keep_dims_0, x = var_3614_cast_fp16)[name = string("op_3616_cast_fp16")];
            fp16 var_3617_to_fp16 = const()[name = string("op_3617_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 4, 16, 1]> var_3618_cast_fp16 = add(x = var_3616_cast_fp16, y = var_3617_to_fp16)[name = string("op_3618_cast_fp16")];
            fp32 var_3619_epsilon_0 = const()[name = string("op_3619_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 4, 16, 1]> var_3619_cast_fp16 = rsqrt(epsilon = var_3619_epsilon_0, x = var_3618_cast_fp16)[name = string("op_3619_cast_fp16")];
            tensor<fp16, [1, 4, 16, 128]> var_3620_cast_fp16 = mul(x = value_151_cast_fp16, y = var_3619_cast_fp16)[name = string("op_3620_cast_fp16")];
            tensor<int32, [4]> key_43_perm_0 = const()[name = string("key_43_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<int32, [4]> value_155_perm_0 = const()[name = string("value_155_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<fp16, [16, 1024]> groups_2_2_in_proj_b_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(163516032))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(163528384))))[name = string("groups_2_2_in_proj_b_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16]> linear_79_bias_0_to_fp16 = const()[name = string("linear_79_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(4862656)))];
            tensor<fp16, [4, 16]> linear_79_cast_fp16 = linear(bias = linear_79_bias_0_to_fp16, weight = groups_2_2_in_proj_b_weight_promoted_to_fp16_palettized, x = input_133_cast_fp16)[name = string("linear_79_cast_fp16")];
            tensor<fp16, [4, 16]> var_3625_cast_fp16 = sigmoid(x = linear_79_cast_fp16)[name = string("op_3625_cast_fp16")];
            tensor<int32, [2]> var_3626_perm_0 = const()[name = string("op_3626_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> beta_17_axes_0 = const()[name = string("beta_17_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 4]> var_3626_cast_fp16 = transpose(perm = var_3626_perm_0, x = var_3625_cast_fp16)[name = string("transpose_65")];
            tensor<fp16, [1, 16, 4]> beta_17_cast_fp16 = expand_dims(axes = beta_17_axes_0, x = var_3626_cast_fp16)[name = string("beta_17_cast_fp16")];
            tensor<fp16, [16, 1024]> op_3632_weight_0_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(163528576))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(163540928))))[name = string("op_3632_weight_0_to_fp16_palettized")];
            tensor<fp16, [16]> var_3632_bias_0_to_fp16 = const()[name = string("op_3632_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(163541120)))];
            tensor<fp16, [4, 16]> var_3632_cast_fp16 = linear(bias = var_3632_bias_0_to_fp16, weight = op_3632_weight_0_to_fp16_palettized, x = input_133_cast_fp16)[name = string("op_3632_cast_fp16")];
            tensor<fp16, [4, 16]> var_3633_cast_fp16 = softplus(x = var_3632_cast_fp16)[name = string("op_3633_cast_fp16")];
            tensor<fp16, [16]> var_3629_promoted_to_fp16 = const()[name = string("op_3629_promoted_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(163541248)))];
            tensor<fp16, [4, 16]> var_3634_cast_fp16 = mul(x = var_3629_promoted_to_fp16, y = var_3633_cast_fp16)[name = string("op_3634_cast_fp16")];
            tensor<int32, [2]> var_3635_perm_0 = const()[name = string("op_3635_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> decay_17_axes_0 = const()[name = string("decay_17_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 4]> var_3635_cast_fp16 = transpose(perm = var_3635_perm_0, x = var_3634_cast_fp16)[name = string("transpose_64")];
            tensor<fp16, [1, 16, 4]> decay_17_cast_fp16 = expand_dims(axes = decay_17_axes_0, x = var_3635_cast_fp16)[name = string("decay_17_cast_fp16")];
            tensor<fp16, [2048, 1024]> groups_2_2_in_proj_z_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [2048, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(163541376))), lut = tensor<fp16, [128, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(165114304))))[name = string("groups_2_2_in_proj_z_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 2048]> linear_81_cast_fp16 = linear(bias = linear_3_bias_0_to_fp16, weight = groups_2_2_in_proj_z_weight_promoted_to_fp16_palettized, x = input_133_cast_fp16)[name = string("linear_81_cast_fp16")];
            tensor<int32, [3]> var_3643_begin_0 = const()[name = string("op_3643_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_3643_end_0 = const()[name = string("op_3643_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_3643_end_mask_0 = const()[name = string("op_3643_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_3643_squeeze_mask_0 = const()[name = string("op_3643_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_3643_cast_fp16 = slice_by_index(begin = var_3643_begin_0, end = var_3643_end_0, end_mask = var_3643_end_mask_0, squeeze_mask = var_3643_squeeze_mask_0, x = decay_17_cast_fp16)[name = string("op_3643_cast_fp16")];
            tensor<fp16, [1, 16]> var_3644_cast_fp16 = exp(x = var_3643_cast_fp16)[name = string("op_3644_cast_fp16")];
            tensor<int32, [1]> var_3645_axes_0 = const()[name = string("op_3645_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_3645_cast_fp16 = expand_dims(axes = var_3645_axes_0, x = var_3644_cast_fp16)[name = string("op_3645_cast_fp16")];
            tensor<int32, [1]> var_3646_axes_0 = const()[name = string("op_3646_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_3646_cast_fp16 = expand_dims(axes = var_3646_axes_0, x = var_3645_cast_fp16)[name = string("op_3646_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_129_cast_fp16 = mul(x = rec10, y = var_3646_cast_fp16)[name = string("state_129_cast_fp16")];
            tensor<int32, [4]> key_token_65_begin_0 = const()[name = string("key_token_65_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> key_token_65_end_0 = const()[name = string("key_token_65_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> key_token_65_end_mask_0 = const()[name = string("key_token_65_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_65_squeeze_mask_0 = const()[name = string("key_token_65_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 4, 128]> key_43_cast_fp16 = transpose(perm = key_43_perm_0, x = var_3620_cast_fp16)[name = string("transpose_63")];
            tensor<fp16, [1, 16, 128]> key_token_65_cast_fp16 = slice_by_index(begin = key_token_65_begin_0, end = key_token_65_end_0, end_mask = key_token_65_end_mask_0, squeeze_mask = key_token_65_squeeze_mask_0, x = key_43_cast_fp16)[name = string("key_token_65_cast_fp16")];
            tensor<int32, [4]> value_token_65_begin_0 = const()[name = string("value_token_65_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> value_token_65_end_0 = const()[name = string("value_token_65_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> value_token_65_end_mask_0 = const()[name = string("value_token_65_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_65_squeeze_mask_0 = const()[name = string("value_token_65_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 4, 128]> value_155_cast_fp16 = transpose(perm = value_155_perm_0, x = value_153_cast_fp16)[name = string("transpose_62")];
            tensor<fp16, [1, 16, 128]> value_token_65_cast_fp16 = slice_by_index(begin = value_token_65_begin_0, end = value_token_65_end_0, end_mask = value_token_65_end_mask_0, squeeze_mask = value_token_65_squeeze_mask_0, x = value_155_cast_fp16)[name = string("value_token_65_cast_fp16")];
            tensor<int32, [1]> var_3654_axes_0 = const()[name = string("op_3654_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_3654_cast_fp16 = expand_dims(axes = var_3654_axes_0, x = key_token_65_cast_fp16)[name = string("op_3654_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3655_cast_fp16 = mul(x = state_129_cast_fp16, y = var_3654_cast_fp16)[name = string("op_3655_cast_fp16")];
            tensor<int32, [1]> memory_65_axes_0 = const()[name = string("memory_65_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_65_keep_dims_0 = const()[name = string("memory_65_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_65_cast_fp16 = reduce_sum(axes = memory_65_axes_0, keep_dims = memory_65_keep_dims_0, x = var_3655_cast_fp16)[name = string("memory_65_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_3658_cast_fp16 = sub(x = value_token_65_cast_fp16, y = memory_65_cast_fp16)[name = string("op_3658_cast_fp16")];
            tensor<int32, [3]> var_3661_begin_0 = const()[name = string("op_3661_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_3661_end_0 = const()[name = string("op_3661_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_3661_end_mask_0 = const()[name = string("op_3661_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_3661_squeeze_mask_0 = const()[name = string("op_3661_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_3661_cast_fp16 = slice_by_index(begin = var_3661_begin_0, end = var_3661_end_0, end_mask = var_3661_end_mask_0, squeeze_mask = var_3661_squeeze_mask_0, x = beta_17_cast_fp16)[name = string("op_3661_cast_fp16")];
            tensor<int32, [1]> var_3662_axes_0 = const()[name = string("op_3662_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_3662_cast_fp16 = expand_dims(axes = var_3662_axes_0, x = var_3661_cast_fp16)[name = string("op_3662_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_65_cast_fp16 = mul(x = var_3658_cast_fp16, y = var_3662_cast_fp16)[name = string("delta_65_cast_fp16")];
            tensor<int32, [1]> var_3665_axes_0 = const()[name = string("op_3665_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_3665_cast_fp16 = expand_dims(axes = var_3665_axes_0, x = delta_65_cast_fp16)[name = string("op_3665_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3666_cast_fp16 = mul(x = var_3654_cast_fp16, y = var_3665_cast_fp16)[name = string("op_3666_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_131_cast_fp16 = add(x = state_129_cast_fp16, y = var_3666_cast_fp16)[name = string("state_131_cast_fp16")];
            tensor<int32, [4]> var_3670_begin_0 = const()[name = string("op_3670_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> var_3670_end_0 = const()[name = string("op_3670_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> var_3670_end_mask_0 = const()[name = string("op_3670_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_3670_squeeze_mask_0 = const()[name = string("op_3670_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_3670_cast_fp16 = slice_by_index(begin = var_3670_begin_0, end = var_3670_end_0, end_mask = var_3670_end_mask_0, squeeze_mask = var_3670_squeeze_mask_0, x = _inversed_query_43_cast_fp16)[name = string("op_3670_cast_fp16")];
            tensor<int32, [1]> var_3671_axes_0 = const()[name = string("op_3671_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_3671_cast_fp16 = expand_dims(axes = var_3671_axes_0, x = var_3670_cast_fp16)[name = string("op_3671_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3672_cast_fp16 = mul(x = state_131_cast_fp16, y = var_3671_cast_fp16)[name = string("op_3672_cast_fp16")];
            tensor<int32, [1]> var_3674_axes_0 = const()[name = string("op_3674_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_3674_keep_dims_0 = const()[name = string("op_3674_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_3674_cast_fp16 = reduce_sum(axes = var_3674_axes_0, keep_dims = var_3674_keep_dims_0, x = var_3672_cast_fp16)[name = string("op_3674_cast_fp16")];
            tensor<int32, [3]> var_3677_begin_0 = const()[name = string("op_3677_begin_0"), val = tensor<int32, [3]>([0, 0, 1])];
            tensor<int32, [3]> var_3677_end_0 = const()[name = string("op_3677_end_0"), val = tensor<int32, [3]>([1, 16, 2])];
            tensor<bool, [3]> var_3677_end_mask_0 = const()[name = string("op_3677_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_3677_squeeze_mask_0 = const()[name = string("op_3677_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_3677_cast_fp16 = slice_by_index(begin = var_3677_begin_0, end = var_3677_end_0, end_mask = var_3677_end_mask_0, squeeze_mask = var_3677_squeeze_mask_0, x = decay_17_cast_fp16)[name = string("op_3677_cast_fp16")];
            tensor<fp16, [1, 16]> var_3678_cast_fp16 = exp(x = var_3677_cast_fp16)[name = string("op_3678_cast_fp16")];
            tensor<int32, [1]> var_3679_axes_0 = const()[name = string("op_3679_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_3679_cast_fp16 = expand_dims(axes = var_3679_axes_0, x = var_3678_cast_fp16)[name = string("op_3679_cast_fp16")];
            tensor<int32, [1]> var_3680_axes_0 = const()[name = string("op_3680_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_3680_cast_fp16 = expand_dims(axes = var_3680_axes_0, x = var_3679_cast_fp16)[name = string("op_3680_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_133_cast_fp16 = mul(x = state_131_cast_fp16, y = var_3680_cast_fp16)[name = string("state_133_cast_fp16")];
            tensor<int32, [4]> key_token_67_begin_0 = const()[name = string("key_token_67_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> key_token_67_end_0 = const()[name = string("key_token_67_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> key_token_67_end_mask_0 = const()[name = string("key_token_67_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_67_squeeze_mask_0 = const()[name = string("key_token_67_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_67_cast_fp16 = slice_by_index(begin = key_token_67_begin_0, end = key_token_67_end_0, end_mask = key_token_67_end_mask_0, squeeze_mask = key_token_67_squeeze_mask_0, x = key_43_cast_fp16)[name = string("key_token_67_cast_fp16")];
            tensor<int32, [4]> value_token_67_begin_0 = const()[name = string("value_token_67_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> value_token_67_end_0 = const()[name = string("value_token_67_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> value_token_67_end_mask_0 = const()[name = string("value_token_67_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_67_squeeze_mask_0 = const()[name = string("value_token_67_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_67_cast_fp16 = slice_by_index(begin = value_token_67_begin_0, end = value_token_67_end_0, end_mask = value_token_67_end_mask_0, squeeze_mask = value_token_67_squeeze_mask_0, x = value_155_cast_fp16)[name = string("value_token_67_cast_fp16")];
            tensor<int32, [1]> var_3688_axes_0 = const()[name = string("op_3688_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_3688_cast_fp16 = expand_dims(axes = var_3688_axes_0, x = key_token_67_cast_fp16)[name = string("op_3688_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3689_cast_fp16 = mul(x = state_133_cast_fp16, y = var_3688_cast_fp16)[name = string("op_3689_cast_fp16")];
            tensor<int32, [1]> memory_67_axes_0 = const()[name = string("memory_67_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_67_keep_dims_0 = const()[name = string("memory_67_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_67_cast_fp16 = reduce_sum(axes = memory_67_axes_0, keep_dims = memory_67_keep_dims_0, x = var_3689_cast_fp16)[name = string("memory_67_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_3692_cast_fp16 = sub(x = value_token_67_cast_fp16, y = memory_67_cast_fp16)[name = string("op_3692_cast_fp16")];
            tensor<int32, [3]> var_3695_begin_0 = const()[name = string("op_3695_begin_0"), val = tensor<int32, [3]>([0, 0, 1])];
            tensor<int32, [3]> var_3695_end_0 = const()[name = string("op_3695_end_0"), val = tensor<int32, [3]>([1, 16, 2])];
            tensor<bool, [3]> var_3695_end_mask_0 = const()[name = string("op_3695_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_3695_squeeze_mask_0 = const()[name = string("op_3695_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_3695_cast_fp16 = slice_by_index(begin = var_3695_begin_0, end = var_3695_end_0, end_mask = var_3695_end_mask_0, squeeze_mask = var_3695_squeeze_mask_0, x = beta_17_cast_fp16)[name = string("op_3695_cast_fp16")];
            tensor<int32, [1]> var_3696_axes_0 = const()[name = string("op_3696_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_3696_cast_fp16 = expand_dims(axes = var_3696_axes_0, x = var_3695_cast_fp16)[name = string("op_3696_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_67_cast_fp16 = mul(x = var_3692_cast_fp16, y = var_3696_cast_fp16)[name = string("delta_67_cast_fp16")];
            tensor<int32, [1]> var_3699_axes_0 = const()[name = string("op_3699_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_3699_cast_fp16 = expand_dims(axes = var_3699_axes_0, x = delta_67_cast_fp16)[name = string("op_3699_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3700_cast_fp16 = mul(x = var_3688_cast_fp16, y = var_3699_cast_fp16)[name = string("op_3700_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_135_cast_fp16 = add(x = state_133_cast_fp16, y = var_3700_cast_fp16)[name = string("state_135_cast_fp16")];
            tensor<int32, [4]> var_3704_begin_0 = const()[name = string("op_3704_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> var_3704_end_0 = const()[name = string("op_3704_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> var_3704_end_mask_0 = const()[name = string("op_3704_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_3704_squeeze_mask_0 = const()[name = string("op_3704_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_3704_cast_fp16 = slice_by_index(begin = var_3704_begin_0, end = var_3704_end_0, end_mask = var_3704_end_mask_0, squeeze_mask = var_3704_squeeze_mask_0, x = _inversed_query_43_cast_fp16)[name = string("op_3704_cast_fp16")];
            tensor<int32, [1]> var_3705_axes_0 = const()[name = string("op_3705_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_3705_cast_fp16 = expand_dims(axes = var_3705_axes_0, x = var_3704_cast_fp16)[name = string("op_3705_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3706_cast_fp16 = mul(x = state_135_cast_fp16, y = var_3705_cast_fp16)[name = string("op_3706_cast_fp16")];
            tensor<int32, [1]> var_3708_axes_0 = const()[name = string("op_3708_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_3708_keep_dims_0 = const()[name = string("op_3708_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_3708_cast_fp16 = reduce_sum(axes = var_3708_axes_0, keep_dims = var_3708_keep_dims_0, x = var_3706_cast_fp16)[name = string("op_3708_cast_fp16")];
            tensor<int32, [3]> var_3711_begin_0 = const()[name = string("op_3711_begin_0"), val = tensor<int32, [3]>([0, 0, 2])];
            tensor<int32, [3]> var_3711_end_0 = const()[name = string("op_3711_end_0"), val = tensor<int32, [3]>([1, 16, 3])];
            tensor<bool, [3]> var_3711_end_mask_0 = const()[name = string("op_3711_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_3711_squeeze_mask_0 = const()[name = string("op_3711_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_3711_cast_fp16 = slice_by_index(begin = var_3711_begin_0, end = var_3711_end_0, end_mask = var_3711_end_mask_0, squeeze_mask = var_3711_squeeze_mask_0, x = decay_17_cast_fp16)[name = string("op_3711_cast_fp16")];
            tensor<fp16, [1, 16]> var_3712_cast_fp16 = exp(x = var_3711_cast_fp16)[name = string("op_3712_cast_fp16")];
            tensor<int32, [1]> var_3713_axes_0 = const()[name = string("op_3713_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_3713_cast_fp16 = expand_dims(axes = var_3713_axes_0, x = var_3712_cast_fp16)[name = string("op_3713_cast_fp16")];
            tensor<int32, [1]> var_3714_axes_0 = const()[name = string("op_3714_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_3714_cast_fp16 = expand_dims(axes = var_3714_axes_0, x = var_3713_cast_fp16)[name = string("op_3714_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_137_cast_fp16 = mul(x = state_135_cast_fp16, y = var_3714_cast_fp16)[name = string("state_137_cast_fp16")];
            tensor<int32, [4]> key_token_69_begin_0 = const()[name = string("key_token_69_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> key_token_69_end_0 = const()[name = string("key_token_69_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> key_token_69_end_mask_0 = const()[name = string("key_token_69_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_69_squeeze_mask_0 = const()[name = string("key_token_69_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_69_cast_fp16 = slice_by_index(begin = key_token_69_begin_0, end = key_token_69_end_0, end_mask = key_token_69_end_mask_0, squeeze_mask = key_token_69_squeeze_mask_0, x = key_43_cast_fp16)[name = string("key_token_69_cast_fp16")];
            tensor<int32, [4]> value_token_69_begin_0 = const()[name = string("value_token_69_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> value_token_69_end_0 = const()[name = string("value_token_69_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> value_token_69_end_mask_0 = const()[name = string("value_token_69_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_69_squeeze_mask_0 = const()[name = string("value_token_69_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_69_cast_fp16 = slice_by_index(begin = value_token_69_begin_0, end = value_token_69_end_0, end_mask = value_token_69_end_mask_0, squeeze_mask = value_token_69_squeeze_mask_0, x = value_155_cast_fp16)[name = string("value_token_69_cast_fp16")];
            tensor<int32, [1]> var_3722_axes_0 = const()[name = string("op_3722_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_3722_cast_fp16 = expand_dims(axes = var_3722_axes_0, x = key_token_69_cast_fp16)[name = string("op_3722_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3723_cast_fp16 = mul(x = state_137_cast_fp16, y = var_3722_cast_fp16)[name = string("op_3723_cast_fp16")];
            tensor<int32, [1]> memory_69_axes_0 = const()[name = string("memory_69_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_69_keep_dims_0 = const()[name = string("memory_69_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_69_cast_fp16 = reduce_sum(axes = memory_69_axes_0, keep_dims = memory_69_keep_dims_0, x = var_3723_cast_fp16)[name = string("memory_69_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_3726_cast_fp16 = sub(x = value_token_69_cast_fp16, y = memory_69_cast_fp16)[name = string("op_3726_cast_fp16")];
            tensor<int32, [3]> var_3729_begin_0 = const()[name = string("op_3729_begin_0"), val = tensor<int32, [3]>([0, 0, 2])];
            tensor<int32, [3]> var_3729_end_0 = const()[name = string("op_3729_end_0"), val = tensor<int32, [3]>([1, 16, 3])];
            tensor<bool, [3]> var_3729_end_mask_0 = const()[name = string("op_3729_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_3729_squeeze_mask_0 = const()[name = string("op_3729_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_3729_cast_fp16 = slice_by_index(begin = var_3729_begin_0, end = var_3729_end_0, end_mask = var_3729_end_mask_0, squeeze_mask = var_3729_squeeze_mask_0, x = beta_17_cast_fp16)[name = string("op_3729_cast_fp16")];
            tensor<int32, [1]> var_3730_axes_0 = const()[name = string("op_3730_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_3730_cast_fp16 = expand_dims(axes = var_3730_axes_0, x = var_3729_cast_fp16)[name = string("op_3730_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_69_cast_fp16 = mul(x = var_3726_cast_fp16, y = var_3730_cast_fp16)[name = string("delta_69_cast_fp16")];
            tensor<int32, [1]> var_3733_axes_0 = const()[name = string("op_3733_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_3733_cast_fp16 = expand_dims(axes = var_3733_axes_0, x = delta_69_cast_fp16)[name = string("op_3733_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3734_cast_fp16 = mul(x = var_3722_cast_fp16, y = var_3733_cast_fp16)[name = string("op_3734_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_139_cast_fp16 = add(x = state_137_cast_fp16, y = var_3734_cast_fp16)[name = string("state_139_cast_fp16")];
            tensor<int32, [4]> var_3738_begin_0 = const()[name = string("op_3738_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> var_3738_end_0 = const()[name = string("op_3738_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> var_3738_end_mask_0 = const()[name = string("op_3738_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_3738_squeeze_mask_0 = const()[name = string("op_3738_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_3738_cast_fp16 = slice_by_index(begin = var_3738_begin_0, end = var_3738_end_0, end_mask = var_3738_end_mask_0, squeeze_mask = var_3738_squeeze_mask_0, x = _inversed_query_43_cast_fp16)[name = string("op_3738_cast_fp16")];
            tensor<int32, [1]> var_3739_axes_0 = const()[name = string("op_3739_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_3739_cast_fp16 = expand_dims(axes = var_3739_axes_0, x = var_3738_cast_fp16)[name = string("op_3739_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3740_cast_fp16 = mul(x = state_139_cast_fp16, y = var_3739_cast_fp16)[name = string("op_3740_cast_fp16")];
            tensor<int32, [1]> var_3742_axes_0 = const()[name = string("op_3742_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_3742_keep_dims_0 = const()[name = string("op_3742_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_3742_cast_fp16 = reduce_sum(axes = var_3742_axes_0, keep_dims = var_3742_keep_dims_0, x = var_3740_cast_fp16)[name = string("op_3742_cast_fp16")];
            tensor<int32, [3]> var_3745_begin_0 = const()[name = string("op_3745_begin_0"), val = tensor<int32, [3]>([0, 0, 3])];
            tensor<int32, [3]> var_3745_end_0 = const()[name = string("op_3745_end_0"), val = tensor<int32, [3]>([1, 16, 4])];
            tensor<bool, [3]> var_3745_end_mask_0 = const()[name = string("op_3745_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_3745_squeeze_mask_0 = const()[name = string("op_3745_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_3745_cast_fp16 = slice_by_index(begin = var_3745_begin_0, end = var_3745_end_0, end_mask = var_3745_end_mask_0, squeeze_mask = var_3745_squeeze_mask_0, x = decay_17_cast_fp16)[name = string("op_3745_cast_fp16")];
            tensor<fp16, [1, 16]> var_3746_cast_fp16 = exp(x = var_3745_cast_fp16)[name = string("op_3746_cast_fp16")];
            tensor<int32, [1]> var_3747_axes_0 = const()[name = string("op_3747_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_3747_cast_fp16 = expand_dims(axes = var_3747_axes_0, x = var_3746_cast_fp16)[name = string("op_3747_cast_fp16")];
            tensor<int32, [1]> var_3748_axes_0 = const()[name = string("op_3748_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_3748_cast_fp16 = expand_dims(axes = var_3748_axes_0, x = var_3747_cast_fp16)[name = string("op_3748_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_141_cast_fp16 = mul(x = state_139_cast_fp16, y = var_3748_cast_fp16)[name = string("state_141_cast_fp16")];
            tensor<int32, [4]> key_token_71_begin_0 = const()[name = string("key_token_71_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> key_token_71_end_0 = const()[name = string("key_token_71_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> key_token_71_end_mask_0 = const()[name = string("key_token_71_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_71_squeeze_mask_0 = const()[name = string("key_token_71_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_71_cast_fp16 = slice_by_index(begin = key_token_71_begin_0, end = key_token_71_end_0, end_mask = key_token_71_end_mask_0, squeeze_mask = key_token_71_squeeze_mask_0, x = key_43_cast_fp16)[name = string("key_token_71_cast_fp16")];
            tensor<int32, [4]> value_token_71_begin_0 = const()[name = string("value_token_71_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> value_token_71_end_0 = const()[name = string("value_token_71_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> value_token_71_end_mask_0 = const()[name = string("value_token_71_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_71_squeeze_mask_0 = const()[name = string("value_token_71_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_71_cast_fp16 = slice_by_index(begin = value_token_71_begin_0, end = value_token_71_end_0, end_mask = value_token_71_end_mask_0, squeeze_mask = value_token_71_squeeze_mask_0, x = value_155_cast_fp16)[name = string("value_token_71_cast_fp16")];
            tensor<int32, [1]> var_3756_axes_0 = const()[name = string("op_3756_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_3756_cast_fp16 = expand_dims(axes = var_3756_axes_0, x = key_token_71_cast_fp16)[name = string("op_3756_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3757_cast_fp16 = mul(x = state_141_cast_fp16, y = var_3756_cast_fp16)[name = string("op_3757_cast_fp16")];
            tensor<int32, [1]> memory_71_axes_0 = const()[name = string("memory_71_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_71_keep_dims_0 = const()[name = string("memory_71_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_71_cast_fp16 = reduce_sum(axes = memory_71_axes_0, keep_dims = memory_71_keep_dims_0, x = var_3757_cast_fp16)[name = string("memory_71_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_3760_cast_fp16 = sub(x = value_token_71_cast_fp16, y = memory_71_cast_fp16)[name = string("op_3760_cast_fp16")];
            tensor<int32, [3]> var_3763_begin_0 = const()[name = string("op_3763_begin_0"), val = tensor<int32, [3]>([0, 0, 3])];
            tensor<int32, [3]> var_3763_end_0 = const()[name = string("op_3763_end_0"), val = tensor<int32, [3]>([1, 16, 4])];
            tensor<bool, [3]> var_3763_end_mask_0 = const()[name = string("op_3763_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_3763_squeeze_mask_0 = const()[name = string("op_3763_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_3763_cast_fp16 = slice_by_index(begin = var_3763_begin_0, end = var_3763_end_0, end_mask = var_3763_end_mask_0, squeeze_mask = var_3763_squeeze_mask_0, x = beta_17_cast_fp16)[name = string("op_3763_cast_fp16")];
            tensor<int32, [1]> var_3764_axes_0 = const()[name = string("op_3764_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_3764_cast_fp16 = expand_dims(axes = var_3764_axes_0, x = var_3763_cast_fp16)[name = string("op_3764_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_71_cast_fp16 = mul(x = var_3760_cast_fp16, y = var_3764_cast_fp16)[name = string("delta_71_cast_fp16")];
            tensor<int32, [1]> var_3767_axes_0 = const()[name = string("op_3767_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_3767_cast_fp16 = expand_dims(axes = var_3767_axes_0, x = delta_71_cast_fp16)[name = string("op_3767_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3768_cast_fp16 = mul(x = var_3756_cast_fp16, y = var_3767_cast_fp16)[name = string("op_3768_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> rec10_out = add(x = state_141_cast_fp16, y = var_3768_cast_fp16)[name = string("state_143_cast_fp16")];
            tensor<int32, [4]> var_3772_begin_0 = const()[name = string("op_3772_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> var_3772_end_0 = const()[name = string("op_3772_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> var_3772_end_mask_0 = const()[name = string("op_3772_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_3772_squeeze_mask_0 = const()[name = string("op_3772_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_3772_cast_fp16 = slice_by_index(begin = var_3772_begin_0, end = var_3772_end_0, end_mask = var_3772_end_mask_0, squeeze_mask = var_3772_squeeze_mask_0, x = _inversed_query_43_cast_fp16)[name = string("op_3772_cast_fp16")];
            tensor<int32, [1]> var_3773_axes_0 = const()[name = string("op_3773_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_3773_cast_fp16 = expand_dims(axes = var_3773_axes_0, x = var_3772_cast_fp16)[name = string("op_3773_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_3774_cast_fp16 = mul(x = rec10_out, y = var_3773_cast_fp16)[name = string("op_3774_cast_fp16")];
            tensor<int32, [1]> var_3776_axes_0 = const()[name = string("op_3776_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_3776_keep_dims_0 = const()[name = string("op_3776_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_3776_cast_fp16 = reduce_sum(axes = var_3776_axes_0, keep_dims = var_3776_keep_dims_0, x = var_3774_cast_fp16)[name = string("op_3776_cast_fp16")];
            int32 attention_81_axis_0 = const()[name = string("attention_81_axis_0"), val = int32(1)];
            tensor<fp16, [1, 4, 16, 128]> attention_81_cast_fp16 = stack(axis = attention_81_axis_0, values = (var_3674_cast_fp16, var_3708_cast_fp16, var_3742_cast_fp16, var_3776_cast_fp16))[name = string("attention_81_cast_fp16")];
            tensor<int32, [2]> var_3779 = const()[name = string("op_3779"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [64, 128]> attention_83_cast_fp16 = reshape(shape = var_3779, x = attention_81_cast_fp16)[name = string("attention_83_cast_fp16")];
            tensor<int32, [2]> var_3781 = const()[name = string("op_3781"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [64, 128]> input_137_cast_fp16 = reshape(shape = var_3781, x = linear_81_cast_fp16)[name = string("input_137_cast_fp16")];
            tensor<fp16, [64, 128]> var_3783_cast_fp16 = mul(x = attention_83_cast_fp16, y = attention_83_cast_fp16)[name = string("op_3783_cast_fp16")];
            tensor<int32, [1]> variance_67_axes_0 = const()[name = string("variance_67_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_67_keep_dims_0 = const()[name = string("variance_67_keep_dims_0"), val = bool(true)];
            tensor<fp16, [64, 1]> variance_67_cast_fp16 = reduce_mean(axes = variance_67_axes_0, keep_dims = variance_67_keep_dims_0, x = var_3783_cast_fp16)[name = string("variance_67_cast_fp16")];
            fp16 var_3786_to_fp16 = const()[name = string("op_3786_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [64, 1]> var_3787_cast_fp16 = add(x = variance_67_cast_fp16, y = var_3786_to_fp16)[name = string("op_3787_cast_fp16")];
            fp32 var_3788_epsilon_0 = const()[name = string("op_3788_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [64, 1]> var_3788_cast_fp16 = rsqrt(epsilon = var_3788_epsilon_0, x = var_3787_cast_fp16)[name = string("op_3788_cast_fp16")];
            tensor<fp16, [64, 128]> attention_85_cast_fp16 = mul(x = attention_83_cast_fp16, y = var_3788_cast_fp16)[name = string("attention_85_cast_fp16")];
            tensor<fp16, [128]> groups_2_2_gated_norm_promoted_to_fp16 = const()[name = string("groups_2_2_gated_norm_promoted_to_fp16"), val = tensor<fp16, [128]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(165130752)))];
            tensor<fp16, [64, 128]> attention_87_cast_fp16 = mul(x = attention_85_cast_fp16, y = groups_2_2_gated_norm_promoted_to_fp16)[name = string("attention_87_cast_fp16")];
            tensor<fp16, [64, 128]> var_3791_cast_fp16 = silu(x = input_137_cast_fp16)[name = string("op_3791_cast_fp16")];
            tensor<fp16, [64, 128]> attention_89_cast_fp16 = mul(x = attention_87_cast_fp16, y = var_3791_cast_fp16)[name = string("attention_89_cast_fp16")];
            tensor<int32, [2]> var_3793 = const()[name = string("op_3793"), val = tensor<int32, [2]>([4, 2048])];
            tensor<fp16, [4, 2048]> input_139_cast_fp16 = reshape(shape = var_3793, x = attention_89_cast_fp16)[name = string("input_139_cast_fp16")];
            tensor<fp16, [1024, 2048]> groups_2_2_out_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 2048]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(165131072))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(166704000))))[name = string("groups_2_2_out_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 1024]> linear_82_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = groups_2_2_out_proj_weight_promoted_to_fp16_palettized, x = input_139_cast_fp16)[name = string("linear_82_cast_fp16")];
            tensor<fp16, [4, 1024]> value_157_cast_fp16 = add(x = value_145_cast_fp16, y = linear_82_cast_fp16)[name = string("value_157_cast_fp16")];
            tensor<fp16, [4, 1024]> var_3798_cast_fp16 = mul(x = value_157_cast_fp16, y = value_157_cast_fp16)[name = string("op_3798_cast_fp16")];
            tensor<int32, [1]> variance_69_axes_0 = const()[name = string("variance_69_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_69_keep_dims_0 = const()[name = string("variance_69_keep_dims_0"), val = bool(true)];
            tensor<fp16, [4, 1]> variance_69_cast_fp16 = reduce_mean(axes = variance_69_axes_0, keep_dims = variance_69_keep_dims_0, x = var_3798_cast_fp16)[name = string("variance_69_cast_fp16")];
            fp16 var_3801_to_fp16 = const()[name = string("op_3801_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [4, 1]> var_3802_cast_fp16 = add(x = variance_69_cast_fp16, y = var_3801_to_fp16)[name = string("op_3802_cast_fp16")];
            fp32 var_3803_epsilon_0 = const()[name = string("op_3803_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [4, 1]> var_3803_cast_fp16 = rsqrt(epsilon = var_3803_epsilon_0, x = var_3802_cast_fp16)[name = string("op_3803_cast_fp16")];
            tensor<fp16, [4, 1024]> var_3804_cast_fp16 = mul(x = value_157_cast_fp16, y = var_3803_cast_fp16)[name = string("op_3804_cast_fp16")];
            tensor<fp16, [1024]> var_3806_to_fp16 = const()[name = string("op_3806_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(166712256)))];
            tensor<fp16, [4, 1024]> input_141_cast_fp16 = mul(x = var_3804_cast_fp16, y = var_3806_to_fp16)[name = string("input_141_cast_fp16")];
            tensor<fp16, [3584, 1024]> groups_2_2_gate_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(166714368))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(169466944))))[name = string("groups_2_2_gate_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 3584]> linear_83_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = groups_2_2_gate_proj_weight_promoted_to_fp16_palettized, x = input_141_cast_fp16)[name = string("linear_83_cast_fp16")];
            tensor<fp16, [4, 3584]> var_3810_cast_fp16 = silu(x = linear_83_cast_fp16)[name = string("op_3810_cast_fp16")];
            tensor<fp16, [3584, 1024]> groups_2_2_up_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(169495680))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(172248256))))[name = string("groups_2_2_up_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 3584]> linear_84_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = groups_2_2_up_proj_weight_promoted_to_fp16_palettized, x = input_141_cast_fp16)[name = string("linear_84_cast_fp16")];
            tensor<fp16, [4, 3584]> input_145_cast_fp16 = mul(x = var_3810_cast_fp16, y = linear_84_cast_fp16)[name = string("input_145_cast_fp16")];
            tensor<fp16, [1024, 3584]> groups_2_2_down_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 3584]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(172276992))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(175029568))))[name = string("groups_2_2_down_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 1024]> linear_85_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = groups_2_2_down_proj_weight_promoted_to_fp16_palettized, x = input_145_cast_fp16)[name = string("linear_85_cast_fp16")];
            tensor<fp16, [4, 1024]> value_159_cast_fp16 = add(x = value_157_cast_fp16, y = linear_85_cast_fp16)[name = string("value_159_cast_fp16")];
            int32 var_3834 = const()[name = string("op_3834"), val = int32(-1)];
            tensor<fp16, [4, 1024]> var_3841_cast_fp16 = mul(x = value_159_cast_fp16, y = value_159_cast_fp16)[name = string("op_3841_cast_fp16")];
            tensor<int32, [1]> variance_71_axes_0 = const()[name = string("variance_71_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_71_keep_dims_0 = const()[name = string("variance_71_keep_dims_0"), val = bool(true)];
            tensor<fp16, [4, 1]> variance_71_cast_fp16 = reduce_mean(axes = variance_71_axes_0, keep_dims = variance_71_keep_dims_0, x = var_3841_cast_fp16)[name = string("variance_71_cast_fp16")];
            fp16 var_3844_to_fp16 = const()[name = string("op_3844_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [4, 1]> var_3845_cast_fp16 = add(x = variance_71_cast_fp16, y = var_3844_to_fp16)[name = string("op_3845_cast_fp16")];
            fp32 var_3846_epsilon_0 = const()[name = string("op_3846_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [4, 1]> var_3846_cast_fp16 = rsqrt(epsilon = var_3846_epsilon_0, x = var_3845_cast_fp16)[name = string("op_3846_cast_fp16")];
            tensor<fp16, [4, 1024]> var_3847_cast_fp16 = mul(x = value_159_cast_fp16, y = var_3846_cast_fp16)[name = string("op_3847_cast_fp16")];
            tensor<fp16, [1024]> var_3849_to_fp16 = const()[name = string("op_3849_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(175037824)))];
            tensor<fp16, [4, 1024]> input_147_cast_fp16 = mul(x = var_3847_cast_fp16, y = var_3849_to_fp16)[name = string("input_147_cast_fp16")];
            tensor<fp16, [4096, 1024]> projections_2_q_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [4096, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(175039936))), lut = tensor<fp16, [256, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(178185728))))[name = string("projections_2_q_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 4096]> linear_86_cast_fp16 = linear(bias = linear_24_bias_0_to_fp16, weight = projections_2_q_proj_weight_promoted_to_fp16_palettized, x = input_147_cast_fp16)[name = string("linear_86_cast_fp16")];
            tensor<int32, [3]> var_3853 = const()[name = string("op_3853"), val = tensor<int32, [3]>([4, 8, 512])];
            tensor<fp16, [4, 8, 512]> query_and_gate_5_cast_fp16 = reshape(shape = var_3853, x = linear_86_cast_fp16)[name = string("query_and_gate_5_cast_fp16")];
            tensor<int32, [2]> var_3855_split_sizes_0 = const()[name = string("op_3855_split_sizes_0"), val = tensor<int32, [2]>([256, 256])];
            int32 var_3855_axis_0 = const()[name = string("op_3855_axis_0"), val = int32(-1)];
            tensor<fp16, [4, 8, 256]> var_3855_cast_fp16_0, tensor<fp16, [4, 8, 256]> var_3855_cast_fp16_1 = split(axis = var_3855_axis_0, split_sizes = var_3855_split_sizes_0, x = query_and_gate_5_cast_fp16)[name = string("op_3855_cast_fp16")];
            tensor<fp16, [512, 1024]> projections_2_k_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [512, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(178218560))), lut = tensor<fp16, [32, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(178611840))))[name = string("projections_2_k_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 512]> linear_87_cast_fp16 = linear(bias = linear_25_bias_0_to_fp16, weight = projections_2_k_proj_weight_promoted_to_fp16_palettized, x = input_147_cast_fp16)[name = string("linear_87_cast_fp16")];
            tensor<int32, [3]> var_3859 = const()[name = string("op_3859"), val = tensor<int32, [3]>([4, 2, 256])];
            tensor<fp16, [4, 2, 256]> value_163_cast_fp16 = reshape(shape = var_3859, x = linear_87_cast_fp16)[name = string("value_163_cast_fp16")];
            tensor<fp16, [512, 1024]> projections_2_v_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [512, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(178616000))), lut = tensor<fp16, [32, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(179009280))))[name = string("projections_2_v_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 512]> linear_88_cast_fp16 = linear(bias = linear_25_bias_0_to_fp16, weight = projections_2_v_proj_weight_promoted_to_fp16_palettized, x = input_147_cast_fp16)[name = string("linear_88_cast_fp16")];
            tensor<int32, [3]> var_3863 = const()[name = string("op_3863"), val = tensor<int32, [3]>([4, 2, 256])];
            tensor<fp16, [4, 2, 256]> value_169_cast_fp16 = reshape(shape = var_3863, x = linear_88_cast_fp16)[name = string("value_169_cast_fp16")];
            tensor<fp16, [4, 8, 256]> var_3865_cast_fp16 = mul(x = var_3855_cast_fp16_0, y = var_3855_cast_fp16_0)[name = string("op_3865_cast_fp16")];
            tensor<int32, [1]> variance_73_axes_0 = const()[name = string("variance_73_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_73_keep_dims_0 = const()[name = string("variance_73_keep_dims_0"), val = bool(true)];
            tensor<fp16, [4, 8, 1]> variance_73_cast_fp16 = reduce_mean(axes = variance_73_axes_0, keep_dims = variance_73_keep_dims_0, x = var_3865_cast_fp16)[name = string("variance_73_cast_fp16")];
            fp16 var_3868_to_fp16 = const()[name = string("op_3868_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [4, 8, 1]> var_3869_cast_fp16 = add(x = variance_73_cast_fp16, y = var_3868_to_fp16)[name = string("op_3869_cast_fp16")];
            fp32 var_3870_epsilon_0 = const()[name = string("op_3870_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [4, 8, 1]> var_3870_cast_fp16 = rsqrt(epsilon = var_3870_epsilon_0, x = var_3869_cast_fp16)[name = string("op_3870_cast_fp16")];
            tensor<fp16, [4, 8, 256]> var_3871_cast_fp16 = mul(x = var_3855_cast_fp16_0, y = var_3870_cast_fp16)[name = string("op_3871_cast_fp16")];
            tensor<fp16, [256]> var_3873_to_fp16 = const()[name = string("op_3873_to_fp16"), val = tensor<fp16, [256]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(179013440)))];
            tensor<fp16, [4, 8, 256]> var_3874_cast_fp16 = mul(x = var_3871_cast_fp16, y = var_3873_to_fp16)[name = string("op_3874_cast_fp16")];
            tensor<int32, [3]> var_3875 = const()[name = string("op_3875"), val = tensor<int32, [3]>([1, 0, 2])];
            tensor<int32, [1]> value_165_axes_0 = const()[name = string("value_165_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [8, 4, 256]> var_3876_cast_fp16 = transpose(perm = var_3875, x = var_3874_cast_fp16)[name = string("transpose_61")];
            tensor<fp16, [1, 8, 4, 256]> value_165_cast_fp16 = expand_dims(axes = value_165_axes_0, x = var_3876_cast_fp16)[name = string("value_165_cast_fp16")];
            tensor<fp16, [4, 2, 256]> var_3878_cast_fp16 = mul(x = value_163_cast_fp16, y = value_163_cast_fp16)[name = string("op_3878_cast_fp16")];
            tensor<int32, [1]> variance_75_axes_0 = const()[name = string("variance_75_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_75_keep_dims_0 = const()[name = string("variance_75_keep_dims_0"), val = bool(true)];
            tensor<fp16, [4, 2, 1]> variance_75_cast_fp16 = reduce_mean(axes = variance_75_axes_0, keep_dims = variance_75_keep_dims_0, x = var_3878_cast_fp16)[name = string("variance_75_cast_fp16")];
            fp16 var_3881_to_fp16 = const()[name = string("op_3881_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [4, 2, 1]> var_3882_cast_fp16 = add(x = variance_75_cast_fp16, y = var_3881_to_fp16)[name = string("op_3882_cast_fp16")];
            fp32 var_3883_epsilon_0 = const()[name = string("op_3883_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [4, 2, 1]> var_3883_cast_fp16 = rsqrt(epsilon = var_3883_epsilon_0, x = var_3882_cast_fp16)[name = string("op_3883_cast_fp16")];
            tensor<fp16, [4, 2, 256]> var_3884_cast_fp16 = mul(x = value_163_cast_fp16, y = var_3883_cast_fp16)[name = string("op_3884_cast_fp16")];
            tensor<fp16, [256]> var_3886_to_fp16 = const()[name = string("op_3886_to_fp16"), val = tensor<fp16, [256]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(179014016)))];
            tensor<fp16, [4, 2, 256]> var_3887_cast_fp16 = mul(x = var_3884_cast_fp16, y = var_3886_to_fp16)[name = string("op_3887_cast_fp16")];
            tensor<int32, [3]> var_3888 = const()[name = string("op_3888"), val = tensor<int32, [3]>([1, 0, 2])];
            tensor<int32, [1]> value_167_axes_0 = const()[name = string("value_167_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [2, 4, 256]> var_3889_cast_fp16 = transpose(perm = var_3888, x = var_3887_cast_fp16)[name = string("transpose_60")];
            tensor<fp16, [1, 2, 4, 256]> value_167_cast_fp16 = expand_dims(axes = value_167_axes_0, x = var_3889_cast_fp16)[name = string("value_167_cast_fp16")];
            tensor<int32, [4]> rotated_9_begin_0 = const()[name = string("rotated_9_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> rotated_9_end_0 = const()[name = string("rotated_9_end_0"), val = tensor<int32, [4]>([1, 8, 4, 64])];
            tensor<bool, [4]> rotated_9_end_mask_0 = const()[name = string("rotated_9_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
            tensor<fp16, [1, 8, 4, 64]> rotated_9_cast_fp16 = slice_by_index(begin = rotated_9_begin_0, end = rotated_9_end_0, end_mask = rotated_9_end_mask_0, x = value_165_cast_fp16)[name = string("rotated_9_cast_fp16")];
            tensor<int32, [4]> passthrough_9_begin_0 = const()[name = string("passthrough_9_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 64])];
            tensor<int32, [4]> passthrough_9_end_0 = const()[name = string("passthrough_9_end_0"), val = tensor<int32, [4]>([1, 8, 4, 256])];
            tensor<bool, [4]> passthrough_9_end_mask_0 = const()[name = string("passthrough_9_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
            tensor<fp16, [1, 8, 4, 192]> passthrough_9_cast_fp16 = slice_by_index(begin = passthrough_9_begin_0, end = passthrough_9_end_0, end_mask = passthrough_9_end_mask_0, x = value_165_cast_fp16)[name = string("passthrough_9_cast_fp16")];
            tensor<int32, [2]> var_3893_split_sizes_0 = const()[name = string("op_3893_split_sizes_0"), val = tensor<int32, [2]>([32, 32])];
            int32 var_3893_axis_0 = const()[name = string("op_3893_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 8, 4, 32]> var_3893_cast_fp16_0, tensor<fp16, [1, 8, 4, 32]> var_3893_cast_fp16_1 = split(axis = var_3893_axis_0, split_sizes = var_3893_split_sizes_0, x = rotated_9_cast_fp16)[name = string("op_3893_cast_fp16")];
            fp16 const_89_promoted_to_fp16 = const()[name = string("const_89_promoted_to_fp16"), val = fp16(-0x1p+0)];
            tensor<fp16, [1, 8, 4, 32]> var_3895_cast_fp16 = mul(x = var_3893_cast_fp16_1, y = const_89_promoted_to_fp16)[name = string("op_3895_cast_fp16")];
            bool rotated_half_9_interleave_0 = const()[name = string("rotated_half_9_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 8, 4, 64]> rotated_half_9_cast_fp16 = concat(axis = var_3834, interleave = rotated_half_9_interleave_0, values = (var_3895_cast_fp16, var_3893_cast_fp16_0))[name = string("rotated_half_9_cast_fp16")];
            tensor<fp16, [1, 8, 4, 64]> var_3898_cast_fp16 = mul(x = rotated_9_cast_fp16, y = cos)[name = string("op_3898_cast_fp16")];
            tensor<fp16, [1, 8, 4, 64]> var_3899_cast_fp16 = mul(x = rotated_half_9_cast_fp16, y = sin)[name = string("op_3899_cast_fp16")];
            tensor<fp16, [1, 8, 4, 64]> var_3900_cast_fp16 = add(x = var_3898_cast_fp16, y = var_3899_cast_fp16)[name = string("op_3900_cast_fp16")];
            bool query_45_interleave_0 = const()[name = string("query_45_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 8, 4, 256]> query_45_cast_fp16 = concat(axis = var_3834, interleave = query_45_interleave_0, values = (var_3900_cast_fp16, passthrough_9_cast_fp16))[name = string("query_45_cast_fp16")];
            tensor<int32, [4]> rotated_11_begin_0 = const()[name = string("rotated_11_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> rotated_11_end_0 = const()[name = string("rotated_11_end_0"), val = tensor<int32, [4]>([1, 2, 4, 64])];
            tensor<bool, [4]> rotated_11_end_mask_0 = const()[name = string("rotated_11_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
            tensor<fp16, [1, 2, 4, 64]> rotated_11_cast_fp16 = slice_by_index(begin = rotated_11_begin_0, end = rotated_11_end_0, end_mask = rotated_11_end_mask_0, x = value_167_cast_fp16)[name = string("rotated_11_cast_fp16")];
            tensor<int32, [4]> passthrough_11_begin_0 = const()[name = string("passthrough_11_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 64])];
            tensor<int32, [4]> passthrough_11_end_0 = const()[name = string("passthrough_11_end_0"), val = tensor<int32, [4]>([1, 2, 4, 256])];
            tensor<bool, [4]> passthrough_11_end_mask_0 = const()[name = string("passthrough_11_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
            tensor<fp16, [1, 2, 4, 192]> passthrough_11_cast_fp16 = slice_by_index(begin = passthrough_11_begin_0, end = passthrough_11_end_0, end_mask = passthrough_11_end_mask_0, x = value_167_cast_fp16)[name = string("passthrough_11_cast_fp16")];
            tensor<int32, [2]> var_3905_split_sizes_0 = const()[name = string("op_3905_split_sizes_0"), val = tensor<int32, [2]>([32, 32])];
            int32 var_3905_axis_0 = const()[name = string("op_3905_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 2, 4, 32]> var_3905_cast_fp16_0, tensor<fp16, [1, 2, 4, 32]> var_3905_cast_fp16_1 = split(axis = var_3905_axis_0, split_sizes = var_3905_split_sizes_0, x = rotated_11_cast_fp16)[name = string("op_3905_cast_fp16")];
            fp16 const_90_promoted_to_fp16 = const()[name = string("const_90_promoted_to_fp16"), val = fp16(-0x1p+0)];
            tensor<fp16, [1, 2, 4, 32]> var_3907_cast_fp16 = mul(x = var_3905_cast_fp16_1, y = const_90_promoted_to_fp16)[name = string("op_3907_cast_fp16")];
            bool rotated_half_11_interleave_0 = const()[name = string("rotated_half_11_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 2, 4, 64]> rotated_half_11_cast_fp16 = concat(axis = var_3834, interleave = rotated_half_11_interleave_0, values = (var_3907_cast_fp16, var_3905_cast_fp16_0))[name = string("rotated_half_11_cast_fp16")];
            tensor<fp16, [1, 2, 4, 64]> var_3910_cast_fp16 = mul(x = rotated_11_cast_fp16, y = cos)[name = string("op_3910_cast_fp16")];
            tensor<fp16, [1, 2, 4, 64]> var_3911_cast_fp16 = mul(x = rotated_half_11_cast_fp16, y = sin)[name = string("op_3911_cast_fp16")];
            tensor<fp16, [1, 2, 4, 64]> var_3912_cast_fp16 = add(x = var_3910_cast_fp16, y = var_3911_cast_fp16)[name = string("op_3912_cast_fp16")];
            bool key_45_interleave_0 = const()[name = string("key_45_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 2, 4, 256]> key_45_cast_fp16 = concat(axis = var_3834, interleave = key_45_interleave_0, values = (var_3912_cast_fp16, passthrough_11_cast_fp16))[name = string("key_45_cast_fp16")];
            tensor<int32, [4]> var_3915 = const()[name = string("op_3915"), val = tensor<int32, [4]>([2, 4, 4, 256])];
            tensor<fp16, [2, 4, 4, 256]> var_3916_cast_fp16 = reshape(shape = var_3915, x = query_45_cast_fp16)[name = string("op_3916_cast_fp16")];
            tensor<int32, [4]> transpose_4_perm_0 = const()[name = string("transpose_4_perm_0"), val = tensor<int32, [4]>([0, 1, 3, 2])];
            tensor<int32, [3]> var_3919 = const()[name = string("op_3919"), val = tensor<int32, [3]>([2, 4, 256])];
            tensor<fp16, [2, 4, 256]> key_47_cast_fp16 = reshape(shape = var_3919, x = key_45_cast_fp16)[name = string("key_47_cast_fp16")];
            tensor<int32, [3]> var_3921 = const()[name = string("op_3921"), val = tensor<int32, [3]>([1, 0, 2])];
            tensor<int32, [2]> var_3923 = const()[name = string("op_3923"), val = tensor<int32, [2]>([4, 2048])];
            tensor<fp16, [4, 2048]> gate_11_cast_fp16 = reshape(shape = var_3923, x = var_3855_cast_fp16_1)[name = string("gate_11_cast_fp16")];
            tensor<int32, [1]> var_3931_axes_0 = const()[name = string("op_3931_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [1, 2, 4, 256]> var_3931_cast_fp16 = expand_dims(axes = var_3931_axes_0, x = key_47_cast_fp16)[name = string("op_3931_cast_fp16")];
            tensor<int32, [1]> var_3933_axes_0 = const()[name = string("op_3933_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [1, 1, 2, 4, 256]> var_3933_cast_fp16 = expand_dims(axes = var_3933_axes_0, x = var_3931_cast_fp16)[name = string("op_3933_cast_fp16")];
            tensor<int32, [1]> expand_dims_32 = const()[name = string("expand_dims_32"), val = tensor<int32, [1]>([2])];
            tensor<int32, [1]> expand_dims_33 = const()[name = string("expand_dims_33"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_34 = const()[name = string("expand_dims_34"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_36 = const()[name = string("expand_dims_36"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_37 = const()[name = string("expand_dims_37"), val = tensor<int32, [1]>([3])];
            tensor<int32, [1]> expand_dims_38 = const()[name = string("expand_dims_38"), val = tensor<int32, [1]>([1])];
            int32 concat_18_axis_0 = const()[name = string("concat_18_axis_0"), val = int32(0)];
            bool concat_18_interleave_0 = const()[name = string("concat_18_interleave_0"), val = bool(false)];
            tensor<int32, [5]> concat_18 = concat(axis = concat_18_axis_0, interleave = concat_18_interleave_0, values = (expand_dims_32, expand_dims_33, expand_dims_34, current_pos, expand_dims_36))[name = string("concat_18")];
            tensor<int32, [1]> concat_19_values2_0 = const()[name = string("concat_19_values2_0"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> concat_19_values4_0 = const()[name = string("concat_19_values4_0"), val = tensor<int32, [1]>([0])];
            int32 concat_19_axis_0 = const()[name = string("concat_19_axis_0"), val = int32(0)];
            bool concat_19_interleave_0 = const()[name = string("concat_19_interleave_0"), val = bool(false)];
            tensor<int32, [5]> concat_19 = concat(axis = concat_19_axis_0, interleave = concat_19_interleave_0, values = (expand_dims_37, expand_dims_38, concat_19_values2_0, var_1074, concat_19_values4_0))[name = string("concat_19")];
            tensor<int32, [5]> kv_cache_internal_tensor_assign_5_stride_0 = const()[name = string("kv_cache_internal_tensor_assign_5_stride_0"), val = tensor<int32, [5]>([1, 1, 1, 1, 1])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_5_begin_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_5_begin_mask_0"), val = tensor<bool, [5]>([false, false, false, false, false])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_5_end_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_5_end_mask_0"), val = tensor<bool, [5]>([false, false, true, false, true])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_5_squeeze_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_5_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, false, false, false])];
            tensor<fp16, [4, 2, 2, 2048, 256]> kv_cache_internal_tensor_assign_5_cast_fp16 = slice_update(begin = concat_18, begin_mask = kv_cache_internal_tensor_assign_5_begin_mask_0, end = concat_19, end_mask = kv_cache_internal_tensor_assign_5_end_mask_0, squeeze_mask = kv_cache_internal_tensor_assign_5_squeeze_mask_0, stride = kv_cache_internal_tensor_assign_5_stride_0, update = var_3933_cast_fp16, x = coreml_update_state_11)[name = string("kv_cache_internal_tensor_assign_5_cast_fp16")];
            write_state(data = kv_cache_internal_tensor_assign_5_cast_fp16, input = kv_cache)[name = string("coreml_update_state_20_write_state")];
            tensor<fp16, [4, 2, 2, 2048, 256]> coreml_update_state_12 = read_state(input = kv_cache)[name = string("coreml_update_state_20")];
            tensor<int32, [1]> var_3964_axes_0 = const()[name = string("op_3964_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [2, 4, 256]> value_171_cast_fp16 = transpose(perm = var_3921, x = value_169_cast_fp16)[name = string("transpose_59")];
            tensor<fp16, [1, 2, 4, 256]> var_3964_cast_fp16 = expand_dims(axes = var_3964_axes_0, x = value_171_cast_fp16)[name = string("op_3964_cast_fp16")];
            tensor<int32, [1]> var_3966_axes_0 = const()[name = string("op_3966_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [1, 1, 2, 4, 256]> var_3966_cast_fp16 = expand_dims(axes = var_3966_axes_0, x = var_3964_cast_fp16)[name = string("op_3966_cast_fp16")];
            tensor<int32, [1]> expand_dims_40 = const()[name = string("expand_dims_40"), val = tensor<int32, [1]>([2])];
            tensor<int32, [1]> expand_dims_41 = const()[name = string("expand_dims_41"), val = tensor<int32, [1]>([1])];
            tensor<int32, [1]> expand_dims_42 = const()[name = string("expand_dims_42"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_44 = const()[name = string("expand_dims_44"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_45 = const()[name = string("expand_dims_45"), val = tensor<int32, [1]>([3])];
            tensor<int32, [1]> expand_dims_46 = const()[name = string("expand_dims_46"), val = tensor<int32, [1]>([2])];
            int32 concat_22_axis_0 = const()[name = string("concat_22_axis_0"), val = int32(0)];
            bool concat_22_interleave_0 = const()[name = string("concat_22_interleave_0"), val = bool(false)];
            tensor<int32, [5]> concat_22 = concat(axis = concat_22_axis_0, interleave = concat_22_interleave_0, values = (expand_dims_40, expand_dims_41, expand_dims_42, current_pos, expand_dims_44))[name = string("concat_22")];
            tensor<int32, [1]> concat_23_values2_0 = const()[name = string("concat_23_values2_0"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> concat_23_values4_0 = const()[name = string("concat_23_values4_0"), val = tensor<int32, [1]>([0])];
            int32 concat_23_axis_0 = const()[name = string("concat_23_axis_0"), val = int32(0)];
            bool concat_23_interleave_0 = const()[name = string("concat_23_interleave_0"), val = bool(false)];
            tensor<int32, [5]> concat_23 = concat(axis = concat_23_axis_0, interleave = concat_23_interleave_0, values = (expand_dims_45, expand_dims_46, concat_23_values2_0, var_1074, concat_23_values4_0))[name = string("concat_23")];
            tensor<int32, [5]> kv_cache_internal_tensor_assign_6_stride_0 = const()[name = string("kv_cache_internal_tensor_assign_6_stride_0"), val = tensor<int32, [5]>([1, 1, 1, 1, 1])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_6_begin_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_6_begin_mask_0"), val = tensor<bool, [5]>([false, false, false, false, false])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_6_end_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_6_end_mask_0"), val = tensor<bool, [5]>([false, false, true, false, true])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_6_squeeze_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_6_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, false, false, false])];
            tensor<fp16, [4, 2, 2, 2048, 256]> kv_cache_internal_tensor_assign_6_cast_fp16 = slice_update(begin = concat_22, begin_mask = kv_cache_internal_tensor_assign_6_begin_mask_0, end = concat_23, end_mask = kv_cache_internal_tensor_assign_6_end_mask_0, squeeze_mask = kv_cache_internal_tensor_assign_6_squeeze_mask_0, stride = kv_cache_internal_tensor_assign_6_stride_0, update = var_3966_cast_fp16, x = coreml_update_state_12)[name = string("kv_cache_internal_tensor_assign_6_cast_fp16")];
            write_state(data = kv_cache_internal_tensor_assign_6_cast_fp16, input = kv_cache)[name = string("coreml_update_state_21_write_state")];
            tensor<fp16, [4, 2, 2, 2048, 256]> coreml_update_state_13 = read_state(input = kv_cache)[name = string("coreml_update_state_21")];
            tensor<int32, [5]> var_3998_begin_0 = const()[name = string("op_3998_begin_0"), val = tensor<int32, [5]>([2, 0, 0, 0, 0])];
            tensor<int32, [5]> var_3998_end_0 = const()[name = string("op_3998_end_0"), val = tensor<int32, [5]>([3, 2, 2, 2048, 256])];
            tensor<bool, [5]> var_3998_end_mask_0 = const()[name = string("op_3998_end_mask_0"), val = tensor<bool, [5]>([false, true, true, true, true])];
            tensor<bool, [5]> var_3998_squeeze_mask_0 = const()[name = string("op_3998_squeeze_mask_0"), val = tensor<bool, [5]>([true, false, false, false, false])];
            tensor<fp16, [2, 2, 2048, 256]> var_3998_cast_fp16 = slice_by_index(begin = var_3998_begin_0, end = var_3998_end_0, end_mask = var_3998_end_mask_0, squeeze_mask = var_3998_squeeze_mask_0, x = coreml_update_state_13)[name = string("op_3998_cast_fp16")];
            tensor<int32, [4]> keys_5_begin_0 = const()[name = string("keys_5_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> keys_5_end_0 = const()[name = string("keys_5_end_0"), val = tensor<int32, [4]>([1, 2, 2048, 256])];
            tensor<bool, [4]> keys_5_end_mask_0 = const()[name = string("keys_5_end_mask_0"), val = tensor<bool, [4]>([false, true, true, true])];
            tensor<bool, [4]> keys_5_squeeze_mask_0 = const()[name = string("keys_5_squeeze_mask_0"), val = tensor<bool, [4]>([true, false, false, false])];
            tensor<fp16, [2, 2048, 256]> keys_5_cast_fp16 = slice_by_index(begin = keys_5_begin_0, end = keys_5_end_0, end_mask = keys_5_end_mask_0, squeeze_mask = keys_5_squeeze_mask_0, x = var_3998_cast_fp16)[name = string("keys_5_cast_fp16")];
            tensor<int32, [4]> values_5_begin_0 = const()[name = string("values_5_begin_0"), val = tensor<int32, [4]>([1, 0, 0, 0])];
            tensor<int32, [4]> values_5_end_0 = const()[name = string("values_5_end_0"), val = tensor<int32, [4]>([2, 2, 2048, 256])];
            tensor<bool, [4]> values_5_end_mask_0 = const()[name = string("values_5_end_mask_0"), val = tensor<bool, [4]>([false, true, true, true])];
            tensor<bool, [4]> values_5_squeeze_mask_0 = const()[name = string("values_5_squeeze_mask_0"), val = tensor<bool, [4]>([true, false, false, false])];
            tensor<fp16, [2, 2048, 256]> values_5_cast_fp16 = slice_by_index(begin = values_5_begin_0, end = values_5_end_0, end_mask = values_5_end_mask_0, squeeze_mask = values_5_squeeze_mask_0, x = var_3998_cast_fp16)[name = string("values_5_cast_fp16")];
            int32 var_4015 = const()[name = string("op_4015"), val = int32(1)];
            tensor<int32, [4]> var_4022 = const()[name = string("op_4022"), val = tensor<int32, [4]>([1, 2048, 1, 4])];
            tensor<fp16, [2, 4, 256, 4]> transpose_4_cast_fp16 = transpose(perm = transpose_4_perm_0, x = var_3916_cast_fp16)[name = string("transpose_58")];
            tensor<fp16, [1, 2048, 1, 4]> var_4023_cast_fp16 = reshape(shape = var_4022, x = transpose_4_cast_fp16)[name = string("op_4023_cast_fp16")];
            tensor<int32, [3]> var_4024 = const()[name = string("op_4024"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [4]> var_4027 = const()[name = string("op_4027"), val = tensor<int32, [4]>([1, 512, 1, 2048])];
            tensor<fp16, [2, 256, 2048]> var_4025_cast_fp16 = transpose(perm = var_4024, x = keys_5_cast_fp16)[name = string("transpose_57")];
            tensor<fp16, [1, 512, 1, 2048]> key_native_5_cast_fp16 = reshape(shape = var_4027, x = var_4025_cast_fp16)[name = string("key_native_5_cast_fp16")];
            tensor<int32, [3]> var_4029 = const()[name = string("op_4029"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [4]> var_4032 = const()[name = string("op_4032"), val = tensor<int32, [4]>([1, 512, 1, 2048])];
            tensor<fp16, [2, 256, 2048]> var_4030_cast_fp16 = transpose(perm = var_4029, x = values_5_cast_fp16)[name = string("transpose_56")];
            tensor<fp16, [1, 512, 1, 2048]> var_4033_cast_fp16 = reshape(shape = var_4032, x = var_4030_cast_fp16)[name = string("op_4033_cast_fp16")];
            tensor<int32, [8]> tile_38 = const()[name = string("tile_38"), val = tensor<int32, [8]>([256, 256, 256, 256, 256, 256, 256, 256])];
            int32 var_4037_axis_0 = const()[name = string("op_4037_axis_0"), val = int32(1)];
            tensor<fp16, [1, 256, 1, 4]> var_4037_cast_fp16_0, tensor<fp16, [1, 256, 1, 4]> var_4037_cast_fp16_1, tensor<fp16, [1, 256, 1, 4]> var_4037_cast_fp16_2, tensor<fp16, [1, 256, 1, 4]> var_4037_cast_fp16_3, tensor<fp16, [1, 256, 1, 4]> var_4037_cast_fp16_4, tensor<fp16, [1, 256, 1, 4]> var_4037_cast_fp16_5, tensor<fp16, [1, 256, 1, 4]> var_4037_cast_fp16_6, tensor<fp16, [1, 256, 1, 4]> var_4037_cast_fp16_7 = split(axis = var_4037_axis_0, split_sizes = tile_38, x = var_4023_cast_fp16)[name = string("op_4037_cast_fp16")];
            tensor<int32, [4]> var_4046_perm_0 = const()[name = string("op_4046_perm_0"), val = tensor<int32, [4]>([0, 3, 2, 1])];
            tensor<int32, [2]> tile_39 = const()[name = string("tile_39"), val = tensor<int32, [2]>([256, 256])];
            int32 var_4047_axis_0 = const()[name = string("op_4047_axis_0"), val = int32(3)];
            tensor<fp16, [1, 2048, 1, 512]> var_4046_cast_fp16 = transpose(perm = var_4046_perm_0, x = key_native_5_cast_fp16)[name = string("transpose_55")];
            tensor<fp16, [1, 2048, 1, 256]> var_4047_cast_fp16_0, tensor<fp16, [1, 2048, 1, 256]> var_4047_cast_fp16_1 = split(axis = var_4047_axis_0, split_sizes = tile_39, x = var_4046_cast_fp16)[name = string("op_4047_cast_fp16")];
            tensor<int32, [2]> tile_40 = const()[name = string("tile_40"), val = tensor<int32, [2]>([256, 256])];
            int32 var_4050_axis_0 = const()[name = string("op_4050_axis_0"), val = int32(1)];
            tensor<fp16, [1, 256, 1, 2048]> var_4050_cast_fp16_0, tensor<fp16, [1, 256, 1, 2048]> var_4050_cast_fp16_1 = split(axis = var_4050_axis_0, split_sizes = tile_40, x = var_4033_cast_fp16)[name = string("op_4050_cast_fp16")];
            string scores_33_equation_0 = const()[name = string("scores_33_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 4]> scores_33_cast_fp16 = einsum(equation = scores_33_equation_0, values = (var_4047_cast_fp16_0, var_4037_cast_fp16_0))[name = string("scores_33_cast_fp16")];
            fp16 var_4055_to_fp16 = const()[name = string("op_4055_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 4]> var_4056_cast_fp16 = mul(x = scores_33_cast_fp16, y = var_4055_to_fp16)[name = string("op_4056_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_4057_cast_fp16 = add(x = var_4056_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_4057_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_4058_cast_fp16 = softmax(axis = var_4015, x = var_4057_cast_fp16)[name = string("op_4058_cast_fp16")];
            tensor<int32, [4]> tile_41 = const()[name = string("tile_41"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_4059_axis_0 = const()[name = string("op_4059_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 4]> var_4059_cast_fp16_0, tensor<fp16, [1, 512, 1, 4]> var_4059_cast_fp16_1, tensor<fp16, [1, 512, 1, 4]> var_4059_cast_fp16_2, tensor<fp16, [1, 512, 1, 4]> var_4059_cast_fp16_3 = split(axis = var_4059_axis_0, split_sizes = tile_41, x = var_4058_cast_fp16)[name = string("op_4059_cast_fp16")];
            tensor<int32, [4]> tile_42 = const()[name = string("tile_42"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_4064_axis_0 = const()[name = string("op_4064_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_4064_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_4064_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_4064_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_4064_cast_fp16_3 = split(axis = var_4064_axis_0, split_sizes = tile_42, x = var_4050_cast_fp16_0)[name = string("op_4064_cast_fp16")];
            fp16 var_4069_to_fp16 = const()[name = string("op_4069_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_4070_cast_fp16 = mul(x = var_4059_cast_fp16_0, y = var_4069_to_fp16)[name = string("op_4070_cast_fp16")];
            string var_4072_equation_0 = const()[name = string("op_4072_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_4072_cast_fp16 = einsum(equation = var_4072_equation_0, values = (var_4064_cast_fp16_0, var_4070_cast_fp16))[name = string("op_4072_cast_fp16")];
            fp16 var_4073_to_fp16 = const()[name = string("op_4073_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_4074_cast_fp16 = mul(x = var_4059_cast_fp16_1, y = var_4073_to_fp16)[name = string("op_4074_cast_fp16")];
            string var_4076_equation_0 = const()[name = string("op_4076_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_4076_cast_fp16 = einsum(equation = var_4076_equation_0, values = (var_4064_cast_fp16_1, var_4074_cast_fp16))[name = string("op_4076_cast_fp16")];
            fp16 var_4077_to_fp16 = const()[name = string("op_4077_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_4078_cast_fp16 = mul(x = var_4059_cast_fp16_2, y = var_4077_to_fp16)[name = string("op_4078_cast_fp16")];
            string var_4080_equation_0 = const()[name = string("op_4080_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_4080_cast_fp16 = einsum(equation = var_4080_equation_0, values = (var_4064_cast_fp16_2, var_4078_cast_fp16))[name = string("op_4080_cast_fp16")];
            fp16 var_4081_to_fp16 = const()[name = string("op_4081_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_4082_cast_fp16 = mul(x = var_4059_cast_fp16_3, y = var_4081_to_fp16)[name = string("op_4082_cast_fp16")];
            string var_4084_equation_0 = const()[name = string("op_4084_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_4084_cast_fp16 = einsum(equation = var_4084_equation_0, values = (var_4064_cast_fp16_3, var_4082_cast_fp16))[name = string("op_4084_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_4085_cast_fp16 = add(x = var_4072_cast_fp16, y = var_4076_cast_fp16)[name = string("op_4085_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_4086_cast_fp16 = add(x = var_4080_cast_fp16, y = var_4084_cast_fp16)[name = string("op_4086_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_4087_cast_fp16 = add(x = var_4085_cast_fp16, y = var_4086_cast_fp16)[name = string("op_4087_cast_fp16")];
            fp16 _inversed_4089_y_0_to_fp16 = const()[name = string("_inversed_4089_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 4]> _inversed_4089_cast_fp16 = mul(x = var_4087_cast_fp16, y = _inversed_4089_y_0_to_fp16)[name = string("_inversed_4089_cast_fp16")];
            string scores_35_equation_0 = const()[name = string("scores_35_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 4]> scores_35_cast_fp16 = einsum(equation = scores_35_equation_0, values = (var_4047_cast_fp16_0, var_4037_cast_fp16_1))[name = string("scores_35_cast_fp16")];
            fp16 var_4092_to_fp16 = const()[name = string("op_4092_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 4]> var_4093_cast_fp16 = mul(x = scores_35_cast_fp16, y = var_4092_to_fp16)[name = string("op_4093_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_4094_cast_fp16 = add(x = var_4093_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_4094_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_4095_cast_fp16 = softmax(axis = var_4015, x = var_4094_cast_fp16)[name = string("op_4095_cast_fp16")];
            tensor<int32, [4]> tile_43 = const()[name = string("tile_43"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_4096_axis_0 = const()[name = string("op_4096_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 4]> var_4096_cast_fp16_0, tensor<fp16, [1, 512, 1, 4]> var_4096_cast_fp16_1, tensor<fp16, [1, 512, 1, 4]> var_4096_cast_fp16_2, tensor<fp16, [1, 512, 1, 4]> var_4096_cast_fp16_3 = split(axis = var_4096_axis_0, split_sizes = tile_43, x = var_4095_cast_fp16)[name = string("op_4096_cast_fp16")];
            tensor<int32, [4]> tile_44 = const()[name = string("tile_44"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_4101_axis_0 = const()[name = string("op_4101_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_4101_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_4101_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_4101_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_4101_cast_fp16_3 = split(axis = var_4101_axis_0, split_sizes = tile_44, x = var_4050_cast_fp16_0)[name = string("op_4101_cast_fp16")];
            fp16 var_4106_to_fp16 = const()[name = string("op_4106_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_4107_cast_fp16 = mul(x = var_4096_cast_fp16_0, y = var_4106_to_fp16)[name = string("op_4107_cast_fp16")];
            string var_4109_equation_0 = const()[name = string("op_4109_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_4109_cast_fp16 = einsum(equation = var_4109_equation_0, values = (var_4101_cast_fp16_0, var_4107_cast_fp16))[name = string("op_4109_cast_fp16")];
            fp16 var_4110_to_fp16 = const()[name = string("op_4110_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_4111_cast_fp16 = mul(x = var_4096_cast_fp16_1, y = var_4110_to_fp16)[name = string("op_4111_cast_fp16")];
            string var_4113_equation_0 = const()[name = string("op_4113_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_4113_cast_fp16 = einsum(equation = var_4113_equation_0, values = (var_4101_cast_fp16_1, var_4111_cast_fp16))[name = string("op_4113_cast_fp16")];
            fp16 var_4114_to_fp16 = const()[name = string("op_4114_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_4115_cast_fp16 = mul(x = var_4096_cast_fp16_2, y = var_4114_to_fp16)[name = string("op_4115_cast_fp16")];
            string var_4117_equation_0 = const()[name = string("op_4117_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_4117_cast_fp16 = einsum(equation = var_4117_equation_0, values = (var_4101_cast_fp16_2, var_4115_cast_fp16))[name = string("op_4117_cast_fp16")];
            fp16 var_4118_to_fp16 = const()[name = string("op_4118_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_4119_cast_fp16 = mul(x = var_4096_cast_fp16_3, y = var_4118_to_fp16)[name = string("op_4119_cast_fp16")];
            string var_4121_equation_0 = const()[name = string("op_4121_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_4121_cast_fp16 = einsum(equation = var_4121_equation_0, values = (var_4101_cast_fp16_3, var_4119_cast_fp16))[name = string("op_4121_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_4122_cast_fp16 = add(x = var_4109_cast_fp16, y = var_4113_cast_fp16)[name = string("op_4122_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_4123_cast_fp16 = add(x = var_4117_cast_fp16, y = var_4121_cast_fp16)[name = string("op_4123_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_4124_cast_fp16 = add(x = var_4122_cast_fp16, y = var_4123_cast_fp16)[name = string("op_4124_cast_fp16")];
            fp16 _inversed_4126_y_0_to_fp16 = const()[name = string("_inversed_4126_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 4]> _inversed_4126_cast_fp16 = mul(x = var_4124_cast_fp16, y = _inversed_4126_y_0_to_fp16)[name = string("_inversed_4126_cast_fp16")];
            string scores_37_equation_0 = const()[name = string("scores_37_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 4]> scores_37_cast_fp16 = einsum(equation = scores_37_equation_0, values = (var_4047_cast_fp16_0, var_4037_cast_fp16_2))[name = string("scores_37_cast_fp16")];
            fp16 var_4129_to_fp16 = const()[name = string("op_4129_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 4]> var_4130_cast_fp16 = mul(x = scores_37_cast_fp16, y = var_4129_to_fp16)[name = string("op_4130_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_4131_cast_fp16 = add(x = var_4130_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_4131_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_4132_cast_fp16 = softmax(axis = var_4015, x = var_4131_cast_fp16)[name = string("op_4132_cast_fp16")];
            tensor<int32, [4]> tile_45 = const()[name = string("tile_45"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_4133_axis_0 = const()[name = string("op_4133_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 4]> var_4133_cast_fp16_0, tensor<fp16, [1, 512, 1, 4]> var_4133_cast_fp16_1, tensor<fp16, [1, 512, 1, 4]> var_4133_cast_fp16_2, tensor<fp16, [1, 512, 1, 4]> var_4133_cast_fp16_3 = split(axis = var_4133_axis_0, split_sizes = tile_45, x = var_4132_cast_fp16)[name = string("op_4133_cast_fp16")];
            tensor<int32, [4]> tile_46 = const()[name = string("tile_46"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_4138_axis_0 = const()[name = string("op_4138_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_4138_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_4138_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_4138_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_4138_cast_fp16_3 = split(axis = var_4138_axis_0, split_sizes = tile_46, x = var_4050_cast_fp16_0)[name = string("op_4138_cast_fp16")];
            fp16 var_4143_to_fp16 = const()[name = string("op_4143_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_4144_cast_fp16 = mul(x = var_4133_cast_fp16_0, y = var_4143_to_fp16)[name = string("op_4144_cast_fp16")];
            string var_4146_equation_0 = const()[name = string("op_4146_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_4146_cast_fp16 = einsum(equation = var_4146_equation_0, values = (var_4138_cast_fp16_0, var_4144_cast_fp16))[name = string("op_4146_cast_fp16")];
            fp16 var_4147_to_fp16 = const()[name = string("op_4147_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_4148_cast_fp16 = mul(x = var_4133_cast_fp16_1, y = var_4147_to_fp16)[name = string("op_4148_cast_fp16")];
            string var_4150_equation_0 = const()[name = string("op_4150_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_4150_cast_fp16 = einsum(equation = var_4150_equation_0, values = (var_4138_cast_fp16_1, var_4148_cast_fp16))[name = string("op_4150_cast_fp16")];
            fp16 var_4151_to_fp16 = const()[name = string("op_4151_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_4152_cast_fp16 = mul(x = var_4133_cast_fp16_2, y = var_4151_to_fp16)[name = string("op_4152_cast_fp16")];
            string var_4154_equation_0 = const()[name = string("op_4154_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_4154_cast_fp16 = einsum(equation = var_4154_equation_0, values = (var_4138_cast_fp16_2, var_4152_cast_fp16))[name = string("op_4154_cast_fp16")];
            fp16 var_4155_to_fp16 = const()[name = string("op_4155_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_4156_cast_fp16 = mul(x = var_4133_cast_fp16_3, y = var_4155_to_fp16)[name = string("op_4156_cast_fp16")];
            string var_4158_equation_0 = const()[name = string("op_4158_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_4158_cast_fp16 = einsum(equation = var_4158_equation_0, values = (var_4138_cast_fp16_3, var_4156_cast_fp16))[name = string("op_4158_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_4159_cast_fp16 = add(x = var_4146_cast_fp16, y = var_4150_cast_fp16)[name = string("op_4159_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_4160_cast_fp16 = add(x = var_4154_cast_fp16, y = var_4158_cast_fp16)[name = string("op_4160_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_4161_cast_fp16 = add(x = var_4159_cast_fp16, y = var_4160_cast_fp16)[name = string("op_4161_cast_fp16")];
            fp16 _inversed_4163_y_0_to_fp16 = const()[name = string("_inversed_4163_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 4]> _inversed_4163_cast_fp16 = mul(x = var_4161_cast_fp16, y = _inversed_4163_y_0_to_fp16)[name = string("_inversed_4163_cast_fp16")];
            string scores_39_equation_0 = const()[name = string("scores_39_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 4]> scores_39_cast_fp16 = einsum(equation = scores_39_equation_0, values = (var_4047_cast_fp16_0, var_4037_cast_fp16_3))[name = string("scores_39_cast_fp16")];
            fp16 var_4166_to_fp16 = const()[name = string("op_4166_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 4]> var_4167_cast_fp16 = mul(x = scores_39_cast_fp16, y = var_4166_to_fp16)[name = string("op_4167_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_4168_cast_fp16 = add(x = var_4167_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_4168_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_4169_cast_fp16 = softmax(axis = var_4015, x = var_4168_cast_fp16)[name = string("op_4169_cast_fp16")];
            tensor<int32, [4]> tile_47 = const()[name = string("tile_47"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_4170_axis_0 = const()[name = string("op_4170_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 4]> var_4170_cast_fp16_0, tensor<fp16, [1, 512, 1, 4]> var_4170_cast_fp16_1, tensor<fp16, [1, 512, 1, 4]> var_4170_cast_fp16_2, tensor<fp16, [1, 512, 1, 4]> var_4170_cast_fp16_3 = split(axis = var_4170_axis_0, split_sizes = tile_47, x = var_4169_cast_fp16)[name = string("op_4170_cast_fp16")];
            tensor<int32, [4]> tile_48 = const()[name = string("tile_48"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_4175_axis_0 = const()[name = string("op_4175_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_4175_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_4175_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_4175_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_4175_cast_fp16_3 = split(axis = var_4175_axis_0, split_sizes = tile_48, x = var_4050_cast_fp16_0)[name = string("op_4175_cast_fp16")];
            fp16 var_4180_to_fp16 = const()[name = string("op_4180_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_4181_cast_fp16 = mul(x = var_4170_cast_fp16_0, y = var_4180_to_fp16)[name = string("op_4181_cast_fp16")];
            string var_4183_equation_0 = const()[name = string("op_4183_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_4183_cast_fp16 = einsum(equation = var_4183_equation_0, values = (var_4175_cast_fp16_0, var_4181_cast_fp16))[name = string("op_4183_cast_fp16")];
            fp16 var_4184_to_fp16 = const()[name = string("op_4184_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_4185_cast_fp16 = mul(x = var_4170_cast_fp16_1, y = var_4184_to_fp16)[name = string("op_4185_cast_fp16")];
            string var_4187_equation_0 = const()[name = string("op_4187_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_4187_cast_fp16 = einsum(equation = var_4187_equation_0, values = (var_4175_cast_fp16_1, var_4185_cast_fp16))[name = string("op_4187_cast_fp16")];
            fp16 var_4188_to_fp16 = const()[name = string("op_4188_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_4189_cast_fp16 = mul(x = var_4170_cast_fp16_2, y = var_4188_to_fp16)[name = string("op_4189_cast_fp16")];
            string var_4191_equation_0 = const()[name = string("op_4191_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_4191_cast_fp16 = einsum(equation = var_4191_equation_0, values = (var_4175_cast_fp16_2, var_4189_cast_fp16))[name = string("op_4191_cast_fp16")];
            fp16 var_4192_to_fp16 = const()[name = string("op_4192_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_4193_cast_fp16 = mul(x = var_4170_cast_fp16_3, y = var_4192_to_fp16)[name = string("op_4193_cast_fp16")];
            string var_4195_equation_0 = const()[name = string("op_4195_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_4195_cast_fp16 = einsum(equation = var_4195_equation_0, values = (var_4175_cast_fp16_3, var_4193_cast_fp16))[name = string("op_4195_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_4196_cast_fp16 = add(x = var_4183_cast_fp16, y = var_4187_cast_fp16)[name = string("op_4196_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_4197_cast_fp16 = add(x = var_4191_cast_fp16, y = var_4195_cast_fp16)[name = string("op_4197_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_4198_cast_fp16 = add(x = var_4196_cast_fp16, y = var_4197_cast_fp16)[name = string("op_4198_cast_fp16")];
            fp16 _inversed_4200_y_0_to_fp16 = const()[name = string("_inversed_4200_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 4]> _inversed_4200_cast_fp16 = mul(x = var_4198_cast_fp16, y = _inversed_4200_y_0_to_fp16)[name = string("_inversed_4200_cast_fp16")];
            string scores_41_equation_0 = const()[name = string("scores_41_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 4]> scores_41_cast_fp16 = einsum(equation = scores_41_equation_0, values = (var_4047_cast_fp16_1, var_4037_cast_fp16_4))[name = string("scores_41_cast_fp16")];
            fp16 var_4203_to_fp16 = const()[name = string("op_4203_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 4]> var_4204_cast_fp16 = mul(x = scores_41_cast_fp16, y = var_4203_to_fp16)[name = string("op_4204_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_4205_cast_fp16 = add(x = var_4204_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_4205_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_4206_cast_fp16 = softmax(axis = var_4015, x = var_4205_cast_fp16)[name = string("op_4206_cast_fp16")];
            tensor<int32, [4]> tile_49 = const()[name = string("tile_49"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_4207_axis_0 = const()[name = string("op_4207_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 4]> var_4207_cast_fp16_0, tensor<fp16, [1, 512, 1, 4]> var_4207_cast_fp16_1, tensor<fp16, [1, 512, 1, 4]> var_4207_cast_fp16_2, tensor<fp16, [1, 512, 1, 4]> var_4207_cast_fp16_3 = split(axis = var_4207_axis_0, split_sizes = tile_49, x = var_4206_cast_fp16)[name = string("op_4207_cast_fp16")];
            tensor<int32, [4]> tile_50 = const()[name = string("tile_50"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_4212_axis_0 = const()[name = string("op_4212_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_4212_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_4212_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_4212_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_4212_cast_fp16_3 = split(axis = var_4212_axis_0, split_sizes = tile_50, x = var_4050_cast_fp16_1)[name = string("op_4212_cast_fp16")];
            fp16 var_4217_to_fp16 = const()[name = string("op_4217_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_4218_cast_fp16 = mul(x = var_4207_cast_fp16_0, y = var_4217_to_fp16)[name = string("op_4218_cast_fp16")];
            string var_4220_equation_0 = const()[name = string("op_4220_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_4220_cast_fp16 = einsum(equation = var_4220_equation_0, values = (var_4212_cast_fp16_0, var_4218_cast_fp16))[name = string("op_4220_cast_fp16")];
            fp16 var_4221_to_fp16 = const()[name = string("op_4221_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_4222_cast_fp16 = mul(x = var_4207_cast_fp16_1, y = var_4221_to_fp16)[name = string("op_4222_cast_fp16")];
            string var_4224_equation_0 = const()[name = string("op_4224_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_4224_cast_fp16 = einsum(equation = var_4224_equation_0, values = (var_4212_cast_fp16_1, var_4222_cast_fp16))[name = string("op_4224_cast_fp16")];
            fp16 var_4225_to_fp16 = const()[name = string("op_4225_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_4226_cast_fp16 = mul(x = var_4207_cast_fp16_2, y = var_4225_to_fp16)[name = string("op_4226_cast_fp16")];
            string var_4228_equation_0 = const()[name = string("op_4228_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_4228_cast_fp16 = einsum(equation = var_4228_equation_0, values = (var_4212_cast_fp16_2, var_4226_cast_fp16))[name = string("op_4228_cast_fp16")];
            fp16 var_4229_to_fp16 = const()[name = string("op_4229_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_4230_cast_fp16 = mul(x = var_4207_cast_fp16_3, y = var_4229_to_fp16)[name = string("op_4230_cast_fp16")];
            string var_4232_equation_0 = const()[name = string("op_4232_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_4232_cast_fp16 = einsum(equation = var_4232_equation_0, values = (var_4212_cast_fp16_3, var_4230_cast_fp16))[name = string("op_4232_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_4233_cast_fp16 = add(x = var_4220_cast_fp16, y = var_4224_cast_fp16)[name = string("op_4233_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_4234_cast_fp16 = add(x = var_4228_cast_fp16, y = var_4232_cast_fp16)[name = string("op_4234_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_4235_cast_fp16 = add(x = var_4233_cast_fp16, y = var_4234_cast_fp16)[name = string("op_4235_cast_fp16")];
            fp16 _inversed_4237_y_0_to_fp16 = const()[name = string("_inversed_4237_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 4]> _inversed_4237_cast_fp16 = mul(x = var_4235_cast_fp16, y = _inversed_4237_y_0_to_fp16)[name = string("_inversed_4237_cast_fp16")];
            string scores_43_equation_0 = const()[name = string("scores_43_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 4]> scores_43_cast_fp16 = einsum(equation = scores_43_equation_0, values = (var_4047_cast_fp16_1, var_4037_cast_fp16_5))[name = string("scores_43_cast_fp16")];
            fp16 var_4240_to_fp16 = const()[name = string("op_4240_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 4]> var_4241_cast_fp16 = mul(x = scores_43_cast_fp16, y = var_4240_to_fp16)[name = string("op_4241_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_4242_cast_fp16 = add(x = var_4241_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_4242_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_4243_cast_fp16 = softmax(axis = var_4015, x = var_4242_cast_fp16)[name = string("op_4243_cast_fp16")];
            tensor<int32, [4]> tile_51 = const()[name = string("tile_51"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_4244_axis_0 = const()[name = string("op_4244_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 4]> var_4244_cast_fp16_0, tensor<fp16, [1, 512, 1, 4]> var_4244_cast_fp16_1, tensor<fp16, [1, 512, 1, 4]> var_4244_cast_fp16_2, tensor<fp16, [1, 512, 1, 4]> var_4244_cast_fp16_3 = split(axis = var_4244_axis_0, split_sizes = tile_51, x = var_4243_cast_fp16)[name = string("op_4244_cast_fp16")];
            tensor<int32, [4]> tile_52 = const()[name = string("tile_52"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_4249_axis_0 = const()[name = string("op_4249_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_4249_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_4249_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_4249_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_4249_cast_fp16_3 = split(axis = var_4249_axis_0, split_sizes = tile_52, x = var_4050_cast_fp16_1)[name = string("op_4249_cast_fp16")];
            fp16 var_4254_to_fp16 = const()[name = string("op_4254_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_4255_cast_fp16 = mul(x = var_4244_cast_fp16_0, y = var_4254_to_fp16)[name = string("op_4255_cast_fp16")];
            string var_4257_equation_0 = const()[name = string("op_4257_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_4257_cast_fp16 = einsum(equation = var_4257_equation_0, values = (var_4249_cast_fp16_0, var_4255_cast_fp16))[name = string("op_4257_cast_fp16")];
            fp16 var_4258_to_fp16 = const()[name = string("op_4258_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_4259_cast_fp16 = mul(x = var_4244_cast_fp16_1, y = var_4258_to_fp16)[name = string("op_4259_cast_fp16")];
            string var_4261_equation_0 = const()[name = string("op_4261_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_4261_cast_fp16 = einsum(equation = var_4261_equation_0, values = (var_4249_cast_fp16_1, var_4259_cast_fp16))[name = string("op_4261_cast_fp16")];
            fp16 var_4262_to_fp16 = const()[name = string("op_4262_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_4263_cast_fp16 = mul(x = var_4244_cast_fp16_2, y = var_4262_to_fp16)[name = string("op_4263_cast_fp16")];
            string var_4265_equation_0 = const()[name = string("op_4265_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_4265_cast_fp16 = einsum(equation = var_4265_equation_0, values = (var_4249_cast_fp16_2, var_4263_cast_fp16))[name = string("op_4265_cast_fp16")];
            fp16 var_4266_to_fp16 = const()[name = string("op_4266_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_4267_cast_fp16 = mul(x = var_4244_cast_fp16_3, y = var_4266_to_fp16)[name = string("op_4267_cast_fp16")];
            string var_4269_equation_0 = const()[name = string("op_4269_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_4269_cast_fp16 = einsum(equation = var_4269_equation_0, values = (var_4249_cast_fp16_3, var_4267_cast_fp16))[name = string("op_4269_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_4270_cast_fp16 = add(x = var_4257_cast_fp16, y = var_4261_cast_fp16)[name = string("op_4270_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_4271_cast_fp16 = add(x = var_4265_cast_fp16, y = var_4269_cast_fp16)[name = string("op_4271_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_4272_cast_fp16 = add(x = var_4270_cast_fp16, y = var_4271_cast_fp16)[name = string("op_4272_cast_fp16")];
            fp16 _inversed_4274_y_0_to_fp16 = const()[name = string("_inversed_4274_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 4]> _inversed_4274_cast_fp16 = mul(x = var_4272_cast_fp16, y = _inversed_4274_y_0_to_fp16)[name = string("_inversed_4274_cast_fp16")];
            string scores_45_equation_0 = const()[name = string("scores_45_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 4]> scores_45_cast_fp16 = einsum(equation = scores_45_equation_0, values = (var_4047_cast_fp16_1, var_4037_cast_fp16_6))[name = string("scores_45_cast_fp16")];
            fp16 var_4277_to_fp16 = const()[name = string("op_4277_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 4]> var_4278_cast_fp16 = mul(x = scores_45_cast_fp16, y = var_4277_to_fp16)[name = string("op_4278_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_4279_cast_fp16 = add(x = var_4278_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_4279_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_4280_cast_fp16 = softmax(axis = var_4015, x = var_4279_cast_fp16)[name = string("op_4280_cast_fp16")];
            tensor<int32, [4]> tile_53 = const()[name = string("tile_53"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_4281_axis_0 = const()[name = string("op_4281_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 4]> var_4281_cast_fp16_0, tensor<fp16, [1, 512, 1, 4]> var_4281_cast_fp16_1, tensor<fp16, [1, 512, 1, 4]> var_4281_cast_fp16_2, tensor<fp16, [1, 512, 1, 4]> var_4281_cast_fp16_3 = split(axis = var_4281_axis_0, split_sizes = tile_53, x = var_4280_cast_fp16)[name = string("op_4281_cast_fp16")];
            tensor<int32, [4]> tile_54 = const()[name = string("tile_54"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_4286_axis_0 = const()[name = string("op_4286_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_4286_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_4286_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_4286_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_4286_cast_fp16_3 = split(axis = var_4286_axis_0, split_sizes = tile_54, x = var_4050_cast_fp16_1)[name = string("op_4286_cast_fp16")];
            fp16 var_4291_to_fp16 = const()[name = string("op_4291_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_4292_cast_fp16 = mul(x = var_4281_cast_fp16_0, y = var_4291_to_fp16)[name = string("op_4292_cast_fp16")];
            string var_4294_equation_0 = const()[name = string("op_4294_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_4294_cast_fp16 = einsum(equation = var_4294_equation_0, values = (var_4286_cast_fp16_0, var_4292_cast_fp16))[name = string("op_4294_cast_fp16")];
            fp16 var_4295_to_fp16 = const()[name = string("op_4295_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_4296_cast_fp16 = mul(x = var_4281_cast_fp16_1, y = var_4295_to_fp16)[name = string("op_4296_cast_fp16")];
            string var_4298_equation_0 = const()[name = string("op_4298_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_4298_cast_fp16 = einsum(equation = var_4298_equation_0, values = (var_4286_cast_fp16_1, var_4296_cast_fp16))[name = string("op_4298_cast_fp16")];
            fp16 var_4299_to_fp16 = const()[name = string("op_4299_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_4300_cast_fp16 = mul(x = var_4281_cast_fp16_2, y = var_4299_to_fp16)[name = string("op_4300_cast_fp16")];
            string var_4302_equation_0 = const()[name = string("op_4302_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_4302_cast_fp16 = einsum(equation = var_4302_equation_0, values = (var_4286_cast_fp16_2, var_4300_cast_fp16))[name = string("op_4302_cast_fp16")];
            fp16 var_4303_to_fp16 = const()[name = string("op_4303_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_4304_cast_fp16 = mul(x = var_4281_cast_fp16_3, y = var_4303_to_fp16)[name = string("op_4304_cast_fp16")];
            string var_4306_equation_0 = const()[name = string("op_4306_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_4306_cast_fp16 = einsum(equation = var_4306_equation_0, values = (var_4286_cast_fp16_3, var_4304_cast_fp16))[name = string("op_4306_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_4307_cast_fp16 = add(x = var_4294_cast_fp16, y = var_4298_cast_fp16)[name = string("op_4307_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_4308_cast_fp16 = add(x = var_4302_cast_fp16, y = var_4306_cast_fp16)[name = string("op_4308_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_4309_cast_fp16 = add(x = var_4307_cast_fp16, y = var_4308_cast_fp16)[name = string("op_4309_cast_fp16")];
            fp16 _inversed_4311_y_0_to_fp16 = const()[name = string("_inversed_4311_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 4]> _inversed_4311_cast_fp16 = mul(x = var_4309_cast_fp16, y = _inversed_4311_y_0_to_fp16)[name = string("_inversed_4311_cast_fp16")];
            string scores_47_equation_0 = const()[name = string("scores_47_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 4]> scores_47_cast_fp16 = einsum(equation = scores_47_equation_0, values = (var_4047_cast_fp16_1, var_4037_cast_fp16_7))[name = string("scores_47_cast_fp16")];
            fp16 var_4314_to_fp16 = const()[name = string("op_4314_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 4]> var_4315_cast_fp16 = mul(x = scores_47_cast_fp16, y = var_4314_to_fp16)[name = string("op_4315_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_4316_cast_fp16 = add(x = var_4315_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_4316_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_4317_cast_fp16 = softmax(axis = var_4015, x = var_4316_cast_fp16)[name = string("op_4317_cast_fp16")];
            tensor<int32, [4]> tile_55 = const()[name = string("tile_55"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_4318_axis_0 = const()[name = string("op_4318_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 4]> var_4318_cast_fp16_0, tensor<fp16, [1, 512, 1, 4]> var_4318_cast_fp16_1, tensor<fp16, [1, 512, 1, 4]> var_4318_cast_fp16_2, tensor<fp16, [1, 512, 1, 4]> var_4318_cast_fp16_3 = split(axis = var_4318_axis_0, split_sizes = tile_55, x = var_4317_cast_fp16)[name = string("op_4318_cast_fp16")];
            tensor<int32, [4]> tile_56 = const()[name = string("tile_56"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_4323_axis_0 = const()[name = string("op_4323_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_4323_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_4323_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_4323_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_4323_cast_fp16_3 = split(axis = var_4323_axis_0, split_sizes = tile_56, x = var_4050_cast_fp16_1)[name = string("op_4323_cast_fp16")];
            fp16 var_4328_to_fp16 = const()[name = string("op_4328_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_4329_cast_fp16 = mul(x = var_4318_cast_fp16_0, y = var_4328_to_fp16)[name = string("op_4329_cast_fp16")];
            string var_4331_equation_0 = const()[name = string("op_4331_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_4331_cast_fp16 = einsum(equation = var_4331_equation_0, values = (var_4323_cast_fp16_0, var_4329_cast_fp16))[name = string("op_4331_cast_fp16")];
            fp16 var_4332_to_fp16 = const()[name = string("op_4332_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_4333_cast_fp16 = mul(x = var_4318_cast_fp16_1, y = var_4332_to_fp16)[name = string("op_4333_cast_fp16")];
            string var_4335_equation_0 = const()[name = string("op_4335_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_4335_cast_fp16 = einsum(equation = var_4335_equation_0, values = (var_4323_cast_fp16_1, var_4333_cast_fp16))[name = string("op_4335_cast_fp16")];
            fp16 var_4336_to_fp16 = const()[name = string("op_4336_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_4337_cast_fp16 = mul(x = var_4318_cast_fp16_2, y = var_4336_to_fp16)[name = string("op_4337_cast_fp16")];
            string var_4339_equation_0 = const()[name = string("op_4339_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_4339_cast_fp16 = einsum(equation = var_4339_equation_0, values = (var_4323_cast_fp16_2, var_4337_cast_fp16))[name = string("op_4339_cast_fp16")];
            fp16 var_4340_to_fp16 = const()[name = string("op_4340_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_4341_cast_fp16 = mul(x = var_4318_cast_fp16_3, y = var_4340_to_fp16)[name = string("op_4341_cast_fp16")];
            string var_4343_equation_0 = const()[name = string("op_4343_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_4343_cast_fp16 = einsum(equation = var_4343_equation_0, values = (var_4323_cast_fp16_3, var_4341_cast_fp16))[name = string("op_4343_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_4344_cast_fp16 = add(x = var_4331_cast_fp16, y = var_4335_cast_fp16)[name = string("op_4344_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_4345_cast_fp16 = add(x = var_4339_cast_fp16, y = var_4343_cast_fp16)[name = string("op_4345_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_4346_cast_fp16 = add(x = var_4344_cast_fp16, y = var_4345_cast_fp16)[name = string("op_4346_cast_fp16")];
            fp16 _inversed_4348_y_0_to_fp16 = const()[name = string("_inversed_4348_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 4]> _inversed_4348_cast_fp16 = mul(x = var_4346_cast_fp16, y = _inversed_4348_y_0_to_fp16)[name = string("_inversed_4348_cast_fp16")];
            bool var_4350_interleave_0 = const()[name = string("op_4350_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 2048, 1, 4]> var_4350_cast_fp16 = concat(axis = var_4015, interleave = var_4350_interleave_0, values = (_inversed_4089_cast_fp16, _inversed_4126_cast_fp16, _inversed_4163_cast_fp16, _inversed_4200_cast_fp16, _inversed_4237_cast_fp16, _inversed_4274_cast_fp16, _inversed_4311_cast_fp16, _inversed_4348_cast_fp16))[name = string("op_4350_cast_fp16")];
            tensor<int32, [4]> var_4351 = const()[name = string("op_4351"), val = tensor<int32, [4]>([2, 4, 256, 4])];
            tensor<fp16, [2, 4, 256, 4]> var_4352_cast_fp16 = reshape(shape = var_4351, x = var_4350_cast_fp16)[name = string("op_4352_cast_fp16")];
            tensor<int32, [4]> transpose_5_perm_0 = const()[name = string("transpose_5_perm_0"), val = tensor<int32, [4]>([3, 0, 1, 2])];
            tensor<int32, [2]> var_4371 = const()[name = string("op_4371"), val = tensor<int32, [2]>([4, 2048])];
            tensor<fp16, [4, 2, 4, 256]> transpose_5_cast_fp16 = transpose(perm = transpose_5_perm_0, x = var_4352_cast_fp16)[name = string("transpose_54")];
            tensor<fp16, [4, 2048]> attention_output_11_cast_fp16 = reshape(shape = var_4371, x = transpose_5_cast_fp16)[name = string("attention_output_11_cast_fp16")];
            tensor<fp16, [4, 2048]> var_4373_cast_fp16 = sigmoid(x = gate_11_cast_fp16)[name = string("op_4373_cast_fp16")];
            tensor<fp16, [4, 2048]> input_149_cast_fp16 = mul(x = attention_output_11_cast_fp16, y = var_4373_cast_fp16)[name = string("input_149_cast_fp16")];
            tensor<fp16, [1024, 2048]> completions_2_o_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 2048]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(179014592))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(180587520))))[name = string("completions_2_o_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 1024]> linear_89_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = completions_2_o_proj_weight_promoted_to_fp16_palettized, x = input_149_cast_fp16)[name = string("linear_89_cast_fp16")];
            tensor<fp16, [4, 1024]> value_173_cast_fp16 = add(x = value_159_cast_fp16, y = linear_89_cast_fp16)[name = string("value_173_cast_fp16")];
            tensor<fp16, [4, 1024]> var_4378_cast_fp16 = mul(x = value_173_cast_fp16, y = value_173_cast_fp16)[name = string("op_4378_cast_fp16")];
            tensor<int32, [1]> variance_77_axes_0 = const()[name = string("variance_77_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_77_keep_dims_0 = const()[name = string("variance_77_keep_dims_0"), val = bool(true)];
            tensor<fp16, [4, 1]> variance_77_cast_fp16 = reduce_mean(axes = variance_77_axes_0, keep_dims = variance_77_keep_dims_0, x = var_4378_cast_fp16)[name = string("variance_77_cast_fp16")];
            fp16 var_4381_to_fp16 = const()[name = string("op_4381_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [4, 1]> var_4382_cast_fp16 = add(x = variance_77_cast_fp16, y = var_4381_to_fp16)[name = string("op_4382_cast_fp16")];
            fp32 var_4383_epsilon_0 = const()[name = string("op_4383_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [4, 1]> var_4383_cast_fp16 = rsqrt(epsilon = var_4383_epsilon_0, x = var_4382_cast_fp16)[name = string("op_4383_cast_fp16")];
            tensor<fp16, [4, 1024]> var_4384_cast_fp16 = mul(x = value_173_cast_fp16, y = var_4383_cast_fp16)[name = string("op_4384_cast_fp16")];
            tensor<fp16, [1024]> var_4386_to_fp16 = const()[name = string("op_4386_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(180595776)))];
            tensor<fp16, [4, 1024]> input_151_cast_fp16 = mul(x = var_4384_cast_fp16, y = var_4386_to_fp16)[name = string("input_151_cast_fp16")];
            tensor<fp16, [3584, 1024]> completions_2_gate_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(180597888))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(183350464))))[name = string("completions_2_gate_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 3584]> linear_90_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = completions_2_gate_proj_weight_promoted_to_fp16_palettized, x = input_151_cast_fp16)[name = string("linear_90_cast_fp16")];
            tensor<fp16, [4, 3584]> var_4390_cast_fp16 = silu(x = linear_90_cast_fp16)[name = string("op_4390_cast_fp16")];
            tensor<fp16, [3584, 1024]> completions_2_up_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(183379200))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(186131776))))[name = string("completions_2_up_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 3584]> linear_91_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = completions_2_up_proj_weight_promoted_to_fp16_palettized, x = input_151_cast_fp16)[name = string("linear_91_cast_fp16")];
            tensor<fp16, [4, 3584]> input_155_cast_fp16 = mul(x = var_4390_cast_fp16, y = linear_91_cast_fp16)[name = string("input_155_cast_fp16")];
            tensor<fp16, [1024, 3584]> completions_2_down_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 3584]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(186160512))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(188913088))))[name = string("completions_2_down_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 1024]> linear_92_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = completions_2_down_proj_weight_promoted_to_fp16_palettized, x = input_155_cast_fp16)[name = string("linear_92_cast_fp16")];
            tensor<fp16, [4, 1024]> value_175_cast_fp16 = add(x = value_173_cast_fp16, y = linear_92_cast_fp16)[name = string("value_175_cast_fp16")];
            int32 var_4413 = const()[name = string("op_4413"), val = int32(-1)];
            tensor<fp16, [4, 1024]> var_4428_cast_fp16 = mul(x = value_175_cast_fp16, y = value_175_cast_fp16)[name = string("op_4428_cast_fp16")];
            tensor<int32, [1]> variance_79_axes_0 = const()[name = string("variance_79_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_79_keep_dims_0 = const()[name = string("variance_79_keep_dims_0"), val = bool(true)];
            tensor<fp16, [4, 1]> variance_79_cast_fp16 = reduce_mean(axes = variance_79_axes_0, keep_dims = variance_79_keep_dims_0, x = var_4428_cast_fp16)[name = string("variance_79_cast_fp16")];
            fp16 var_4431_to_fp16 = const()[name = string("op_4431_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [4, 1]> var_4432_cast_fp16 = add(x = variance_79_cast_fp16, y = var_4431_to_fp16)[name = string("op_4432_cast_fp16")];
            fp32 var_4433_epsilon_0 = const()[name = string("op_4433_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [4, 1]> var_4433_cast_fp16 = rsqrt(epsilon = var_4433_epsilon_0, x = var_4432_cast_fp16)[name = string("op_4433_cast_fp16")];
            tensor<fp16, [4, 1024]> var_4434_cast_fp16 = mul(x = value_175_cast_fp16, y = var_4433_cast_fp16)[name = string("op_4434_cast_fp16")];
            tensor<fp16, [1024]> var_4436_to_fp16 = const()[name = string("op_4436_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(188921344)))];
            tensor<fp16, [4, 1024]> input_157_cast_fp16 = mul(x = var_4434_cast_fp16, y = var_4436_to_fp16)[name = string("input_157_cast_fp16")];
            tensor<fp16, [6144, 1024]> groups_3_0_in_proj_qkv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(188923456))), lut = tensor<fp16, [384, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(193642112))))[name = string("groups_3_0_in_proj_qkv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 6144]> linear_93_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = groups_3_0_in_proj_qkv_weight_promoted_to_fp16_palettized, x = input_157_cast_fp16)[name = string("linear_93_cast_fp16")];
            tensor<int32, [2]> var_4440_perm_0 = const()[name = string("op_4440_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> mixed_19_axes_0 = const()[name = string("mixed_19_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [6144, 4]> var_4440_cast_fp16 = transpose(perm = var_4440_perm_0, x = linear_93_cast_fp16)[name = string("transpose_53")];
            tensor<fp16, [1, 6144, 4]> mixed_19_cast_fp16 = expand_dims(axes = mixed_19_axes_0, x = var_4440_cast_fp16)[name = string("mixed_19_cast_fp16")];
            bool convolution_input_19_interleave_0 = const()[name = string("convolution_input_19_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 6144, 7]> convolution_input_19_cast_fp16 = concat(axis = var_4413, interleave = convolution_input_19_interleave_0, values = (conv12, mixed_19_cast_fp16))[name = string("convolution_input_19_cast_fp16")];
            string input_159_pad_type_0 = const()[name = string("input_159_pad_type_0"), val = string("valid")];
            int32 input_159_groups_0 = const()[name = string("input_159_groups_0"), val = int32(6144)];
            tensor<int32, [1]> input_159_strides_0 = const()[name = string("input_159_strides_0"), val = tensor<int32, [1]>([1])];
            tensor<int32, [2]> input_159_pad_0 = const()[name = string("input_159_pad_0"), val = tensor<int32, [2]>([0, 0])];
            tensor<int32, [1]> input_159_dilations_0 = const()[name = string("input_159_dilations_0"), val = tensor<int32, [1]>([1])];
            tensor<fp16, [6144, 1, 4]> groups_3_0_conv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1, 4]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(193691328))), lut = tensor<fp16, [384, 1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(193709824))))[name = string("groups_3_0_conv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 6144, 4]> input_159_cast_fp16 = conv(dilations = input_159_dilations_0, groups = input_159_groups_0, pad = input_159_pad_0, pad_type = input_159_pad_type_0, strides = input_159_strides_0, weight = groups_3_0_conv_weight_promoted_to_fp16_palettized, x = convolution_input_19_cast_fp16)[name = string("input_159_cast_fp16")];
            tensor<fp16, [1, 6144, 4]> var_4449_cast_fp16 = silu(x = input_159_cast_fp16)[name = string("op_4449_cast_fp16")];
            tensor<int32, [3]> var_4450_perm_0 = const()[name = string("op_4450_perm_0"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [3]> var_4453_begin_0 = const()[name = string("op_4453_begin_0"), val = tensor<int32, [3]>([0, 0, 4])];
            tensor<int32, [3]> var_4453_end_0 = const()[name = string("op_4453_end_0"), val = tensor<int32, [3]>([1, 6144, 7])];
            tensor<bool, [3]> var_4453_end_mask_0 = const()[name = string("op_4453_end_mask_0"), val = tensor<bool, [3]>([true, true, true])];
            tensor<fp16, [1, 6144, 3]> conv12_out = slice_by_index(begin = var_4453_begin_0, end = var_4453_end_0, end_mask = var_4453_end_mask_0, x = convolution_input_19_cast_fp16)[name = string("op_4453_cast_fp16")];
            tensor<int32, [3]> var_4454 = const()[name = string("op_4454"), val = tensor<int32, [3]>([2048, 2048, 2048])];
            int32 var_4455_axis_0 = const()[name = string("op_4455_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 4, 6144]> var_4450_cast_fp16 = transpose(perm = var_4450_perm_0, x = var_4449_cast_fp16)[name = string("transpose_52")];
            tensor<fp16, [1, 4, 2048]> var_4455_cast_fp16_0, tensor<fp16, [1, 4, 2048]> var_4455_cast_fp16_1, tensor<fp16, [1, 4, 2048]> var_4455_cast_fp16_2 = split(axis = var_4455_axis_0, split_sizes = var_4454, x = var_4450_cast_fp16)[name = string("op_4455_cast_fp16")];
            tensor<int32, [4]> var_4459 = const()[name = string("op_4459"), val = tensor<int32, [4]>([1, 4, 16, 128])];
            tensor<fp16, [1, 4, 16, 128]> value_179_cast_fp16 = reshape(shape = var_4459, x = var_4455_cast_fp16_0)[name = string("value_179_cast_fp16")];
            tensor<int32, [4]> var_4461 = const()[name = string("op_4461"), val = tensor<int32, [4]>([1, 4, 16, 128])];
            tensor<fp16, [1, 4, 16, 128]> value_181_cast_fp16 = reshape(shape = var_4461, x = var_4455_cast_fp16_1)[name = string("value_181_cast_fp16")];
            tensor<int32, [4]> var_4463 = const()[name = string("op_4463"), val = tensor<int32, [4]>([1, 4, 16, 128])];
            tensor<fp16, [1, 4, 16, 128]> value_183_cast_fp16 = reshape(shape = var_4463, x = var_4455_cast_fp16_2)[name = string("value_183_cast_fp16")];
            tensor<fp16, [1, 4, 16, 128]> var_4465_cast_fp16 = mul(x = value_179_cast_fp16, y = value_179_cast_fp16)[name = string("op_4465_cast_fp16")];
            tensor<int32, [1]> var_4467_axes_0 = const()[name = string("op_4467_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_4467_keep_dims_0 = const()[name = string("op_4467_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 4, 16, 1]> var_4467_cast_fp16 = reduce_sum(axes = var_4467_axes_0, keep_dims = var_4467_keep_dims_0, x = var_4465_cast_fp16)[name = string("op_4467_cast_fp16")];
            fp16 var_4468_to_fp16 = const()[name = string("op_4468_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 4, 16, 1]> var_4469_cast_fp16 = add(x = var_4467_cast_fp16, y = var_4468_to_fp16)[name = string("op_4469_cast_fp16")];
            fp32 var_4470_epsilon_0 = const()[name = string("op_4470_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 4, 16, 1]> var_4470_cast_fp16 = rsqrt(epsilon = var_4470_epsilon_0, x = var_4469_cast_fp16)[name = string("op_4470_cast_fp16")];
            tensor<fp16, [1, 4, 16, 128]> var_4471_cast_fp16 = mul(x = value_179_cast_fp16, y = var_4470_cast_fp16)[name = string("op_4471_cast_fp16")];
            tensor<int32, [4]> var_4472_perm_0 = const()[name = string("op_4472_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            fp16 _inversed_query_51_y_0_to_fp16 = const()[name = string("_inversed_query_51_y_0_to_fp16"), val = fp16(0x1.6ap-4)];
            tensor<fp16, [1, 16, 4, 128]> var_4472_cast_fp16 = transpose(perm = var_4472_perm_0, x = var_4471_cast_fp16)[name = string("transpose_51")];
            tensor<fp16, [1, 16, 4, 128]> _inversed_query_51_cast_fp16 = mul(x = var_4472_cast_fp16, y = _inversed_query_51_y_0_to_fp16)[name = string("_inversed_query_51_cast_fp16")];
            tensor<fp16, [1, 4, 16, 128]> var_4475_cast_fp16 = mul(x = value_181_cast_fp16, y = value_181_cast_fp16)[name = string("op_4475_cast_fp16")];
            tensor<int32, [1]> var_4477_axes_0 = const()[name = string("op_4477_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_4477_keep_dims_0 = const()[name = string("op_4477_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 4, 16, 1]> var_4477_cast_fp16 = reduce_sum(axes = var_4477_axes_0, keep_dims = var_4477_keep_dims_0, x = var_4475_cast_fp16)[name = string("op_4477_cast_fp16")];
            fp16 var_4478_to_fp16 = const()[name = string("op_4478_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 4, 16, 1]> var_4479_cast_fp16 = add(x = var_4477_cast_fp16, y = var_4478_to_fp16)[name = string("op_4479_cast_fp16")];
            fp32 var_4480_epsilon_0 = const()[name = string("op_4480_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 4, 16, 1]> var_4480_cast_fp16 = rsqrt(epsilon = var_4480_epsilon_0, x = var_4479_cast_fp16)[name = string("op_4480_cast_fp16")];
            tensor<fp16, [1, 4, 16, 128]> var_4481_cast_fp16 = mul(x = value_181_cast_fp16, y = var_4480_cast_fp16)[name = string("op_4481_cast_fp16")];
            tensor<int32, [4]> key_51_perm_0 = const()[name = string("key_51_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<int32, [4]> value_185_perm_0 = const()[name = string("value_185_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<fp16, [16, 1024]> groups_3_0_in_proj_b_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(193759040))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(193771392))))[name = string("groups_3_0_in_proj_b_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16]> linear_94_bias_0_to_fp16 = const()[name = string("linear_94_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(4862656)))];
            tensor<fp16, [4, 16]> linear_94_cast_fp16 = linear(bias = linear_94_bias_0_to_fp16, weight = groups_3_0_in_proj_b_weight_promoted_to_fp16_palettized, x = input_157_cast_fp16)[name = string("linear_94_cast_fp16")];
            tensor<fp16, [4, 16]> var_4486_cast_fp16 = sigmoid(x = linear_94_cast_fp16)[name = string("op_4486_cast_fp16")];
            tensor<int32, [2]> var_4487_perm_0 = const()[name = string("op_4487_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> beta_19_axes_0 = const()[name = string("beta_19_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 4]> var_4487_cast_fp16 = transpose(perm = var_4487_perm_0, x = var_4486_cast_fp16)[name = string("transpose_50")];
            tensor<fp16, [1, 16, 4]> beta_19_cast_fp16 = expand_dims(axes = beta_19_axes_0, x = var_4487_cast_fp16)[name = string("beta_19_cast_fp16")];
            tensor<fp16, [16, 1024]> op_4493_weight_0_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(193771584))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(193783936))))[name = string("op_4493_weight_0_to_fp16_palettized")];
            tensor<fp16, [16]> var_4493_bias_0_to_fp16 = const()[name = string("op_4493_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(193784128)))];
            tensor<fp16, [4, 16]> var_4493_cast_fp16 = linear(bias = var_4493_bias_0_to_fp16, weight = op_4493_weight_0_to_fp16_palettized, x = input_157_cast_fp16)[name = string("op_4493_cast_fp16")];
            tensor<fp16, [4, 16]> var_4494_cast_fp16 = softplus(x = var_4493_cast_fp16)[name = string("op_4494_cast_fp16")];
            tensor<fp16, [16]> var_4490_promoted_to_fp16 = const()[name = string("op_4490_promoted_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(193784256)))];
            tensor<fp16, [4, 16]> var_4495_cast_fp16 = mul(x = var_4490_promoted_to_fp16, y = var_4494_cast_fp16)[name = string("op_4495_cast_fp16")];
            tensor<int32, [2]> var_4496_perm_0 = const()[name = string("op_4496_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> decay_19_axes_0 = const()[name = string("decay_19_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 4]> var_4496_cast_fp16 = transpose(perm = var_4496_perm_0, x = var_4495_cast_fp16)[name = string("transpose_49")];
            tensor<fp16, [1, 16, 4]> decay_19_cast_fp16 = expand_dims(axes = decay_19_axes_0, x = var_4496_cast_fp16)[name = string("decay_19_cast_fp16")];
            tensor<fp16, [2048, 1024]> groups_3_0_in_proj_z_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [2048, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(193784384))), lut = tensor<fp16, [128, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(195357312))))[name = string("groups_3_0_in_proj_z_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 2048]> linear_96_cast_fp16 = linear(bias = linear_3_bias_0_to_fp16, weight = groups_3_0_in_proj_z_weight_promoted_to_fp16_palettized, x = input_157_cast_fp16)[name = string("linear_96_cast_fp16")];
            tensor<int32, [3]> var_4504_begin_0 = const()[name = string("op_4504_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_4504_end_0 = const()[name = string("op_4504_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_4504_end_mask_0 = const()[name = string("op_4504_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_4504_squeeze_mask_0 = const()[name = string("op_4504_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_4504_cast_fp16 = slice_by_index(begin = var_4504_begin_0, end = var_4504_end_0, end_mask = var_4504_end_mask_0, squeeze_mask = var_4504_squeeze_mask_0, x = decay_19_cast_fp16)[name = string("op_4504_cast_fp16")];
            tensor<fp16, [1, 16]> var_4505_cast_fp16 = exp(x = var_4504_cast_fp16)[name = string("op_4505_cast_fp16")];
            tensor<int32, [1]> var_4506_axes_0 = const()[name = string("op_4506_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_4506_cast_fp16 = expand_dims(axes = var_4506_axes_0, x = var_4505_cast_fp16)[name = string("op_4506_cast_fp16")];
            tensor<int32, [1]> var_4507_axes_0 = const()[name = string("op_4507_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_4507_cast_fp16 = expand_dims(axes = var_4507_axes_0, x = var_4506_cast_fp16)[name = string("op_4507_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_145_cast_fp16 = mul(x = rec12, y = var_4507_cast_fp16)[name = string("state_145_cast_fp16")];
            tensor<int32, [4]> key_token_73_begin_0 = const()[name = string("key_token_73_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> key_token_73_end_0 = const()[name = string("key_token_73_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> key_token_73_end_mask_0 = const()[name = string("key_token_73_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_73_squeeze_mask_0 = const()[name = string("key_token_73_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 4, 128]> key_51_cast_fp16 = transpose(perm = key_51_perm_0, x = var_4481_cast_fp16)[name = string("transpose_48")];
            tensor<fp16, [1, 16, 128]> key_token_73_cast_fp16 = slice_by_index(begin = key_token_73_begin_0, end = key_token_73_end_0, end_mask = key_token_73_end_mask_0, squeeze_mask = key_token_73_squeeze_mask_0, x = key_51_cast_fp16)[name = string("key_token_73_cast_fp16")];
            tensor<int32, [4]> value_token_73_begin_0 = const()[name = string("value_token_73_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> value_token_73_end_0 = const()[name = string("value_token_73_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> value_token_73_end_mask_0 = const()[name = string("value_token_73_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_73_squeeze_mask_0 = const()[name = string("value_token_73_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 4, 128]> value_185_cast_fp16 = transpose(perm = value_185_perm_0, x = value_183_cast_fp16)[name = string("transpose_47")];
            tensor<fp16, [1, 16, 128]> value_token_73_cast_fp16 = slice_by_index(begin = value_token_73_begin_0, end = value_token_73_end_0, end_mask = value_token_73_end_mask_0, squeeze_mask = value_token_73_squeeze_mask_0, x = value_185_cast_fp16)[name = string("value_token_73_cast_fp16")];
            tensor<int32, [1]> var_4515_axes_0 = const()[name = string("op_4515_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_4515_cast_fp16 = expand_dims(axes = var_4515_axes_0, x = key_token_73_cast_fp16)[name = string("op_4515_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4516_cast_fp16 = mul(x = state_145_cast_fp16, y = var_4515_cast_fp16)[name = string("op_4516_cast_fp16")];
            tensor<int32, [1]> memory_73_axes_0 = const()[name = string("memory_73_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_73_keep_dims_0 = const()[name = string("memory_73_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_73_cast_fp16 = reduce_sum(axes = memory_73_axes_0, keep_dims = memory_73_keep_dims_0, x = var_4516_cast_fp16)[name = string("memory_73_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_4519_cast_fp16 = sub(x = value_token_73_cast_fp16, y = memory_73_cast_fp16)[name = string("op_4519_cast_fp16")];
            tensor<int32, [3]> var_4522_begin_0 = const()[name = string("op_4522_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_4522_end_0 = const()[name = string("op_4522_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_4522_end_mask_0 = const()[name = string("op_4522_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_4522_squeeze_mask_0 = const()[name = string("op_4522_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_4522_cast_fp16 = slice_by_index(begin = var_4522_begin_0, end = var_4522_end_0, end_mask = var_4522_end_mask_0, squeeze_mask = var_4522_squeeze_mask_0, x = beta_19_cast_fp16)[name = string("op_4522_cast_fp16")];
            tensor<int32, [1]> var_4523_axes_0 = const()[name = string("op_4523_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_4523_cast_fp16 = expand_dims(axes = var_4523_axes_0, x = var_4522_cast_fp16)[name = string("op_4523_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_73_cast_fp16 = mul(x = var_4519_cast_fp16, y = var_4523_cast_fp16)[name = string("delta_73_cast_fp16")];
            tensor<int32, [1]> var_4526_axes_0 = const()[name = string("op_4526_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_4526_cast_fp16 = expand_dims(axes = var_4526_axes_0, x = delta_73_cast_fp16)[name = string("op_4526_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4527_cast_fp16 = mul(x = var_4515_cast_fp16, y = var_4526_cast_fp16)[name = string("op_4527_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_147_cast_fp16 = add(x = state_145_cast_fp16, y = var_4527_cast_fp16)[name = string("state_147_cast_fp16")];
            tensor<int32, [4]> var_4531_begin_0 = const()[name = string("op_4531_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> var_4531_end_0 = const()[name = string("op_4531_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> var_4531_end_mask_0 = const()[name = string("op_4531_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_4531_squeeze_mask_0 = const()[name = string("op_4531_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_4531_cast_fp16 = slice_by_index(begin = var_4531_begin_0, end = var_4531_end_0, end_mask = var_4531_end_mask_0, squeeze_mask = var_4531_squeeze_mask_0, x = _inversed_query_51_cast_fp16)[name = string("op_4531_cast_fp16")];
            tensor<int32, [1]> var_4532_axes_0 = const()[name = string("op_4532_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_4532_cast_fp16 = expand_dims(axes = var_4532_axes_0, x = var_4531_cast_fp16)[name = string("op_4532_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4533_cast_fp16 = mul(x = state_147_cast_fp16, y = var_4532_cast_fp16)[name = string("op_4533_cast_fp16")];
            tensor<int32, [1]> var_4535_axes_0 = const()[name = string("op_4535_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_4535_keep_dims_0 = const()[name = string("op_4535_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_4535_cast_fp16 = reduce_sum(axes = var_4535_axes_0, keep_dims = var_4535_keep_dims_0, x = var_4533_cast_fp16)[name = string("op_4535_cast_fp16")];
            tensor<int32, [3]> var_4538_begin_0 = const()[name = string("op_4538_begin_0"), val = tensor<int32, [3]>([0, 0, 1])];
            tensor<int32, [3]> var_4538_end_0 = const()[name = string("op_4538_end_0"), val = tensor<int32, [3]>([1, 16, 2])];
            tensor<bool, [3]> var_4538_end_mask_0 = const()[name = string("op_4538_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_4538_squeeze_mask_0 = const()[name = string("op_4538_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_4538_cast_fp16 = slice_by_index(begin = var_4538_begin_0, end = var_4538_end_0, end_mask = var_4538_end_mask_0, squeeze_mask = var_4538_squeeze_mask_0, x = decay_19_cast_fp16)[name = string("op_4538_cast_fp16")];
            tensor<fp16, [1, 16]> var_4539_cast_fp16 = exp(x = var_4538_cast_fp16)[name = string("op_4539_cast_fp16")];
            tensor<int32, [1]> var_4540_axes_0 = const()[name = string("op_4540_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_4540_cast_fp16 = expand_dims(axes = var_4540_axes_0, x = var_4539_cast_fp16)[name = string("op_4540_cast_fp16")];
            tensor<int32, [1]> var_4541_axes_0 = const()[name = string("op_4541_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_4541_cast_fp16 = expand_dims(axes = var_4541_axes_0, x = var_4540_cast_fp16)[name = string("op_4541_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_149_cast_fp16 = mul(x = state_147_cast_fp16, y = var_4541_cast_fp16)[name = string("state_149_cast_fp16")];
            tensor<int32, [4]> key_token_75_begin_0 = const()[name = string("key_token_75_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> key_token_75_end_0 = const()[name = string("key_token_75_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> key_token_75_end_mask_0 = const()[name = string("key_token_75_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_75_squeeze_mask_0 = const()[name = string("key_token_75_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_75_cast_fp16 = slice_by_index(begin = key_token_75_begin_0, end = key_token_75_end_0, end_mask = key_token_75_end_mask_0, squeeze_mask = key_token_75_squeeze_mask_0, x = key_51_cast_fp16)[name = string("key_token_75_cast_fp16")];
            tensor<int32, [4]> value_token_75_begin_0 = const()[name = string("value_token_75_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> value_token_75_end_0 = const()[name = string("value_token_75_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> value_token_75_end_mask_0 = const()[name = string("value_token_75_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_75_squeeze_mask_0 = const()[name = string("value_token_75_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_75_cast_fp16 = slice_by_index(begin = value_token_75_begin_0, end = value_token_75_end_0, end_mask = value_token_75_end_mask_0, squeeze_mask = value_token_75_squeeze_mask_0, x = value_185_cast_fp16)[name = string("value_token_75_cast_fp16")];
            tensor<int32, [1]> var_4549_axes_0 = const()[name = string("op_4549_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_4549_cast_fp16 = expand_dims(axes = var_4549_axes_0, x = key_token_75_cast_fp16)[name = string("op_4549_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4550_cast_fp16 = mul(x = state_149_cast_fp16, y = var_4549_cast_fp16)[name = string("op_4550_cast_fp16")];
            tensor<int32, [1]> memory_75_axes_0 = const()[name = string("memory_75_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_75_keep_dims_0 = const()[name = string("memory_75_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_75_cast_fp16 = reduce_sum(axes = memory_75_axes_0, keep_dims = memory_75_keep_dims_0, x = var_4550_cast_fp16)[name = string("memory_75_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_4553_cast_fp16 = sub(x = value_token_75_cast_fp16, y = memory_75_cast_fp16)[name = string("op_4553_cast_fp16")];
            tensor<int32, [3]> var_4556_begin_0 = const()[name = string("op_4556_begin_0"), val = tensor<int32, [3]>([0, 0, 1])];
            tensor<int32, [3]> var_4556_end_0 = const()[name = string("op_4556_end_0"), val = tensor<int32, [3]>([1, 16, 2])];
            tensor<bool, [3]> var_4556_end_mask_0 = const()[name = string("op_4556_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_4556_squeeze_mask_0 = const()[name = string("op_4556_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_4556_cast_fp16 = slice_by_index(begin = var_4556_begin_0, end = var_4556_end_0, end_mask = var_4556_end_mask_0, squeeze_mask = var_4556_squeeze_mask_0, x = beta_19_cast_fp16)[name = string("op_4556_cast_fp16")];
            tensor<int32, [1]> var_4557_axes_0 = const()[name = string("op_4557_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_4557_cast_fp16 = expand_dims(axes = var_4557_axes_0, x = var_4556_cast_fp16)[name = string("op_4557_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_75_cast_fp16 = mul(x = var_4553_cast_fp16, y = var_4557_cast_fp16)[name = string("delta_75_cast_fp16")];
            tensor<int32, [1]> var_4560_axes_0 = const()[name = string("op_4560_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_4560_cast_fp16 = expand_dims(axes = var_4560_axes_0, x = delta_75_cast_fp16)[name = string("op_4560_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4561_cast_fp16 = mul(x = var_4549_cast_fp16, y = var_4560_cast_fp16)[name = string("op_4561_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_151_cast_fp16 = add(x = state_149_cast_fp16, y = var_4561_cast_fp16)[name = string("state_151_cast_fp16")];
            tensor<int32, [4]> var_4565_begin_0 = const()[name = string("op_4565_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> var_4565_end_0 = const()[name = string("op_4565_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> var_4565_end_mask_0 = const()[name = string("op_4565_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_4565_squeeze_mask_0 = const()[name = string("op_4565_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_4565_cast_fp16 = slice_by_index(begin = var_4565_begin_0, end = var_4565_end_0, end_mask = var_4565_end_mask_0, squeeze_mask = var_4565_squeeze_mask_0, x = _inversed_query_51_cast_fp16)[name = string("op_4565_cast_fp16")];
            tensor<int32, [1]> var_4566_axes_0 = const()[name = string("op_4566_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_4566_cast_fp16 = expand_dims(axes = var_4566_axes_0, x = var_4565_cast_fp16)[name = string("op_4566_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4567_cast_fp16 = mul(x = state_151_cast_fp16, y = var_4566_cast_fp16)[name = string("op_4567_cast_fp16")];
            tensor<int32, [1]> var_4569_axes_0 = const()[name = string("op_4569_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_4569_keep_dims_0 = const()[name = string("op_4569_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_4569_cast_fp16 = reduce_sum(axes = var_4569_axes_0, keep_dims = var_4569_keep_dims_0, x = var_4567_cast_fp16)[name = string("op_4569_cast_fp16")];
            tensor<int32, [3]> var_4572_begin_0 = const()[name = string("op_4572_begin_0"), val = tensor<int32, [3]>([0, 0, 2])];
            tensor<int32, [3]> var_4572_end_0 = const()[name = string("op_4572_end_0"), val = tensor<int32, [3]>([1, 16, 3])];
            tensor<bool, [3]> var_4572_end_mask_0 = const()[name = string("op_4572_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_4572_squeeze_mask_0 = const()[name = string("op_4572_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_4572_cast_fp16 = slice_by_index(begin = var_4572_begin_0, end = var_4572_end_0, end_mask = var_4572_end_mask_0, squeeze_mask = var_4572_squeeze_mask_0, x = decay_19_cast_fp16)[name = string("op_4572_cast_fp16")];
            tensor<fp16, [1, 16]> var_4573_cast_fp16 = exp(x = var_4572_cast_fp16)[name = string("op_4573_cast_fp16")];
            tensor<int32, [1]> var_4574_axes_0 = const()[name = string("op_4574_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_4574_cast_fp16 = expand_dims(axes = var_4574_axes_0, x = var_4573_cast_fp16)[name = string("op_4574_cast_fp16")];
            tensor<int32, [1]> var_4575_axes_0 = const()[name = string("op_4575_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_4575_cast_fp16 = expand_dims(axes = var_4575_axes_0, x = var_4574_cast_fp16)[name = string("op_4575_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_153_cast_fp16 = mul(x = state_151_cast_fp16, y = var_4575_cast_fp16)[name = string("state_153_cast_fp16")];
            tensor<int32, [4]> key_token_77_begin_0 = const()[name = string("key_token_77_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> key_token_77_end_0 = const()[name = string("key_token_77_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> key_token_77_end_mask_0 = const()[name = string("key_token_77_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_77_squeeze_mask_0 = const()[name = string("key_token_77_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_77_cast_fp16 = slice_by_index(begin = key_token_77_begin_0, end = key_token_77_end_0, end_mask = key_token_77_end_mask_0, squeeze_mask = key_token_77_squeeze_mask_0, x = key_51_cast_fp16)[name = string("key_token_77_cast_fp16")];
            tensor<int32, [4]> value_token_77_begin_0 = const()[name = string("value_token_77_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> value_token_77_end_0 = const()[name = string("value_token_77_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> value_token_77_end_mask_0 = const()[name = string("value_token_77_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_77_squeeze_mask_0 = const()[name = string("value_token_77_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_77_cast_fp16 = slice_by_index(begin = value_token_77_begin_0, end = value_token_77_end_0, end_mask = value_token_77_end_mask_0, squeeze_mask = value_token_77_squeeze_mask_0, x = value_185_cast_fp16)[name = string("value_token_77_cast_fp16")];
            tensor<int32, [1]> var_4583_axes_0 = const()[name = string("op_4583_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_4583_cast_fp16 = expand_dims(axes = var_4583_axes_0, x = key_token_77_cast_fp16)[name = string("op_4583_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4584_cast_fp16 = mul(x = state_153_cast_fp16, y = var_4583_cast_fp16)[name = string("op_4584_cast_fp16")];
            tensor<int32, [1]> memory_77_axes_0 = const()[name = string("memory_77_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_77_keep_dims_0 = const()[name = string("memory_77_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_77_cast_fp16 = reduce_sum(axes = memory_77_axes_0, keep_dims = memory_77_keep_dims_0, x = var_4584_cast_fp16)[name = string("memory_77_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_4587_cast_fp16 = sub(x = value_token_77_cast_fp16, y = memory_77_cast_fp16)[name = string("op_4587_cast_fp16")];
            tensor<int32, [3]> var_4590_begin_0 = const()[name = string("op_4590_begin_0"), val = tensor<int32, [3]>([0, 0, 2])];
            tensor<int32, [3]> var_4590_end_0 = const()[name = string("op_4590_end_0"), val = tensor<int32, [3]>([1, 16, 3])];
            tensor<bool, [3]> var_4590_end_mask_0 = const()[name = string("op_4590_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_4590_squeeze_mask_0 = const()[name = string("op_4590_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_4590_cast_fp16 = slice_by_index(begin = var_4590_begin_0, end = var_4590_end_0, end_mask = var_4590_end_mask_0, squeeze_mask = var_4590_squeeze_mask_0, x = beta_19_cast_fp16)[name = string("op_4590_cast_fp16")];
            tensor<int32, [1]> var_4591_axes_0 = const()[name = string("op_4591_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_4591_cast_fp16 = expand_dims(axes = var_4591_axes_0, x = var_4590_cast_fp16)[name = string("op_4591_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_77_cast_fp16 = mul(x = var_4587_cast_fp16, y = var_4591_cast_fp16)[name = string("delta_77_cast_fp16")];
            tensor<int32, [1]> var_4594_axes_0 = const()[name = string("op_4594_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_4594_cast_fp16 = expand_dims(axes = var_4594_axes_0, x = delta_77_cast_fp16)[name = string("op_4594_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4595_cast_fp16 = mul(x = var_4583_cast_fp16, y = var_4594_cast_fp16)[name = string("op_4595_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_155_cast_fp16 = add(x = state_153_cast_fp16, y = var_4595_cast_fp16)[name = string("state_155_cast_fp16")];
            tensor<int32, [4]> var_4599_begin_0 = const()[name = string("op_4599_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> var_4599_end_0 = const()[name = string("op_4599_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> var_4599_end_mask_0 = const()[name = string("op_4599_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_4599_squeeze_mask_0 = const()[name = string("op_4599_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_4599_cast_fp16 = slice_by_index(begin = var_4599_begin_0, end = var_4599_end_0, end_mask = var_4599_end_mask_0, squeeze_mask = var_4599_squeeze_mask_0, x = _inversed_query_51_cast_fp16)[name = string("op_4599_cast_fp16")];
            tensor<int32, [1]> var_4600_axes_0 = const()[name = string("op_4600_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_4600_cast_fp16 = expand_dims(axes = var_4600_axes_0, x = var_4599_cast_fp16)[name = string("op_4600_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4601_cast_fp16 = mul(x = state_155_cast_fp16, y = var_4600_cast_fp16)[name = string("op_4601_cast_fp16")];
            tensor<int32, [1]> var_4603_axes_0 = const()[name = string("op_4603_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_4603_keep_dims_0 = const()[name = string("op_4603_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_4603_cast_fp16 = reduce_sum(axes = var_4603_axes_0, keep_dims = var_4603_keep_dims_0, x = var_4601_cast_fp16)[name = string("op_4603_cast_fp16")];
            tensor<int32, [3]> var_4606_begin_0 = const()[name = string("op_4606_begin_0"), val = tensor<int32, [3]>([0, 0, 3])];
            tensor<int32, [3]> var_4606_end_0 = const()[name = string("op_4606_end_0"), val = tensor<int32, [3]>([1, 16, 4])];
            tensor<bool, [3]> var_4606_end_mask_0 = const()[name = string("op_4606_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_4606_squeeze_mask_0 = const()[name = string("op_4606_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_4606_cast_fp16 = slice_by_index(begin = var_4606_begin_0, end = var_4606_end_0, end_mask = var_4606_end_mask_0, squeeze_mask = var_4606_squeeze_mask_0, x = decay_19_cast_fp16)[name = string("op_4606_cast_fp16")];
            tensor<fp16, [1, 16]> var_4607_cast_fp16 = exp(x = var_4606_cast_fp16)[name = string("op_4607_cast_fp16")];
            tensor<int32, [1]> var_4608_axes_0 = const()[name = string("op_4608_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_4608_cast_fp16 = expand_dims(axes = var_4608_axes_0, x = var_4607_cast_fp16)[name = string("op_4608_cast_fp16")];
            tensor<int32, [1]> var_4609_axes_0 = const()[name = string("op_4609_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_4609_cast_fp16 = expand_dims(axes = var_4609_axes_0, x = var_4608_cast_fp16)[name = string("op_4609_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_157_cast_fp16 = mul(x = state_155_cast_fp16, y = var_4609_cast_fp16)[name = string("state_157_cast_fp16")];
            tensor<int32, [4]> key_token_79_begin_0 = const()[name = string("key_token_79_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> key_token_79_end_0 = const()[name = string("key_token_79_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> key_token_79_end_mask_0 = const()[name = string("key_token_79_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_79_squeeze_mask_0 = const()[name = string("key_token_79_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_79_cast_fp16 = slice_by_index(begin = key_token_79_begin_0, end = key_token_79_end_0, end_mask = key_token_79_end_mask_0, squeeze_mask = key_token_79_squeeze_mask_0, x = key_51_cast_fp16)[name = string("key_token_79_cast_fp16")];
            tensor<int32, [4]> value_token_79_begin_0 = const()[name = string("value_token_79_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> value_token_79_end_0 = const()[name = string("value_token_79_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> value_token_79_end_mask_0 = const()[name = string("value_token_79_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_79_squeeze_mask_0 = const()[name = string("value_token_79_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_79_cast_fp16 = slice_by_index(begin = value_token_79_begin_0, end = value_token_79_end_0, end_mask = value_token_79_end_mask_0, squeeze_mask = value_token_79_squeeze_mask_0, x = value_185_cast_fp16)[name = string("value_token_79_cast_fp16")];
            tensor<int32, [1]> var_4617_axes_0 = const()[name = string("op_4617_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_4617_cast_fp16 = expand_dims(axes = var_4617_axes_0, x = key_token_79_cast_fp16)[name = string("op_4617_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4618_cast_fp16 = mul(x = state_157_cast_fp16, y = var_4617_cast_fp16)[name = string("op_4618_cast_fp16")];
            tensor<int32, [1]> memory_79_axes_0 = const()[name = string("memory_79_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_79_keep_dims_0 = const()[name = string("memory_79_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_79_cast_fp16 = reduce_sum(axes = memory_79_axes_0, keep_dims = memory_79_keep_dims_0, x = var_4618_cast_fp16)[name = string("memory_79_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_4621_cast_fp16 = sub(x = value_token_79_cast_fp16, y = memory_79_cast_fp16)[name = string("op_4621_cast_fp16")];
            tensor<int32, [3]> var_4624_begin_0 = const()[name = string("op_4624_begin_0"), val = tensor<int32, [3]>([0, 0, 3])];
            tensor<int32, [3]> var_4624_end_0 = const()[name = string("op_4624_end_0"), val = tensor<int32, [3]>([1, 16, 4])];
            tensor<bool, [3]> var_4624_end_mask_0 = const()[name = string("op_4624_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_4624_squeeze_mask_0 = const()[name = string("op_4624_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_4624_cast_fp16 = slice_by_index(begin = var_4624_begin_0, end = var_4624_end_0, end_mask = var_4624_end_mask_0, squeeze_mask = var_4624_squeeze_mask_0, x = beta_19_cast_fp16)[name = string("op_4624_cast_fp16")];
            tensor<int32, [1]> var_4625_axes_0 = const()[name = string("op_4625_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_4625_cast_fp16 = expand_dims(axes = var_4625_axes_0, x = var_4624_cast_fp16)[name = string("op_4625_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_79_cast_fp16 = mul(x = var_4621_cast_fp16, y = var_4625_cast_fp16)[name = string("delta_79_cast_fp16")];
            tensor<int32, [1]> var_4628_axes_0 = const()[name = string("op_4628_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_4628_cast_fp16 = expand_dims(axes = var_4628_axes_0, x = delta_79_cast_fp16)[name = string("op_4628_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4629_cast_fp16 = mul(x = var_4617_cast_fp16, y = var_4628_cast_fp16)[name = string("op_4629_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> rec12_out = add(x = state_157_cast_fp16, y = var_4629_cast_fp16)[name = string("state_159_cast_fp16")];
            tensor<int32, [4]> var_4633_begin_0 = const()[name = string("op_4633_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> var_4633_end_0 = const()[name = string("op_4633_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> var_4633_end_mask_0 = const()[name = string("op_4633_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_4633_squeeze_mask_0 = const()[name = string("op_4633_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_4633_cast_fp16 = slice_by_index(begin = var_4633_begin_0, end = var_4633_end_0, end_mask = var_4633_end_mask_0, squeeze_mask = var_4633_squeeze_mask_0, x = _inversed_query_51_cast_fp16)[name = string("op_4633_cast_fp16")];
            tensor<int32, [1]> var_4634_axes_0 = const()[name = string("op_4634_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_4634_cast_fp16 = expand_dims(axes = var_4634_axes_0, x = var_4633_cast_fp16)[name = string("op_4634_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4635_cast_fp16 = mul(x = rec12_out, y = var_4634_cast_fp16)[name = string("op_4635_cast_fp16")];
            tensor<int32, [1]> var_4637_axes_0 = const()[name = string("op_4637_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_4637_keep_dims_0 = const()[name = string("op_4637_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_4637_cast_fp16 = reduce_sum(axes = var_4637_axes_0, keep_dims = var_4637_keep_dims_0, x = var_4635_cast_fp16)[name = string("op_4637_cast_fp16")];
            int32 attention_91_axis_0 = const()[name = string("attention_91_axis_0"), val = int32(1)];
            tensor<fp16, [1, 4, 16, 128]> attention_91_cast_fp16 = stack(axis = attention_91_axis_0, values = (var_4535_cast_fp16, var_4569_cast_fp16, var_4603_cast_fp16, var_4637_cast_fp16))[name = string("attention_91_cast_fp16")];
            tensor<int32, [2]> var_4640 = const()[name = string("op_4640"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [64, 128]> attention_93_cast_fp16 = reshape(shape = var_4640, x = attention_91_cast_fp16)[name = string("attention_93_cast_fp16")];
            tensor<int32, [2]> var_4642 = const()[name = string("op_4642"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [64, 128]> input_161_cast_fp16 = reshape(shape = var_4642, x = linear_96_cast_fp16)[name = string("input_161_cast_fp16")];
            tensor<fp16, [64, 128]> var_4644_cast_fp16 = mul(x = attention_93_cast_fp16, y = attention_93_cast_fp16)[name = string("op_4644_cast_fp16")];
            tensor<int32, [1]> variance_81_axes_0 = const()[name = string("variance_81_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_81_keep_dims_0 = const()[name = string("variance_81_keep_dims_0"), val = bool(true)];
            tensor<fp16, [64, 1]> variance_81_cast_fp16 = reduce_mean(axes = variance_81_axes_0, keep_dims = variance_81_keep_dims_0, x = var_4644_cast_fp16)[name = string("variance_81_cast_fp16")];
            fp16 var_4647_to_fp16 = const()[name = string("op_4647_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [64, 1]> var_4648_cast_fp16 = add(x = variance_81_cast_fp16, y = var_4647_to_fp16)[name = string("op_4648_cast_fp16")];
            fp32 var_4649_epsilon_0 = const()[name = string("op_4649_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [64, 1]> var_4649_cast_fp16 = rsqrt(epsilon = var_4649_epsilon_0, x = var_4648_cast_fp16)[name = string("op_4649_cast_fp16")];
            tensor<fp16, [64, 128]> attention_95_cast_fp16 = mul(x = attention_93_cast_fp16, y = var_4649_cast_fp16)[name = string("attention_95_cast_fp16")];
            tensor<fp16, [128]> groups_3_0_gated_norm_promoted_to_fp16 = const()[name = string("groups_3_0_gated_norm_promoted_to_fp16"), val = tensor<fp16, [128]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(195373760)))];
            tensor<fp16, [64, 128]> attention_97_cast_fp16 = mul(x = attention_95_cast_fp16, y = groups_3_0_gated_norm_promoted_to_fp16)[name = string("attention_97_cast_fp16")];
            tensor<fp16, [64, 128]> var_4652_cast_fp16 = silu(x = input_161_cast_fp16)[name = string("op_4652_cast_fp16")];
            tensor<fp16, [64, 128]> attention_99_cast_fp16 = mul(x = attention_97_cast_fp16, y = var_4652_cast_fp16)[name = string("attention_99_cast_fp16")];
            tensor<int32, [2]> var_4654 = const()[name = string("op_4654"), val = tensor<int32, [2]>([4, 2048])];
            tensor<fp16, [4, 2048]> input_163_cast_fp16 = reshape(shape = var_4654, x = attention_99_cast_fp16)[name = string("input_163_cast_fp16")];
            tensor<fp16, [1024, 2048]> groups_3_0_out_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 2048]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(195374080))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(196947008))))[name = string("groups_3_0_out_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 1024]> linear_97_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = groups_3_0_out_proj_weight_promoted_to_fp16_palettized, x = input_163_cast_fp16)[name = string("linear_97_cast_fp16")];
            tensor<fp16, [4, 1024]> value_187_cast_fp16 = add(x = value_175_cast_fp16, y = linear_97_cast_fp16)[name = string("value_187_cast_fp16")];
            tensor<fp16, [4, 1024]> var_4659_cast_fp16 = mul(x = value_187_cast_fp16, y = value_187_cast_fp16)[name = string("op_4659_cast_fp16")];
            tensor<int32, [1]> variance_83_axes_0 = const()[name = string("variance_83_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_83_keep_dims_0 = const()[name = string("variance_83_keep_dims_0"), val = bool(true)];
            tensor<fp16, [4, 1]> variance_83_cast_fp16 = reduce_mean(axes = variance_83_axes_0, keep_dims = variance_83_keep_dims_0, x = var_4659_cast_fp16)[name = string("variance_83_cast_fp16")];
            fp16 var_4662_to_fp16 = const()[name = string("op_4662_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [4, 1]> var_4663_cast_fp16 = add(x = variance_83_cast_fp16, y = var_4662_to_fp16)[name = string("op_4663_cast_fp16")];
            fp32 var_4664_epsilon_0 = const()[name = string("op_4664_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [4, 1]> var_4664_cast_fp16 = rsqrt(epsilon = var_4664_epsilon_0, x = var_4663_cast_fp16)[name = string("op_4664_cast_fp16")];
            tensor<fp16, [4, 1024]> var_4665_cast_fp16 = mul(x = value_187_cast_fp16, y = var_4664_cast_fp16)[name = string("op_4665_cast_fp16")];
            tensor<fp16, [1024]> var_4667_to_fp16 = const()[name = string("op_4667_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(196955264)))];
            tensor<fp16, [4, 1024]> input_165_cast_fp16 = mul(x = var_4665_cast_fp16, y = var_4667_to_fp16)[name = string("input_165_cast_fp16")];
            tensor<fp16, [3584, 1024]> groups_3_0_gate_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(196957376))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(199709952))))[name = string("groups_3_0_gate_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 3584]> linear_98_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = groups_3_0_gate_proj_weight_promoted_to_fp16_palettized, x = input_165_cast_fp16)[name = string("linear_98_cast_fp16")];
            tensor<fp16, [4, 3584]> var_4671_cast_fp16 = silu(x = linear_98_cast_fp16)[name = string("op_4671_cast_fp16")];
            tensor<fp16, [3584, 1024]> groups_3_0_up_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(199738688))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(202491264))))[name = string("groups_3_0_up_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 3584]> linear_99_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = groups_3_0_up_proj_weight_promoted_to_fp16_palettized, x = input_165_cast_fp16)[name = string("linear_99_cast_fp16")];
            tensor<fp16, [4, 3584]> input_169_cast_fp16 = mul(x = var_4671_cast_fp16, y = linear_99_cast_fp16)[name = string("input_169_cast_fp16")];
            tensor<fp16, [1024, 3584]> groups_3_0_down_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 3584]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(202520000))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(205272576))))[name = string("groups_3_0_down_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 1024]> linear_100_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = groups_3_0_down_proj_weight_promoted_to_fp16_palettized, x = input_169_cast_fp16)[name = string("linear_100_cast_fp16")];
            tensor<fp16, [4, 1024]> value_189_cast_fp16 = add(x = value_187_cast_fp16, y = linear_100_cast_fp16)[name = string("value_189_cast_fp16")];
            int32 var_4698 = const()[name = string("op_4698"), val = int32(-1)];
            tensor<fp16, [4, 1024]> var_4713_cast_fp16 = mul(x = value_189_cast_fp16, y = value_189_cast_fp16)[name = string("op_4713_cast_fp16")];
            tensor<int32, [1]> variance_85_axes_0 = const()[name = string("variance_85_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_85_keep_dims_0 = const()[name = string("variance_85_keep_dims_0"), val = bool(true)];
            tensor<fp16, [4, 1]> variance_85_cast_fp16 = reduce_mean(axes = variance_85_axes_0, keep_dims = variance_85_keep_dims_0, x = var_4713_cast_fp16)[name = string("variance_85_cast_fp16")];
            fp16 var_4716_to_fp16 = const()[name = string("op_4716_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [4, 1]> var_4717_cast_fp16 = add(x = variance_85_cast_fp16, y = var_4716_to_fp16)[name = string("op_4717_cast_fp16")];
            fp32 var_4718_epsilon_0 = const()[name = string("op_4718_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [4, 1]> var_4718_cast_fp16 = rsqrt(epsilon = var_4718_epsilon_0, x = var_4717_cast_fp16)[name = string("op_4718_cast_fp16")];
            tensor<fp16, [4, 1024]> var_4719_cast_fp16 = mul(x = value_189_cast_fp16, y = var_4718_cast_fp16)[name = string("op_4719_cast_fp16")];
            tensor<fp16, [1024]> var_4721_to_fp16 = const()[name = string("op_4721_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(205280832)))];
            tensor<fp16, [4, 1024]> input_171_cast_fp16 = mul(x = var_4719_cast_fp16, y = var_4721_to_fp16)[name = string("input_171_cast_fp16")];
            tensor<fp16, [6144, 1024]> groups_3_1_in_proj_qkv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(205282944))), lut = tensor<fp16, [384, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(210001600))))[name = string("groups_3_1_in_proj_qkv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 6144]> linear_101_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = groups_3_1_in_proj_qkv_weight_promoted_to_fp16_palettized, x = input_171_cast_fp16)[name = string("linear_101_cast_fp16")];
            tensor<int32, [2]> var_4725_perm_0 = const()[name = string("op_4725_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> mixed_21_axes_0 = const()[name = string("mixed_21_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [6144, 4]> var_4725_cast_fp16 = transpose(perm = var_4725_perm_0, x = linear_101_cast_fp16)[name = string("transpose_46")];
            tensor<fp16, [1, 6144, 4]> mixed_21_cast_fp16 = expand_dims(axes = mixed_21_axes_0, x = var_4725_cast_fp16)[name = string("mixed_21_cast_fp16")];
            bool convolution_input_21_interleave_0 = const()[name = string("convolution_input_21_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 6144, 7]> convolution_input_21_cast_fp16 = concat(axis = var_4698, interleave = convolution_input_21_interleave_0, values = (conv13, mixed_21_cast_fp16))[name = string("convolution_input_21_cast_fp16")];
            string input_173_pad_type_0 = const()[name = string("input_173_pad_type_0"), val = string("valid")];
            int32 input_173_groups_0 = const()[name = string("input_173_groups_0"), val = int32(6144)];
            tensor<int32, [1]> input_173_strides_0 = const()[name = string("input_173_strides_0"), val = tensor<int32, [1]>([1])];
            tensor<int32, [2]> input_173_pad_0 = const()[name = string("input_173_pad_0"), val = tensor<int32, [2]>([0, 0])];
            tensor<int32, [1]> input_173_dilations_0 = const()[name = string("input_173_dilations_0"), val = tensor<int32, [1]>([1])];
            tensor<fp16, [6144, 1, 4]> groups_3_1_conv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1, 4]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(210050816))), lut = tensor<fp16, [384, 1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(210069312))))[name = string("groups_3_1_conv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 6144, 4]> input_173_cast_fp16 = conv(dilations = input_173_dilations_0, groups = input_173_groups_0, pad = input_173_pad_0, pad_type = input_173_pad_type_0, strides = input_173_strides_0, weight = groups_3_1_conv_weight_promoted_to_fp16_palettized, x = convolution_input_21_cast_fp16)[name = string("input_173_cast_fp16")];
            tensor<fp16, [1, 6144, 4]> var_4734_cast_fp16 = silu(x = input_173_cast_fp16)[name = string("op_4734_cast_fp16")];
            tensor<int32, [3]> var_4735_perm_0 = const()[name = string("op_4735_perm_0"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [3]> var_4738_begin_0 = const()[name = string("op_4738_begin_0"), val = tensor<int32, [3]>([0, 0, 4])];
            tensor<int32, [3]> var_4738_end_0 = const()[name = string("op_4738_end_0"), val = tensor<int32, [3]>([1, 6144, 7])];
            tensor<bool, [3]> var_4738_end_mask_0 = const()[name = string("op_4738_end_mask_0"), val = tensor<bool, [3]>([true, true, true])];
            tensor<fp16, [1, 6144, 3]> conv13_out = slice_by_index(begin = var_4738_begin_0, end = var_4738_end_0, end_mask = var_4738_end_mask_0, x = convolution_input_21_cast_fp16)[name = string("op_4738_cast_fp16")];
            tensor<int32, [3]> var_4739 = const()[name = string("op_4739"), val = tensor<int32, [3]>([2048, 2048, 2048])];
            int32 var_4740_axis_0 = const()[name = string("op_4740_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 4, 6144]> var_4735_cast_fp16 = transpose(perm = var_4735_perm_0, x = var_4734_cast_fp16)[name = string("transpose_45")];
            tensor<fp16, [1, 4, 2048]> var_4740_cast_fp16_0, tensor<fp16, [1, 4, 2048]> var_4740_cast_fp16_1, tensor<fp16, [1, 4, 2048]> var_4740_cast_fp16_2 = split(axis = var_4740_axis_0, split_sizes = var_4739, x = var_4735_cast_fp16)[name = string("op_4740_cast_fp16")];
            tensor<int32, [4]> var_4744 = const()[name = string("op_4744"), val = tensor<int32, [4]>([1, 4, 16, 128])];
            tensor<fp16, [1, 4, 16, 128]> value_193_cast_fp16 = reshape(shape = var_4744, x = var_4740_cast_fp16_0)[name = string("value_193_cast_fp16")];
            tensor<int32, [4]> var_4746 = const()[name = string("op_4746"), val = tensor<int32, [4]>([1, 4, 16, 128])];
            tensor<fp16, [1, 4, 16, 128]> value_195_cast_fp16 = reshape(shape = var_4746, x = var_4740_cast_fp16_1)[name = string("value_195_cast_fp16")];
            tensor<int32, [4]> var_4748 = const()[name = string("op_4748"), val = tensor<int32, [4]>([1, 4, 16, 128])];
            tensor<fp16, [1, 4, 16, 128]> value_197_cast_fp16 = reshape(shape = var_4748, x = var_4740_cast_fp16_2)[name = string("value_197_cast_fp16")];
            tensor<fp16, [1, 4, 16, 128]> var_4750_cast_fp16 = mul(x = value_193_cast_fp16, y = value_193_cast_fp16)[name = string("op_4750_cast_fp16")];
            tensor<int32, [1]> var_4752_axes_0 = const()[name = string("op_4752_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_4752_keep_dims_0 = const()[name = string("op_4752_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 4, 16, 1]> var_4752_cast_fp16 = reduce_sum(axes = var_4752_axes_0, keep_dims = var_4752_keep_dims_0, x = var_4750_cast_fp16)[name = string("op_4752_cast_fp16")];
            fp16 var_4753_to_fp16 = const()[name = string("op_4753_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 4, 16, 1]> var_4754_cast_fp16 = add(x = var_4752_cast_fp16, y = var_4753_to_fp16)[name = string("op_4754_cast_fp16")];
            fp32 var_4755_epsilon_0 = const()[name = string("op_4755_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 4, 16, 1]> var_4755_cast_fp16 = rsqrt(epsilon = var_4755_epsilon_0, x = var_4754_cast_fp16)[name = string("op_4755_cast_fp16")];
            tensor<fp16, [1, 4, 16, 128]> var_4756_cast_fp16 = mul(x = value_193_cast_fp16, y = var_4755_cast_fp16)[name = string("op_4756_cast_fp16")];
            tensor<int32, [4]> var_4757_perm_0 = const()[name = string("op_4757_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            fp16 _inversed_query_55_y_0_to_fp16 = const()[name = string("_inversed_query_55_y_0_to_fp16"), val = fp16(0x1.6ap-4)];
            tensor<fp16, [1, 16, 4, 128]> var_4757_cast_fp16 = transpose(perm = var_4757_perm_0, x = var_4756_cast_fp16)[name = string("transpose_44")];
            tensor<fp16, [1, 16, 4, 128]> _inversed_query_55_cast_fp16 = mul(x = var_4757_cast_fp16, y = _inversed_query_55_y_0_to_fp16)[name = string("_inversed_query_55_cast_fp16")];
            tensor<fp16, [1, 4, 16, 128]> var_4760_cast_fp16 = mul(x = value_195_cast_fp16, y = value_195_cast_fp16)[name = string("op_4760_cast_fp16")];
            tensor<int32, [1]> var_4762_axes_0 = const()[name = string("op_4762_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_4762_keep_dims_0 = const()[name = string("op_4762_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 4, 16, 1]> var_4762_cast_fp16 = reduce_sum(axes = var_4762_axes_0, keep_dims = var_4762_keep_dims_0, x = var_4760_cast_fp16)[name = string("op_4762_cast_fp16")];
            fp16 var_4763_to_fp16 = const()[name = string("op_4763_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 4, 16, 1]> var_4764_cast_fp16 = add(x = var_4762_cast_fp16, y = var_4763_to_fp16)[name = string("op_4764_cast_fp16")];
            fp32 var_4765_epsilon_0 = const()[name = string("op_4765_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 4, 16, 1]> var_4765_cast_fp16 = rsqrt(epsilon = var_4765_epsilon_0, x = var_4764_cast_fp16)[name = string("op_4765_cast_fp16")];
            tensor<fp16, [1, 4, 16, 128]> var_4766_cast_fp16 = mul(x = value_195_cast_fp16, y = var_4765_cast_fp16)[name = string("op_4766_cast_fp16")];
            tensor<int32, [4]> key_55_perm_0 = const()[name = string("key_55_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<int32, [4]> value_199_perm_0 = const()[name = string("value_199_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<fp16, [16, 1024]> groups_3_1_in_proj_b_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(210118528))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(210130880))))[name = string("groups_3_1_in_proj_b_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16]> linear_102_bias_0_to_fp16 = const()[name = string("linear_102_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(4862656)))];
            tensor<fp16, [4, 16]> linear_102_cast_fp16 = linear(bias = linear_102_bias_0_to_fp16, weight = groups_3_1_in_proj_b_weight_promoted_to_fp16_palettized, x = input_171_cast_fp16)[name = string("linear_102_cast_fp16")];
            tensor<fp16, [4, 16]> var_4771_cast_fp16 = sigmoid(x = linear_102_cast_fp16)[name = string("op_4771_cast_fp16")];
            tensor<int32, [2]> var_4772_perm_0 = const()[name = string("op_4772_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> beta_21_axes_0 = const()[name = string("beta_21_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 4]> var_4772_cast_fp16 = transpose(perm = var_4772_perm_0, x = var_4771_cast_fp16)[name = string("transpose_43")];
            tensor<fp16, [1, 16, 4]> beta_21_cast_fp16 = expand_dims(axes = beta_21_axes_0, x = var_4772_cast_fp16)[name = string("beta_21_cast_fp16")];
            tensor<fp16, [16, 1024]> op_4778_weight_0_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(210131072))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(210143424))))[name = string("op_4778_weight_0_to_fp16_palettized")];
            tensor<fp16, [16]> var_4778_bias_0_to_fp16 = const()[name = string("op_4778_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(210143616)))];
            tensor<fp16, [4, 16]> var_4778_cast_fp16 = linear(bias = var_4778_bias_0_to_fp16, weight = op_4778_weight_0_to_fp16_palettized, x = input_171_cast_fp16)[name = string("op_4778_cast_fp16")];
            tensor<fp16, [4, 16]> var_4779_cast_fp16 = softplus(x = var_4778_cast_fp16)[name = string("op_4779_cast_fp16")];
            tensor<fp16, [16]> var_4775_promoted_to_fp16 = const()[name = string("op_4775_promoted_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(210143744)))];
            tensor<fp16, [4, 16]> var_4780_cast_fp16 = mul(x = var_4775_promoted_to_fp16, y = var_4779_cast_fp16)[name = string("op_4780_cast_fp16")];
            tensor<int32, [2]> var_4781_perm_0 = const()[name = string("op_4781_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> decay_21_axes_0 = const()[name = string("decay_21_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 4]> var_4781_cast_fp16 = transpose(perm = var_4781_perm_0, x = var_4780_cast_fp16)[name = string("transpose_42")];
            tensor<fp16, [1, 16, 4]> decay_21_cast_fp16 = expand_dims(axes = decay_21_axes_0, x = var_4781_cast_fp16)[name = string("decay_21_cast_fp16")];
            tensor<fp16, [2048, 1024]> groups_3_1_in_proj_z_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [2048, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(210143872))), lut = tensor<fp16, [128, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(211716800))))[name = string("groups_3_1_in_proj_z_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 2048]> linear_104_cast_fp16 = linear(bias = linear_3_bias_0_to_fp16, weight = groups_3_1_in_proj_z_weight_promoted_to_fp16_palettized, x = input_171_cast_fp16)[name = string("linear_104_cast_fp16")];
            tensor<int32, [3]> var_4789_begin_0 = const()[name = string("op_4789_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_4789_end_0 = const()[name = string("op_4789_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_4789_end_mask_0 = const()[name = string("op_4789_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_4789_squeeze_mask_0 = const()[name = string("op_4789_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_4789_cast_fp16 = slice_by_index(begin = var_4789_begin_0, end = var_4789_end_0, end_mask = var_4789_end_mask_0, squeeze_mask = var_4789_squeeze_mask_0, x = decay_21_cast_fp16)[name = string("op_4789_cast_fp16")];
            tensor<fp16, [1, 16]> var_4790_cast_fp16 = exp(x = var_4789_cast_fp16)[name = string("op_4790_cast_fp16")];
            tensor<int32, [1]> var_4791_axes_0 = const()[name = string("op_4791_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_4791_cast_fp16 = expand_dims(axes = var_4791_axes_0, x = var_4790_cast_fp16)[name = string("op_4791_cast_fp16")];
            tensor<int32, [1]> var_4792_axes_0 = const()[name = string("op_4792_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_4792_cast_fp16 = expand_dims(axes = var_4792_axes_0, x = var_4791_cast_fp16)[name = string("op_4792_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_161_cast_fp16 = mul(x = rec13, y = var_4792_cast_fp16)[name = string("state_161_cast_fp16")];
            tensor<int32, [4]> key_token_81_begin_0 = const()[name = string("key_token_81_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> key_token_81_end_0 = const()[name = string("key_token_81_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> key_token_81_end_mask_0 = const()[name = string("key_token_81_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_81_squeeze_mask_0 = const()[name = string("key_token_81_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 4, 128]> key_55_cast_fp16 = transpose(perm = key_55_perm_0, x = var_4766_cast_fp16)[name = string("transpose_41")];
            tensor<fp16, [1, 16, 128]> key_token_81_cast_fp16 = slice_by_index(begin = key_token_81_begin_0, end = key_token_81_end_0, end_mask = key_token_81_end_mask_0, squeeze_mask = key_token_81_squeeze_mask_0, x = key_55_cast_fp16)[name = string("key_token_81_cast_fp16")];
            tensor<int32, [4]> value_token_81_begin_0 = const()[name = string("value_token_81_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> value_token_81_end_0 = const()[name = string("value_token_81_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> value_token_81_end_mask_0 = const()[name = string("value_token_81_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_81_squeeze_mask_0 = const()[name = string("value_token_81_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 4, 128]> value_199_cast_fp16 = transpose(perm = value_199_perm_0, x = value_197_cast_fp16)[name = string("transpose_40")];
            tensor<fp16, [1, 16, 128]> value_token_81_cast_fp16 = slice_by_index(begin = value_token_81_begin_0, end = value_token_81_end_0, end_mask = value_token_81_end_mask_0, squeeze_mask = value_token_81_squeeze_mask_0, x = value_199_cast_fp16)[name = string("value_token_81_cast_fp16")];
            tensor<int32, [1]> var_4800_axes_0 = const()[name = string("op_4800_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_4800_cast_fp16 = expand_dims(axes = var_4800_axes_0, x = key_token_81_cast_fp16)[name = string("op_4800_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4801_cast_fp16 = mul(x = state_161_cast_fp16, y = var_4800_cast_fp16)[name = string("op_4801_cast_fp16")];
            tensor<int32, [1]> memory_81_axes_0 = const()[name = string("memory_81_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_81_keep_dims_0 = const()[name = string("memory_81_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_81_cast_fp16 = reduce_sum(axes = memory_81_axes_0, keep_dims = memory_81_keep_dims_0, x = var_4801_cast_fp16)[name = string("memory_81_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_4804_cast_fp16 = sub(x = value_token_81_cast_fp16, y = memory_81_cast_fp16)[name = string("op_4804_cast_fp16")];
            tensor<int32, [3]> var_4807_begin_0 = const()[name = string("op_4807_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_4807_end_0 = const()[name = string("op_4807_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_4807_end_mask_0 = const()[name = string("op_4807_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_4807_squeeze_mask_0 = const()[name = string("op_4807_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_4807_cast_fp16 = slice_by_index(begin = var_4807_begin_0, end = var_4807_end_0, end_mask = var_4807_end_mask_0, squeeze_mask = var_4807_squeeze_mask_0, x = beta_21_cast_fp16)[name = string("op_4807_cast_fp16")];
            tensor<int32, [1]> var_4808_axes_0 = const()[name = string("op_4808_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_4808_cast_fp16 = expand_dims(axes = var_4808_axes_0, x = var_4807_cast_fp16)[name = string("op_4808_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_81_cast_fp16 = mul(x = var_4804_cast_fp16, y = var_4808_cast_fp16)[name = string("delta_81_cast_fp16")];
            tensor<int32, [1]> var_4811_axes_0 = const()[name = string("op_4811_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_4811_cast_fp16 = expand_dims(axes = var_4811_axes_0, x = delta_81_cast_fp16)[name = string("op_4811_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4812_cast_fp16 = mul(x = var_4800_cast_fp16, y = var_4811_cast_fp16)[name = string("op_4812_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_163_cast_fp16 = add(x = state_161_cast_fp16, y = var_4812_cast_fp16)[name = string("state_163_cast_fp16")];
            tensor<int32, [4]> var_4816_begin_0 = const()[name = string("op_4816_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> var_4816_end_0 = const()[name = string("op_4816_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> var_4816_end_mask_0 = const()[name = string("op_4816_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_4816_squeeze_mask_0 = const()[name = string("op_4816_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_4816_cast_fp16 = slice_by_index(begin = var_4816_begin_0, end = var_4816_end_0, end_mask = var_4816_end_mask_0, squeeze_mask = var_4816_squeeze_mask_0, x = _inversed_query_55_cast_fp16)[name = string("op_4816_cast_fp16")];
            tensor<int32, [1]> var_4817_axes_0 = const()[name = string("op_4817_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_4817_cast_fp16 = expand_dims(axes = var_4817_axes_0, x = var_4816_cast_fp16)[name = string("op_4817_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4818_cast_fp16 = mul(x = state_163_cast_fp16, y = var_4817_cast_fp16)[name = string("op_4818_cast_fp16")];
            tensor<int32, [1]> var_4820_axes_0 = const()[name = string("op_4820_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_4820_keep_dims_0 = const()[name = string("op_4820_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_4820_cast_fp16 = reduce_sum(axes = var_4820_axes_0, keep_dims = var_4820_keep_dims_0, x = var_4818_cast_fp16)[name = string("op_4820_cast_fp16")];
            tensor<int32, [3]> var_4823_begin_0 = const()[name = string("op_4823_begin_0"), val = tensor<int32, [3]>([0, 0, 1])];
            tensor<int32, [3]> var_4823_end_0 = const()[name = string("op_4823_end_0"), val = tensor<int32, [3]>([1, 16, 2])];
            tensor<bool, [3]> var_4823_end_mask_0 = const()[name = string("op_4823_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_4823_squeeze_mask_0 = const()[name = string("op_4823_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_4823_cast_fp16 = slice_by_index(begin = var_4823_begin_0, end = var_4823_end_0, end_mask = var_4823_end_mask_0, squeeze_mask = var_4823_squeeze_mask_0, x = decay_21_cast_fp16)[name = string("op_4823_cast_fp16")];
            tensor<fp16, [1, 16]> var_4824_cast_fp16 = exp(x = var_4823_cast_fp16)[name = string("op_4824_cast_fp16")];
            tensor<int32, [1]> var_4825_axes_0 = const()[name = string("op_4825_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_4825_cast_fp16 = expand_dims(axes = var_4825_axes_0, x = var_4824_cast_fp16)[name = string("op_4825_cast_fp16")];
            tensor<int32, [1]> var_4826_axes_0 = const()[name = string("op_4826_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_4826_cast_fp16 = expand_dims(axes = var_4826_axes_0, x = var_4825_cast_fp16)[name = string("op_4826_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_165_cast_fp16 = mul(x = state_163_cast_fp16, y = var_4826_cast_fp16)[name = string("state_165_cast_fp16")];
            tensor<int32, [4]> key_token_83_begin_0 = const()[name = string("key_token_83_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> key_token_83_end_0 = const()[name = string("key_token_83_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> key_token_83_end_mask_0 = const()[name = string("key_token_83_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_83_squeeze_mask_0 = const()[name = string("key_token_83_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_83_cast_fp16 = slice_by_index(begin = key_token_83_begin_0, end = key_token_83_end_0, end_mask = key_token_83_end_mask_0, squeeze_mask = key_token_83_squeeze_mask_0, x = key_55_cast_fp16)[name = string("key_token_83_cast_fp16")];
            tensor<int32, [4]> value_token_83_begin_0 = const()[name = string("value_token_83_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> value_token_83_end_0 = const()[name = string("value_token_83_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> value_token_83_end_mask_0 = const()[name = string("value_token_83_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_83_squeeze_mask_0 = const()[name = string("value_token_83_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_83_cast_fp16 = slice_by_index(begin = value_token_83_begin_0, end = value_token_83_end_0, end_mask = value_token_83_end_mask_0, squeeze_mask = value_token_83_squeeze_mask_0, x = value_199_cast_fp16)[name = string("value_token_83_cast_fp16")];
            tensor<int32, [1]> var_4834_axes_0 = const()[name = string("op_4834_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_4834_cast_fp16 = expand_dims(axes = var_4834_axes_0, x = key_token_83_cast_fp16)[name = string("op_4834_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4835_cast_fp16 = mul(x = state_165_cast_fp16, y = var_4834_cast_fp16)[name = string("op_4835_cast_fp16")];
            tensor<int32, [1]> memory_83_axes_0 = const()[name = string("memory_83_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_83_keep_dims_0 = const()[name = string("memory_83_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_83_cast_fp16 = reduce_sum(axes = memory_83_axes_0, keep_dims = memory_83_keep_dims_0, x = var_4835_cast_fp16)[name = string("memory_83_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_4838_cast_fp16 = sub(x = value_token_83_cast_fp16, y = memory_83_cast_fp16)[name = string("op_4838_cast_fp16")];
            tensor<int32, [3]> var_4841_begin_0 = const()[name = string("op_4841_begin_0"), val = tensor<int32, [3]>([0, 0, 1])];
            tensor<int32, [3]> var_4841_end_0 = const()[name = string("op_4841_end_0"), val = tensor<int32, [3]>([1, 16, 2])];
            tensor<bool, [3]> var_4841_end_mask_0 = const()[name = string("op_4841_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_4841_squeeze_mask_0 = const()[name = string("op_4841_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_4841_cast_fp16 = slice_by_index(begin = var_4841_begin_0, end = var_4841_end_0, end_mask = var_4841_end_mask_0, squeeze_mask = var_4841_squeeze_mask_0, x = beta_21_cast_fp16)[name = string("op_4841_cast_fp16")];
            tensor<int32, [1]> var_4842_axes_0 = const()[name = string("op_4842_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_4842_cast_fp16 = expand_dims(axes = var_4842_axes_0, x = var_4841_cast_fp16)[name = string("op_4842_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_83_cast_fp16 = mul(x = var_4838_cast_fp16, y = var_4842_cast_fp16)[name = string("delta_83_cast_fp16")];
            tensor<int32, [1]> var_4845_axes_0 = const()[name = string("op_4845_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_4845_cast_fp16 = expand_dims(axes = var_4845_axes_0, x = delta_83_cast_fp16)[name = string("op_4845_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4846_cast_fp16 = mul(x = var_4834_cast_fp16, y = var_4845_cast_fp16)[name = string("op_4846_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_167_cast_fp16 = add(x = state_165_cast_fp16, y = var_4846_cast_fp16)[name = string("state_167_cast_fp16")];
            tensor<int32, [4]> var_4850_begin_0 = const()[name = string("op_4850_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> var_4850_end_0 = const()[name = string("op_4850_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> var_4850_end_mask_0 = const()[name = string("op_4850_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_4850_squeeze_mask_0 = const()[name = string("op_4850_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_4850_cast_fp16 = slice_by_index(begin = var_4850_begin_0, end = var_4850_end_0, end_mask = var_4850_end_mask_0, squeeze_mask = var_4850_squeeze_mask_0, x = _inversed_query_55_cast_fp16)[name = string("op_4850_cast_fp16")];
            tensor<int32, [1]> var_4851_axes_0 = const()[name = string("op_4851_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_4851_cast_fp16 = expand_dims(axes = var_4851_axes_0, x = var_4850_cast_fp16)[name = string("op_4851_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4852_cast_fp16 = mul(x = state_167_cast_fp16, y = var_4851_cast_fp16)[name = string("op_4852_cast_fp16")];
            tensor<int32, [1]> var_4854_axes_0 = const()[name = string("op_4854_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_4854_keep_dims_0 = const()[name = string("op_4854_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_4854_cast_fp16 = reduce_sum(axes = var_4854_axes_0, keep_dims = var_4854_keep_dims_0, x = var_4852_cast_fp16)[name = string("op_4854_cast_fp16")];
            tensor<int32, [3]> var_4857_begin_0 = const()[name = string("op_4857_begin_0"), val = tensor<int32, [3]>([0, 0, 2])];
            tensor<int32, [3]> var_4857_end_0 = const()[name = string("op_4857_end_0"), val = tensor<int32, [3]>([1, 16, 3])];
            tensor<bool, [3]> var_4857_end_mask_0 = const()[name = string("op_4857_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_4857_squeeze_mask_0 = const()[name = string("op_4857_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_4857_cast_fp16 = slice_by_index(begin = var_4857_begin_0, end = var_4857_end_0, end_mask = var_4857_end_mask_0, squeeze_mask = var_4857_squeeze_mask_0, x = decay_21_cast_fp16)[name = string("op_4857_cast_fp16")];
            tensor<fp16, [1, 16]> var_4858_cast_fp16 = exp(x = var_4857_cast_fp16)[name = string("op_4858_cast_fp16")];
            tensor<int32, [1]> var_4859_axes_0 = const()[name = string("op_4859_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_4859_cast_fp16 = expand_dims(axes = var_4859_axes_0, x = var_4858_cast_fp16)[name = string("op_4859_cast_fp16")];
            tensor<int32, [1]> var_4860_axes_0 = const()[name = string("op_4860_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_4860_cast_fp16 = expand_dims(axes = var_4860_axes_0, x = var_4859_cast_fp16)[name = string("op_4860_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_169_cast_fp16 = mul(x = state_167_cast_fp16, y = var_4860_cast_fp16)[name = string("state_169_cast_fp16")];
            tensor<int32, [4]> key_token_85_begin_0 = const()[name = string("key_token_85_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> key_token_85_end_0 = const()[name = string("key_token_85_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> key_token_85_end_mask_0 = const()[name = string("key_token_85_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_85_squeeze_mask_0 = const()[name = string("key_token_85_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_85_cast_fp16 = slice_by_index(begin = key_token_85_begin_0, end = key_token_85_end_0, end_mask = key_token_85_end_mask_0, squeeze_mask = key_token_85_squeeze_mask_0, x = key_55_cast_fp16)[name = string("key_token_85_cast_fp16")];
            tensor<int32, [4]> value_token_85_begin_0 = const()[name = string("value_token_85_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> value_token_85_end_0 = const()[name = string("value_token_85_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> value_token_85_end_mask_0 = const()[name = string("value_token_85_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_85_squeeze_mask_0 = const()[name = string("value_token_85_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_85_cast_fp16 = slice_by_index(begin = value_token_85_begin_0, end = value_token_85_end_0, end_mask = value_token_85_end_mask_0, squeeze_mask = value_token_85_squeeze_mask_0, x = value_199_cast_fp16)[name = string("value_token_85_cast_fp16")];
            tensor<int32, [1]> var_4868_axes_0 = const()[name = string("op_4868_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_4868_cast_fp16 = expand_dims(axes = var_4868_axes_0, x = key_token_85_cast_fp16)[name = string("op_4868_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4869_cast_fp16 = mul(x = state_169_cast_fp16, y = var_4868_cast_fp16)[name = string("op_4869_cast_fp16")];
            tensor<int32, [1]> memory_85_axes_0 = const()[name = string("memory_85_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_85_keep_dims_0 = const()[name = string("memory_85_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_85_cast_fp16 = reduce_sum(axes = memory_85_axes_0, keep_dims = memory_85_keep_dims_0, x = var_4869_cast_fp16)[name = string("memory_85_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_4872_cast_fp16 = sub(x = value_token_85_cast_fp16, y = memory_85_cast_fp16)[name = string("op_4872_cast_fp16")];
            tensor<int32, [3]> var_4875_begin_0 = const()[name = string("op_4875_begin_0"), val = tensor<int32, [3]>([0, 0, 2])];
            tensor<int32, [3]> var_4875_end_0 = const()[name = string("op_4875_end_0"), val = tensor<int32, [3]>([1, 16, 3])];
            tensor<bool, [3]> var_4875_end_mask_0 = const()[name = string("op_4875_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_4875_squeeze_mask_0 = const()[name = string("op_4875_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_4875_cast_fp16 = slice_by_index(begin = var_4875_begin_0, end = var_4875_end_0, end_mask = var_4875_end_mask_0, squeeze_mask = var_4875_squeeze_mask_0, x = beta_21_cast_fp16)[name = string("op_4875_cast_fp16")];
            tensor<int32, [1]> var_4876_axes_0 = const()[name = string("op_4876_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_4876_cast_fp16 = expand_dims(axes = var_4876_axes_0, x = var_4875_cast_fp16)[name = string("op_4876_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_85_cast_fp16 = mul(x = var_4872_cast_fp16, y = var_4876_cast_fp16)[name = string("delta_85_cast_fp16")];
            tensor<int32, [1]> var_4879_axes_0 = const()[name = string("op_4879_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_4879_cast_fp16 = expand_dims(axes = var_4879_axes_0, x = delta_85_cast_fp16)[name = string("op_4879_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4880_cast_fp16 = mul(x = var_4868_cast_fp16, y = var_4879_cast_fp16)[name = string("op_4880_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_171_cast_fp16 = add(x = state_169_cast_fp16, y = var_4880_cast_fp16)[name = string("state_171_cast_fp16")];
            tensor<int32, [4]> var_4884_begin_0 = const()[name = string("op_4884_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> var_4884_end_0 = const()[name = string("op_4884_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> var_4884_end_mask_0 = const()[name = string("op_4884_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_4884_squeeze_mask_0 = const()[name = string("op_4884_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_4884_cast_fp16 = slice_by_index(begin = var_4884_begin_0, end = var_4884_end_0, end_mask = var_4884_end_mask_0, squeeze_mask = var_4884_squeeze_mask_0, x = _inversed_query_55_cast_fp16)[name = string("op_4884_cast_fp16")];
            tensor<int32, [1]> var_4885_axes_0 = const()[name = string("op_4885_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_4885_cast_fp16 = expand_dims(axes = var_4885_axes_0, x = var_4884_cast_fp16)[name = string("op_4885_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4886_cast_fp16 = mul(x = state_171_cast_fp16, y = var_4885_cast_fp16)[name = string("op_4886_cast_fp16")];
            tensor<int32, [1]> var_4888_axes_0 = const()[name = string("op_4888_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_4888_keep_dims_0 = const()[name = string("op_4888_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_4888_cast_fp16 = reduce_sum(axes = var_4888_axes_0, keep_dims = var_4888_keep_dims_0, x = var_4886_cast_fp16)[name = string("op_4888_cast_fp16")];
            tensor<int32, [3]> var_4891_begin_0 = const()[name = string("op_4891_begin_0"), val = tensor<int32, [3]>([0, 0, 3])];
            tensor<int32, [3]> var_4891_end_0 = const()[name = string("op_4891_end_0"), val = tensor<int32, [3]>([1, 16, 4])];
            tensor<bool, [3]> var_4891_end_mask_0 = const()[name = string("op_4891_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_4891_squeeze_mask_0 = const()[name = string("op_4891_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_4891_cast_fp16 = slice_by_index(begin = var_4891_begin_0, end = var_4891_end_0, end_mask = var_4891_end_mask_0, squeeze_mask = var_4891_squeeze_mask_0, x = decay_21_cast_fp16)[name = string("op_4891_cast_fp16")];
            tensor<fp16, [1, 16]> var_4892_cast_fp16 = exp(x = var_4891_cast_fp16)[name = string("op_4892_cast_fp16")];
            tensor<int32, [1]> var_4893_axes_0 = const()[name = string("op_4893_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_4893_cast_fp16 = expand_dims(axes = var_4893_axes_0, x = var_4892_cast_fp16)[name = string("op_4893_cast_fp16")];
            tensor<int32, [1]> var_4894_axes_0 = const()[name = string("op_4894_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_4894_cast_fp16 = expand_dims(axes = var_4894_axes_0, x = var_4893_cast_fp16)[name = string("op_4894_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_173_cast_fp16 = mul(x = state_171_cast_fp16, y = var_4894_cast_fp16)[name = string("state_173_cast_fp16")];
            tensor<int32, [4]> key_token_87_begin_0 = const()[name = string("key_token_87_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> key_token_87_end_0 = const()[name = string("key_token_87_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> key_token_87_end_mask_0 = const()[name = string("key_token_87_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_87_squeeze_mask_0 = const()[name = string("key_token_87_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_87_cast_fp16 = slice_by_index(begin = key_token_87_begin_0, end = key_token_87_end_0, end_mask = key_token_87_end_mask_0, squeeze_mask = key_token_87_squeeze_mask_0, x = key_55_cast_fp16)[name = string("key_token_87_cast_fp16")];
            tensor<int32, [4]> value_token_87_begin_0 = const()[name = string("value_token_87_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> value_token_87_end_0 = const()[name = string("value_token_87_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> value_token_87_end_mask_0 = const()[name = string("value_token_87_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_87_squeeze_mask_0 = const()[name = string("value_token_87_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_87_cast_fp16 = slice_by_index(begin = value_token_87_begin_0, end = value_token_87_end_0, end_mask = value_token_87_end_mask_0, squeeze_mask = value_token_87_squeeze_mask_0, x = value_199_cast_fp16)[name = string("value_token_87_cast_fp16")];
            tensor<int32, [1]> var_4902_axes_0 = const()[name = string("op_4902_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_4902_cast_fp16 = expand_dims(axes = var_4902_axes_0, x = key_token_87_cast_fp16)[name = string("op_4902_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4903_cast_fp16 = mul(x = state_173_cast_fp16, y = var_4902_cast_fp16)[name = string("op_4903_cast_fp16")];
            tensor<int32, [1]> memory_87_axes_0 = const()[name = string("memory_87_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_87_keep_dims_0 = const()[name = string("memory_87_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_87_cast_fp16 = reduce_sum(axes = memory_87_axes_0, keep_dims = memory_87_keep_dims_0, x = var_4903_cast_fp16)[name = string("memory_87_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_4906_cast_fp16 = sub(x = value_token_87_cast_fp16, y = memory_87_cast_fp16)[name = string("op_4906_cast_fp16")];
            tensor<int32, [3]> var_4909_begin_0 = const()[name = string("op_4909_begin_0"), val = tensor<int32, [3]>([0, 0, 3])];
            tensor<int32, [3]> var_4909_end_0 = const()[name = string("op_4909_end_0"), val = tensor<int32, [3]>([1, 16, 4])];
            tensor<bool, [3]> var_4909_end_mask_0 = const()[name = string("op_4909_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_4909_squeeze_mask_0 = const()[name = string("op_4909_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_4909_cast_fp16 = slice_by_index(begin = var_4909_begin_0, end = var_4909_end_0, end_mask = var_4909_end_mask_0, squeeze_mask = var_4909_squeeze_mask_0, x = beta_21_cast_fp16)[name = string("op_4909_cast_fp16")];
            tensor<int32, [1]> var_4910_axes_0 = const()[name = string("op_4910_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_4910_cast_fp16 = expand_dims(axes = var_4910_axes_0, x = var_4909_cast_fp16)[name = string("op_4910_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_87_cast_fp16 = mul(x = var_4906_cast_fp16, y = var_4910_cast_fp16)[name = string("delta_87_cast_fp16")];
            tensor<int32, [1]> var_4913_axes_0 = const()[name = string("op_4913_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_4913_cast_fp16 = expand_dims(axes = var_4913_axes_0, x = delta_87_cast_fp16)[name = string("op_4913_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4914_cast_fp16 = mul(x = var_4902_cast_fp16, y = var_4913_cast_fp16)[name = string("op_4914_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> rec13_out = add(x = state_173_cast_fp16, y = var_4914_cast_fp16)[name = string("state_175_cast_fp16")];
            tensor<int32, [4]> var_4918_begin_0 = const()[name = string("op_4918_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> var_4918_end_0 = const()[name = string("op_4918_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> var_4918_end_mask_0 = const()[name = string("op_4918_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_4918_squeeze_mask_0 = const()[name = string("op_4918_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_4918_cast_fp16 = slice_by_index(begin = var_4918_begin_0, end = var_4918_end_0, end_mask = var_4918_end_mask_0, squeeze_mask = var_4918_squeeze_mask_0, x = _inversed_query_55_cast_fp16)[name = string("op_4918_cast_fp16")];
            tensor<int32, [1]> var_4919_axes_0 = const()[name = string("op_4919_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_4919_cast_fp16 = expand_dims(axes = var_4919_axes_0, x = var_4918_cast_fp16)[name = string("op_4919_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_4920_cast_fp16 = mul(x = rec13_out, y = var_4919_cast_fp16)[name = string("op_4920_cast_fp16")];
            tensor<int32, [1]> var_4922_axes_0 = const()[name = string("op_4922_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_4922_keep_dims_0 = const()[name = string("op_4922_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_4922_cast_fp16 = reduce_sum(axes = var_4922_axes_0, keep_dims = var_4922_keep_dims_0, x = var_4920_cast_fp16)[name = string("op_4922_cast_fp16")];
            int32 attention_101_axis_0 = const()[name = string("attention_101_axis_0"), val = int32(1)];
            tensor<fp16, [1, 4, 16, 128]> attention_101_cast_fp16 = stack(axis = attention_101_axis_0, values = (var_4820_cast_fp16, var_4854_cast_fp16, var_4888_cast_fp16, var_4922_cast_fp16))[name = string("attention_101_cast_fp16")];
            tensor<int32, [2]> var_4925 = const()[name = string("op_4925"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [64, 128]> attention_103_cast_fp16 = reshape(shape = var_4925, x = attention_101_cast_fp16)[name = string("attention_103_cast_fp16")];
            tensor<int32, [2]> var_4927 = const()[name = string("op_4927"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [64, 128]> input_175_cast_fp16 = reshape(shape = var_4927, x = linear_104_cast_fp16)[name = string("input_175_cast_fp16")];
            tensor<fp16, [64, 128]> var_4929_cast_fp16 = mul(x = attention_103_cast_fp16, y = attention_103_cast_fp16)[name = string("op_4929_cast_fp16")];
            tensor<int32, [1]> variance_87_axes_0 = const()[name = string("variance_87_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_87_keep_dims_0 = const()[name = string("variance_87_keep_dims_0"), val = bool(true)];
            tensor<fp16, [64, 1]> variance_87_cast_fp16 = reduce_mean(axes = variance_87_axes_0, keep_dims = variance_87_keep_dims_0, x = var_4929_cast_fp16)[name = string("variance_87_cast_fp16")];
            fp16 var_4932_to_fp16 = const()[name = string("op_4932_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [64, 1]> var_4933_cast_fp16 = add(x = variance_87_cast_fp16, y = var_4932_to_fp16)[name = string("op_4933_cast_fp16")];
            fp32 var_4934_epsilon_0 = const()[name = string("op_4934_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [64, 1]> var_4934_cast_fp16 = rsqrt(epsilon = var_4934_epsilon_0, x = var_4933_cast_fp16)[name = string("op_4934_cast_fp16")];
            tensor<fp16, [64, 128]> attention_105_cast_fp16 = mul(x = attention_103_cast_fp16, y = var_4934_cast_fp16)[name = string("attention_105_cast_fp16")];
            tensor<fp16, [128]> groups_3_1_gated_norm_promoted_to_fp16 = const()[name = string("groups_3_1_gated_norm_promoted_to_fp16"), val = tensor<fp16, [128]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(211733248)))];
            tensor<fp16, [64, 128]> attention_107_cast_fp16 = mul(x = attention_105_cast_fp16, y = groups_3_1_gated_norm_promoted_to_fp16)[name = string("attention_107_cast_fp16")];
            tensor<fp16, [64, 128]> var_4937_cast_fp16 = silu(x = input_175_cast_fp16)[name = string("op_4937_cast_fp16")];
            tensor<fp16, [64, 128]> attention_109_cast_fp16 = mul(x = attention_107_cast_fp16, y = var_4937_cast_fp16)[name = string("attention_109_cast_fp16")];
            tensor<int32, [2]> var_4939 = const()[name = string("op_4939"), val = tensor<int32, [2]>([4, 2048])];
            tensor<fp16, [4, 2048]> input_177_cast_fp16 = reshape(shape = var_4939, x = attention_109_cast_fp16)[name = string("input_177_cast_fp16")];
            tensor<fp16, [1024, 2048]> groups_3_1_out_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 2048]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(211733568))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(213306496))))[name = string("groups_3_1_out_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 1024]> linear_105_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = groups_3_1_out_proj_weight_promoted_to_fp16_palettized, x = input_177_cast_fp16)[name = string("linear_105_cast_fp16")];
            tensor<fp16, [4, 1024]> value_201_cast_fp16 = add(x = value_189_cast_fp16, y = linear_105_cast_fp16)[name = string("value_201_cast_fp16")];
            tensor<fp16, [4, 1024]> var_4944_cast_fp16 = mul(x = value_201_cast_fp16, y = value_201_cast_fp16)[name = string("op_4944_cast_fp16")];
            tensor<int32, [1]> variance_89_axes_0 = const()[name = string("variance_89_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_89_keep_dims_0 = const()[name = string("variance_89_keep_dims_0"), val = bool(true)];
            tensor<fp16, [4, 1]> variance_89_cast_fp16 = reduce_mean(axes = variance_89_axes_0, keep_dims = variance_89_keep_dims_0, x = var_4944_cast_fp16)[name = string("variance_89_cast_fp16")];
            fp16 var_4947_to_fp16 = const()[name = string("op_4947_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [4, 1]> var_4948_cast_fp16 = add(x = variance_89_cast_fp16, y = var_4947_to_fp16)[name = string("op_4948_cast_fp16")];
            fp32 var_4949_epsilon_0 = const()[name = string("op_4949_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [4, 1]> var_4949_cast_fp16 = rsqrt(epsilon = var_4949_epsilon_0, x = var_4948_cast_fp16)[name = string("op_4949_cast_fp16")];
            tensor<fp16, [4, 1024]> var_4950_cast_fp16 = mul(x = value_201_cast_fp16, y = var_4949_cast_fp16)[name = string("op_4950_cast_fp16")];
            tensor<fp16, [1024]> var_4952_to_fp16 = const()[name = string("op_4952_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(213314752)))];
            tensor<fp16, [4, 1024]> input_179_cast_fp16 = mul(x = var_4950_cast_fp16, y = var_4952_to_fp16)[name = string("input_179_cast_fp16")];
            tensor<fp16, [3584, 1024]> groups_3_1_gate_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(213316864))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(216069440))))[name = string("groups_3_1_gate_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 3584]> linear_106_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = groups_3_1_gate_proj_weight_promoted_to_fp16_palettized, x = input_179_cast_fp16)[name = string("linear_106_cast_fp16")];
            tensor<fp16, [4, 3584]> var_4956_cast_fp16 = silu(x = linear_106_cast_fp16)[name = string("op_4956_cast_fp16")];
            tensor<fp16, [3584, 1024]> groups_3_1_up_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(216098176))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(218850752))))[name = string("groups_3_1_up_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 3584]> linear_107_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = groups_3_1_up_proj_weight_promoted_to_fp16_palettized, x = input_179_cast_fp16)[name = string("linear_107_cast_fp16")];
            tensor<fp16, [4, 3584]> input_183_cast_fp16 = mul(x = var_4956_cast_fp16, y = linear_107_cast_fp16)[name = string("input_183_cast_fp16")];
            tensor<fp16, [1024, 3584]> groups_3_1_down_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 3584]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(218879488))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(221632064))))[name = string("groups_3_1_down_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 1024]> linear_108_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = groups_3_1_down_proj_weight_promoted_to_fp16_palettized, x = input_183_cast_fp16)[name = string("linear_108_cast_fp16")];
            tensor<fp16, [4, 1024]> value_203_cast_fp16 = add(x = value_201_cast_fp16, y = linear_108_cast_fp16)[name = string("value_203_cast_fp16")];
            int32 var_4983 = const()[name = string("op_4983"), val = int32(-1)];
            tensor<fp16, [4, 1024]> var_4998_cast_fp16 = mul(x = value_203_cast_fp16, y = value_203_cast_fp16)[name = string("op_4998_cast_fp16")];
            tensor<int32, [1]> variance_91_axes_0 = const()[name = string("variance_91_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_91_keep_dims_0 = const()[name = string("variance_91_keep_dims_0"), val = bool(true)];
            tensor<fp16, [4, 1]> variance_91_cast_fp16 = reduce_mean(axes = variance_91_axes_0, keep_dims = variance_91_keep_dims_0, x = var_4998_cast_fp16)[name = string("variance_91_cast_fp16")];
            fp16 var_5001_to_fp16 = const()[name = string("op_5001_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [4, 1]> var_5002_cast_fp16 = add(x = variance_91_cast_fp16, y = var_5001_to_fp16)[name = string("op_5002_cast_fp16")];
            fp32 var_5003_epsilon_0 = const()[name = string("op_5003_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [4, 1]> var_5003_cast_fp16 = rsqrt(epsilon = var_5003_epsilon_0, x = var_5002_cast_fp16)[name = string("op_5003_cast_fp16")];
            tensor<fp16, [4, 1024]> var_5004_cast_fp16 = mul(x = value_203_cast_fp16, y = var_5003_cast_fp16)[name = string("op_5004_cast_fp16")];
            tensor<fp16, [1024]> var_5006_to_fp16 = const()[name = string("op_5006_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(221640320)))];
            tensor<fp16, [4, 1024]> input_185_cast_fp16 = mul(x = var_5004_cast_fp16, y = var_5006_to_fp16)[name = string("input_185_cast_fp16")];
            tensor<fp16, [6144, 1024]> groups_3_2_in_proj_qkv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(221642432))), lut = tensor<fp16, [384, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(226361088))))[name = string("groups_3_2_in_proj_qkv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 6144]> linear_109_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = groups_3_2_in_proj_qkv_weight_promoted_to_fp16_palettized, x = input_185_cast_fp16)[name = string("linear_109_cast_fp16")];
            tensor<int32, [2]> var_5010_perm_0 = const()[name = string("op_5010_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> mixed_23_axes_0 = const()[name = string("mixed_23_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [6144, 4]> var_5010_cast_fp16 = transpose(perm = var_5010_perm_0, x = linear_109_cast_fp16)[name = string("transpose_39")];
            tensor<fp16, [1, 6144, 4]> mixed_23_cast_fp16 = expand_dims(axes = mixed_23_axes_0, x = var_5010_cast_fp16)[name = string("mixed_23_cast_fp16")];
            bool convolution_input_23_interleave_0 = const()[name = string("convolution_input_23_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 6144, 7]> convolution_input_23_cast_fp16 = concat(axis = var_4983, interleave = convolution_input_23_interleave_0, values = (conv14, mixed_23_cast_fp16))[name = string("convolution_input_23_cast_fp16")];
            string input_187_pad_type_0 = const()[name = string("input_187_pad_type_0"), val = string("valid")];
            int32 input_187_groups_0 = const()[name = string("input_187_groups_0"), val = int32(6144)];
            tensor<int32, [1]> input_187_strides_0 = const()[name = string("input_187_strides_0"), val = tensor<int32, [1]>([1])];
            tensor<int32, [2]> input_187_pad_0 = const()[name = string("input_187_pad_0"), val = tensor<int32, [2]>([0, 0])];
            tensor<int32, [1]> input_187_dilations_0 = const()[name = string("input_187_dilations_0"), val = tensor<int32, [1]>([1])];
            tensor<fp16, [6144, 1, 4]> groups_3_2_conv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1, 4]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(226410304))), lut = tensor<fp16, [384, 1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(226428800))))[name = string("groups_3_2_conv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 6144, 4]> input_187_cast_fp16 = conv(dilations = input_187_dilations_0, groups = input_187_groups_0, pad = input_187_pad_0, pad_type = input_187_pad_type_0, strides = input_187_strides_0, weight = groups_3_2_conv_weight_promoted_to_fp16_palettized, x = convolution_input_23_cast_fp16)[name = string("input_187_cast_fp16")];
            tensor<fp16, [1, 6144, 4]> var_5019_cast_fp16 = silu(x = input_187_cast_fp16)[name = string("op_5019_cast_fp16")];
            tensor<int32, [3]> var_5020_perm_0 = const()[name = string("op_5020_perm_0"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [3]> var_5023_begin_0 = const()[name = string("op_5023_begin_0"), val = tensor<int32, [3]>([0, 0, 4])];
            tensor<int32, [3]> var_5023_end_0 = const()[name = string("op_5023_end_0"), val = tensor<int32, [3]>([1, 6144, 7])];
            tensor<bool, [3]> var_5023_end_mask_0 = const()[name = string("op_5023_end_mask_0"), val = tensor<bool, [3]>([true, true, true])];
            tensor<fp16, [1, 6144, 3]> conv14_out = slice_by_index(begin = var_5023_begin_0, end = var_5023_end_0, end_mask = var_5023_end_mask_0, x = convolution_input_23_cast_fp16)[name = string("op_5023_cast_fp16")];
            tensor<int32, [3]> var_5024 = const()[name = string("op_5024"), val = tensor<int32, [3]>([2048, 2048, 2048])];
            int32 var_5025_axis_0 = const()[name = string("op_5025_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 4, 6144]> var_5020_cast_fp16 = transpose(perm = var_5020_perm_0, x = var_5019_cast_fp16)[name = string("transpose_38")];
            tensor<fp16, [1, 4, 2048]> var_5025_cast_fp16_0, tensor<fp16, [1, 4, 2048]> var_5025_cast_fp16_1, tensor<fp16, [1, 4, 2048]> var_5025_cast_fp16_2 = split(axis = var_5025_axis_0, split_sizes = var_5024, x = var_5020_cast_fp16)[name = string("op_5025_cast_fp16")];
            tensor<int32, [4]> var_5029 = const()[name = string("op_5029"), val = tensor<int32, [4]>([1, 4, 16, 128])];
            tensor<fp16, [1, 4, 16, 128]> value_207_cast_fp16 = reshape(shape = var_5029, x = var_5025_cast_fp16_0)[name = string("value_207_cast_fp16")];
            tensor<int32, [4]> var_5031 = const()[name = string("op_5031"), val = tensor<int32, [4]>([1, 4, 16, 128])];
            tensor<fp16, [1, 4, 16, 128]> value_209_cast_fp16 = reshape(shape = var_5031, x = var_5025_cast_fp16_1)[name = string("value_209_cast_fp16")];
            tensor<int32, [4]> var_5033 = const()[name = string("op_5033"), val = tensor<int32, [4]>([1, 4, 16, 128])];
            tensor<fp16, [1, 4, 16, 128]> value_211_cast_fp16 = reshape(shape = var_5033, x = var_5025_cast_fp16_2)[name = string("value_211_cast_fp16")];
            tensor<fp16, [1, 4, 16, 128]> var_5035_cast_fp16 = mul(x = value_207_cast_fp16, y = value_207_cast_fp16)[name = string("op_5035_cast_fp16")];
            tensor<int32, [1]> var_5037_axes_0 = const()[name = string("op_5037_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_5037_keep_dims_0 = const()[name = string("op_5037_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 4, 16, 1]> var_5037_cast_fp16 = reduce_sum(axes = var_5037_axes_0, keep_dims = var_5037_keep_dims_0, x = var_5035_cast_fp16)[name = string("op_5037_cast_fp16")];
            fp16 var_5038_to_fp16 = const()[name = string("op_5038_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 4, 16, 1]> var_5039_cast_fp16 = add(x = var_5037_cast_fp16, y = var_5038_to_fp16)[name = string("op_5039_cast_fp16")];
            fp32 var_5040_epsilon_0 = const()[name = string("op_5040_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 4, 16, 1]> var_5040_cast_fp16 = rsqrt(epsilon = var_5040_epsilon_0, x = var_5039_cast_fp16)[name = string("op_5040_cast_fp16")];
            tensor<fp16, [1, 4, 16, 128]> var_5041_cast_fp16 = mul(x = value_207_cast_fp16, y = var_5040_cast_fp16)[name = string("op_5041_cast_fp16")];
            tensor<int32, [4]> var_5042_perm_0 = const()[name = string("op_5042_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            fp16 _inversed_query_59_y_0_to_fp16 = const()[name = string("_inversed_query_59_y_0_to_fp16"), val = fp16(0x1.6ap-4)];
            tensor<fp16, [1, 16, 4, 128]> var_5042_cast_fp16 = transpose(perm = var_5042_perm_0, x = var_5041_cast_fp16)[name = string("transpose_37")];
            tensor<fp16, [1, 16, 4, 128]> _inversed_query_59_cast_fp16 = mul(x = var_5042_cast_fp16, y = _inversed_query_59_y_0_to_fp16)[name = string("_inversed_query_59_cast_fp16")];
            tensor<fp16, [1, 4, 16, 128]> var_5045_cast_fp16 = mul(x = value_209_cast_fp16, y = value_209_cast_fp16)[name = string("op_5045_cast_fp16")];
            tensor<int32, [1]> var_5047_axes_0 = const()[name = string("op_5047_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_5047_keep_dims_0 = const()[name = string("op_5047_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 4, 16, 1]> var_5047_cast_fp16 = reduce_sum(axes = var_5047_axes_0, keep_dims = var_5047_keep_dims_0, x = var_5045_cast_fp16)[name = string("op_5047_cast_fp16")];
            fp16 var_5048_to_fp16 = const()[name = string("op_5048_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 4, 16, 1]> var_5049_cast_fp16 = add(x = var_5047_cast_fp16, y = var_5048_to_fp16)[name = string("op_5049_cast_fp16")];
            fp32 var_5050_epsilon_0 = const()[name = string("op_5050_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 4, 16, 1]> var_5050_cast_fp16 = rsqrt(epsilon = var_5050_epsilon_0, x = var_5049_cast_fp16)[name = string("op_5050_cast_fp16")];
            tensor<fp16, [1, 4, 16, 128]> var_5051_cast_fp16 = mul(x = value_209_cast_fp16, y = var_5050_cast_fp16)[name = string("op_5051_cast_fp16")];
            tensor<int32, [4]> key_59_perm_0 = const()[name = string("key_59_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<int32, [4]> value_213_perm_0 = const()[name = string("value_213_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<fp16, [16, 1024]> groups_3_2_in_proj_b_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(226478016))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(226490368))))[name = string("groups_3_2_in_proj_b_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16]> linear_110_bias_0_to_fp16 = const()[name = string("linear_110_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(4862656)))];
            tensor<fp16, [4, 16]> linear_110_cast_fp16 = linear(bias = linear_110_bias_0_to_fp16, weight = groups_3_2_in_proj_b_weight_promoted_to_fp16_palettized, x = input_185_cast_fp16)[name = string("linear_110_cast_fp16")];
            tensor<fp16, [4, 16]> var_5056_cast_fp16 = sigmoid(x = linear_110_cast_fp16)[name = string("op_5056_cast_fp16")];
            tensor<int32, [2]> var_5057_perm_0 = const()[name = string("op_5057_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> beta_23_axes_0 = const()[name = string("beta_23_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 4]> var_5057_cast_fp16 = transpose(perm = var_5057_perm_0, x = var_5056_cast_fp16)[name = string("transpose_36")];
            tensor<fp16, [1, 16, 4]> beta_23_cast_fp16 = expand_dims(axes = beta_23_axes_0, x = var_5057_cast_fp16)[name = string("beta_23_cast_fp16")];
            tensor<fp16, [16, 1024]> op_5063_weight_0_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(226490560))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(226502912))))[name = string("op_5063_weight_0_to_fp16_palettized")];
            tensor<fp16, [16]> var_5063_bias_0_to_fp16 = const()[name = string("op_5063_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(226503104)))];
            tensor<fp16, [4, 16]> var_5063_cast_fp16 = linear(bias = var_5063_bias_0_to_fp16, weight = op_5063_weight_0_to_fp16_palettized, x = input_185_cast_fp16)[name = string("op_5063_cast_fp16")];
            tensor<fp16, [4, 16]> var_5064_cast_fp16 = softplus(x = var_5063_cast_fp16)[name = string("op_5064_cast_fp16")];
            tensor<fp16, [16]> var_5060_promoted_to_fp16 = const()[name = string("op_5060_promoted_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(226503232)))];
            tensor<fp16, [4, 16]> var_5065_cast_fp16 = mul(x = var_5060_promoted_to_fp16, y = var_5064_cast_fp16)[name = string("op_5065_cast_fp16")];
            tensor<int32, [2]> var_5066_perm_0 = const()[name = string("op_5066_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> decay_23_axes_0 = const()[name = string("decay_23_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 4]> var_5066_cast_fp16 = transpose(perm = var_5066_perm_0, x = var_5065_cast_fp16)[name = string("transpose_35")];
            tensor<fp16, [1, 16, 4]> decay_23_cast_fp16 = expand_dims(axes = decay_23_axes_0, x = var_5066_cast_fp16)[name = string("decay_23_cast_fp16")];
            tensor<fp16, [2048, 1024]> groups_3_2_in_proj_z_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [2048, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(226503360))), lut = tensor<fp16, [128, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(228076288))))[name = string("groups_3_2_in_proj_z_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 2048]> linear_112_cast_fp16 = linear(bias = linear_3_bias_0_to_fp16, weight = groups_3_2_in_proj_z_weight_promoted_to_fp16_palettized, x = input_185_cast_fp16)[name = string("linear_112_cast_fp16")];
            tensor<int32, [3]> var_5074_begin_0 = const()[name = string("op_5074_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_5074_end_0 = const()[name = string("op_5074_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_5074_end_mask_0 = const()[name = string("op_5074_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_5074_squeeze_mask_0 = const()[name = string("op_5074_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_5074_cast_fp16 = slice_by_index(begin = var_5074_begin_0, end = var_5074_end_0, end_mask = var_5074_end_mask_0, squeeze_mask = var_5074_squeeze_mask_0, x = decay_23_cast_fp16)[name = string("op_5074_cast_fp16")];
            tensor<fp16, [1, 16]> var_5075_cast_fp16 = exp(x = var_5074_cast_fp16)[name = string("op_5075_cast_fp16")];
            tensor<int32, [1]> var_5076_axes_0 = const()[name = string("op_5076_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_5076_cast_fp16 = expand_dims(axes = var_5076_axes_0, x = var_5075_cast_fp16)[name = string("op_5076_cast_fp16")];
            tensor<int32, [1]> var_5077_axes_0 = const()[name = string("op_5077_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_5077_cast_fp16 = expand_dims(axes = var_5077_axes_0, x = var_5076_cast_fp16)[name = string("op_5077_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_177_cast_fp16 = mul(x = rec14, y = var_5077_cast_fp16)[name = string("state_177_cast_fp16")];
            tensor<int32, [4]> key_token_89_begin_0 = const()[name = string("key_token_89_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> key_token_89_end_0 = const()[name = string("key_token_89_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> key_token_89_end_mask_0 = const()[name = string("key_token_89_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_89_squeeze_mask_0 = const()[name = string("key_token_89_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 4, 128]> key_59_cast_fp16 = transpose(perm = key_59_perm_0, x = var_5051_cast_fp16)[name = string("transpose_34")];
            tensor<fp16, [1, 16, 128]> key_token_89_cast_fp16 = slice_by_index(begin = key_token_89_begin_0, end = key_token_89_end_0, end_mask = key_token_89_end_mask_0, squeeze_mask = key_token_89_squeeze_mask_0, x = key_59_cast_fp16)[name = string("key_token_89_cast_fp16")];
            tensor<int32, [4]> value_token_89_begin_0 = const()[name = string("value_token_89_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> value_token_89_end_0 = const()[name = string("value_token_89_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> value_token_89_end_mask_0 = const()[name = string("value_token_89_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_89_squeeze_mask_0 = const()[name = string("value_token_89_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 4, 128]> value_213_cast_fp16 = transpose(perm = value_213_perm_0, x = value_211_cast_fp16)[name = string("transpose_33")];
            tensor<fp16, [1, 16, 128]> value_token_89_cast_fp16 = slice_by_index(begin = value_token_89_begin_0, end = value_token_89_end_0, end_mask = value_token_89_end_mask_0, squeeze_mask = value_token_89_squeeze_mask_0, x = value_213_cast_fp16)[name = string("value_token_89_cast_fp16")];
            tensor<int32, [1]> var_5085_axes_0 = const()[name = string("op_5085_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_5085_cast_fp16 = expand_dims(axes = var_5085_axes_0, x = key_token_89_cast_fp16)[name = string("op_5085_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_5086_cast_fp16 = mul(x = state_177_cast_fp16, y = var_5085_cast_fp16)[name = string("op_5086_cast_fp16")];
            tensor<int32, [1]> memory_89_axes_0 = const()[name = string("memory_89_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_89_keep_dims_0 = const()[name = string("memory_89_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_89_cast_fp16 = reduce_sum(axes = memory_89_axes_0, keep_dims = memory_89_keep_dims_0, x = var_5086_cast_fp16)[name = string("memory_89_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_5089_cast_fp16 = sub(x = value_token_89_cast_fp16, y = memory_89_cast_fp16)[name = string("op_5089_cast_fp16")];
            tensor<int32, [3]> var_5092_begin_0 = const()[name = string("op_5092_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_5092_end_0 = const()[name = string("op_5092_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_5092_end_mask_0 = const()[name = string("op_5092_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_5092_squeeze_mask_0 = const()[name = string("op_5092_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_5092_cast_fp16 = slice_by_index(begin = var_5092_begin_0, end = var_5092_end_0, end_mask = var_5092_end_mask_0, squeeze_mask = var_5092_squeeze_mask_0, x = beta_23_cast_fp16)[name = string("op_5092_cast_fp16")];
            tensor<int32, [1]> var_5093_axes_0 = const()[name = string("op_5093_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_5093_cast_fp16 = expand_dims(axes = var_5093_axes_0, x = var_5092_cast_fp16)[name = string("op_5093_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_89_cast_fp16 = mul(x = var_5089_cast_fp16, y = var_5093_cast_fp16)[name = string("delta_89_cast_fp16")];
            tensor<int32, [1]> var_5096_axes_0 = const()[name = string("op_5096_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_5096_cast_fp16 = expand_dims(axes = var_5096_axes_0, x = delta_89_cast_fp16)[name = string("op_5096_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_5097_cast_fp16 = mul(x = var_5085_cast_fp16, y = var_5096_cast_fp16)[name = string("op_5097_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_179_cast_fp16 = add(x = state_177_cast_fp16, y = var_5097_cast_fp16)[name = string("state_179_cast_fp16")];
            tensor<int32, [4]> var_5101_begin_0 = const()[name = string("op_5101_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> var_5101_end_0 = const()[name = string("op_5101_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> var_5101_end_mask_0 = const()[name = string("op_5101_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_5101_squeeze_mask_0 = const()[name = string("op_5101_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_5101_cast_fp16 = slice_by_index(begin = var_5101_begin_0, end = var_5101_end_0, end_mask = var_5101_end_mask_0, squeeze_mask = var_5101_squeeze_mask_0, x = _inversed_query_59_cast_fp16)[name = string("op_5101_cast_fp16")];
            tensor<int32, [1]> var_5102_axes_0 = const()[name = string("op_5102_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_5102_cast_fp16 = expand_dims(axes = var_5102_axes_0, x = var_5101_cast_fp16)[name = string("op_5102_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_5103_cast_fp16 = mul(x = state_179_cast_fp16, y = var_5102_cast_fp16)[name = string("op_5103_cast_fp16")];
            tensor<int32, [1]> var_5105_axes_0 = const()[name = string("op_5105_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_5105_keep_dims_0 = const()[name = string("op_5105_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_5105_cast_fp16 = reduce_sum(axes = var_5105_axes_0, keep_dims = var_5105_keep_dims_0, x = var_5103_cast_fp16)[name = string("op_5105_cast_fp16")];
            tensor<int32, [3]> var_5108_begin_0 = const()[name = string("op_5108_begin_0"), val = tensor<int32, [3]>([0, 0, 1])];
            tensor<int32, [3]> var_5108_end_0 = const()[name = string("op_5108_end_0"), val = tensor<int32, [3]>([1, 16, 2])];
            tensor<bool, [3]> var_5108_end_mask_0 = const()[name = string("op_5108_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_5108_squeeze_mask_0 = const()[name = string("op_5108_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_5108_cast_fp16 = slice_by_index(begin = var_5108_begin_0, end = var_5108_end_0, end_mask = var_5108_end_mask_0, squeeze_mask = var_5108_squeeze_mask_0, x = decay_23_cast_fp16)[name = string("op_5108_cast_fp16")];
            tensor<fp16, [1, 16]> var_5109_cast_fp16 = exp(x = var_5108_cast_fp16)[name = string("op_5109_cast_fp16")];
            tensor<int32, [1]> var_5110_axes_0 = const()[name = string("op_5110_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_5110_cast_fp16 = expand_dims(axes = var_5110_axes_0, x = var_5109_cast_fp16)[name = string("op_5110_cast_fp16")];
            tensor<int32, [1]> var_5111_axes_0 = const()[name = string("op_5111_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_5111_cast_fp16 = expand_dims(axes = var_5111_axes_0, x = var_5110_cast_fp16)[name = string("op_5111_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_181_cast_fp16 = mul(x = state_179_cast_fp16, y = var_5111_cast_fp16)[name = string("state_181_cast_fp16")];
            tensor<int32, [4]> key_token_91_begin_0 = const()[name = string("key_token_91_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> key_token_91_end_0 = const()[name = string("key_token_91_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> key_token_91_end_mask_0 = const()[name = string("key_token_91_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_91_squeeze_mask_0 = const()[name = string("key_token_91_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_91_cast_fp16 = slice_by_index(begin = key_token_91_begin_0, end = key_token_91_end_0, end_mask = key_token_91_end_mask_0, squeeze_mask = key_token_91_squeeze_mask_0, x = key_59_cast_fp16)[name = string("key_token_91_cast_fp16")];
            tensor<int32, [4]> value_token_91_begin_0 = const()[name = string("value_token_91_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> value_token_91_end_0 = const()[name = string("value_token_91_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> value_token_91_end_mask_0 = const()[name = string("value_token_91_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_91_squeeze_mask_0 = const()[name = string("value_token_91_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_91_cast_fp16 = slice_by_index(begin = value_token_91_begin_0, end = value_token_91_end_0, end_mask = value_token_91_end_mask_0, squeeze_mask = value_token_91_squeeze_mask_0, x = value_213_cast_fp16)[name = string("value_token_91_cast_fp16")];
            tensor<int32, [1]> var_5119_axes_0 = const()[name = string("op_5119_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_5119_cast_fp16 = expand_dims(axes = var_5119_axes_0, x = key_token_91_cast_fp16)[name = string("op_5119_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_5120_cast_fp16 = mul(x = state_181_cast_fp16, y = var_5119_cast_fp16)[name = string("op_5120_cast_fp16")];
            tensor<int32, [1]> memory_91_axes_0 = const()[name = string("memory_91_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_91_keep_dims_0 = const()[name = string("memory_91_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_91_cast_fp16 = reduce_sum(axes = memory_91_axes_0, keep_dims = memory_91_keep_dims_0, x = var_5120_cast_fp16)[name = string("memory_91_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_5123_cast_fp16 = sub(x = value_token_91_cast_fp16, y = memory_91_cast_fp16)[name = string("op_5123_cast_fp16")];
            tensor<int32, [3]> var_5126_begin_0 = const()[name = string("op_5126_begin_0"), val = tensor<int32, [3]>([0, 0, 1])];
            tensor<int32, [3]> var_5126_end_0 = const()[name = string("op_5126_end_0"), val = tensor<int32, [3]>([1, 16, 2])];
            tensor<bool, [3]> var_5126_end_mask_0 = const()[name = string("op_5126_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_5126_squeeze_mask_0 = const()[name = string("op_5126_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_5126_cast_fp16 = slice_by_index(begin = var_5126_begin_0, end = var_5126_end_0, end_mask = var_5126_end_mask_0, squeeze_mask = var_5126_squeeze_mask_0, x = beta_23_cast_fp16)[name = string("op_5126_cast_fp16")];
            tensor<int32, [1]> var_5127_axes_0 = const()[name = string("op_5127_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_5127_cast_fp16 = expand_dims(axes = var_5127_axes_0, x = var_5126_cast_fp16)[name = string("op_5127_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_91_cast_fp16 = mul(x = var_5123_cast_fp16, y = var_5127_cast_fp16)[name = string("delta_91_cast_fp16")];
            tensor<int32, [1]> var_5130_axes_0 = const()[name = string("op_5130_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_5130_cast_fp16 = expand_dims(axes = var_5130_axes_0, x = delta_91_cast_fp16)[name = string("op_5130_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_5131_cast_fp16 = mul(x = var_5119_cast_fp16, y = var_5130_cast_fp16)[name = string("op_5131_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_183_cast_fp16 = add(x = state_181_cast_fp16, y = var_5131_cast_fp16)[name = string("state_183_cast_fp16")];
            tensor<int32, [4]> var_5135_begin_0 = const()[name = string("op_5135_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> var_5135_end_0 = const()[name = string("op_5135_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> var_5135_end_mask_0 = const()[name = string("op_5135_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_5135_squeeze_mask_0 = const()[name = string("op_5135_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_5135_cast_fp16 = slice_by_index(begin = var_5135_begin_0, end = var_5135_end_0, end_mask = var_5135_end_mask_0, squeeze_mask = var_5135_squeeze_mask_0, x = _inversed_query_59_cast_fp16)[name = string("op_5135_cast_fp16")];
            tensor<int32, [1]> var_5136_axes_0 = const()[name = string("op_5136_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_5136_cast_fp16 = expand_dims(axes = var_5136_axes_0, x = var_5135_cast_fp16)[name = string("op_5136_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_5137_cast_fp16 = mul(x = state_183_cast_fp16, y = var_5136_cast_fp16)[name = string("op_5137_cast_fp16")];
            tensor<int32, [1]> var_5139_axes_0 = const()[name = string("op_5139_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_5139_keep_dims_0 = const()[name = string("op_5139_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_5139_cast_fp16 = reduce_sum(axes = var_5139_axes_0, keep_dims = var_5139_keep_dims_0, x = var_5137_cast_fp16)[name = string("op_5139_cast_fp16")];
            tensor<int32, [3]> var_5142_begin_0 = const()[name = string("op_5142_begin_0"), val = tensor<int32, [3]>([0, 0, 2])];
            tensor<int32, [3]> var_5142_end_0 = const()[name = string("op_5142_end_0"), val = tensor<int32, [3]>([1, 16, 3])];
            tensor<bool, [3]> var_5142_end_mask_0 = const()[name = string("op_5142_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_5142_squeeze_mask_0 = const()[name = string("op_5142_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_5142_cast_fp16 = slice_by_index(begin = var_5142_begin_0, end = var_5142_end_0, end_mask = var_5142_end_mask_0, squeeze_mask = var_5142_squeeze_mask_0, x = decay_23_cast_fp16)[name = string("op_5142_cast_fp16")];
            tensor<fp16, [1, 16]> var_5143_cast_fp16 = exp(x = var_5142_cast_fp16)[name = string("op_5143_cast_fp16")];
            tensor<int32, [1]> var_5144_axes_0 = const()[name = string("op_5144_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_5144_cast_fp16 = expand_dims(axes = var_5144_axes_0, x = var_5143_cast_fp16)[name = string("op_5144_cast_fp16")];
            tensor<int32, [1]> var_5145_axes_0 = const()[name = string("op_5145_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_5145_cast_fp16 = expand_dims(axes = var_5145_axes_0, x = var_5144_cast_fp16)[name = string("op_5145_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_185_cast_fp16 = mul(x = state_183_cast_fp16, y = var_5145_cast_fp16)[name = string("state_185_cast_fp16")];
            tensor<int32, [4]> key_token_93_begin_0 = const()[name = string("key_token_93_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> key_token_93_end_0 = const()[name = string("key_token_93_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> key_token_93_end_mask_0 = const()[name = string("key_token_93_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_93_squeeze_mask_0 = const()[name = string("key_token_93_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_93_cast_fp16 = slice_by_index(begin = key_token_93_begin_0, end = key_token_93_end_0, end_mask = key_token_93_end_mask_0, squeeze_mask = key_token_93_squeeze_mask_0, x = key_59_cast_fp16)[name = string("key_token_93_cast_fp16")];
            tensor<int32, [4]> value_token_93_begin_0 = const()[name = string("value_token_93_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> value_token_93_end_0 = const()[name = string("value_token_93_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> value_token_93_end_mask_0 = const()[name = string("value_token_93_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_93_squeeze_mask_0 = const()[name = string("value_token_93_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_93_cast_fp16 = slice_by_index(begin = value_token_93_begin_0, end = value_token_93_end_0, end_mask = value_token_93_end_mask_0, squeeze_mask = value_token_93_squeeze_mask_0, x = value_213_cast_fp16)[name = string("value_token_93_cast_fp16")];
            tensor<int32, [1]> var_5153_axes_0 = const()[name = string("op_5153_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_5153_cast_fp16 = expand_dims(axes = var_5153_axes_0, x = key_token_93_cast_fp16)[name = string("op_5153_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_5154_cast_fp16 = mul(x = state_185_cast_fp16, y = var_5153_cast_fp16)[name = string("op_5154_cast_fp16")];
            tensor<int32, [1]> memory_93_axes_0 = const()[name = string("memory_93_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_93_keep_dims_0 = const()[name = string("memory_93_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_93_cast_fp16 = reduce_sum(axes = memory_93_axes_0, keep_dims = memory_93_keep_dims_0, x = var_5154_cast_fp16)[name = string("memory_93_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_5157_cast_fp16 = sub(x = value_token_93_cast_fp16, y = memory_93_cast_fp16)[name = string("op_5157_cast_fp16")];
            tensor<int32, [3]> var_5160_begin_0 = const()[name = string("op_5160_begin_0"), val = tensor<int32, [3]>([0, 0, 2])];
            tensor<int32, [3]> var_5160_end_0 = const()[name = string("op_5160_end_0"), val = tensor<int32, [3]>([1, 16, 3])];
            tensor<bool, [3]> var_5160_end_mask_0 = const()[name = string("op_5160_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_5160_squeeze_mask_0 = const()[name = string("op_5160_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_5160_cast_fp16 = slice_by_index(begin = var_5160_begin_0, end = var_5160_end_0, end_mask = var_5160_end_mask_0, squeeze_mask = var_5160_squeeze_mask_0, x = beta_23_cast_fp16)[name = string("op_5160_cast_fp16")];
            tensor<int32, [1]> var_5161_axes_0 = const()[name = string("op_5161_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_5161_cast_fp16 = expand_dims(axes = var_5161_axes_0, x = var_5160_cast_fp16)[name = string("op_5161_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_93_cast_fp16 = mul(x = var_5157_cast_fp16, y = var_5161_cast_fp16)[name = string("delta_93_cast_fp16")];
            tensor<int32, [1]> var_5164_axes_0 = const()[name = string("op_5164_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_5164_cast_fp16 = expand_dims(axes = var_5164_axes_0, x = delta_93_cast_fp16)[name = string("op_5164_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_5165_cast_fp16 = mul(x = var_5153_cast_fp16, y = var_5164_cast_fp16)[name = string("op_5165_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_187_cast_fp16 = add(x = state_185_cast_fp16, y = var_5165_cast_fp16)[name = string("state_187_cast_fp16")];
            tensor<int32, [4]> var_5169_begin_0 = const()[name = string("op_5169_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> var_5169_end_0 = const()[name = string("op_5169_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> var_5169_end_mask_0 = const()[name = string("op_5169_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_5169_squeeze_mask_0 = const()[name = string("op_5169_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_5169_cast_fp16 = slice_by_index(begin = var_5169_begin_0, end = var_5169_end_0, end_mask = var_5169_end_mask_0, squeeze_mask = var_5169_squeeze_mask_0, x = _inversed_query_59_cast_fp16)[name = string("op_5169_cast_fp16")];
            tensor<int32, [1]> var_5170_axes_0 = const()[name = string("op_5170_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_5170_cast_fp16 = expand_dims(axes = var_5170_axes_0, x = var_5169_cast_fp16)[name = string("op_5170_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_5171_cast_fp16 = mul(x = state_187_cast_fp16, y = var_5170_cast_fp16)[name = string("op_5171_cast_fp16")];
            tensor<int32, [1]> var_5173_axes_0 = const()[name = string("op_5173_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_5173_keep_dims_0 = const()[name = string("op_5173_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_5173_cast_fp16 = reduce_sum(axes = var_5173_axes_0, keep_dims = var_5173_keep_dims_0, x = var_5171_cast_fp16)[name = string("op_5173_cast_fp16")];
            tensor<int32, [3]> var_5176_begin_0 = const()[name = string("op_5176_begin_0"), val = tensor<int32, [3]>([0, 0, 3])];
            tensor<int32, [3]> var_5176_end_0 = const()[name = string("op_5176_end_0"), val = tensor<int32, [3]>([1, 16, 4])];
            tensor<bool, [3]> var_5176_end_mask_0 = const()[name = string("op_5176_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_5176_squeeze_mask_0 = const()[name = string("op_5176_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_5176_cast_fp16 = slice_by_index(begin = var_5176_begin_0, end = var_5176_end_0, end_mask = var_5176_end_mask_0, squeeze_mask = var_5176_squeeze_mask_0, x = decay_23_cast_fp16)[name = string("op_5176_cast_fp16")];
            tensor<fp16, [1, 16]> var_5177_cast_fp16 = exp(x = var_5176_cast_fp16)[name = string("op_5177_cast_fp16")];
            tensor<int32, [1]> var_5178_axes_0 = const()[name = string("op_5178_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_5178_cast_fp16 = expand_dims(axes = var_5178_axes_0, x = var_5177_cast_fp16)[name = string("op_5178_cast_fp16")];
            tensor<int32, [1]> var_5179_axes_0 = const()[name = string("op_5179_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_5179_cast_fp16 = expand_dims(axes = var_5179_axes_0, x = var_5178_cast_fp16)[name = string("op_5179_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_189_cast_fp16 = mul(x = state_187_cast_fp16, y = var_5179_cast_fp16)[name = string("state_189_cast_fp16")];
            tensor<int32, [4]> key_token_95_begin_0 = const()[name = string("key_token_95_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> key_token_95_end_0 = const()[name = string("key_token_95_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> key_token_95_end_mask_0 = const()[name = string("key_token_95_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_95_squeeze_mask_0 = const()[name = string("key_token_95_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_95_cast_fp16 = slice_by_index(begin = key_token_95_begin_0, end = key_token_95_end_0, end_mask = key_token_95_end_mask_0, squeeze_mask = key_token_95_squeeze_mask_0, x = key_59_cast_fp16)[name = string("key_token_95_cast_fp16")];
            tensor<int32, [4]> value_token_95_begin_0 = const()[name = string("value_token_95_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> value_token_95_end_0 = const()[name = string("value_token_95_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> value_token_95_end_mask_0 = const()[name = string("value_token_95_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_95_squeeze_mask_0 = const()[name = string("value_token_95_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_95_cast_fp16 = slice_by_index(begin = value_token_95_begin_0, end = value_token_95_end_0, end_mask = value_token_95_end_mask_0, squeeze_mask = value_token_95_squeeze_mask_0, x = value_213_cast_fp16)[name = string("value_token_95_cast_fp16")];
            tensor<int32, [1]> var_5187_axes_0 = const()[name = string("op_5187_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_5187_cast_fp16 = expand_dims(axes = var_5187_axes_0, x = key_token_95_cast_fp16)[name = string("op_5187_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_5188_cast_fp16 = mul(x = state_189_cast_fp16, y = var_5187_cast_fp16)[name = string("op_5188_cast_fp16")];
            tensor<int32, [1]> memory_95_axes_0 = const()[name = string("memory_95_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_95_keep_dims_0 = const()[name = string("memory_95_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_95_cast_fp16 = reduce_sum(axes = memory_95_axes_0, keep_dims = memory_95_keep_dims_0, x = var_5188_cast_fp16)[name = string("memory_95_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_5191_cast_fp16 = sub(x = value_token_95_cast_fp16, y = memory_95_cast_fp16)[name = string("op_5191_cast_fp16")];
            tensor<int32, [3]> var_5194_begin_0 = const()[name = string("op_5194_begin_0"), val = tensor<int32, [3]>([0, 0, 3])];
            tensor<int32, [3]> var_5194_end_0 = const()[name = string("op_5194_end_0"), val = tensor<int32, [3]>([1, 16, 4])];
            tensor<bool, [3]> var_5194_end_mask_0 = const()[name = string("op_5194_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_5194_squeeze_mask_0 = const()[name = string("op_5194_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_5194_cast_fp16 = slice_by_index(begin = var_5194_begin_0, end = var_5194_end_0, end_mask = var_5194_end_mask_0, squeeze_mask = var_5194_squeeze_mask_0, x = beta_23_cast_fp16)[name = string("op_5194_cast_fp16")];
            tensor<int32, [1]> var_5195_axes_0 = const()[name = string("op_5195_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_5195_cast_fp16 = expand_dims(axes = var_5195_axes_0, x = var_5194_cast_fp16)[name = string("op_5195_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_95_cast_fp16 = mul(x = var_5191_cast_fp16, y = var_5195_cast_fp16)[name = string("delta_95_cast_fp16")];
            tensor<int32, [1]> var_5198_axes_0 = const()[name = string("op_5198_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_5198_cast_fp16 = expand_dims(axes = var_5198_axes_0, x = delta_95_cast_fp16)[name = string("op_5198_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_5199_cast_fp16 = mul(x = var_5187_cast_fp16, y = var_5198_cast_fp16)[name = string("op_5199_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> rec14_out = add(x = state_189_cast_fp16, y = var_5199_cast_fp16)[name = string("state_191_cast_fp16")];
            tensor<int32, [4]> var_5203_begin_0 = const()[name = string("op_5203_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> var_5203_end_0 = const()[name = string("op_5203_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> var_5203_end_mask_0 = const()[name = string("op_5203_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_5203_squeeze_mask_0 = const()[name = string("op_5203_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_5203_cast_fp16 = slice_by_index(begin = var_5203_begin_0, end = var_5203_end_0, end_mask = var_5203_end_mask_0, squeeze_mask = var_5203_squeeze_mask_0, x = _inversed_query_59_cast_fp16)[name = string("op_5203_cast_fp16")];
            tensor<int32, [1]> var_5204_axes_0 = const()[name = string("op_5204_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_5204_cast_fp16 = expand_dims(axes = var_5204_axes_0, x = var_5203_cast_fp16)[name = string("op_5204_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_5205_cast_fp16 = mul(x = rec14_out, y = var_5204_cast_fp16)[name = string("op_5205_cast_fp16")];
            tensor<int32, [1]> var_5207_axes_0 = const()[name = string("op_5207_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_5207_keep_dims_0 = const()[name = string("op_5207_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_5207_cast_fp16 = reduce_sum(axes = var_5207_axes_0, keep_dims = var_5207_keep_dims_0, x = var_5205_cast_fp16)[name = string("op_5207_cast_fp16")];
            int32 attention_111_axis_0 = const()[name = string("attention_111_axis_0"), val = int32(1)];
            tensor<fp16, [1, 4, 16, 128]> attention_111_cast_fp16 = stack(axis = attention_111_axis_0, values = (var_5105_cast_fp16, var_5139_cast_fp16, var_5173_cast_fp16, var_5207_cast_fp16))[name = string("attention_111_cast_fp16")];
            tensor<int32, [2]> var_5210 = const()[name = string("op_5210"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [64, 128]> attention_113_cast_fp16 = reshape(shape = var_5210, x = attention_111_cast_fp16)[name = string("attention_113_cast_fp16")];
            tensor<int32, [2]> var_5212 = const()[name = string("op_5212"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [64, 128]> input_189_cast_fp16 = reshape(shape = var_5212, x = linear_112_cast_fp16)[name = string("input_189_cast_fp16")];
            tensor<fp16, [64, 128]> var_5214_cast_fp16 = mul(x = attention_113_cast_fp16, y = attention_113_cast_fp16)[name = string("op_5214_cast_fp16")];
            tensor<int32, [1]> variance_93_axes_0 = const()[name = string("variance_93_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_93_keep_dims_0 = const()[name = string("variance_93_keep_dims_0"), val = bool(true)];
            tensor<fp16, [64, 1]> variance_93_cast_fp16 = reduce_mean(axes = variance_93_axes_0, keep_dims = variance_93_keep_dims_0, x = var_5214_cast_fp16)[name = string("variance_93_cast_fp16")];
            fp16 var_5217_to_fp16 = const()[name = string("op_5217_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [64, 1]> var_5218_cast_fp16 = add(x = variance_93_cast_fp16, y = var_5217_to_fp16)[name = string("op_5218_cast_fp16")];
            fp32 var_5219_epsilon_0 = const()[name = string("op_5219_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [64, 1]> var_5219_cast_fp16 = rsqrt(epsilon = var_5219_epsilon_0, x = var_5218_cast_fp16)[name = string("op_5219_cast_fp16")];
            tensor<fp16, [64, 128]> attention_115_cast_fp16 = mul(x = attention_113_cast_fp16, y = var_5219_cast_fp16)[name = string("attention_115_cast_fp16")];
            tensor<fp16, [128]> groups_3_2_gated_norm_promoted_to_fp16 = const()[name = string("groups_3_2_gated_norm_promoted_to_fp16"), val = tensor<fp16, [128]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(228092736)))];
            tensor<fp16, [64, 128]> attention_117_cast_fp16 = mul(x = attention_115_cast_fp16, y = groups_3_2_gated_norm_promoted_to_fp16)[name = string("attention_117_cast_fp16")];
            tensor<fp16, [64, 128]> var_5222_cast_fp16 = silu(x = input_189_cast_fp16)[name = string("op_5222_cast_fp16")];
            tensor<fp16, [64, 128]> attention_119_cast_fp16 = mul(x = attention_117_cast_fp16, y = var_5222_cast_fp16)[name = string("attention_119_cast_fp16")];
            tensor<int32, [2]> var_5224 = const()[name = string("op_5224"), val = tensor<int32, [2]>([4, 2048])];
            tensor<fp16, [4, 2048]> input_191_cast_fp16 = reshape(shape = var_5224, x = attention_119_cast_fp16)[name = string("input_191_cast_fp16")];
            tensor<fp16, [1024, 2048]> groups_3_2_out_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 2048]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(228093056))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(229665984))))[name = string("groups_3_2_out_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 1024]> linear_113_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = groups_3_2_out_proj_weight_promoted_to_fp16_palettized, x = input_191_cast_fp16)[name = string("linear_113_cast_fp16")];
            tensor<fp16, [4, 1024]> value_215_cast_fp16 = add(x = value_203_cast_fp16, y = linear_113_cast_fp16)[name = string("value_215_cast_fp16")];
            tensor<fp16, [4, 1024]> var_5229_cast_fp16 = mul(x = value_215_cast_fp16, y = value_215_cast_fp16)[name = string("op_5229_cast_fp16")];
            tensor<int32, [1]> variance_95_axes_0 = const()[name = string("variance_95_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_95_keep_dims_0 = const()[name = string("variance_95_keep_dims_0"), val = bool(true)];
            tensor<fp16, [4, 1]> variance_95_cast_fp16 = reduce_mean(axes = variance_95_axes_0, keep_dims = variance_95_keep_dims_0, x = var_5229_cast_fp16)[name = string("variance_95_cast_fp16")];
            fp16 var_5232_to_fp16 = const()[name = string("op_5232_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [4, 1]> var_5233_cast_fp16 = add(x = variance_95_cast_fp16, y = var_5232_to_fp16)[name = string("op_5233_cast_fp16")];
            fp32 var_5234_epsilon_0 = const()[name = string("op_5234_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [4, 1]> var_5234_cast_fp16 = rsqrt(epsilon = var_5234_epsilon_0, x = var_5233_cast_fp16)[name = string("op_5234_cast_fp16")];
            tensor<fp16, [4, 1024]> var_5235_cast_fp16 = mul(x = value_215_cast_fp16, y = var_5234_cast_fp16)[name = string("op_5235_cast_fp16")];
            tensor<fp16, [1024]> var_5237_to_fp16 = const()[name = string("op_5237_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(229674240)))];
            tensor<fp16, [4, 1024]> input_193_cast_fp16 = mul(x = var_5235_cast_fp16, y = var_5237_to_fp16)[name = string("input_193_cast_fp16")];
            tensor<fp16, [3584, 1024]> groups_3_2_gate_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(229676352))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(232428928))))[name = string("groups_3_2_gate_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 3584]> linear_114_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = groups_3_2_gate_proj_weight_promoted_to_fp16_palettized, x = input_193_cast_fp16)[name = string("linear_114_cast_fp16")];
            tensor<fp16, [4, 3584]> var_5241_cast_fp16 = silu(x = linear_114_cast_fp16)[name = string("op_5241_cast_fp16")];
            tensor<fp16, [3584, 1024]> groups_3_2_up_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(232457664))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(235210240))))[name = string("groups_3_2_up_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 3584]> linear_115_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = groups_3_2_up_proj_weight_promoted_to_fp16_palettized, x = input_193_cast_fp16)[name = string("linear_115_cast_fp16")];
            tensor<fp16, [4, 3584]> input_197_cast_fp16 = mul(x = var_5241_cast_fp16, y = linear_115_cast_fp16)[name = string("input_197_cast_fp16")];
            tensor<fp16, [1024, 3584]> groups_3_2_down_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 3584]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(235238976))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(237991552))))[name = string("groups_3_2_down_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 1024]> linear_116_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = groups_3_2_down_proj_weight_promoted_to_fp16_palettized, x = input_197_cast_fp16)[name = string("linear_116_cast_fp16")];
            tensor<fp16, [4, 1024]> value_217_cast_fp16 = add(x = value_215_cast_fp16, y = linear_116_cast_fp16)[name = string("value_217_cast_fp16")];
            int32 var_5265 = const()[name = string("op_5265"), val = int32(-1)];
            tensor<fp16, [4, 1024]> var_5272_cast_fp16 = mul(x = value_217_cast_fp16, y = value_217_cast_fp16)[name = string("op_5272_cast_fp16")];
            tensor<int32, [1]> variance_97_axes_0 = const()[name = string("variance_97_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_97_keep_dims_0 = const()[name = string("variance_97_keep_dims_0"), val = bool(true)];
            tensor<fp16, [4, 1]> variance_97_cast_fp16 = reduce_mean(axes = variance_97_axes_0, keep_dims = variance_97_keep_dims_0, x = var_5272_cast_fp16)[name = string("variance_97_cast_fp16")];
            fp16 var_5275_to_fp16 = const()[name = string("op_5275_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [4, 1]> var_5276_cast_fp16 = add(x = variance_97_cast_fp16, y = var_5275_to_fp16)[name = string("op_5276_cast_fp16")];
            fp32 var_5277_epsilon_0 = const()[name = string("op_5277_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [4, 1]> var_5277_cast_fp16 = rsqrt(epsilon = var_5277_epsilon_0, x = var_5276_cast_fp16)[name = string("op_5277_cast_fp16")];
            tensor<fp16, [4, 1024]> var_5278_cast_fp16 = mul(x = value_217_cast_fp16, y = var_5277_cast_fp16)[name = string("op_5278_cast_fp16")];
            tensor<fp16, [1024]> var_5280_to_fp16 = const()[name = string("op_5280_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(237999808)))];
            tensor<fp16, [4, 1024]> input_199_cast_fp16 = mul(x = var_5278_cast_fp16, y = var_5280_to_fp16)[name = string("input_199_cast_fp16")];
            tensor<fp16, [4096, 1024]> projections_3_q_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [4096, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(238001920))), lut = tensor<fp16, [256, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(241147712))))[name = string("projections_3_q_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 4096]> linear_117_cast_fp16 = linear(bias = linear_24_bias_0_to_fp16, weight = projections_3_q_proj_weight_promoted_to_fp16_palettized, x = input_199_cast_fp16)[name = string("linear_117_cast_fp16")];
            tensor<int32, [3]> var_5284 = const()[name = string("op_5284"), val = tensor<int32, [3]>([4, 8, 512])];
            tensor<fp16, [4, 8, 512]> query_and_gate_7_cast_fp16 = reshape(shape = var_5284, x = linear_117_cast_fp16)[name = string("query_and_gate_7_cast_fp16")];
            tensor<int32, [2]> var_5286_split_sizes_0 = const()[name = string("op_5286_split_sizes_0"), val = tensor<int32, [2]>([256, 256])];
            int32 var_5286_axis_0 = const()[name = string("op_5286_axis_0"), val = int32(-1)];
            tensor<fp16, [4, 8, 256]> var_5286_cast_fp16_0, tensor<fp16, [4, 8, 256]> var_5286_cast_fp16_1 = split(axis = var_5286_axis_0, split_sizes = var_5286_split_sizes_0, x = query_and_gate_7_cast_fp16)[name = string("op_5286_cast_fp16")];
            tensor<fp16, [512, 1024]> projections_3_k_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [512, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(241180544))), lut = tensor<fp16, [32, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(241573824))))[name = string("projections_3_k_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 512]> linear_118_cast_fp16 = linear(bias = linear_25_bias_0_to_fp16, weight = projections_3_k_proj_weight_promoted_to_fp16_palettized, x = input_199_cast_fp16)[name = string("linear_118_cast_fp16")];
            tensor<int32, [3]> var_5290 = const()[name = string("op_5290"), val = tensor<int32, [3]>([4, 2, 256])];
            tensor<fp16, [4, 2, 256]> value_221_cast_fp16 = reshape(shape = var_5290, x = linear_118_cast_fp16)[name = string("value_221_cast_fp16")];
            tensor<fp16, [512, 1024]> projections_3_v_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [512, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(241577984))), lut = tensor<fp16, [32, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(241971264))))[name = string("projections_3_v_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 512]> linear_119_cast_fp16 = linear(bias = linear_25_bias_0_to_fp16, weight = projections_3_v_proj_weight_promoted_to_fp16_palettized, x = input_199_cast_fp16)[name = string("linear_119_cast_fp16")];
            tensor<int32, [3]> var_5294 = const()[name = string("op_5294"), val = tensor<int32, [3]>([4, 2, 256])];
            tensor<fp16, [4, 2, 256]> value_227_cast_fp16 = reshape(shape = var_5294, x = linear_119_cast_fp16)[name = string("value_227_cast_fp16")];
            tensor<fp16, [4, 8, 256]> var_5296_cast_fp16 = mul(x = var_5286_cast_fp16_0, y = var_5286_cast_fp16_0)[name = string("op_5296_cast_fp16")];
            tensor<int32, [1]> variance_99_axes_0 = const()[name = string("variance_99_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_99_keep_dims_0 = const()[name = string("variance_99_keep_dims_0"), val = bool(true)];
            tensor<fp16, [4, 8, 1]> variance_99_cast_fp16 = reduce_mean(axes = variance_99_axes_0, keep_dims = variance_99_keep_dims_0, x = var_5296_cast_fp16)[name = string("variance_99_cast_fp16")];
            fp16 var_5299_to_fp16 = const()[name = string("op_5299_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [4, 8, 1]> var_5300_cast_fp16 = add(x = variance_99_cast_fp16, y = var_5299_to_fp16)[name = string("op_5300_cast_fp16")];
            fp32 var_5301_epsilon_0 = const()[name = string("op_5301_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [4, 8, 1]> var_5301_cast_fp16 = rsqrt(epsilon = var_5301_epsilon_0, x = var_5300_cast_fp16)[name = string("op_5301_cast_fp16")];
            tensor<fp16, [4, 8, 256]> var_5302_cast_fp16 = mul(x = var_5286_cast_fp16_0, y = var_5301_cast_fp16)[name = string("op_5302_cast_fp16")];
            tensor<fp16, [256]> var_5304_to_fp16 = const()[name = string("op_5304_to_fp16"), val = tensor<fp16, [256]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(241975424)))];
            tensor<fp16, [4, 8, 256]> var_5305_cast_fp16 = mul(x = var_5302_cast_fp16, y = var_5304_to_fp16)[name = string("op_5305_cast_fp16")];
            tensor<int32, [3]> var_5306 = const()[name = string("op_5306"), val = tensor<int32, [3]>([1, 0, 2])];
            tensor<int32, [1]> value_223_axes_0 = const()[name = string("value_223_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [8, 4, 256]> var_5307_cast_fp16 = transpose(perm = var_5306, x = var_5305_cast_fp16)[name = string("transpose_32")];
            tensor<fp16, [1, 8, 4, 256]> value_223_cast_fp16 = expand_dims(axes = value_223_axes_0, x = var_5307_cast_fp16)[name = string("value_223_cast_fp16")];
            tensor<fp16, [4, 2, 256]> var_5309_cast_fp16 = mul(x = value_221_cast_fp16, y = value_221_cast_fp16)[name = string("op_5309_cast_fp16")];
            tensor<int32, [1]> variance_101_axes_0 = const()[name = string("variance_101_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_101_keep_dims_0 = const()[name = string("variance_101_keep_dims_0"), val = bool(true)];
            tensor<fp16, [4, 2, 1]> variance_101_cast_fp16 = reduce_mean(axes = variance_101_axes_0, keep_dims = variance_101_keep_dims_0, x = var_5309_cast_fp16)[name = string("variance_101_cast_fp16")];
            fp16 var_5312_to_fp16 = const()[name = string("op_5312_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [4, 2, 1]> var_5313_cast_fp16 = add(x = variance_101_cast_fp16, y = var_5312_to_fp16)[name = string("op_5313_cast_fp16")];
            fp32 var_5314_epsilon_0 = const()[name = string("op_5314_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [4, 2, 1]> var_5314_cast_fp16 = rsqrt(epsilon = var_5314_epsilon_0, x = var_5313_cast_fp16)[name = string("op_5314_cast_fp16")];
            tensor<fp16, [4, 2, 256]> var_5315_cast_fp16 = mul(x = value_221_cast_fp16, y = var_5314_cast_fp16)[name = string("op_5315_cast_fp16")];
            tensor<fp16, [256]> var_5317_to_fp16 = const()[name = string("op_5317_to_fp16"), val = tensor<fp16, [256]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(241976000)))];
            tensor<fp16, [4, 2, 256]> var_5318_cast_fp16 = mul(x = var_5315_cast_fp16, y = var_5317_to_fp16)[name = string("op_5318_cast_fp16")];
            tensor<int32, [3]> var_5319 = const()[name = string("op_5319"), val = tensor<int32, [3]>([1, 0, 2])];
            tensor<int32, [1]> value_225_axes_0 = const()[name = string("value_225_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [2, 4, 256]> var_5320_cast_fp16 = transpose(perm = var_5319, x = var_5318_cast_fp16)[name = string("transpose_31")];
            tensor<fp16, [1, 2, 4, 256]> value_225_cast_fp16 = expand_dims(axes = value_225_axes_0, x = var_5320_cast_fp16)[name = string("value_225_cast_fp16")];
            tensor<int32, [4]> rotated_13_begin_0 = const()[name = string("rotated_13_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> rotated_13_end_0 = const()[name = string("rotated_13_end_0"), val = tensor<int32, [4]>([1, 8, 4, 64])];
            tensor<bool, [4]> rotated_13_end_mask_0 = const()[name = string("rotated_13_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
            tensor<fp16, [1, 8, 4, 64]> rotated_13_cast_fp16 = slice_by_index(begin = rotated_13_begin_0, end = rotated_13_end_0, end_mask = rotated_13_end_mask_0, x = value_223_cast_fp16)[name = string("rotated_13_cast_fp16")];
            tensor<int32, [4]> passthrough_13_begin_0 = const()[name = string("passthrough_13_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 64])];
            tensor<int32, [4]> passthrough_13_end_0 = const()[name = string("passthrough_13_end_0"), val = tensor<int32, [4]>([1, 8, 4, 256])];
            tensor<bool, [4]> passthrough_13_end_mask_0 = const()[name = string("passthrough_13_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
            tensor<fp16, [1, 8, 4, 192]> passthrough_13_cast_fp16 = slice_by_index(begin = passthrough_13_begin_0, end = passthrough_13_end_0, end_mask = passthrough_13_end_mask_0, x = value_223_cast_fp16)[name = string("passthrough_13_cast_fp16")];
            tensor<int32, [2]> var_5324_split_sizes_0 = const()[name = string("op_5324_split_sizes_0"), val = tensor<int32, [2]>([32, 32])];
            int32 var_5324_axis_0 = const()[name = string("op_5324_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 8, 4, 32]> var_5324_cast_fp16_0, tensor<fp16, [1, 8, 4, 32]> var_5324_cast_fp16_1 = split(axis = var_5324_axis_0, split_sizes = var_5324_split_sizes_0, x = rotated_13_cast_fp16)[name = string("op_5324_cast_fp16")];
            fp16 const_132_promoted_to_fp16 = const()[name = string("const_132_promoted_to_fp16"), val = fp16(-0x1p+0)];
            tensor<fp16, [1, 8, 4, 32]> var_5326_cast_fp16 = mul(x = var_5324_cast_fp16_1, y = const_132_promoted_to_fp16)[name = string("op_5326_cast_fp16")];
            bool rotated_half_13_interleave_0 = const()[name = string("rotated_half_13_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 8, 4, 64]> rotated_half_13_cast_fp16 = concat(axis = var_5265, interleave = rotated_half_13_interleave_0, values = (var_5326_cast_fp16, var_5324_cast_fp16_0))[name = string("rotated_half_13_cast_fp16")];
            tensor<fp16, [1, 8, 4, 64]> var_5329_cast_fp16 = mul(x = rotated_13_cast_fp16, y = cos)[name = string("op_5329_cast_fp16")];
            tensor<fp16, [1, 8, 4, 64]> var_5330_cast_fp16 = mul(x = rotated_half_13_cast_fp16, y = sin)[name = string("op_5330_cast_fp16")];
            tensor<fp16, [1, 8, 4, 64]> var_5331_cast_fp16 = add(x = var_5329_cast_fp16, y = var_5330_cast_fp16)[name = string("op_5331_cast_fp16")];
            bool query_61_interleave_0 = const()[name = string("query_61_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 8, 4, 256]> query_61_cast_fp16 = concat(axis = var_5265, interleave = query_61_interleave_0, values = (var_5331_cast_fp16, passthrough_13_cast_fp16))[name = string("query_61_cast_fp16")];
            tensor<int32, [4]> rotated_15_begin_0 = const()[name = string("rotated_15_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> rotated_15_end_0 = const()[name = string("rotated_15_end_0"), val = tensor<int32, [4]>([1, 2, 4, 64])];
            tensor<bool, [4]> rotated_15_end_mask_0 = const()[name = string("rotated_15_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
            tensor<fp16, [1, 2, 4, 64]> rotated_15_cast_fp16 = slice_by_index(begin = rotated_15_begin_0, end = rotated_15_end_0, end_mask = rotated_15_end_mask_0, x = value_225_cast_fp16)[name = string("rotated_15_cast_fp16")];
            tensor<int32, [4]> passthrough_15_begin_0 = const()[name = string("passthrough_15_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 64])];
            tensor<int32, [4]> passthrough_15_end_0 = const()[name = string("passthrough_15_end_0"), val = tensor<int32, [4]>([1, 2, 4, 256])];
            tensor<bool, [4]> passthrough_15_end_mask_0 = const()[name = string("passthrough_15_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
            tensor<fp16, [1, 2, 4, 192]> passthrough_15_cast_fp16 = slice_by_index(begin = passthrough_15_begin_0, end = passthrough_15_end_0, end_mask = passthrough_15_end_mask_0, x = value_225_cast_fp16)[name = string("passthrough_15_cast_fp16")];
            tensor<int32, [2]> var_5336_split_sizes_0 = const()[name = string("op_5336_split_sizes_0"), val = tensor<int32, [2]>([32, 32])];
            int32 var_5336_axis_0 = const()[name = string("op_5336_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 2, 4, 32]> var_5336_cast_fp16_0, tensor<fp16, [1, 2, 4, 32]> var_5336_cast_fp16_1 = split(axis = var_5336_axis_0, split_sizes = var_5336_split_sizes_0, x = rotated_15_cast_fp16)[name = string("op_5336_cast_fp16")];
            fp16 const_133_promoted_to_fp16 = const()[name = string("const_133_promoted_to_fp16"), val = fp16(-0x1p+0)];
            tensor<fp16, [1, 2, 4, 32]> var_5338_cast_fp16 = mul(x = var_5336_cast_fp16_1, y = const_133_promoted_to_fp16)[name = string("op_5338_cast_fp16")];
            bool rotated_half_15_interleave_0 = const()[name = string("rotated_half_15_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 2, 4, 64]> rotated_half_15_cast_fp16 = concat(axis = var_5265, interleave = rotated_half_15_interleave_0, values = (var_5338_cast_fp16, var_5336_cast_fp16_0))[name = string("rotated_half_15_cast_fp16")];
            tensor<fp16, [1, 2, 4, 64]> var_5341_cast_fp16 = mul(x = rotated_15_cast_fp16, y = cos)[name = string("op_5341_cast_fp16")];
            tensor<fp16, [1, 2, 4, 64]> var_5342_cast_fp16 = mul(x = rotated_half_15_cast_fp16, y = sin)[name = string("op_5342_cast_fp16")];
            tensor<fp16, [1, 2, 4, 64]> var_5343_cast_fp16 = add(x = var_5341_cast_fp16, y = var_5342_cast_fp16)[name = string("op_5343_cast_fp16")];
            bool key_61_interleave_0 = const()[name = string("key_61_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 2, 4, 256]> key_61_cast_fp16 = concat(axis = var_5265, interleave = key_61_interleave_0, values = (var_5343_cast_fp16, passthrough_15_cast_fp16))[name = string("key_61_cast_fp16")];
            tensor<int32, [4]> var_5346 = const()[name = string("op_5346"), val = tensor<int32, [4]>([2, 4, 4, 256])];
            tensor<fp16, [2, 4, 4, 256]> var_5347_cast_fp16 = reshape(shape = var_5346, x = query_61_cast_fp16)[name = string("op_5347_cast_fp16")];
            tensor<int32, [4]> transpose_6_perm_0 = const()[name = string("transpose_6_perm_0"), val = tensor<int32, [4]>([0, 1, 3, 2])];
            tensor<int32, [3]> var_5350 = const()[name = string("op_5350"), val = tensor<int32, [3]>([2, 4, 256])];
            tensor<fp16, [2, 4, 256]> key_63_cast_fp16 = reshape(shape = var_5350, x = key_61_cast_fp16)[name = string("key_63_cast_fp16")];
            tensor<int32, [3]> var_5352 = const()[name = string("op_5352"), val = tensor<int32, [3]>([1, 0, 2])];
            tensor<int32, [2]> var_5354 = const()[name = string("op_5354"), val = tensor<int32, [2]>([4, 2048])];
            tensor<fp16, [4, 2048]> gate_15_cast_fp16 = reshape(shape = var_5354, x = var_5286_cast_fp16_1)[name = string("gate_15_cast_fp16")];
            tensor<int32, [1]> var_5362_axes_0 = const()[name = string("op_5362_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [1, 2, 4, 256]> var_5362_cast_fp16 = expand_dims(axes = var_5362_axes_0, x = key_63_cast_fp16)[name = string("op_5362_cast_fp16")];
            tensor<int32, [1]> var_5364_axes_0 = const()[name = string("op_5364_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [1, 1, 2, 4, 256]> var_5364_cast_fp16 = expand_dims(axes = var_5364_axes_0, x = var_5362_cast_fp16)[name = string("op_5364_cast_fp16")];
            tensor<int32, [1]> expand_dims_48 = const()[name = string("expand_dims_48"), val = tensor<int32, [1]>([3])];
            tensor<int32, [1]> expand_dims_49 = const()[name = string("expand_dims_49"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_50 = const()[name = string("expand_dims_50"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_52 = const()[name = string("expand_dims_52"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_53 = const()[name = string("expand_dims_53"), val = tensor<int32, [1]>([4])];
            tensor<int32, [1]> expand_dims_54 = const()[name = string("expand_dims_54"), val = tensor<int32, [1]>([1])];
            int32 concat_26_axis_0 = const()[name = string("concat_26_axis_0"), val = int32(0)];
            bool concat_26_interleave_0 = const()[name = string("concat_26_interleave_0"), val = bool(false)];
            tensor<int32, [5]> concat_26 = concat(axis = concat_26_axis_0, interleave = concat_26_interleave_0, values = (expand_dims_48, expand_dims_49, expand_dims_50, current_pos, expand_dims_52))[name = string("concat_26")];
            tensor<int32, [1]> concat_27_values2_0 = const()[name = string("concat_27_values2_0"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> concat_27_values4_0 = const()[name = string("concat_27_values4_0"), val = tensor<int32, [1]>([0])];
            int32 concat_27_axis_0 = const()[name = string("concat_27_axis_0"), val = int32(0)];
            bool concat_27_interleave_0 = const()[name = string("concat_27_interleave_0"), val = bool(false)];
            tensor<int32, [5]> concat_27 = concat(axis = concat_27_axis_0, interleave = concat_27_interleave_0, values = (expand_dims_53, expand_dims_54, concat_27_values2_0, var_1074, concat_27_values4_0))[name = string("concat_27")];
            tensor<int32, [5]> kv_cache_internal_tensor_assign_7_stride_0 = const()[name = string("kv_cache_internal_tensor_assign_7_stride_0"), val = tensor<int32, [5]>([1, 1, 1, 1, 1])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_7_begin_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_7_begin_mask_0"), val = tensor<bool, [5]>([false, false, false, false, false])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_7_end_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_7_end_mask_0"), val = tensor<bool, [5]>([false, false, true, false, true])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_7_squeeze_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_7_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, false, false, false])];
            tensor<fp16, [4, 2, 2, 2048, 256]> kv_cache_internal_tensor_assign_7_cast_fp16 = slice_update(begin = concat_26, begin_mask = kv_cache_internal_tensor_assign_7_begin_mask_0, end = concat_27, end_mask = kv_cache_internal_tensor_assign_7_end_mask_0, squeeze_mask = kv_cache_internal_tensor_assign_7_squeeze_mask_0, stride = kv_cache_internal_tensor_assign_7_stride_0, update = var_5364_cast_fp16, x = coreml_update_state_13)[name = string("kv_cache_internal_tensor_assign_7_cast_fp16")];
            write_state(data = kv_cache_internal_tensor_assign_7_cast_fp16, input = kv_cache)[name = string("coreml_update_state_22_write_state")];
            tensor<fp16, [4, 2, 2, 2048, 256]> coreml_update_state_14 = read_state(input = kv_cache)[name = string("coreml_update_state_22")];
            tensor<int32, [1]> var_5395_axes_0 = const()[name = string("op_5395_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [2, 4, 256]> value_229_cast_fp16 = transpose(perm = var_5352, x = value_227_cast_fp16)[name = string("transpose_30")];
            tensor<fp16, [1, 2, 4, 256]> var_5395_cast_fp16 = expand_dims(axes = var_5395_axes_0, x = value_229_cast_fp16)[name = string("op_5395_cast_fp16")];
            tensor<int32, [1]> var_5397_axes_0 = const()[name = string("op_5397_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [1, 1, 2, 4, 256]> var_5397_cast_fp16 = expand_dims(axes = var_5397_axes_0, x = var_5395_cast_fp16)[name = string("op_5397_cast_fp16")];
            tensor<int32, [1]> expand_dims_56 = const()[name = string("expand_dims_56"), val = tensor<int32, [1]>([3])];
            tensor<int32, [1]> expand_dims_57 = const()[name = string("expand_dims_57"), val = tensor<int32, [1]>([1])];
            tensor<int32, [1]> expand_dims_58 = const()[name = string("expand_dims_58"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_60 = const()[name = string("expand_dims_60"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_61 = const()[name = string("expand_dims_61"), val = tensor<int32, [1]>([4])];
            tensor<int32, [1]> expand_dims_62 = const()[name = string("expand_dims_62"), val = tensor<int32, [1]>([2])];
            int32 concat_30_axis_0 = const()[name = string("concat_30_axis_0"), val = int32(0)];
            bool concat_30_interleave_0 = const()[name = string("concat_30_interleave_0"), val = bool(false)];
            tensor<int32, [5]> concat_30 = concat(axis = concat_30_axis_0, interleave = concat_30_interleave_0, values = (expand_dims_56, expand_dims_57, expand_dims_58, current_pos, expand_dims_60))[name = string("concat_30")];
            tensor<int32, [1]> concat_31_values2_0 = const()[name = string("concat_31_values2_0"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> concat_31_values4_0 = const()[name = string("concat_31_values4_0"), val = tensor<int32, [1]>([0])];
            int32 concat_31_axis_0 = const()[name = string("concat_31_axis_0"), val = int32(0)];
            bool concat_31_interleave_0 = const()[name = string("concat_31_interleave_0"), val = bool(false)];
            tensor<int32, [5]> concat_31 = concat(axis = concat_31_axis_0, interleave = concat_31_interleave_0, values = (expand_dims_61, expand_dims_62, concat_31_values2_0, var_1074, concat_31_values4_0))[name = string("concat_31")];
            tensor<int32, [5]> kv_cache_internal_tensor_assign_8_stride_0 = const()[name = string("kv_cache_internal_tensor_assign_8_stride_0"), val = tensor<int32, [5]>([1, 1, 1, 1, 1])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_8_begin_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_8_begin_mask_0"), val = tensor<bool, [5]>([false, false, false, false, false])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_8_end_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_8_end_mask_0"), val = tensor<bool, [5]>([false, false, true, false, true])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_8_squeeze_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_8_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, false, false, false])];
            tensor<fp16, [4, 2, 2, 2048, 256]> kv_cache_internal_tensor_assign_8_cast_fp16 = slice_update(begin = concat_30, begin_mask = kv_cache_internal_tensor_assign_8_begin_mask_0, end = concat_31, end_mask = kv_cache_internal_tensor_assign_8_end_mask_0, squeeze_mask = kv_cache_internal_tensor_assign_8_squeeze_mask_0, stride = kv_cache_internal_tensor_assign_8_stride_0, update = var_5397_cast_fp16, x = coreml_update_state_14)[name = string("kv_cache_internal_tensor_assign_8_cast_fp16")];
            write_state(data = kv_cache_internal_tensor_assign_8_cast_fp16, input = kv_cache)[name = string("coreml_update_state_23_write_state")];
            tensor<fp16, [4, 2, 2, 2048, 256]> coreml_update_state_15 = read_state(input = kv_cache)[name = string("coreml_update_state_23")];
            tensor<int32, [5]> var_5429_begin_0 = const()[name = string("op_5429_begin_0"), val = tensor<int32, [5]>([3, 0, 0, 0, 0])];
            tensor<int32, [5]> var_5429_end_0 = const()[name = string("op_5429_end_0"), val = tensor<int32, [5]>([4, 2, 2, 2048, 256])];
            tensor<bool, [5]> var_5429_end_mask_0 = const()[name = string("op_5429_end_mask_0"), val = tensor<bool, [5]>([false, true, true, true, true])];
            tensor<bool, [5]> var_5429_squeeze_mask_0 = const()[name = string("op_5429_squeeze_mask_0"), val = tensor<bool, [5]>([true, false, false, false, false])];
            tensor<fp16, [2, 2, 2048, 256]> var_5429_cast_fp16 = slice_by_index(begin = var_5429_begin_0, end = var_5429_end_0, end_mask = var_5429_end_mask_0, squeeze_mask = var_5429_squeeze_mask_0, x = coreml_update_state_15)[name = string("op_5429_cast_fp16")];
            tensor<int32, [4]> keys_begin_0 = const()[name = string("keys_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> keys_end_0 = const()[name = string("keys_end_0"), val = tensor<int32, [4]>([1, 2, 2048, 256])];
            tensor<bool, [4]> keys_end_mask_0 = const()[name = string("keys_end_mask_0"), val = tensor<bool, [4]>([false, true, true, true])];
            tensor<bool, [4]> keys_squeeze_mask_0 = const()[name = string("keys_squeeze_mask_0"), val = tensor<bool, [4]>([true, false, false, false])];
            tensor<fp16, [2, 2048, 256]> keys_cast_fp16 = slice_by_index(begin = keys_begin_0, end = keys_end_0, end_mask = keys_end_mask_0, squeeze_mask = keys_squeeze_mask_0, x = var_5429_cast_fp16)[name = string("keys_cast_fp16")];
            tensor<int32, [4]> values_begin_0 = const()[name = string("values_begin_0"), val = tensor<int32, [4]>([1, 0, 0, 0])];
            tensor<int32, [4]> values_end_0 = const()[name = string("values_end_0"), val = tensor<int32, [4]>([2, 2, 2048, 256])];
            tensor<bool, [4]> values_end_mask_0 = const()[name = string("values_end_mask_0"), val = tensor<bool, [4]>([false, true, true, true])];
            tensor<bool, [4]> values_squeeze_mask_0 = const()[name = string("values_squeeze_mask_0"), val = tensor<bool, [4]>([true, false, false, false])];
            tensor<fp16, [2, 2048, 256]> values_cast_fp16 = slice_by_index(begin = values_begin_0, end = values_end_0, end_mask = values_end_mask_0, squeeze_mask = values_squeeze_mask_0, x = var_5429_cast_fp16)[name = string("values_cast_fp16")];
            int32 var_5446 = const()[name = string("op_5446"), val = int32(1)];
            tensor<int32, [4]> var_5453 = const()[name = string("op_5453"), val = tensor<int32, [4]>([1, 2048, 1, 4])];
            tensor<fp16, [2, 4, 256, 4]> transpose_6_cast_fp16 = transpose(perm = transpose_6_perm_0, x = var_5347_cast_fp16)[name = string("transpose_29")];
            tensor<fp16, [1, 2048, 1, 4]> var_5454_cast_fp16 = reshape(shape = var_5453, x = transpose_6_cast_fp16)[name = string("op_5454_cast_fp16")];
            tensor<int32, [3]> var_5455 = const()[name = string("op_5455"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [4]> var_5458 = const()[name = string("op_5458"), val = tensor<int32, [4]>([1, 512, 1, 2048])];
            tensor<fp16, [2, 256, 2048]> var_5456_cast_fp16 = transpose(perm = var_5455, x = keys_cast_fp16)[name = string("transpose_28")];
            tensor<fp16, [1, 512, 1, 2048]> key_native_cast_fp16 = reshape(shape = var_5458, x = var_5456_cast_fp16)[name = string("key_native_cast_fp16")];
            tensor<int32, [3]> var_5460 = const()[name = string("op_5460"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [4]> var_5463 = const()[name = string("op_5463"), val = tensor<int32, [4]>([1, 512, 1, 2048])];
            tensor<fp16, [2, 256, 2048]> var_5461_cast_fp16 = transpose(perm = var_5460, x = values_cast_fp16)[name = string("transpose_27")];
            tensor<fp16, [1, 512, 1, 2048]> var_5464_cast_fp16 = reshape(shape = var_5463, x = var_5461_cast_fp16)[name = string("op_5464_cast_fp16")];
            tensor<int32, [8]> tile_57 = const()[name = string("tile_57"), val = tensor<int32, [8]>([256, 256, 256, 256, 256, 256, 256, 256])];
            int32 var_5468_axis_0 = const()[name = string("op_5468_axis_0"), val = int32(1)];
            tensor<fp16, [1, 256, 1, 4]> var_5468_cast_fp16_0, tensor<fp16, [1, 256, 1, 4]> var_5468_cast_fp16_1, tensor<fp16, [1, 256, 1, 4]> var_5468_cast_fp16_2, tensor<fp16, [1, 256, 1, 4]> var_5468_cast_fp16_3, tensor<fp16, [1, 256, 1, 4]> var_5468_cast_fp16_4, tensor<fp16, [1, 256, 1, 4]> var_5468_cast_fp16_5, tensor<fp16, [1, 256, 1, 4]> var_5468_cast_fp16_6, tensor<fp16, [1, 256, 1, 4]> var_5468_cast_fp16_7 = split(axis = var_5468_axis_0, split_sizes = tile_57, x = var_5454_cast_fp16)[name = string("op_5468_cast_fp16")];
            tensor<int32, [4]> var_5477_perm_0 = const()[name = string("op_5477_perm_0"), val = tensor<int32, [4]>([0, 3, 2, 1])];
            tensor<int32, [2]> tile_58 = const()[name = string("tile_58"), val = tensor<int32, [2]>([256, 256])];
            int32 var_5478_axis_0 = const()[name = string("op_5478_axis_0"), val = int32(3)];
            tensor<fp16, [1, 2048, 1, 512]> var_5477_cast_fp16 = transpose(perm = var_5477_perm_0, x = key_native_cast_fp16)[name = string("transpose_26")];
            tensor<fp16, [1, 2048, 1, 256]> var_5478_cast_fp16_0, tensor<fp16, [1, 2048, 1, 256]> var_5478_cast_fp16_1 = split(axis = var_5478_axis_0, split_sizes = tile_58, x = var_5477_cast_fp16)[name = string("op_5478_cast_fp16")];
            tensor<int32, [2]> tile_59 = const()[name = string("tile_59"), val = tensor<int32, [2]>([256, 256])];
            int32 var_5481_axis_0 = const()[name = string("op_5481_axis_0"), val = int32(1)];
            tensor<fp16, [1, 256, 1, 2048]> var_5481_cast_fp16_0, tensor<fp16, [1, 256, 1, 2048]> var_5481_cast_fp16_1 = split(axis = var_5481_axis_0, split_sizes = tile_59, x = var_5464_cast_fp16)[name = string("op_5481_cast_fp16")];
            string scores_49_equation_0 = const()[name = string("scores_49_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 4]> scores_49_cast_fp16 = einsum(equation = scores_49_equation_0, values = (var_5478_cast_fp16_0, var_5468_cast_fp16_0))[name = string("scores_49_cast_fp16")];
            fp16 var_5486_to_fp16 = const()[name = string("op_5486_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 4]> var_5487_cast_fp16 = mul(x = scores_49_cast_fp16, y = var_5486_to_fp16)[name = string("op_5487_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_5488_cast_fp16 = add(x = var_5487_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_5488_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_5489_cast_fp16 = softmax(axis = var_5446, x = var_5488_cast_fp16)[name = string("op_5489_cast_fp16")];
            tensor<int32, [4]> tile_60 = const()[name = string("tile_60"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_5490_axis_0 = const()[name = string("op_5490_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 4]> var_5490_cast_fp16_0, tensor<fp16, [1, 512, 1, 4]> var_5490_cast_fp16_1, tensor<fp16, [1, 512, 1, 4]> var_5490_cast_fp16_2, tensor<fp16, [1, 512, 1, 4]> var_5490_cast_fp16_3 = split(axis = var_5490_axis_0, split_sizes = tile_60, x = var_5489_cast_fp16)[name = string("op_5490_cast_fp16")];
            tensor<int32, [4]> tile_61 = const()[name = string("tile_61"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_5495_axis_0 = const()[name = string("op_5495_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_5495_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_5495_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_5495_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_5495_cast_fp16_3 = split(axis = var_5495_axis_0, split_sizes = tile_61, x = var_5481_cast_fp16_0)[name = string("op_5495_cast_fp16")];
            fp16 var_5500_to_fp16 = const()[name = string("op_5500_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_5501_cast_fp16 = mul(x = var_5490_cast_fp16_0, y = var_5500_to_fp16)[name = string("op_5501_cast_fp16")];
            string var_5503_equation_0 = const()[name = string("op_5503_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_5503_cast_fp16 = einsum(equation = var_5503_equation_0, values = (var_5495_cast_fp16_0, var_5501_cast_fp16))[name = string("op_5503_cast_fp16")];
            fp16 var_5504_to_fp16 = const()[name = string("op_5504_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_5505_cast_fp16 = mul(x = var_5490_cast_fp16_1, y = var_5504_to_fp16)[name = string("op_5505_cast_fp16")];
            string var_5507_equation_0 = const()[name = string("op_5507_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_5507_cast_fp16 = einsum(equation = var_5507_equation_0, values = (var_5495_cast_fp16_1, var_5505_cast_fp16))[name = string("op_5507_cast_fp16")];
            fp16 var_5508_to_fp16 = const()[name = string("op_5508_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_5509_cast_fp16 = mul(x = var_5490_cast_fp16_2, y = var_5508_to_fp16)[name = string("op_5509_cast_fp16")];
            string var_5511_equation_0 = const()[name = string("op_5511_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_5511_cast_fp16 = einsum(equation = var_5511_equation_0, values = (var_5495_cast_fp16_2, var_5509_cast_fp16))[name = string("op_5511_cast_fp16")];
            fp16 var_5512_to_fp16 = const()[name = string("op_5512_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_5513_cast_fp16 = mul(x = var_5490_cast_fp16_3, y = var_5512_to_fp16)[name = string("op_5513_cast_fp16")];
            string var_5515_equation_0 = const()[name = string("op_5515_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_5515_cast_fp16 = einsum(equation = var_5515_equation_0, values = (var_5495_cast_fp16_3, var_5513_cast_fp16))[name = string("op_5515_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_5516_cast_fp16 = add(x = var_5503_cast_fp16, y = var_5507_cast_fp16)[name = string("op_5516_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_5517_cast_fp16 = add(x = var_5511_cast_fp16, y = var_5515_cast_fp16)[name = string("op_5517_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_5518_cast_fp16 = add(x = var_5516_cast_fp16, y = var_5517_cast_fp16)[name = string("op_5518_cast_fp16")];
            fp16 _inversed_5520_y_0_to_fp16 = const()[name = string("_inversed_5520_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 4]> _inversed_5520_cast_fp16 = mul(x = var_5518_cast_fp16, y = _inversed_5520_y_0_to_fp16)[name = string("_inversed_5520_cast_fp16")];
            string scores_51_equation_0 = const()[name = string("scores_51_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 4]> scores_51_cast_fp16 = einsum(equation = scores_51_equation_0, values = (var_5478_cast_fp16_0, var_5468_cast_fp16_1))[name = string("scores_51_cast_fp16")];
            fp16 var_5523_to_fp16 = const()[name = string("op_5523_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 4]> var_5524_cast_fp16 = mul(x = scores_51_cast_fp16, y = var_5523_to_fp16)[name = string("op_5524_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_5525_cast_fp16 = add(x = var_5524_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_5525_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_5526_cast_fp16 = softmax(axis = var_5446, x = var_5525_cast_fp16)[name = string("op_5526_cast_fp16")];
            tensor<int32, [4]> tile_62 = const()[name = string("tile_62"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_5527_axis_0 = const()[name = string("op_5527_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 4]> var_5527_cast_fp16_0, tensor<fp16, [1, 512, 1, 4]> var_5527_cast_fp16_1, tensor<fp16, [1, 512, 1, 4]> var_5527_cast_fp16_2, tensor<fp16, [1, 512, 1, 4]> var_5527_cast_fp16_3 = split(axis = var_5527_axis_0, split_sizes = tile_62, x = var_5526_cast_fp16)[name = string("op_5527_cast_fp16")];
            tensor<int32, [4]> tile_63 = const()[name = string("tile_63"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_5532_axis_0 = const()[name = string("op_5532_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_5532_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_5532_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_5532_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_5532_cast_fp16_3 = split(axis = var_5532_axis_0, split_sizes = tile_63, x = var_5481_cast_fp16_0)[name = string("op_5532_cast_fp16")];
            fp16 var_5537_to_fp16 = const()[name = string("op_5537_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_5538_cast_fp16 = mul(x = var_5527_cast_fp16_0, y = var_5537_to_fp16)[name = string("op_5538_cast_fp16")];
            string var_5540_equation_0 = const()[name = string("op_5540_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_5540_cast_fp16 = einsum(equation = var_5540_equation_0, values = (var_5532_cast_fp16_0, var_5538_cast_fp16))[name = string("op_5540_cast_fp16")];
            fp16 var_5541_to_fp16 = const()[name = string("op_5541_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_5542_cast_fp16 = mul(x = var_5527_cast_fp16_1, y = var_5541_to_fp16)[name = string("op_5542_cast_fp16")];
            string var_5544_equation_0 = const()[name = string("op_5544_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_5544_cast_fp16 = einsum(equation = var_5544_equation_0, values = (var_5532_cast_fp16_1, var_5542_cast_fp16))[name = string("op_5544_cast_fp16")];
            fp16 var_5545_to_fp16 = const()[name = string("op_5545_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_5546_cast_fp16 = mul(x = var_5527_cast_fp16_2, y = var_5545_to_fp16)[name = string("op_5546_cast_fp16")];
            string var_5548_equation_0 = const()[name = string("op_5548_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_5548_cast_fp16 = einsum(equation = var_5548_equation_0, values = (var_5532_cast_fp16_2, var_5546_cast_fp16))[name = string("op_5548_cast_fp16")];
            fp16 var_5549_to_fp16 = const()[name = string("op_5549_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_5550_cast_fp16 = mul(x = var_5527_cast_fp16_3, y = var_5549_to_fp16)[name = string("op_5550_cast_fp16")];
            string var_5552_equation_0 = const()[name = string("op_5552_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_5552_cast_fp16 = einsum(equation = var_5552_equation_0, values = (var_5532_cast_fp16_3, var_5550_cast_fp16))[name = string("op_5552_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_5553_cast_fp16 = add(x = var_5540_cast_fp16, y = var_5544_cast_fp16)[name = string("op_5553_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_5554_cast_fp16 = add(x = var_5548_cast_fp16, y = var_5552_cast_fp16)[name = string("op_5554_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_5555_cast_fp16 = add(x = var_5553_cast_fp16, y = var_5554_cast_fp16)[name = string("op_5555_cast_fp16")];
            fp16 _inversed_5557_y_0_to_fp16 = const()[name = string("_inversed_5557_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 4]> _inversed_5557_cast_fp16 = mul(x = var_5555_cast_fp16, y = _inversed_5557_y_0_to_fp16)[name = string("_inversed_5557_cast_fp16")];
            string scores_53_equation_0 = const()[name = string("scores_53_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 4]> scores_53_cast_fp16 = einsum(equation = scores_53_equation_0, values = (var_5478_cast_fp16_0, var_5468_cast_fp16_2))[name = string("scores_53_cast_fp16")];
            fp16 var_5560_to_fp16 = const()[name = string("op_5560_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 4]> var_5561_cast_fp16 = mul(x = scores_53_cast_fp16, y = var_5560_to_fp16)[name = string("op_5561_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_5562_cast_fp16 = add(x = var_5561_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_5562_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_5563_cast_fp16 = softmax(axis = var_5446, x = var_5562_cast_fp16)[name = string("op_5563_cast_fp16")];
            tensor<int32, [4]> tile_64 = const()[name = string("tile_64"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_5564_axis_0 = const()[name = string("op_5564_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 4]> var_5564_cast_fp16_0, tensor<fp16, [1, 512, 1, 4]> var_5564_cast_fp16_1, tensor<fp16, [1, 512, 1, 4]> var_5564_cast_fp16_2, tensor<fp16, [1, 512, 1, 4]> var_5564_cast_fp16_3 = split(axis = var_5564_axis_0, split_sizes = tile_64, x = var_5563_cast_fp16)[name = string("op_5564_cast_fp16")];
            tensor<int32, [4]> tile_65 = const()[name = string("tile_65"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_5569_axis_0 = const()[name = string("op_5569_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_5569_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_5569_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_5569_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_5569_cast_fp16_3 = split(axis = var_5569_axis_0, split_sizes = tile_65, x = var_5481_cast_fp16_0)[name = string("op_5569_cast_fp16")];
            fp16 var_5574_to_fp16 = const()[name = string("op_5574_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_5575_cast_fp16 = mul(x = var_5564_cast_fp16_0, y = var_5574_to_fp16)[name = string("op_5575_cast_fp16")];
            string var_5577_equation_0 = const()[name = string("op_5577_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_5577_cast_fp16 = einsum(equation = var_5577_equation_0, values = (var_5569_cast_fp16_0, var_5575_cast_fp16))[name = string("op_5577_cast_fp16")];
            fp16 var_5578_to_fp16 = const()[name = string("op_5578_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_5579_cast_fp16 = mul(x = var_5564_cast_fp16_1, y = var_5578_to_fp16)[name = string("op_5579_cast_fp16")];
            string var_5581_equation_0 = const()[name = string("op_5581_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_5581_cast_fp16 = einsum(equation = var_5581_equation_0, values = (var_5569_cast_fp16_1, var_5579_cast_fp16))[name = string("op_5581_cast_fp16")];
            fp16 var_5582_to_fp16 = const()[name = string("op_5582_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_5583_cast_fp16 = mul(x = var_5564_cast_fp16_2, y = var_5582_to_fp16)[name = string("op_5583_cast_fp16")];
            string var_5585_equation_0 = const()[name = string("op_5585_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_5585_cast_fp16 = einsum(equation = var_5585_equation_0, values = (var_5569_cast_fp16_2, var_5583_cast_fp16))[name = string("op_5585_cast_fp16")];
            fp16 var_5586_to_fp16 = const()[name = string("op_5586_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_5587_cast_fp16 = mul(x = var_5564_cast_fp16_3, y = var_5586_to_fp16)[name = string("op_5587_cast_fp16")];
            string var_5589_equation_0 = const()[name = string("op_5589_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_5589_cast_fp16 = einsum(equation = var_5589_equation_0, values = (var_5569_cast_fp16_3, var_5587_cast_fp16))[name = string("op_5589_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_5590_cast_fp16 = add(x = var_5577_cast_fp16, y = var_5581_cast_fp16)[name = string("op_5590_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_5591_cast_fp16 = add(x = var_5585_cast_fp16, y = var_5589_cast_fp16)[name = string("op_5591_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_5592_cast_fp16 = add(x = var_5590_cast_fp16, y = var_5591_cast_fp16)[name = string("op_5592_cast_fp16")];
            fp16 _inversed_5594_y_0_to_fp16 = const()[name = string("_inversed_5594_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 4]> _inversed_5594_cast_fp16 = mul(x = var_5592_cast_fp16, y = _inversed_5594_y_0_to_fp16)[name = string("_inversed_5594_cast_fp16")];
            string scores_55_equation_0 = const()[name = string("scores_55_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 4]> scores_55_cast_fp16 = einsum(equation = scores_55_equation_0, values = (var_5478_cast_fp16_0, var_5468_cast_fp16_3))[name = string("scores_55_cast_fp16")];
            fp16 var_5597_to_fp16 = const()[name = string("op_5597_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 4]> var_5598_cast_fp16 = mul(x = scores_55_cast_fp16, y = var_5597_to_fp16)[name = string("op_5598_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_5599_cast_fp16 = add(x = var_5598_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_5599_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_5600_cast_fp16 = softmax(axis = var_5446, x = var_5599_cast_fp16)[name = string("op_5600_cast_fp16")];
            tensor<int32, [4]> tile_66 = const()[name = string("tile_66"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_5601_axis_0 = const()[name = string("op_5601_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 4]> var_5601_cast_fp16_0, tensor<fp16, [1, 512, 1, 4]> var_5601_cast_fp16_1, tensor<fp16, [1, 512, 1, 4]> var_5601_cast_fp16_2, tensor<fp16, [1, 512, 1, 4]> var_5601_cast_fp16_3 = split(axis = var_5601_axis_0, split_sizes = tile_66, x = var_5600_cast_fp16)[name = string("op_5601_cast_fp16")];
            tensor<int32, [4]> tile_67 = const()[name = string("tile_67"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_5606_axis_0 = const()[name = string("op_5606_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_5606_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_5606_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_5606_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_5606_cast_fp16_3 = split(axis = var_5606_axis_0, split_sizes = tile_67, x = var_5481_cast_fp16_0)[name = string("op_5606_cast_fp16")];
            fp16 var_5611_to_fp16 = const()[name = string("op_5611_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_5612_cast_fp16 = mul(x = var_5601_cast_fp16_0, y = var_5611_to_fp16)[name = string("op_5612_cast_fp16")];
            string var_5614_equation_0 = const()[name = string("op_5614_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_5614_cast_fp16 = einsum(equation = var_5614_equation_0, values = (var_5606_cast_fp16_0, var_5612_cast_fp16))[name = string("op_5614_cast_fp16")];
            fp16 var_5615_to_fp16 = const()[name = string("op_5615_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_5616_cast_fp16 = mul(x = var_5601_cast_fp16_1, y = var_5615_to_fp16)[name = string("op_5616_cast_fp16")];
            string var_5618_equation_0 = const()[name = string("op_5618_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_5618_cast_fp16 = einsum(equation = var_5618_equation_0, values = (var_5606_cast_fp16_1, var_5616_cast_fp16))[name = string("op_5618_cast_fp16")];
            fp16 var_5619_to_fp16 = const()[name = string("op_5619_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_5620_cast_fp16 = mul(x = var_5601_cast_fp16_2, y = var_5619_to_fp16)[name = string("op_5620_cast_fp16")];
            string var_5622_equation_0 = const()[name = string("op_5622_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_5622_cast_fp16 = einsum(equation = var_5622_equation_0, values = (var_5606_cast_fp16_2, var_5620_cast_fp16))[name = string("op_5622_cast_fp16")];
            fp16 var_5623_to_fp16 = const()[name = string("op_5623_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_5624_cast_fp16 = mul(x = var_5601_cast_fp16_3, y = var_5623_to_fp16)[name = string("op_5624_cast_fp16")];
            string var_5626_equation_0 = const()[name = string("op_5626_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_5626_cast_fp16 = einsum(equation = var_5626_equation_0, values = (var_5606_cast_fp16_3, var_5624_cast_fp16))[name = string("op_5626_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_5627_cast_fp16 = add(x = var_5614_cast_fp16, y = var_5618_cast_fp16)[name = string("op_5627_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_5628_cast_fp16 = add(x = var_5622_cast_fp16, y = var_5626_cast_fp16)[name = string("op_5628_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_5629_cast_fp16 = add(x = var_5627_cast_fp16, y = var_5628_cast_fp16)[name = string("op_5629_cast_fp16")];
            fp16 _inversed_5631_y_0_to_fp16 = const()[name = string("_inversed_5631_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 4]> _inversed_5631_cast_fp16 = mul(x = var_5629_cast_fp16, y = _inversed_5631_y_0_to_fp16)[name = string("_inversed_5631_cast_fp16")];
            string scores_57_equation_0 = const()[name = string("scores_57_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 4]> scores_57_cast_fp16 = einsum(equation = scores_57_equation_0, values = (var_5478_cast_fp16_1, var_5468_cast_fp16_4))[name = string("scores_57_cast_fp16")];
            fp16 var_5634_to_fp16 = const()[name = string("op_5634_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 4]> var_5635_cast_fp16 = mul(x = scores_57_cast_fp16, y = var_5634_to_fp16)[name = string("op_5635_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_5636_cast_fp16 = add(x = var_5635_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_5636_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_5637_cast_fp16 = softmax(axis = var_5446, x = var_5636_cast_fp16)[name = string("op_5637_cast_fp16")];
            tensor<int32, [4]> tile_68 = const()[name = string("tile_68"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_5638_axis_0 = const()[name = string("op_5638_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 4]> var_5638_cast_fp16_0, tensor<fp16, [1, 512, 1, 4]> var_5638_cast_fp16_1, tensor<fp16, [1, 512, 1, 4]> var_5638_cast_fp16_2, tensor<fp16, [1, 512, 1, 4]> var_5638_cast_fp16_3 = split(axis = var_5638_axis_0, split_sizes = tile_68, x = var_5637_cast_fp16)[name = string("op_5638_cast_fp16")];
            tensor<int32, [4]> tile_69 = const()[name = string("tile_69"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_5643_axis_0 = const()[name = string("op_5643_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_5643_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_5643_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_5643_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_5643_cast_fp16_3 = split(axis = var_5643_axis_0, split_sizes = tile_69, x = var_5481_cast_fp16_1)[name = string("op_5643_cast_fp16")];
            fp16 var_5648_to_fp16 = const()[name = string("op_5648_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_5649_cast_fp16 = mul(x = var_5638_cast_fp16_0, y = var_5648_to_fp16)[name = string("op_5649_cast_fp16")];
            string var_5651_equation_0 = const()[name = string("op_5651_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_5651_cast_fp16 = einsum(equation = var_5651_equation_0, values = (var_5643_cast_fp16_0, var_5649_cast_fp16))[name = string("op_5651_cast_fp16")];
            fp16 var_5652_to_fp16 = const()[name = string("op_5652_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_5653_cast_fp16 = mul(x = var_5638_cast_fp16_1, y = var_5652_to_fp16)[name = string("op_5653_cast_fp16")];
            string var_5655_equation_0 = const()[name = string("op_5655_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_5655_cast_fp16 = einsum(equation = var_5655_equation_0, values = (var_5643_cast_fp16_1, var_5653_cast_fp16))[name = string("op_5655_cast_fp16")];
            fp16 var_5656_to_fp16 = const()[name = string("op_5656_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_5657_cast_fp16 = mul(x = var_5638_cast_fp16_2, y = var_5656_to_fp16)[name = string("op_5657_cast_fp16")];
            string var_5659_equation_0 = const()[name = string("op_5659_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_5659_cast_fp16 = einsum(equation = var_5659_equation_0, values = (var_5643_cast_fp16_2, var_5657_cast_fp16))[name = string("op_5659_cast_fp16")];
            fp16 var_5660_to_fp16 = const()[name = string("op_5660_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_5661_cast_fp16 = mul(x = var_5638_cast_fp16_3, y = var_5660_to_fp16)[name = string("op_5661_cast_fp16")];
            string var_5663_equation_0 = const()[name = string("op_5663_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_5663_cast_fp16 = einsum(equation = var_5663_equation_0, values = (var_5643_cast_fp16_3, var_5661_cast_fp16))[name = string("op_5663_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_5664_cast_fp16 = add(x = var_5651_cast_fp16, y = var_5655_cast_fp16)[name = string("op_5664_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_5665_cast_fp16 = add(x = var_5659_cast_fp16, y = var_5663_cast_fp16)[name = string("op_5665_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_5666_cast_fp16 = add(x = var_5664_cast_fp16, y = var_5665_cast_fp16)[name = string("op_5666_cast_fp16")];
            fp16 _inversed_5668_y_0_to_fp16 = const()[name = string("_inversed_5668_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 4]> _inversed_5668_cast_fp16 = mul(x = var_5666_cast_fp16, y = _inversed_5668_y_0_to_fp16)[name = string("_inversed_5668_cast_fp16")];
            string scores_59_equation_0 = const()[name = string("scores_59_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 4]> scores_59_cast_fp16 = einsum(equation = scores_59_equation_0, values = (var_5478_cast_fp16_1, var_5468_cast_fp16_5))[name = string("scores_59_cast_fp16")];
            fp16 var_5671_to_fp16 = const()[name = string("op_5671_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 4]> var_5672_cast_fp16 = mul(x = scores_59_cast_fp16, y = var_5671_to_fp16)[name = string("op_5672_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_5673_cast_fp16 = add(x = var_5672_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_5673_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_5674_cast_fp16 = softmax(axis = var_5446, x = var_5673_cast_fp16)[name = string("op_5674_cast_fp16")];
            tensor<int32, [4]> tile_70 = const()[name = string("tile_70"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_5675_axis_0 = const()[name = string("op_5675_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 4]> var_5675_cast_fp16_0, tensor<fp16, [1, 512, 1, 4]> var_5675_cast_fp16_1, tensor<fp16, [1, 512, 1, 4]> var_5675_cast_fp16_2, tensor<fp16, [1, 512, 1, 4]> var_5675_cast_fp16_3 = split(axis = var_5675_axis_0, split_sizes = tile_70, x = var_5674_cast_fp16)[name = string("op_5675_cast_fp16")];
            tensor<int32, [4]> tile_71 = const()[name = string("tile_71"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_5680_axis_0 = const()[name = string("op_5680_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_5680_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_5680_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_5680_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_5680_cast_fp16_3 = split(axis = var_5680_axis_0, split_sizes = tile_71, x = var_5481_cast_fp16_1)[name = string("op_5680_cast_fp16")];
            fp16 var_5685_to_fp16 = const()[name = string("op_5685_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_5686_cast_fp16 = mul(x = var_5675_cast_fp16_0, y = var_5685_to_fp16)[name = string("op_5686_cast_fp16")];
            string var_5688_equation_0 = const()[name = string("op_5688_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_5688_cast_fp16 = einsum(equation = var_5688_equation_0, values = (var_5680_cast_fp16_0, var_5686_cast_fp16))[name = string("op_5688_cast_fp16")];
            fp16 var_5689_to_fp16 = const()[name = string("op_5689_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_5690_cast_fp16 = mul(x = var_5675_cast_fp16_1, y = var_5689_to_fp16)[name = string("op_5690_cast_fp16")];
            string var_5692_equation_0 = const()[name = string("op_5692_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_5692_cast_fp16 = einsum(equation = var_5692_equation_0, values = (var_5680_cast_fp16_1, var_5690_cast_fp16))[name = string("op_5692_cast_fp16")];
            fp16 var_5693_to_fp16 = const()[name = string("op_5693_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_5694_cast_fp16 = mul(x = var_5675_cast_fp16_2, y = var_5693_to_fp16)[name = string("op_5694_cast_fp16")];
            string var_5696_equation_0 = const()[name = string("op_5696_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_5696_cast_fp16 = einsum(equation = var_5696_equation_0, values = (var_5680_cast_fp16_2, var_5694_cast_fp16))[name = string("op_5696_cast_fp16")];
            fp16 var_5697_to_fp16 = const()[name = string("op_5697_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_5698_cast_fp16 = mul(x = var_5675_cast_fp16_3, y = var_5697_to_fp16)[name = string("op_5698_cast_fp16")];
            string var_5700_equation_0 = const()[name = string("op_5700_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_5700_cast_fp16 = einsum(equation = var_5700_equation_0, values = (var_5680_cast_fp16_3, var_5698_cast_fp16))[name = string("op_5700_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_5701_cast_fp16 = add(x = var_5688_cast_fp16, y = var_5692_cast_fp16)[name = string("op_5701_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_5702_cast_fp16 = add(x = var_5696_cast_fp16, y = var_5700_cast_fp16)[name = string("op_5702_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_5703_cast_fp16 = add(x = var_5701_cast_fp16, y = var_5702_cast_fp16)[name = string("op_5703_cast_fp16")];
            fp16 _inversed_5705_y_0_to_fp16 = const()[name = string("_inversed_5705_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 4]> _inversed_5705_cast_fp16 = mul(x = var_5703_cast_fp16, y = _inversed_5705_y_0_to_fp16)[name = string("_inversed_5705_cast_fp16")];
            string scores_61_equation_0 = const()[name = string("scores_61_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 4]> scores_61_cast_fp16 = einsum(equation = scores_61_equation_0, values = (var_5478_cast_fp16_1, var_5468_cast_fp16_6))[name = string("scores_61_cast_fp16")];
            fp16 var_5708_to_fp16 = const()[name = string("op_5708_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 4]> var_5709_cast_fp16 = mul(x = scores_61_cast_fp16, y = var_5708_to_fp16)[name = string("op_5709_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_5710_cast_fp16 = add(x = var_5709_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_5710_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_5711_cast_fp16 = softmax(axis = var_5446, x = var_5710_cast_fp16)[name = string("op_5711_cast_fp16")];
            tensor<int32, [4]> tile_72 = const()[name = string("tile_72"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_5712_axis_0 = const()[name = string("op_5712_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 4]> var_5712_cast_fp16_0, tensor<fp16, [1, 512, 1, 4]> var_5712_cast_fp16_1, tensor<fp16, [1, 512, 1, 4]> var_5712_cast_fp16_2, tensor<fp16, [1, 512, 1, 4]> var_5712_cast_fp16_3 = split(axis = var_5712_axis_0, split_sizes = tile_72, x = var_5711_cast_fp16)[name = string("op_5712_cast_fp16")];
            tensor<int32, [4]> tile_73 = const()[name = string("tile_73"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_5717_axis_0 = const()[name = string("op_5717_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_5717_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_5717_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_5717_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_5717_cast_fp16_3 = split(axis = var_5717_axis_0, split_sizes = tile_73, x = var_5481_cast_fp16_1)[name = string("op_5717_cast_fp16")];
            fp16 var_5722_to_fp16 = const()[name = string("op_5722_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_5723_cast_fp16 = mul(x = var_5712_cast_fp16_0, y = var_5722_to_fp16)[name = string("op_5723_cast_fp16")];
            string var_5725_equation_0 = const()[name = string("op_5725_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_5725_cast_fp16 = einsum(equation = var_5725_equation_0, values = (var_5717_cast_fp16_0, var_5723_cast_fp16))[name = string("op_5725_cast_fp16")];
            fp16 var_5726_to_fp16 = const()[name = string("op_5726_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_5727_cast_fp16 = mul(x = var_5712_cast_fp16_1, y = var_5726_to_fp16)[name = string("op_5727_cast_fp16")];
            string var_5729_equation_0 = const()[name = string("op_5729_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_5729_cast_fp16 = einsum(equation = var_5729_equation_0, values = (var_5717_cast_fp16_1, var_5727_cast_fp16))[name = string("op_5729_cast_fp16")];
            fp16 var_5730_to_fp16 = const()[name = string("op_5730_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_5731_cast_fp16 = mul(x = var_5712_cast_fp16_2, y = var_5730_to_fp16)[name = string("op_5731_cast_fp16")];
            string var_5733_equation_0 = const()[name = string("op_5733_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_5733_cast_fp16 = einsum(equation = var_5733_equation_0, values = (var_5717_cast_fp16_2, var_5731_cast_fp16))[name = string("op_5733_cast_fp16")];
            fp16 var_5734_to_fp16 = const()[name = string("op_5734_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_5735_cast_fp16 = mul(x = var_5712_cast_fp16_3, y = var_5734_to_fp16)[name = string("op_5735_cast_fp16")];
            string var_5737_equation_0 = const()[name = string("op_5737_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_5737_cast_fp16 = einsum(equation = var_5737_equation_0, values = (var_5717_cast_fp16_3, var_5735_cast_fp16))[name = string("op_5737_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_5738_cast_fp16 = add(x = var_5725_cast_fp16, y = var_5729_cast_fp16)[name = string("op_5738_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_5739_cast_fp16 = add(x = var_5733_cast_fp16, y = var_5737_cast_fp16)[name = string("op_5739_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_5740_cast_fp16 = add(x = var_5738_cast_fp16, y = var_5739_cast_fp16)[name = string("op_5740_cast_fp16")];
            fp16 _inversed_5742_y_0_to_fp16 = const()[name = string("_inversed_5742_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 4]> _inversed_5742_cast_fp16 = mul(x = var_5740_cast_fp16, y = _inversed_5742_y_0_to_fp16)[name = string("_inversed_5742_cast_fp16")];
            string scores_equation_0 = const()[name = string("scores_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 4]> scores_cast_fp16 = einsum(equation = scores_equation_0, values = (var_5478_cast_fp16_1, var_5468_cast_fp16_7))[name = string("scores_cast_fp16")];
            fp16 var_5745_to_fp16 = const()[name = string("op_5745_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 4]> var_5746_cast_fp16 = mul(x = scores_cast_fp16, y = var_5745_to_fp16)[name = string("op_5746_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_5747_cast_fp16 = add(x = var_5746_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_5747_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_5748_cast_fp16 = softmax(axis = var_5446, x = var_5747_cast_fp16)[name = string("op_5748_cast_fp16")];
            tensor<int32, [4]> tile_74 = const()[name = string("tile_74"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_5749_axis_0 = const()[name = string("op_5749_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 4]> var_5749_cast_fp16_0, tensor<fp16, [1, 512, 1, 4]> var_5749_cast_fp16_1, tensor<fp16, [1, 512, 1, 4]> var_5749_cast_fp16_2, tensor<fp16, [1, 512, 1, 4]> var_5749_cast_fp16_3 = split(axis = var_5749_axis_0, split_sizes = tile_74, x = var_5748_cast_fp16)[name = string("op_5749_cast_fp16")];
            tensor<int32, [4]> tile_75 = const()[name = string("tile_75"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_5754_axis_0 = const()[name = string("op_5754_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_5754_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_5754_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_5754_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_5754_cast_fp16_3 = split(axis = var_5754_axis_0, split_sizes = tile_75, x = var_5481_cast_fp16_1)[name = string("op_5754_cast_fp16")];
            fp16 var_5759_to_fp16 = const()[name = string("op_5759_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_5760_cast_fp16 = mul(x = var_5749_cast_fp16_0, y = var_5759_to_fp16)[name = string("op_5760_cast_fp16")];
            string var_5762_equation_0 = const()[name = string("op_5762_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_5762_cast_fp16 = einsum(equation = var_5762_equation_0, values = (var_5754_cast_fp16_0, var_5760_cast_fp16))[name = string("op_5762_cast_fp16")];
            fp16 var_5763_to_fp16 = const()[name = string("op_5763_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_5764_cast_fp16 = mul(x = var_5749_cast_fp16_1, y = var_5763_to_fp16)[name = string("op_5764_cast_fp16")];
            string var_5766_equation_0 = const()[name = string("op_5766_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_5766_cast_fp16 = einsum(equation = var_5766_equation_0, values = (var_5754_cast_fp16_1, var_5764_cast_fp16))[name = string("op_5766_cast_fp16")];
            fp16 var_5767_to_fp16 = const()[name = string("op_5767_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_5768_cast_fp16 = mul(x = var_5749_cast_fp16_2, y = var_5767_to_fp16)[name = string("op_5768_cast_fp16")];
            string var_5770_equation_0 = const()[name = string("op_5770_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_5770_cast_fp16 = einsum(equation = var_5770_equation_0, values = (var_5754_cast_fp16_2, var_5768_cast_fp16))[name = string("op_5770_cast_fp16")];
            fp16 var_5771_to_fp16 = const()[name = string("op_5771_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_5772_cast_fp16 = mul(x = var_5749_cast_fp16_3, y = var_5771_to_fp16)[name = string("op_5772_cast_fp16")];
            string var_5774_equation_0 = const()[name = string("op_5774_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_5774_cast_fp16 = einsum(equation = var_5774_equation_0, values = (var_5754_cast_fp16_3, var_5772_cast_fp16))[name = string("op_5774_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_5775_cast_fp16 = add(x = var_5762_cast_fp16, y = var_5766_cast_fp16)[name = string("op_5775_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_5776_cast_fp16 = add(x = var_5770_cast_fp16, y = var_5774_cast_fp16)[name = string("op_5776_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_5777_cast_fp16 = add(x = var_5775_cast_fp16, y = var_5776_cast_fp16)[name = string("op_5777_cast_fp16")];
            fp16 _inversed_5779_y_0_to_fp16 = const()[name = string("_inversed_5779_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 4]> _inversed_5779_cast_fp16 = mul(x = var_5777_cast_fp16, y = _inversed_5779_y_0_to_fp16)[name = string("_inversed_5779_cast_fp16")];
            bool var_5781_interleave_0 = const()[name = string("op_5781_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 2048, 1, 4]> var_5781_cast_fp16 = concat(axis = var_5446, interleave = var_5781_interleave_0, values = (_inversed_5520_cast_fp16, _inversed_5557_cast_fp16, _inversed_5594_cast_fp16, _inversed_5631_cast_fp16, _inversed_5668_cast_fp16, _inversed_5705_cast_fp16, _inversed_5742_cast_fp16, _inversed_5779_cast_fp16))[name = string("op_5781_cast_fp16")];
            tensor<int32, [4]> var_5782 = const()[name = string("op_5782"), val = tensor<int32, [4]>([2, 4, 256, 4])];
            tensor<fp16, [2, 4, 256, 4]> var_5783_cast_fp16 = reshape(shape = var_5782, x = var_5781_cast_fp16)[name = string("op_5783_cast_fp16")];
            tensor<int32, [4]> transpose_7_perm_0 = const()[name = string("transpose_7_perm_0"), val = tensor<int32, [4]>([3, 0, 1, 2])];
            tensor<int32, [2]> var_5802 = const()[name = string("op_5802"), val = tensor<int32, [2]>([4, 2048])];
            tensor<fp16, [4, 2, 4, 256]> transpose_7_cast_fp16 = transpose(perm = transpose_7_perm_0, x = var_5783_cast_fp16)[name = string("transpose_25")];
            tensor<fp16, [4, 2048]> attention_output_cast_fp16 = reshape(shape = var_5802, x = transpose_7_cast_fp16)[name = string("attention_output_cast_fp16")];
            tensor<fp16, [4, 2048]> var_5804_cast_fp16 = sigmoid(x = gate_15_cast_fp16)[name = string("op_5804_cast_fp16")];
            tensor<fp16, [4, 2048]> input_201_cast_fp16 = mul(x = attention_output_cast_fp16, y = var_5804_cast_fp16)[name = string("input_201_cast_fp16")];
            tensor<fp16, [1024, 2048]> completions_3_o_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 2048]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(241976576))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(243549504))))[name = string("completions_3_o_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 1024]> linear_120_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = completions_3_o_proj_weight_promoted_to_fp16_palettized, x = input_201_cast_fp16)[name = string("linear_120_cast_fp16")];
            tensor<fp16, [4, 1024]> value_231_cast_fp16 = add(x = value_217_cast_fp16, y = linear_120_cast_fp16)[name = string("value_231_cast_fp16")];
            tensor<fp16, [4, 1024]> var_5809_cast_fp16 = mul(x = value_231_cast_fp16, y = value_231_cast_fp16)[name = string("op_5809_cast_fp16")];
            tensor<int32, [1]> variance_103_axes_0 = const()[name = string("variance_103_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_103_keep_dims_0 = const()[name = string("variance_103_keep_dims_0"), val = bool(true)];
            tensor<fp16, [4, 1]> variance_103_cast_fp16 = reduce_mean(axes = variance_103_axes_0, keep_dims = variance_103_keep_dims_0, x = var_5809_cast_fp16)[name = string("variance_103_cast_fp16")];
            fp16 var_5812_to_fp16 = const()[name = string("op_5812_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [4, 1]> var_5813_cast_fp16 = add(x = variance_103_cast_fp16, y = var_5812_to_fp16)[name = string("op_5813_cast_fp16")];
            fp32 var_5814_epsilon_0 = const()[name = string("op_5814_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [4, 1]> var_5814_cast_fp16 = rsqrt(epsilon = var_5814_epsilon_0, x = var_5813_cast_fp16)[name = string("op_5814_cast_fp16")];
            tensor<fp16, [4, 1024]> var_5815_cast_fp16 = mul(x = value_231_cast_fp16, y = var_5814_cast_fp16)[name = string("op_5815_cast_fp16")];
            tensor<fp16, [1024]> var_5817_to_fp16 = const()[name = string("op_5817_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(243557760)))];
            tensor<fp16, [4, 1024]> input_203_cast_fp16 = mul(x = var_5815_cast_fp16, y = var_5817_to_fp16)[name = string("input_203_cast_fp16")];
            tensor<fp16, [3584, 1024]> completions_3_gate_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(243559872))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(246312448))))[name = string("completions_3_gate_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 3584]> linear_121_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = completions_3_gate_proj_weight_promoted_to_fp16_palettized, x = input_203_cast_fp16)[name = string("linear_121_cast_fp16")];
            tensor<fp16, [4, 3584]> var_5821_cast_fp16 = silu(x = linear_121_cast_fp16)[name = string("op_5821_cast_fp16")];
            tensor<fp16, [3584, 1024]> completions_3_up_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(246341184))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(249093760))))[name = string("completions_3_up_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 3584]> linear_122_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = completions_3_up_proj_weight_promoted_to_fp16_palettized, x = input_203_cast_fp16)[name = string("linear_122_cast_fp16")];
            tensor<fp16, [4, 3584]> input_207_cast_fp16 = mul(x = var_5821_cast_fp16, y = linear_122_cast_fp16)[name = string("input_207_cast_fp16")];
            tensor<fp16, [1024, 3584]> completions_3_down_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 3584]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(249122496))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(251875072))))[name = string("completions_3_down_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 1024]> linear_123_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = completions_3_down_proj_weight_promoted_to_fp16_palettized, x = input_207_cast_fp16)[name = string("linear_123_cast_fp16")];
            tensor<fp16, [4, 1024]> value_233_cast_fp16 = add(x = value_231_cast_fp16, y = linear_123_cast_fp16)[name = string("value_233_cast_fp16")];
            int32 var_5844 = const()[name = string("op_5844"), val = int32(-1)];
            tensor<fp16, [4, 1024]> var_5859_cast_fp16 = mul(x = value_233_cast_fp16, y = value_233_cast_fp16)[name = string("op_5859_cast_fp16")];
            tensor<int32, [1]> variance_105_axes_0 = const()[name = string("variance_105_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_105_keep_dims_0 = const()[name = string("variance_105_keep_dims_0"), val = bool(true)];
            tensor<fp16, [4, 1]> variance_105_cast_fp16 = reduce_mean(axes = variance_105_axes_0, keep_dims = variance_105_keep_dims_0, x = var_5859_cast_fp16)[name = string("variance_105_cast_fp16")];
            fp16 var_5862_to_fp16 = const()[name = string("op_5862_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [4, 1]> var_5863_cast_fp16 = add(x = variance_105_cast_fp16, y = var_5862_to_fp16)[name = string("op_5863_cast_fp16")];
            fp32 var_5864_epsilon_0 = const()[name = string("op_5864_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [4, 1]> var_5864_cast_fp16 = rsqrt(epsilon = var_5864_epsilon_0, x = var_5863_cast_fp16)[name = string("op_5864_cast_fp16")];
            tensor<fp16, [4, 1024]> var_5865_cast_fp16 = mul(x = value_233_cast_fp16, y = var_5864_cast_fp16)[name = string("op_5865_cast_fp16")];
            tensor<fp16, [1024]> var_5867_to_fp16 = const()[name = string("op_5867_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(251883328)))];
            tensor<fp16, [4, 1024]> input_209_cast_fp16 = mul(x = var_5865_cast_fp16, y = var_5867_to_fp16)[name = string("input_209_cast_fp16")];
            tensor<fp16, [6144, 1024]> final_group_0_in_proj_qkv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(251885440))), lut = tensor<fp16, [384, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(256604096))))[name = string("final_group_0_in_proj_qkv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 6144]> linear_124_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = final_group_0_in_proj_qkv_weight_promoted_to_fp16_palettized, x = input_209_cast_fp16)[name = string("linear_124_cast_fp16")];
            tensor<int32, [2]> var_5871_perm_0 = const()[name = string("op_5871_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> mixed_25_axes_0 = const()[name = string("mixed_25_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [6144, 4]> var_5871_cast_fp16 = transpose(perm = var_5871_perm_0, x = linear_124_cast_fp16)[name = string("transpose_24")];
            tensor<fp16, [1, 6144, 4]> mixed_25_cast_fp16 = expand_dims(axes = mixed_25_axes_0, x = var_5871_cast_fp16)[name = string("mixed_25_cast_fp16")];
            bool convolution_input_25_interleave_0 = const()[name = string("convolution_input_25_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 6144, 7]> convolution_input_25_cast_fp16 = concat(axis = var_5844, interleave = convolution_input_25_interleave_0, values = (conv16, mixed_25_cast_fp16))[name = string("convolution_input_25_cast_fp16")];
            string input_211_pad_type_0 = const()[name = string("input_211_pad_type_0"), val = string("valid")];
            int32 input_211_groups_0 = const()[name = string("input_211_groups_0"), val = int32(6144)];
            tensor<int32, [1]> input_211_strides_0 = const()[name = string("input_211_strides_0"), val = tensor<int32, [1]>([1])];
            tensor<int32, [2]> input_211_pad_0 = const()[name = string("input_211_pad_0"), val = tensor<int32, [2]>([0, 0])];
            tensor<int32, [1]> input_211_dilations_0 = const()[name = string("input_211_dilations_0"), val = tensor<int32, [1]>([1])];
            tensor<fp16, [6144, 1, 4]> final_group_0_conv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1, 4]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(256653312))), lut = tensor<fp16, [384, 1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(256671808))))[name = string("final_group_0_conv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 6144, 4]> input_211_cast_fp16 = conv(dilations = input_211_dilations_0, groups = input_211_groups_0, pad = input_211_pad_0, pad_type = input_211_pad_type_0, strides = input_211_strides_0, weight = final_group_0_conv_weight_promoted_to_fp16_palettized, x = convolution_input_25_cast_fp16)[name = string("input_211_cast_fp16")];
            tensor<fp16, [1, 6144, 4]> var_5880_cast_fp16 = silu(x = input_211_cast_fp16)[name = string("op_5880_cast_fp16")];
            tensor<int32, [3]> var_5881_perm_0 = const()[name = string("op_5881_perm_0"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [3]> var_5884_begin_0 = const()[name = string("op_5884_begin_0"), val = tensor<int32, [3]>([0, 0, 4])];
            tensor<int32, [3]> var_5884_end_0 = const()[name = string("op_5884_end_0"), val = tensor<int32, [3]>([1, 6144, 7])];
            tensor<bool, [3]> var_5884_end_mask_0 = const()[name = string("op_5884_end_mask_0"), val = tensor<bool, [3]>([true, true, true])];
            tensor<fp16, [1, 6144, 3]> conv16_out = slice_by_index(begin = var_5884_begin_0, end = var_5884_end_0, end_mask = var_5884_end_mask_0, x = convolution_input_25_cast_fp16)[name = string("op_5884_cast_fp16")];
            tensor<int32, [3]> var_5885 = const()[name = string("op_5885"), val = tensor<int32, [3]>([2048, 2048, 2048])];
            int32 var_5886_axis_0 = const()[name = string("op_5886_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 4, 6144]> var_5881_cast_fp16 = transpose(perm = var_5881_perm_0, x = var_5880_cast_fp16)[name = string("transpose_23")];
            tensor<fp16, [1, 4, 2048]> var_5886_cast_fp16_0, tensor<fp16, [1, 4, 2048]> var_5886_cast_fp16_1, tensor<fp16, [1, 4, 2048]> var_5886_cast_fp16_2 = split(axis = var_5886_axis_0, split_sizes = var_5885, x = var_5881_cast_fp16)[name = string("op_5886_cast_fp16")];
            tensor<int32, [4]> var_5890 = const()[name = string("op_5890"), val = tensor<int32, [4]>([1, 4, 16, 128])];
            tensor<fp16, [1, 4, 16, 128]> value_237_cast_fp16 = reshape(shape = var_5890, x = var_5886_cast_fp16_0)[name = string("value_237_cast_fp16")];
            tensor<int32, [4]> var_5892 = const()[name = string("op_5892"), val = tensor<int32, [4]>([1, 4, 16, 128])];
            tensor<fp16, [1, 4, 16, 128]> value_239_cast_fp16 = reshape(shape = var_5892, x = var_5886_cast_fp16_1)[name = string("value_239_cast_fp16")];
            tensor<int32, [4]> var_5894 = const()[name = string("op_5894"), val = tensor<int32, [4]>([1, 4, 16, 128])];
            tensor<fp16, [1, 4, 16, 128]> value_241_cast_fp16 = reshape(shape = var_5894, x = var_5886_cast_fp16_2)[name = string("value_241_cast_fp16")];
            tensor<fp16, [1, 4, 16, 128]> var_5896_cast_fp16 = mul(x = value_237_cast_fp16, y = value_237_cast_fp16)[name = string("op_5896_cast_fp16")];
            tensor<int32, [1]> var_5898_axes_0 = const()[name = string("op_5898_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_5898_keep_dims_0 = const()[name = string("op_5898_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 4, 16, 1]> var_5898_cast_fp16 = reduce_sum(axes = var_5898_axes_0, keep_dims = var_5898_keep_dims_0, x = var_5896_cast_fp16)[name = string("op_5898_cast_fp16")];
            fp16 var_5899_to_fp16 = const()[name = string("op_5899_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 4, 16, 1]> var_5900_cast_fp16 = add(x = var_5898_cast_fp16, y = var_5899_to_fp16)[name = string("op_5900_cast_fp16")];
            fp32 var_5901_epsilon_0 = const()[name = string("op_5901_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 4, 16, 1]> var_5901_cast_fp16 = rsqrt(epsilon = var_5901_epsilon_0, x = var_5900_cast_fp16)[name = string("op_5901_cast_fp16")];
            tensor<fp16, [1, 4, 16, 128]> var_5902_cast_fp16 = mul(x = value_237_cast_fp16, y = var_5901_cast_fp16)[name = string("op_5902_cast_fp16")];
            tensor<int32, [4]> var_5903_perm_0 = const()[name = string("op_5903_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            fp16 _inversed_query_67_y_0_to_fp16 = const()[name = string("_inversed_query_67_y_0_to_fp16"), val = fp16(0x1.6ap-4)];
            tensor<fp16, [1, 16, 4, 128]> var_5903_cast_fp16 = transpose(perm = var_5903_perm_0, x = var_5902_cast_fp16)[name = string("transpose_22")];
            tensor<fp16, [1, 16, 4, 128]> _inversed_query_67_cast_fp16 = mul(x = var_5903_cast_fp16, y = _inversed_query_67_y_0_to_fp16)[name = string("_inversed_query_67_cast_fp16")];
            tensor<fp16, [1, 4, 16, 128]> var_5906_cast_fp16 = mul(x = value_239_cast_fp16, y = value_239_cast_fp16)[name = string("op_5906_cast_fp16")];
            tensor<int32, [1]> var_5908_axes_0 = const()[name = string("op_5908_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_5908_keep_dims_0 = const()[name = string("op_5908_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 4, 16, 1]> var_5908_cast_fp16 = reduce_sum(axes = var_5908_axes_0, keep_dims = var_5908_keep_dims_0, x = var_5906_cast_fp16)[name = string("op_5908_cast_fp16")];
            fp16 var_5909_to_fp16 = const()[name = string("op_5909_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 4, 16, 1]> var_5910_cast_fp16 = add(x = var_5908_cast_fp16, y = var_5909_to_fp16)[name = string("op_5910_cast_fp16")];
            fp32 var_5911_epsilon_0 = const()[name = string("op_5911_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 4, 16, 1]> var_5911_cast_fp16 = rsqrt(epsilon = var_5911_epsilon_0, x = var_5910_cast_fp16)[name = string("op_5911_cast_fp16")];
            tensor<fp16, [1, 4, 16, 128]> var_5912_cast_fp16 = mul(x = value_239_cast_fp16, y = var_5911_cast_fp16)[name = string("op_5912_cast_fp16")];
            tensor<int32, [4]> key_67_perm_0 = const()[name = string("key_67_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<int32, [4]> value_243_perm_0 = const()[name = string("value_243_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<fp16, [16, 1024]> final_group_0_in_proj_b_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(256721024))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(256733376))))[name = string("final_group_0_in_proj_b_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16]> linear_125_bias_0_to_fp16 = const()[name = string("linear_125_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(4862656)))];
            tensor<fp16, [4, 16]> linear_125_cast_fp16 = linear(bias = linear_125_bias_0_to_fp16, weight = final_group_0_in_proj_b_weight_promoted_to_fp16_palettized, x = input_209_cast_fp16)[name = string("linear_125_cast_fp16")];
            tensor<fp16, [4, 16]> var_5917_cast_fp16 = sigmoid(x = linear_125_cast_fp16)[name = string("op_5917_cast_fp16")];
            tensor<int32, [2]> var_5918_perm_0 = const()[name = string("op_5918_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> beta_25_axes_0 = const()[name = string("beta_25_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 4]> var_5918_cast_fp16 = transpose(perm = var_5918_perm_0, x = var_5917_cast_fp16)[name = string("transpose_21")];
            tensor<fp16, [1, 16, 4]> beta_25_cast_fp16 = expand_dims(axes = beta_25_axes_0, x = var_5918_cast_fp16)[name = string("beta_25_cast_fp16")];
            tensor<fp16, [16, 1024]> op_5924_weight_0_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(256733568))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(256745920))))[name = string("op_5924_weight_0_to_fp16_palettized")];
            tensor<fp16, [16]> var_5924_bias_0_to_fp16 = const()[name = string("op_5924_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(256746112)))];
            tensor<fp16, [4, 16]> var_5924_cast_fp16 = linear(bias = var_5924_bias_0_to_fp16, weight = op_5924_weight_0_to_fp16_palettized, x = input_209_cast_fp16)[name = string("op_5924_cast_fp16")];
            tensor<fp16, [4, 16]> var_5925_cast_fp16 = softplus(x = var_5924_cast_fp16)[name = string("op_5925_cast_fp16")];
            tensor<fp16, [16]> var_5921_promoted_to_fp16 = const()[name = string("op_5921_promoted_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(256746240)))];
            tensor<fp16, [4, 16]> var_5926_cast_fp16 = mul(x = var_5921_promoted_to_fp16, y = var_5925_cast_fp16)[name = string("op_5926_cast_fp16")];
            tensor<int32, [2]> var_5927_perm_0 = const()[name = string("op_5927_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> decay_25_axes_0 = const()[name = string("decay_25_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 4]> var_5927_cast_fp16 = transpose(perm = var_5927_perm_0, x = var_5926_cast_fp16)[name = string("transpose_20")];
            tensor<fp16, [1, 16, 4]> decay_25_cast_fp16 = expand_dims(axes = decay_25_axes_0, x = var_5927_cast_fp16)[name = string("decay_25_cast_fp16")];
            tensor<fp16, [2048, 1024]> final_group_0_in_proj_z_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [2048, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(256746368))), lut = tensor<fp16, [128, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(258319296))))[name = string("final_group_0_in_proj_z_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 2048]> linear_127_cast_fp16 = linear(bias = linear_3_bias_0_to_fp16, weight = final_group_0_in_proj_z_weight_promoted_to_fp16_palettized, x = input_209_cast_fp16)[name = string("linear_127_cast_fp16")];
            tensor<int32, [3]> var_5935_begin_0 = const()[name = string("op_5935_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_5935_end_0 = const()[name = string("op_5935_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_5935_end_mask_0 = const()[name = string("op_5935_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_5935_squeeze_mask_0 = const()[name = string("op_5935_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_5935_cast_fp16 = slice_by_index(begin = var_5935_begin_0, end = var_5935_end_0, end_mask = var_5935_end_mask_0, squeeze_mask = var_5935_squeeze_mask_0, x = decay_25_cast_fp16)[name = string("op_5935_cast_fp16")];
            tensor<fp16, [1, 16]> var_5936_cast_fp16 = exp(x = var_5935_cast_fp16)[name = string("op_5936_cast_fp16")];
            tensor<int32, [1]> var_5937_axes_0 = const()[name = string("op_5937_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_5937_cast_fp16 = expand_dims(axes = var_5937_axes_0, x = var_5936_cast_fp16)[name = string("op_5937_cast_fp16")];
            tensor<int32, [1]> var_5938_axes_0 = const()[name = string("op_5938_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_5938_cast_fp16 = expand_dims(axes = var_5938_axes_0, x = var_5937_cast_fp16)[name = string("op_5938_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_193_cast_fp16 = mul(x = rec16, y = var_5938_cast_fp16)[name = string("state_193_cast_fp16")];
            tensor<int32, [4]> key_token_97_begin_0 = const()[name = string("key_token_97_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> key_token_97_end_0 = const()[name = string("key_token_97_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> key_token_97_end_mask_0 = const()[name = string("key_token_97_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_97_squeeze_mask_0 = const()[name = string("key_token_97_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 4, 128]> key_67_cast_fp16 = transpose(perm = key_67_perm_0, x = var_5912_cast_fp16)[name = string("transpose_19")];
            tensor<fp16, [1, 16, 128]> key_token_97_cast_fp16 = slice_by_index(begin = key_token_97_begin_0, end = key_token_97_end_0, end_mask = key_token_97_end_mask_0, squeeze_mask = key_token_97_squeeze_mask_0, x = key_67_cast_fp16)[name = string("key_token_97_cast_fp16")];
            tensor<int32, [4]> value_token_97_begin_0 = const()[name = string("value_token_97_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> value_token_97_end_0 = const()[name = string("value_token_97_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> value_token_97_end_mask_0 = const()[name = string("value_token_97_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_97_squeeze_mask_0 = const()[name = string("value_token_97_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 4, 128]> value_243_cast_fp16 = transpose(perm = value_243_perm_0, x = value_241_cast_fp16)[name = string("transpose_18")];
            tensor<fp16, [1, 16, 128]> value_token_97_cast_fp16 = slice_by_index(begin = value_token_97_begin_0, end = value_token_97_end_0, end_mask = value_token_97_end_mask_0, squeeze_mask = value_token_97_squeeze_mask_0, x = value_243_cast_fp16)[name = string("value_token_97_cast_fp16")];
            tensor<int32, [1]> var_5946_axes_0 = const()[name = string("op_5946_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_5946_cast_fp16 = expand_dims(axes = var_5946_axes_0, x = key_token_97_cast_fp16)[name = string("op_5946_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_5947_cast_fp16 = mul(x = state_193_cast_fp16, y = var_5946_cast_fp16)[name = string("op_5947_cast_fp16")];
            tensor<int32, [1]> memory_97_axes_0 = const()[name = string("memory_97_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_97_keep_dims_0 = const()[name = string("memory_97_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_97_cast_fp16 = reduce_sum(axes = memory_97_axes_0, keep_dims = memory_97_keep_dims_0, x = var_5947_cast_fp16)[name = string("memory_97_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_5950_cast_fp16 = sub(x = value_token_97_cast_fp16, y = memory_97_cast_fp16)[name = string("op_5950_cast_fp16")];
            tensor<int32, [3]> var_5953_begin_0 = const()[name = string("op_5953_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_5953_end_0 = const()[name = string("op_5953_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_5953_end_mask_0 = const()[name = string("op_5953_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_5953_squeeze_mask_0 = const()[name = string("op_5953_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_5953_cast_fp16 = slice_by_index(begin = var_5953_begin_0, end = var_5953_end_0, end_mask = var_5953_end_mask_0, squeeze_mask = var_5953_squeeze_mask_0, x = beta_25_cast_fp16)[name = string("op_5953_cast_fp16")];
            tensor<int32, [1]> var_5954_axes_0 = const()[name = string("op_5954_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_5954_cast_fp16 = expand_dims(axes = var_5954_axes_0, x = var_5953_cast_fp16)[name = string("op_5954_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_97_cast_fp16 = mul(x = var_5950_cast_fp16, y = var_5954_cast_fp16)[name = string("delta_97_cast_fp16")];
            tensor<int32, [1]> var_5957_axes_0 = const()[name = string("op_5957_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_5957_cast_fp16 = expand_dims(axes = var_5957_axes_0, x = delta_97_cast_fp16)[name = string("op_5957_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_5958_cast_fp16 = mul(x = var_5946_cast_fp16, y = var_5957_cast_fp16)[name = string("op_5958_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_195_cast_fp16 = add(x = state_193_cast_fp16, y = var_5958_cast_fp16)[name = string("state_195_cast_fp16")];
            tensor<int32, [4]> var_5962_begin_0 = const()[name = string("op_5962_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> var_5962_end_0 = const()[name = string("op_5962_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> var_5962_end_mask_0 = const()[name = string("op_5962_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_5962_squeeze_mask_0 = const()[name = string("op_5962_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_5962_cast_fp16 = slice_by_index(begin = var_5962_begin_0, end = var_5962_end_0, end_mask = var_5962_end_mask_0, squeeze_mask = var_5962_squeeze_mask_0, x = _inversed_query_67_cast_fp16)[name = string("op_5962_cast_fp16")];
            tensor<int32, [1]> var_5963_axes_0 = const()[name = string("op_5963_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_5963_cast_fp16 = expand_dims(axes = var_5963_axes_0, x = var_5962_cast_fp16)[name = string("op_5963_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_5964_cast_fp16 = mul(x = state_195_cast_fp16, y = var_5963_cast_fp16)[name = string("op_5964_cast_fp16")];
            tensor<int32, [1]> var_5966_axes_0 = const()[name = string("op_5966_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_5966_keep_dims_0 = const()[name = string("op_5966_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_5966_cast_fp16 = reduce_sum(axes = var_5966_axes_0, keep_dims = var_5966_keep_dims_0, x = var_5964_cast_fp16)[name = string("op_5966_cast_fp16")];
            tensor<int32, [3]> var_5969_begin_0 = const()[name = string("op_5969_begin_0"), val = tensor<int32, [3]>([0, 0, 1])];
            tensor<int32, [3]> var_5969_end_0 = const()[name = string("op_5969_end_0"), val = tensor<int32, [3]>([1, 16, 2])];
            tensor<bool, [3]> var_5969_end_mask_0 = const()[name = string("op_5969_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_5969_squeeze_mask_0 = const()[name = string("op_5969_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_5969_cast_fp16 = slice_by_index(begin = var_5969_begin_0, end = var_5969_end_0, end_mask = var_5969_end_mask_0, squeeze_mask = var_5969_squeeze_mask_0, x = decay_25_cast_fp16)[name = string("op_5969_cast_fp16")];
            tensor<fp16, [1, 16]> var_5970_cast_fp16 = exp(x = var_5969_cast_fp16)[name = string("op_5970_cast_fp16")];
            tensor<int32, [1]> var_5971_axes_0 = const()[name = string("op_5971_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_5971_cast_fp16 = expand_dims(axes = var_5971_axes_0, x = var_5970_cast_fp16)[name = string("op_5971_cast_fp16")];
            tensor<int32, [1]> var_5972_axes_0 = const()[name = string("op_5972_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_5972_cast_fp16 = expand_dims(axes = var_5972_axes_0, x = var_5971_cast_fp16)[name = string("op_5972_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_197_cast_fp16 = mul(x = state_195_cast_fp16, y = var_5972_cast_fp16)[name = string("state_197_cast_fp16")];
            tensor<int32, [4]> key_token_99_begin_0 = const()[name = string("key_token_99_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> key_token_99_end_0 = const()[name = string("key_token_99_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> key_token_99_end_mask_0 = const()[name = string("key_token_99_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_99_squeeze_mask_0 = const()[name = string("key_token_99_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_99_cast_fp16 = slice_by_index(begin = key_token_99_begin_0, end = key_token_99_end_0, end_mask = key_token_99_end_mask_0, squeeze_mask = key_token_99_squeeze_mask_0, x = key_67_cast_fp16)[name = string("key_token_99_cast_fp16")];
            tensor<int32, [4]> value_token_99_begin_0 = const()[name = string("value_token_99_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> value_token_99_end_0 = const()[name = string("value_token_99_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> value_token_99_end_mask_0 = const()[name = string("value_token_99_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_99_squeeze_mask_0 = const()[name = string("value_token_99_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_99_cast_fp16 = slice_by_index(begin = value_token_99_begin_0, end = value_token_99_end_0, end_mask = value_token_99_end_mask_0, squeeze_mask = value_token_99_squeeze_mask_0, x = value_243_cast_fp16)[name = string("value_token_99_cast_fp16")];
            tensor<int32, [1]> var_5980_axes_0 = const()[name = string("op_5980_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_5980_cast_fp16 = expand_dims(axes = var_5980_axes_0, x = key_token_99_cast_fp16)[name = string("op_5980_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_5981_cast_fp16 = mul(x = state_197_cast_fp16, y = var_5980_cast_fp16)[name = string("op_5981_cast_fp16")];
            tensor<int32, [1]> memory_99_axes_0 = const()[name = string("memory_99_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_99_keep_dims_0 = const()[name = string("memory_99_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_99_cast_fp16 = reduce_sum(axes = memory_99_axes_0, keep_dims = memory_99_keep_dims_0, x = var_5981_cast_fp16)[name = string("memory_99_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_5984_cast_fp16 = sub(x = value_token_99_cast_fp16, y = memory_99_cast_fp16)[name = string("op_5984_cast_fp16")];
            tensor<int32, [3]> var_5987_begin_0 = const()[name = string("op_5987_begin_0"), val = tensor<int32, [3]>([0, 0, 1])];
            tensor<int32, [3]> var_5987_end_0 = const()[name = string("op_5987_end_0"), val = tensor<int32, [3]>([1, 16, 2])];
            tensor<bool, [3]> var_5987_end_mask_0 = const()[name = string("op_5987_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_5987_squeeze_mask_0 = const()[name = string("op_5987_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_5987_cast_fp16 = slice_by_index(begin = var_5987_begin_0, end = var_5987_end_0, end_mask = var_5987_end_mask_0, squeeze_mask = var_5987_squeeze_mask_0, x = beta_25_cast_fp16)[name = string("op_5987_cast_fp16")];
            tensor<int32, [1]> var_5988_axes_0 = const()[name = string("op_5988_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_5988_cast_fp16 = expand_dims(axes = var_5988_axes_0, x = var_5987_cast_fp16)[name = string("op_5988_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_99_cast_fp16 = mul(x = var_5984_cast_fp16, y = var_5988_cast_fp16)[name = string("delta_99_cast_fp16")];
            tensor<int32, [1]> var_5991_axes_0 = const()[name = string("op_5991_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_5991_cast_fp16 = expand_dims(axes = var_5991_axes_0, x = delta_99_cast_fp16)[name = string("op_5991_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_5992_cast_fp16 = mul(x = var_5980_cast_fp16, y = var_5991_cast_fp16)[name = string("op_5992_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_199_cast_fp16 = add(x = state_197_cast_fp16, y = var_5992_cast_fp16)[name = string("state_199_cast_fp16")];
            tensor<int32, [4]> var_5996_begin_0 = const()[name = string("op_5996_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> var_5996_end_0 = const()[name = string("op_5996_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> var_5996_end_mask_0 = const()[name = string("op_5996_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_5996_squeeze_mask_0 = const()[name = string("op_5996_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_5996_cast_fp16 = slice_by_index(begin = var_5996_begin_0, end = var_5996_end_0, end_mask = var_5996_end_mask_0, squeeze_mask = var_5996_squeeze_mask_0, x = _inversed_query_67_cast_fp16)[name = string("op_5996_cast_fp16")];
            tensor<int32, [1]> var_5997_axes_0 = const()[name = string("op_5997_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_5997_cast_fp16 = expand_dims(axes = var_5997_axes_0, x = var_5996_cast_fp16)[name = string("op_5997_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_5998_cast_fp16 = mul(x = state_199_cast_fp16, y = var_5997_cast_fp16)[name = string("op_5998_cast_fp16")];
            tensor<int32, [1]> var_6000_axes_0 = const()[name = string("op_6000_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_6000_keep_dims_0 = const()[name = string("op_6000_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_6000_cast_fp16 = reduce_sum(axes = var_6000_axes_0, keep_dims = var_6000_keep_dims_0, x = var_5998_cast_fp16)[name = string("op_6000_cast_fp16")];
            tensor<int32, [3]> var_6003_begin_0 = const()[name = string("op_6003_begin_0"), val = tensor<int32, [3]>([0, 0, 2])];
            tensor<int32, [3]> var_6003_end_0 = const()[name = string("op_6003_end_0"), val = tensor<int32, [3]>([1, 16, 3])];
            tensor<bool, [3]> var_6003_end_mask_0 = const()[name = string("op_6003_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_6003_squeeze_mask_0 = const()[name = string("op_6003_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_6003_cast_fp16 = slice_by_index(begin = var_6003_begin_0, end = var_6003_end_0, end_mask = var_6003_end_mask_0, squeeze_mask = var_6003_squeeze_mask_0, x = decay_25_cast_fp16)[name = string("op_6003_cast_fp16")];
            tensor<fp16, [1, 16]> var_6004_cast_fp16 = exp(x = var_6003_cast_fp16)[name = string("op_6004_cast_fp16")];
            tensor<int32, [1]> var_6005_axes_0 = const()[name = string("op_6005_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_6005_cast_fp16 = expand_dims(axes = var_6005_axes_0, x = var_6004_cast_fp16)[name = string("op_6005_cast_fp16")];
            tensor<int32, [1]> var_6006_axes_0 = const()[name = string("op_6006_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_6006_cast_fp16 = expand_dims(axes = var_6006_axes_0, x = var_6005_cast_fp16)[name = string("op_6006_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_201_cast_fp16 = mul(x = state_199_cast_fp16, y = var_6006_cast_fp16)[name = string("state_201_cast_fp16")];
            tensor<int32, [4]> key_token_101_begin_0 = const()[name = string("key_token_101_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> key_token_101_end_0 = const()[name = string("key_token_101_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> key_token_101_end_mask_0 = const()[name = string("key_token_101_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_101_squeeze_mask_0 = const()[name = string("key_token_101_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_101_cast_fp16 = slice_by_index(begin = key_token_101_begin_0, end = key_token_101_end_0, end_mask = key_token_101_end_mask_0, squeeze_mask = key_token_101_squeeze_mask_0, x = key_67_cast_fp16)[name = string("key_token_101_cast_fp16")];
            tensor<int32, [4]> value_token_101_begin_0 = const()[name = string("value_token_101_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> value_token_101_end_0 = const()[name = string("value_token_101_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> value_token_101_end_mask_0 = const()[name = string("value_token_101_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_101_squeeze_mask_0 = const()[name = string("value_token_101_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_101_cast_fp16 = slice_by_index(begin = value_token_101_begin_0, end = value_token_101_end_0, end_mask = value_token_101_end_mask_0, squeeze_mask = value_token_101_squeeze_mask_0, x = value_243_cast_fp16)[name = string("value_token_101_cast_fp16")];
            tensor<int32, [1]> var_6014_axes_0 = const()[name = string("op_6014_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_6014_cast_fp16 = expand_dims(axes = var_6014_axes_0, x = key_token_101_cast_fp16)[name = string("op_6014_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6015_cast_fp16 = mul(x = state_201_cast_fp16, y = var_6014_cast_fp16)[name = string("op_6015_cast_fp16")];
            tensor<int32, [1]> memory_101_axes_0 = const()[name = string("memory_101_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_101_keep_dims_0 = const()[name = string("memory_101_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_101_cast_fp16 = reduce_sum(axes = memory_101_axes_0, keep_dims = memory_101_keep_dims_0, x = var_6015_cast_fp16)[name = string("memory_101_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_6018_cast_fp16 = sub(x = value_token_101_cast_fp16, y = memory_101_cast_fp16)[name = string("op_6018_cast_fp16")];
            tensor<int32, [3]> var_6021_begin_0 = const()[name = string("op_6021_begin_0"), val = tensor<int32, [3]>([0, 0, 2])];
            tensor<int32, [3]> var_6021_end_0 = const()[name = string("op_6021_end_0"), val = tensor<int32, [3]>([1, 16, 3])];
            tensor<bool, [3]> var_6021_end_mask_0 = const()[name = string("op_6021_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_6021_squeeze_mask_0 = const()[name = string("op_6021_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_6021_cast_fp16 = slice_by_index(begin = var_6021_begin_0, end = var_6021_end_0, end_mask = var_6021_end_mask_0, squeeze_mask = var_6021_squeeze_mask_0, x = beta_25_cast_fp16)[name = string("op_6021_cast_fp16")];
            tensor<int32, [1]> var_6022_axes_0 = const()[name = string("op_6022_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_6022_cast_fp16 = expand_dims(axes = var_6022_axes_0, x = var_6021_cast_fp16)[name = string("op_6022_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_101_cast_fp16 = mul(x = var_6018_cast_fp16, y = var_6022_cast_fp16)[name = string("delta_101_cast_fp16")];
            tensor<int32, [1]> var_6025_axes_0 = const()[name = string("op_6025_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_6025_cast_fp16 = expand_dims(axes = var_6025_axes_0, x = delta_101_cast_fp16)[name = string("op_6025_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6026_cast_fp16 = mul(x = var_6014_cast_fp16, y = var_6025_cast_fp16)[name = string("op_6026_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_203_cast_fp16 = add(x = state_201_cast_fp16, y = var_6026_cast_fp16)[name = string("state_203_cast_fp16")];
            tensor<int32, [4]> var_6030_begin_0 = const()[name = string("op_6030_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> var_6030_end_0 = const()[name = string("op_6030_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> var_6030_end_mask_0 = const()[name = string("op_6030_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_6030_squeeze_mask_0 = const()[name = string("op_6030_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_6030_cast_fp16 = slice_by_index(begin = var_6030_begin_0, end = var_6030_end_0, end_mask = var_6030_end_mask_0, squeeze_mask = var_6030_squeeze_mask_0, x = _inversed_query_67_cast_fp16)[name = string("op_6030_cast_fp16")];
            tensor<int32, [1]> var_6031_axes_0 = const()[name = string("op_6031_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_6031_cast_fp16 = expand_dims(axes = var_6031_axes_0, x = var_6030_cast_fp16)[name = string("op_6031_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6032_cast_fp16 = mul(x = state_203_cast_fp16, y = var_6031_cast_fp16)[name = string("op_6032_cast_fp16")];
            tensor<int32, [1]> var_6034_axes_0 = const()[name = string("op_6034_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_6034_keep_dims_0 = const()[name = string("op_6034_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_6034_cast_fp16 = reduce_sum(axes = var_6034_axes_0, keep_dims = var_6034_keep_dims_0, x = var_6032_cast_fp16)[name = string("op_6034_cast_fp16")];
            tensor<int32, [3]> var_6037_begin_0 = const()[name = string("op_6037_begin_0"), val = tensor<int32, [3]>([0, 0, 3])];
            tensor<int32, [3]> var_6037_end_0 = const()[name = string("op_6037_end_0"), val = tensor<int32, [3]>([1, 16, 4])];
            tensor<bool, [3]> var_6037_end_mask_0 = const()[name = string("op_6037_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_6037_squeeze_mask_0 = const()[name = string("op_6037_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_6037_cast_fp16 = slice_by_index(begin = var_6037_begin_0, end = var_6037_end_0, end_mask = var_6037_end_mask_0, squeeze_mask = var_6037_squeeze_mask_0, x = decay_25_cast_fp16)[name = string("op_6037_cast_fp16")];
            tensor<fp16, [1, 16]> var_6038_cast_fp16 = exp(x = var_6037_cast_fp16)[name = string("op_6038_cast_fp16")];
            tensor<int32, [1]> var_6039_axes_0 = const()[name = string("op_6039_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_6039_cast_fp16 = expand_dims(axes = var_6039_axes_0, x = var_6038_cast_fp16)[name = string("op_6039_cast_fp16")];
            tensor<int32, [1]> var_6040_axes_0 = const()[name = string("op_6040_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_6040_cast_fp16 = expand_dims(axes = var_6040_axes_0, x = var_6039_cast_fp16)[name = string("op_6040_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_205_cast_fp16 = mul(x = state_203_cast_fp16, y = var_6040_cast_fp16)[name = string("state_205_cast_fp16")];
            tensor<int32, [4]> key_token_103_begin_0 = const()[name = string("key_token_103_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> key_token_103_end_0 = const()[name = string("key_token_103_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> key_token_103_end_mask_0 = const()[name = string("key_token_103_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_103_squeeze_mask_0 = const()[name = string("key_token_103_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_103_cast_fp16 = slice_by_index(begin = key_token_103_begin_0, end = key_token_103_end_0, end_mask = key_token_103_end_mask_0, squeeze_mask = key_token_103_squeeze_mask_0, x = key_67_cast_fp16)[name = string("key_token_103_cast_fp16")];
            tensor<int32, [4]> value_token_103_begin_0 = const()[name = string("value_token_103_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> value_token_103_end_0 = const()[name = string("value_token_103_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> value_token_103_end_mask_0 = const()[name = string("value_token_103_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_103_squeeze_mask_0 = const()[name = string("value_token_103_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_103_cast_fp16 = slice_by_index(begin = value_token_103_begin_0, end = value_token_103_end_0, end_mask = value_token_103_end_mask_0, squeeze_mask = value_token_103_squeeze_mask_0, x = value_243_cast_fp16)[name = string("value_token_103_cast_fp16")];
            tensor<int32, [1]> var_6048_axes_0 = const()[name = string("op_6048_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_6048_cast_fp16 = expand_dims(axes = var_6048_axes_0, x = key_token_103_cast_fp16)[name = string("op_6048_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6049_cast_fp16 = mul(x = state_205_cast_fp16, y = var_6048_cast_fp16)[name = string("op_6049_cast_fp16")];
            tensor<int32, [1]> memory_103_axes_0 = const()[name = string("memory_103_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_103_keep_dims_0 = const()[name = string("memory_103_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_103_cast_fp16 = reduce_sum(axes = memory_103_axes_0, keep_dims = memory_103_keep_dims_0, x = var_6049_cast_fp16)[name = string("memory_103_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_6052_cast_fp16 = sub(x = value_token_103_cast_fp16, y = memory_103_cast_fp16)[name = string("op_6052_cast_fp16")];
            tensor<int32, [3]> var_6055_begin_0 = const()[name = string("op_6055_begin_0"), val = tensor<int32, [3]>([0, 0, 3])];
            tensor<int32, [3]> var_6055_end_0 = const()[name = string("op_6055_end_0"), val = tensor<int32, [3]>([1, 16, 4])];
            tensor<bool, [3]> var_6055_end_mask_0 = const()[name = string("op_6055_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_6055_squeeze_mask_0 = const()[name = string("op_6055_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_6055_cast_fp16 = slice_by_index(begin = var_6055_begin_0, end = var_6055_end_0, end_mask = var_6055_end_mask_0, squeeze_mask = var_6055_squeeze_mask_0, x = beta_25_cast_fp16)[name = string("op_6055_cast_fp16")];
            tensor<int32, [1]> var_6056_axes_0 = const()[name = string("op_6056_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_6056_cast_fp16 = expand_dims(axes = var_6056_axes_0, x = var_6055_cast_fp16)[name = string("op_6056_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_103_cast_fp16 = mul(x = var_6052_cast_fp16, y = var_6056_cast_fp16)[name = string("delta_103_cast_fp16")];
            tensor<int32, [1]> var_6059_axes_0 = const()[name = string("op_6059_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_6059_cast_fp16 = expand_dims(axes = var_6059_axes_0, x = delta_103_cast_fp16)[name = string("op_6059_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6060_cast_fp16 = mul(x = var_6048_cast_fp16, y = var_6059_cast_fp16)[name = string("op_6060_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> rec16_out = add(x = state_205_cast_fp16, y = var_6060_cast_fp16)[name = string("state_207_cast_fp16")];
            tensor<int32, [4]> var_6064_begin_0 = const()[name = string("op_6064_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> var_6064_end_0 = const()[name = string("op_6064_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> var_6064_end_mask_0 = const()[name = string("op_6064_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_6064_squeeze_mask_0 = const()[name = string("op_6064_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_6064_cast_fp16 = slice_by_index(begin = var_6064_begin_0, end = var_6064_end_0, end_mask = var_6064_end_mask_0, squeeze_mask = var_6064_squeeze_mask_0, x = _inversed_query_67_cast_fp16)[name = string("op_6064_cast_fp16")];
            tensor<int32, [1]> var_6065_axes_0 = const()[name = string("op_6065_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_6065_cast_fp16 = expand_dims(axes = var_6065_axes_0, x = var_6064_cast_fp16)[name = string("op_6065_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6066_cast_fp16 = mul(x = rec16_out, y = var_6065_cast_fp16)[name = string("op_6066_cast_fp16")];
            tensor<int32, [1]> var_6068_axes_0 = const()[name = string("op_6068_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_6068_keep_dims_0 = const()[name = string("op_6068_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_6068_cast_fp16 = reduce_sum(axes = var_6068_axes_0, keep_dims = var_6068_keep_dims_0, x = var_6066_cast_fp16)[name = string("op_6068_cast_fp16")];
            int32 attention_121_axis_0 = const()[name = string("attention_121_axis_0"), val = int32(1)];
            tensor<fp16, [1, 4, 16, 128]> attention_121_cast_fp16 = stack(axis = attention_121_axis_0, values = (var_5966_cast_fp16, var_6000_cast_fp16, var_6034_cast_fp16, var_6068_cast_fp16))[name = string("attention_121_cast_fp16")];
            tensor<int32, [2]> var_6071 = const()[name = string("op_6071"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [64, 128]> attention_123_cast_fp16 = reshape(shape = var_6071, x = attention_121_cast_fp16)[name = string("attention_123_cast_fp16")];
            tensor<int32, [2]> var_6073 = const()[name = string("op_6073"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [64, 128]> input_213_cast_fp16 = reshape(shape = var_6073, x = linear_127_cast_fp16)[name = string("input_213_cast_fp16")];
            tensor<fp16, [64, 128]> var_6075_cast_fp16 = mul(x = attention_123_cast_fp16, y = attention_123_cast_fp16)[name = string("op_6075_cast_fp16")];
            tensor<int32, [1]> variance_107_axes_0 = const()[name = string("variance_107_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_107_keep_dims_0 = const()[name = string("variance_107_keep_dims_0"), val = bool(true)];
            tensor<fp16, [64, 1]> variance_107_cast_fp16 = reduce_mean(axes = variance_107_axes_0, keep_dims = variance_107_keep_dims_0, x = var_6075_cast_fp16)[name = string("variance_107_cast_fp16")];
            fp16 var_6078_to_fp16 = const()[name = string("op_6078_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [64, 1]> var_6079_cast_fp16 = add(x = variance_107_cast_fp16, y = var_6078_to_fp16)[name = string("op_6079_cast_fp16")];
            fp32 var_6080_epsilon_0 = const()[name = string("op_6080_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [64, 1]> var_6080_cast_fp16 = rsqrt(epsilon = var_6080_epsilon_0, x = var_6079_cast_fp16)[name = string("op_6080_cast_fp16")];
            tensor<fp16, [64, 128]> attention_125_cast_fp16 = mul(x = attention_123_cast_fp16, y = var_6080_cast_fp16)[name = string("attention_125_cast_fp16")];
            tensor<fp16, [128]> final_group_0_gated_norm_promoted_to_fp16 = const()[name = string("final_group_0_gated_norm_promoted_to_fp16"), val = tensor<fp16, [128]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(258335744)))];
            tensor<fp16, [64, 128]> attention_127_cast_fp16 = mul(x = attention_125_cast_fp16, y = final_group_0_gated_norm_promoted_to_fp16)[name = string("attention_127_cast_fp16")];
            tensor<fp16, [64, 128]> var_6083_cast_fp16 = silu(x = input_213_cast_fp16)[name = string("op_6083_cast_fp16")];
            tensor<fp16, [64, 128]> attention_129_cast_fp16 = mul(x = attention_127_cast_fp16, y = var_6083_cast_fp16)[name = string("attention_129_cast_fp16")];
            tensor<int32, [2]> var_6085 = const()[name = string("op_6085"), val = tensor<int32, [2]>([4, 2048])];
            tensor<fp16, [4, 2048]> input_215_cast_fp16 = reshape(shape = var_6085, x = attention_129_cast_fp16)[name = string("input_215_cast_fp16")];
            tensor<fp16, [1024, 2048]> final_group_0_out_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 2048]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(258336064))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(259908992))))[name = string("final_group_0_out_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 1024]> linear_128_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = final_group_0_out_proj_weight_promoted_to_fp16_palettized, x = input_215_cast_fp16)[name = string("linear_128_cast_fp16")];
            tensor<fp16, [4, 1024]> value_245_cast_fp16 = add(x = value_233_cast_fp16, y = linear_128_cast_fp16)[name = string("value_245_cast_fp16")];
            tensor<fp16, [4, 1024]> var_6090_cast_fp16 = mul(x = value_245_cast_fp16, y = value_245_cast_fp16)[name = string("op_6090_cast_fp16")];
            tensor<int32, [1]> variance_109_axes_0 = const()[name = string("variance_109_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_109_keep_dims_0 = const()[name = string("variance_109_keep_dims_0"), val = bool(true)];
            tensor<fp16, [4, 1]> variance_109_cast_fp16 = reduce_mean(axes = variance_109_axes_0, keep_dims = variance_109_keep_dims_0, x = var_6090_cast_fp16)[name = string("variance_109_cast_fp16")];
            fp16 var_6093_to_fp16 = const()[name = string("op_6093_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [4, 1]> var_6094_cast_fp16 = add(x = variance_109_cast_fp16, y = var_6093_to_fp16)[name = string("op_6094_cast_fp16")];
            fp32 var_6095_epsilon_0 = const()[name = string("op_6095_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [4, 1]> var_6095_cast_fp16 = rsqrt(epsilon = var_6095_epsilon_0, x = var_6094_cast_fp16)[name = string("op_6095_cast_fp16")];
            tensor<fp16, [4, 1024]> var_6096_cast_fp16 = mul(x = value_245_cast_fp16, y = var_6095_cast_fp16)[name = string("op_6096_cast_fp16")];
            tensor<fp16, [1024]> var_6098_to_fp16 = const()[name = string("op_6098_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(259917248)))];
            tensor<fp16, [4, 1024]> input_217_cast_fp16 = mul(x = var_6096_cast_fp16, y = var_6098_to_fp16)[name = string("input_217_cast_fp16")];
            tensor<fp16, [3584, 1024]> final_group_0_gate_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(259919360))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(262671936))))[name = string("final_group_0_gate_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 3584]> linear_129_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = final_group_0_gate_proj_weight_promoted_to_fp16_palettized, x = input_217_cast_fp16)[name = string("linear_129_cast_fp16")];
            tensor<fp16, [4, 3584]> var_6102_cast_fp16 = silu(x = linear_129_cast_fp16)[name = string("op_6102_cast_fp16")];
            tensor<fp16, [3584, 1024]> final_group_0_up_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(262700672))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(265453248))))[name = string("final_group_0_up_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 3584]> linear_130_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = final_group_0_up_proj_weight_promoted_to_fp16_palettized, x = input_217_cast_fp16)[name = string("linear_130_cast_fp16")];
            tensor<fp16, [4, 3584]> input_221_cast_fp16 = mul(x = var_6102_cast_fp16, y = linear_130_cast_fp16)[name = string("input_221_cast_fp16")];
            tensor<fp16, [1024, 3584]> final_group_0_down_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 3584]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(265481984))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(268234560))))[name = string("final_group_0_down_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 1024]> linear_131_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = final_group_0_down_proj_weight_promoted_to_fp16_palettized, x = input_221_cast_fp16)[name = string("linear_131_cast_fp16")];
            tensor<fp16, [4, 1024]> value_247_cast_fp16 = add(x = value_245_cast_fp16, y = linear_131_cast_fp16)[name = string("value_247_cast_fp16")];
            int32 var_6129 = const()[name = string("op_6129"), val = int32(-1)];
            tensor<fp16, [4, 1024]> var_6144_cast_fp16 = mul(x = value_247_cast_fp16, y = value_247_cast_fp16)[name = string("op_6144_cast_fp16")];
            tensor<int32, [1]> variance_111_axes_0 = const()[name = string("variance_111_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_111_keep_dims_0 = const()[name = string("variance_111_keep_dims_0"), val = bool(true)];
            tensor<fp16, [4, 1]> variance_111_cast_fp16 = reduce_mean(axes = variance_111_axes_0, keep_dims = variance_111_keep_dims_0, x = var_6144_cast_fp16)[name = string("variance_111_cast_fp16")];
            fp16 var_6147_to_fp16 = const()[name = string("op_6147_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [4, 1]> var_6148_cast_fp16 = add(x = variance_111_cast_fp16, y = var_6147_to_fp16)[name = string("op_6148_cast_fp16")];
            fp32 var_6149_epsilon_0 = const()[name = string("op_6149_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [4, 1]> var_6149_cast_fp16 = rsqrt(epsilon = var_6149_epsilon_0, x = var_6148_cast_fp16)[name = string("op_6149_cast_fp16")];
            tensor<fp16, [4, 1024]> var_6150_cast_fp16 = mul(x = value_247_cast_fp16, y = var_6149_cast_fp16)[name = string("op_6150_cast_fp16")];
            tensor<fp16, [1024]> var_6152_to_fp16 = const()[name = string("op_6152_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(268242816)))];
            tensor<fp16, [4, 1024]> input_223_cast_fp16 = mul(x = var_6150_cast_fp16, y = var_6152_to_fp16)[name = string("input_223_cast_fp16")];
            tensor<fp16, [6144, 1024]> final_group_1_in_proj_qkv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(268244928))), lut = tensor<fp16, [384, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(272963584))))[name = string("final_group_1_in_proj_qkv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 6144]> linear_132_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = final_group_1_in_proj_qkv_weight_promoted_to_fp16_palettized, x = input_223_cast_fp16)[name = string("linear_132_cast_fp16")];
            tensor<int32, [2]> var_6156_perm_0 = const()[name = string("op_6156_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> mixed_27_axes_0 = const()[name = string("mixed_27_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [6144, 4]> var_6156_cast_fp16 = transpose(perm = var_6156_perm_0, x = linear_132_cast_fp16)[name = string("transpose_17")];
            tensor<fp16, [1, 6144, 4]> mixed_27_cast_fp16 = expand_dims(axes = mixed_27_axes_0, x = var_6156_cast_fp16)[name = string("mixed_27_cast_fp16")];
            bool convolution_input_27_interleave_0 = const()[name = string("convolution_input_27_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 6144, 7]> convolution_input_27_cast_fp16 = concat(axis = var_6129, interleave = convolution_input_27_interleave_0, values = (conv17, mixed_27_cast_fp16))[name = string("convolution_input_27_cast_fp16")];
            string input_225_pad_type_0 = const()[name = string("input_225_pad_type_0"), val = string("valid")];
            int32 input_225_groups_0 = const()[name = string("input_225_groups_0"), val = int32(6144)];
            tensor<int32, [1]> input_225_strides_0 = const()[name = string("input_225_strides_0"), val = tensor<int32, [1]>([1])];
            tensor<int32, [2]> input_225_pad_0 = const()[name = string("input_225_pad_0"), val = tensor<int32, [2]>([0, 0])];
            tensor<int32, [1]> input_225_dilations_0 = const()[name = string("input_225_dilations_0"), val = tensor<int32, [1]>([1])];
            tensor<fp16, [6144, 1, 4]> final_group_1_conv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1, 4]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(273012800))), lut = tensor<fp16, [384, 1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(273031296))))[name = string("final_group_1_conv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 6144, 4]> input_225_cast_fp16 = conv(dilations = input_225_dilations_0, groups = input_225_groups_0, pad = input_225_pad_0, pad_type = input_225_pad_type_0, strides = input_225_strides_0, weight = final_group_1_conv_weight_promoted_to_fp16_palettized, x = convolution_input_27_cast_fp16)[name = string("input_225_cast_fp16")];
            tensor<fp16, [1, 6144, 4]> var_6165_cast_fp16 = silu(x = input_225_cast_fp16)[name = string("op_6165_cast_fp16")];
            tensor<int32, [3]> var_6166_perm_0 = const()[name = string("op_6166_perm_0"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [3]> var_6169_begin_0 = const()[name = string("op_6169_begin_0"), val = tensor<int32, [3]>([0, 0, 4])];
            tensor<int32, [3]> var_6169_end_0 = const()[name = string("op_6169_end_0"), val = tensor<int32, [3]>([1, 6144, 7])];
            tensor<bool, [3]> var_6169_end_mask_0 = const()[name = string("op_6169_end_mask_0"), val = tensor<bool, [3]>([true, true, true])];
            tensor<fp16, [1, 6144, 3]> conv17_out = slice_by_index(begin = var_6169_begin_0, end = var_6169_end_0, end_mask = var_6169_end_mask_0, x = convolution_input_27_cast_fp16)[name = string("op_6169_cast_fp16")];
            tensor<int32, [3]> var_6170 = const()[name = string("op_6170"), val = tensor<int32, [3]>([2048, 2048, 2048])];
            int32 var_6171_axis_0 = const()[name = string("op_6171_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 4, 6144]> var_6166_cast_fp16 = transpose(perm = var_6166_perm_0, x = var_6165_cast_fp16)[name = string("transpose_16")];
            tensor<fp16, [1, 4, 2048]> var_6171_cast_fp16_0, tensor<fp16, [1, 4, 2048]> var_6171_cast_fp16_1, tensor<fp16, [1, 4, 2048]> var_6171_cast_fp16_2 = split(axis = var_6171_axis_0, split_sizes = var_6170, x = var_6166_cast_fp16)[name = string("op_6171_cast_fp16")];
            tensor<int32, [4]> var_6175 = const()[name = string("op_6175"), val = tensor<int32, [4]>([1, 4, 16, 128])];
            tensor<fp16, [1, 4, 16, 128]> value_251_cast_fp16 = reshape(shape = var_6175, x = var_6171_cast_fp16_0)[name = string("value_251_cast_fp16")];
            tensor<int32, [4]> var_6177 = const()[name = string("op_6177"), val = tensor<int32, [4]>([1, 4, 16, 128])];
            tensor<fp16, [1, 4, 16, 128]> value_253_cast_fp16 = reshape(shape = var_6177, x = var_6171_cast_fp16_1)[name = string("value_253_cast_fp16")];
            tensor<int32, [4]> var_6179 = const()[name = string("op_6179"), val = tensor<int32, [4]>([1, 4, 16, 128])];
            tensor<fp16, [1, 4, 16, 128]> value_255_cast_fp16 = reshape(shape = var_6179, x = var_6171_cast_fp16_2)[name = string("value_255_cast_fp16")];
            tensor<fp16, [1, 4, 16, 128]> var_6181_cast_fp16 = mul(x = value_251_cast_fp16, y = value_251_cast_fp16)[name = string("op_6181_cast_fp16")];
            tensor<int32, [1]> var_6183_axes_0 = const()[name = string("op_6183_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_6183_keep_dims_0 = const()[name = string("op_6183_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 4, 16, 1]> var_6183_cast_fp16 = reduce_sum(axes = var_6183_axes_0, keep_dims = var_6183_keep_dims_0, x = var_6181_cast_fp16)[name = string("op_6183_cast_fp16")];
            fp16 var_6184_to_fp16 = const()[name = string("op_6184_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 4, 16, 1]> var_6185_cast_fp16 = add(x = var_6183_cast_fp16, y = var_6184_to_fp16)[name = string("op_6185_cast_fp16")];
            fp32 var_6186_epsilon_0 = const()[name = string("op_6186_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 4, 16, 1]> var_6186_cast_fp16 = rsqrt(epsilon = var_6186_epsilon_0, x = var_6185_cast_fp16)[name = string("op_6186_cast_fp16")];
            tensor<fp16, [1, 4, 16, 128]> var_6187_cast_fp16 = mul(x = value_251_cast_fp16, y = var_6186_cast_fp16)[name = string("op_6187_cast_fp16")];
            tensor<int32, [4]> var_6188_perm_0 = const()[name = string("op_6188_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            fp16 _inversed_query_71_y_0_to_fp16 = const()[name = string("_inversed_query_71_y_0_to_fp16"), val = fp16(0x1.6ap-4)];
            tensor<fp16, [1, 16, 4, 128]> var_6188_cast_fp16 = transpose(perm = var_6188_perm_0, x = var_6187_cast_fp16)[name = string("transpose_15")];
            tensor<fp16, [1, 16, 4, 128]> _inversed_query_71_cast_fp16 = mul(x = var_6188_cast_fp16, y = _inversed_query_71_y_0_to_fp16)[name = string("_inversed_query_71_cast_fp16")];
            tensor<fp16, [1, 4, 16, 128]> var_6191_cast_fp16 = mul(x = value_253_cast_fp16, y = value_253_cast_fp16)[name = string("op_6191_cast_fp16")];
            tensor<int32, [1]> var_6193_axes_0 = const()[name = string("op_6193_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_6193_keep_dims_0 = const()[name = string("op_6193_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 4, 16, 1]> var_6193_cast_fp16 = reduce_sum(axes = var_6193_axes_0, keep_dims = var_6193_keep_dims_0, x = var_6191_cast_fp16)[name = string("op_6193_cast_fp16")];
            fp16 var_6194_to_fp16 = const()[name = string("op_6194_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 4, 16, 1]> var_6195_cast_fp16 = add(x = var_6193_cast_fp16, y = var_6194_to_fp16)[name = string("op_6195_cast_fp16")];
            fp32 var_6196_epsilon_0 = const()[name = string("op_6196_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 4, 16, 1]> var_6196_cast_fp16 = rsqrt(epsilon = var_6196_epsilon_0, x = var_6195_cast_fp16)[name = string("op_6196_cast_fp16")];
            tensor<fp16, [1, 4, 16, 128]> var_6197_cast_fp16 = mul(x = value_253_cast_fp16, y = var_6196_cast_fp16)[name = string("op_6197_cast_fp16")];
            tensor<int32, [4]> key_71_perm_0 = const()[name = string("key_71_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<int32, [4]> value_257_perm_0 = const()[name = string("value_257_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<fp16, [16, 1024]> final_group_1_in_proj_b_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(273080512))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(273092864))))[name = string("final_group_1_in_proj_b_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16]> linear_133_bias_0_to_fp16 = const()[name = string("linear_133_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(4862656)))];
            tensor<fp16, [4, 16]> linear_133_cast_fp16 = linear(bias = linear_133_bias_0_to_fp16, weight = final_group_1_in_proj_b_weight_promoted_to_fp16_palettized, x = input_223_cast_fp16)[name = string("linear_133_cast_fp16")];
            tensor<fp16, [4, 16]> var_6202_cast_fp16 = sigmoid(x = linear_133_cast_fp16)[name = string("op_6202_cast_fp16")];
            tensor<int32, [2]> var_6203_perm_0 = const()[name = string("op_6203_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> beta_27_axes_0 = const()[name = string("beta_27_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 4]> var_6203_cast_fp16 = transpose(perm = var_6203_perm_0, x = var_6202_cast_fp16)[name = string("transpose_14")];
            tensor<fp16, [1, 16, 4]> beta_27_cast_fp16 = expand_dims(axes = beta_27_axes_0, x = var_6203_cast_fp16)[name = string("beta_27_cast_fp16")];
            tensor<fp16, [16, 1024]> op_6209_weight_0_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(273093056))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(273105408))))[name = string("op_6209_weight_0_to_fp16_palettized")];
            tensor<fp16, [16]> var_6209_bias_0_to_fp16 = const()[name = string("op_6209_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(273105600)))];
            tensor<fp16, [4, 16]> var_6209_cast_fp16 = linear(bias = var_6209_bias_0_to_fp16, weight = op_6209_weight_0_to_fp16_palettized, x = input_223_cast_fp16)[name = string("op_6209_cast_fp16")];
            tensor<fp16, [4, 16]> var_6210_cast_fp16 = softplus(x = var_6209_cast_fp16)[name = string("op_6210_cast_fp16")];
            tensor<fp16, [16]> var_6206_promoted_to_fp16 = const()[name = string("op_6206_promoted_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(273105728)))];
            tensor<fp16, [4, 16]> var_6211_cast_fp16 = mul(x = var_6206_promoted_to_fp16, y = var_6210_cast_fp16)[name = string("op_6211_cast_fp16")];
            tensor<int32, [2]> var_6212_perm_0 = const()[name = string("op_6212_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> decay_27_axes_0 = const()[name = string("decay_27_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 4]> var_6212_cast_fp16 = transpose(perm = var_6212_perm_0, x = var_6211_cast_fp16)[name = string("transpose_13")];
            tensor<fp16, [1, 16, 4]> decay_27_cast_fp16 = expand_dims(axes = decay_27_axes_0, x = var_6212_cast_fp16)[name = string("decay_27_cast_fp16")];
            tensor<fp16, [2048, 1024]> final_group_1_in_proj_z_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [2048, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(273105856))), lut = tensor<fp16, [128, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(274678784))))[name = string("final_group_1_in_proj_z_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 2048]> linear_135_cast_fp16 = linear(bias = linear_3_bias_0_to_fp16, weight = final_group_1_in_proj_z_weight_promoted_to_fp16_palettized, x = input_223_cast_fp16)[name = string("linear_135_cast_fp16")];
            tensor<int32, [3]> var_6220_begin_0 = const()[name = string("op_6220_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_6220_end_0 = const()[name = string("op_6220_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_6220_end_mask_0 = const()[name = string("op_6220_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_6220_squeeze_mask_0 = const()[name = string("op_6220_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_6220_cast_fp16 = slice_by_index(begin = var_6220_begin_0, end = var_6220_end_0, end_mask = var_6220_end_mask_0, squeeze_mask = var_6220_squeeze_mask_0, x = decay_27_cast_fp16)[name = string("op_6220_cast_fp16")];
            tensor<fp16, [1, 16]> var_6221_cast_fp16 = exp(x = var_6220_cast_fp16)[name = string("op_6221_cast_fp16")];
            tensor<int32, [1]> var_6222_axes_0 = const()[name = string("op_6222_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_6222_cast_fp16 = expand_dims(axes = var_6222_axes_0, x = var_6221_cast_fp16)[name = string("op_6222_cast_fp16")];
            tensor<int32, [1]> var_6223_axes_0 = const()[name = string("op_6223_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_6223_cast_fp16 = expand_dims(axes = var_6223_axes_0, x = var_6222_cast_fp16)[name = string("op_6223_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_209_cast_fp16 = mul(x = rec17, y = var_6223_cast_fp16)[name = string("state_209_cast_fp16")];
            tensor<int32, [4]> key_token_105_begin_0 = const()[name = string("key_token_105_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> key_token_105_end_0 = const()[name = string("key_token_105_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> key_token_105_end_mask_0 = const()[name = string("key_token_105_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_105_squeeze_mask_0 = const()[name = string("key_token_105_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 4, 128]> key_71_cast_fp16 = transpose(perm = key_71_perm_0, x = var_6197_cast_fp16)[name = string("transpose_12")];
            tensor<fp16, [1, 16, 128]> key_token_105_cast_fp16 = slice_by_index(begin = key_token_105_begin_0, end = key_token_105_end_0, end_mask = key_token_105_end_mask_0, squeeze_mask = key_token_105_squeeze_mask_0, x = key_71_cast_fp16)[name = string("key_token_105_cast_fp16")];
            tensor<int32, [4]> value_token_105_begin_0 = const()[name = string("value_token_105_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> value_token_105_end_0 = const()[name = string("value_token_105_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> value_token_105_end_mask_0 = const()[name = string("value_token_105_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_105_squeeze_mask_0 = const()[name = string("value_token_105_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 4, 128]> value_257_cast_fp16 = transpose(perm = value_257_perm_0, x = value_255_cast_fp16)[name = string("transpose_11")];
            tensor<fp16, [1, 16, 128]> value_token_105_cast_fp16 = slice_by_index(begin = value_token_105_begin_0, end = value_token_105_end_0, end_mask = value_token_105_end_mask_0, squeeze_mask = value_token_105_squeeze_mask_0, x = value_257_cast_fp16)[name = string("value_token_105_cast_fp16")];
            tensor<int32, [1]> var_6231_axes_0 = const()[name = string("op_6231_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_6231_cast_fp16 = expand_dims(axes = var_6231_axes_0, x = key_token_105_cast_fp16)[name = string("op_6231_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6232_cast_fp16 = mul(x = state_209_cast_fp16, y = var_6231_cast_fp16)[name = string("op_6232_cast_fp16")];
            tensor<int32, [1]> memory_105_axes_0 = const()[name = string("memory_105_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_105_keep_dims_0 = const()[name = string("memory_105_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_105_cast_fp16 = reduce_sum(axes = memory_105_axes_0, keep_dims = memory_105_keep_dims_0, x = var_6232_cast_fp16)[name = string("memory_105_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_6235_cast_fp16 = sub(x = value_token_105_cast_fp16, y = memory_105_cast_fp16)[name = string("op_6235_cast_fp16")];
            tensor<int32, [3]> var_6238_begin_0 = const()[name = string("op_6238_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_6238_end_0 = const()[name = string("op_6238_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_6238_end_mask_0 = const()[name = string("op_6238_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_6238_squeeze_mask_0 = const()[name = string("op_6238_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_6238_cast_fp16 = slice_by_index(begin = var_6238_begin_0, end = var_6238_end_0, end_mask = var_6238_end_mask_0, squeeze_mask = var_6238_squeeze_mask_0, x = beta_27_cast_fp16)[name = string("op_6238_cast_fp16")];
            tensor<int32, [1]> var_6239_axes_0 = const()[name = string("op_6239_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_6239_cast_fp16 = expand_dims(axes = var_6239_axes_0, x = var_6238_cast_fp16)[name = string("op_6239_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_105_cast_fp16 = mul(x = var_6235_cast_fp16, y = var_6239_cast_fp16)[name = string("delta_105_cast_fp16")];
            tensor<int32, [1]> var_6242_axes_0 = const()[name = string("op_6242_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_6242_cast_fp16 = expand_dims(axes = var_6242_axes_0, x = delta_105_cast_fp16)[name = string("op_6242_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6243_cast_fp16 = mul(x = var_6231_cast_fp16, y = var_6242_cast_fp16)[name = string("op_6243_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_211_cast_fp16 = add(x = state_209_cast_fp16, y = var_6243_cast_fp16)[name = string("state_211_cast_fp16")];
            tensor<int32, [4]> var_6247_begin_0 = const()[name = string("op_6247_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> var_6247_end_0 = const()[name = string("op_6247_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> var_6247_end_mask_0 = const()[name = string("op_6247_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_6247_squeeze_mask_0 = const()[name = string("op_6247_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_6247_cast_fp16 = slice_by_index(begin = var_6247_begin_0, end = var_6247_end_0, end_mask = var_6247_end_mask_0, squeeze_mask = var_6247_squeeze_mask_0, x = _inversed_query_71_cast_fp16)[name = string("op_6247_cast_fp16")];
            tensor<int32, [1]> var_6248_axes_0 = const()[name = string("op_6248_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_6248_cast_fp16 = expand_dims(axes = var_6248_axes_0, x = var_6247_cast_fp16)[name = string("op_6248_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6249_cast_fp16 = mul(x = state_211_cast_fp16, y = var_6248_cast_fp16)[name = string("op_6249_cast_fp16")];
            tensor<int32, [1]> var_6251_axes_0 = const()[name = string("op_6251_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_6251_keep_dims_0 = const()[name = string("op_6251_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_6251_cast_fp16 = reduce_sum(axes = var_6251_axes_0, keep_dims = var_6251_keep_dims_0, x = var_6249_cast_fp16)[name = string("op_6251_cast_fp16")];
            tensor<int32, [3]> var_6254_begin_0 = const()[name = string("op_6254_begin_0"), val = tensor<int32, [3]>([0, 0, 1])];
            tensor<int32, [3]> var_6254_end_0 = const()[name = string("op_6254_end_0"), val = tensor<int32, [3]>([1, 16, 2])];
            tensor<bool, [3]> var_6254_end_mask_0 = const()[name = string("op_6254_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_6254_squeeze_mask_0 = const()[name = string("op_6254_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_6254_cast_fp16 = slice_by_index(begin = var_6254_begin_0, end = var_6254_end_0, end_mask = var_6254_end_mask_0, squeeze_mask = var_6254_squeeze_mask_0, x = decay_27_cast_fp16)[name = string("op_6254_cast_fp16")];
            tensor<fp16, [1, 16]> var_6255_cast_fp16 = exp(x = var_6254_cast_fp16)[name = string("op_6255_cast_fp16")];
            tensor<int32, [1]> var_6256_axes_0 = const()[name = string("op_6256_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_6256_cast_fp16 = expand_dims(axes = var_6256_axes_0, x = var_6255_cast_fp16)[name = string("op_6256_cast_fp16")];
            tensor<int32, [1]> var_6257_axes_0 = const()[name = string("op_6257_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_6257_cast_fp16 = expand_dims(axes = var_6257_axes_0, x = var_6256_cast_fp16)[name = string("op_6257_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_213_cast_fp16 = mul(x = state_211_cast_fp16, y = var_6257_cast_fp16)[name = string("state_213_cast_fp16")];
            tensor<int32, [4]> key_token_107_begin_0 = const()[name = string("key_token_107_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> key_token_107_end_0 = const()[name = string("key_token_107_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> key_token_107_end_mask_0 = const()[name = string("key_token_107_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_107_squeeze_mask_0 = const()[name = string("key_token_107_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_107_cast_fp16 = slice_by_index(begin = key_token_107_begin_0, end = key_token_107_end_0, end_mask = key_token_107_end_mask_0, squeeze_mask = key_token_107_squeeze_mask_0, x = key_71_cast_fp16)[name = string("key_token_107_cast_fp16")];
            tensor<int32, [4]> value_token_107_begin_0 = const()[name = string("value_token_107_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> value_token_107_end_0 = const()[name = string("value_token_107_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> value_token_107_end_mask_0 = const()[name = string("value_token_107_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_107_squeeze_mask_0 = const()[name = string("value_token_107_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_107_cast_fp16 = slice_by_index(begin = value_token_107_begin_0, end = value_token_107_end_0, end_mask = value_token_107_end_mask_0, squeeze_mask = value_token_107_squeeze_mask_0, x = value_257_cast_fp16)[name = string("value_token_107_cast_fp16")];
            tensor<int32, [1]> var_6265_axes_0 = const()[name = string("op_6265_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_6265_cast_fp16 = expand_dims(axes = var_6265_axes_0, x = key_token_107_cast_fp16)[name = string("op_6265_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6266_cast_fp16 = mul(x = state_213_cast_fp16, y = var_6265_cast_fp16)[name = string("op_6266_cast_fp16")];
            tensor<int32, [1]> memory_107_axes_0 = const()[name = string("memory_107_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_107_keep_dims_0 = const()[name = string("memory_107_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_107_cast_fp16 = reduce_sum(axes = memory_107_axes_0, keep_dims = memory_107_keep_dims_0, x = var_6266_cast_fp16)[name = string("memory_107_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_6269_cast_fp16 = sub(x = value_token_107_cast_fp16, y = memory_107_cast_fp16)[name = string("op_6269_cast_fp16")];
            tensor<int32, [3]> var_6272_begin_0 = const()[name = string("op_6272_begin_0"), val = tensor<int32, [3]>([0, 0, 1])];
            tensor<int32, [3]> var_6272_end_0 = const()[name = string("op_6272_end_0"), val = tensor<int32, [3]>([1, 16, 2])];
            tensor<bool, [3]> var_6272_end_mask_0 = const()[name = string("op_6272_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_6272_squeeze_mask_0 = const()[name = string("op_6272_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_6272_cast_fp16 = slice_by_index(begin = var_6272_begin_0, end = var_6272_end_0, end_mask = var_6272_end_mask_0, squeeze_mask = var_6272_squeeze_mask_0, x = beta_27_cast_fp16)[name = string("op_6272_cast_fp16")];
            tensor<int32, [1]> var_6273_axes_0 = const()[name = string("op_6273_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_6273_cast_fp16 = expand_dims(axes = var_6273_axes_0, x = var_6272_cast_fp16)[name = string("op_6273_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_107_cast_fp16 = mul(x = var_6269_cast_fp16, y = var_6273_cast_fp16)[name = string("delta_107_cast_fp16")];
            tensor<int32, [1]> var_6276_axes_0 = const()[name = string("op_6276_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_6276_cast_fp16 = expand_dims(axes = var_6276_axes_0, x = delta_107_cast_fp16)[name = string("op_6276_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6277_cast_fp16 = mul(x = var_6265_cast_fp16, y = var_6276_cast_fp16)[name = string("op_6277_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_215_cast_fp16 = add(x = state_213_cast_fp16, y = var_6277_cast_fp16)[name = string("state_215_cast_fp16")];
            tensor<int32, [4]> var_6281_begin_0 = const()[name = string("op_6281_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> var_6281_end_0 = const()[name = string("op_6281_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> var_6281_end_mask_0 = const()[name = string("op_6281_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_6281_squeeze_mask_0 = const()[name = string("op_6281_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_6281_cast_fp16 = slice_by_index(begin = var_6281_begin_0, end = var_6281_end_0, end_mask = var_6281_end_mask_0, squeeze_mask = var_6281_squeeze_mask_0, x = _inversed_query_71_cast_fp16)[name = string("op_6281_cast_fp16")];
            tensor<int32, [1]> var_6282_axes_0 = const()[name = string("op_6282_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_6282_cast_fp16 = expand_dims(axes = var_6282_axes_0, x = var_6281_cast_fp16)[name = string("op_6282_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6283_cast_fp16 = mul(x = state_215_cast_fp16, y = var_6282_cast_fp16)[name = string("op_6283_cast_fp16")];
            tensor<int32, [1]> var_6285_axes_0 = const()[name = string("op_6285_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_6285_keep_dims_0 = const()[name = string("op_6285_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_6285_cast_fp16 = reduce_sum(axes = var_6285_axes_0, keep_dims = var_6285_keep_dims_0, x = var_6283_cast_fp16)[name = string("op_6285_cast_fp16")];
            tensor<int32, [3]> var_6288_begin_0 = const()[name = string("op_6288_begin_0"), val = tensor<int32, [3]>([0, 0, 2])];
            tensor<int32, [3]> var_6288_end_0 = const()[name = string("op_6288_end_0"), val = tensor<int32, [3]>([1, 16, 3])];
            tensor<bool, [3]> var_6288_end_mask_0 = const()[name = string("op_6288_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_6288_squeeze_mask_0 = const()[name = string("op_6288_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_6288_cast_fp16 = slice_by_index(begin = var_6288_begin_0, end = var_6288_end_0, end_mask = var_6288_end_mask_0, squeeze_mask = var_6288_squeeze_mask_0, x = decay_27_cast_fp16)[name = string("op_6288_cast_fp16")];
            tensor<fp16, [1, 16]> var_6289_cast_fp16 = exp(x = var_6288_cast_fp16)[name = string("op_6289_cast_fp16")];
            tensor<int32, [1]> var_6290_axes_0 = const()[name = string("op_6290_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_6290_cast_fp16 = expand_dims(axes = var_6290_axes_0, x = var_6289_cast_fp16)[name = string("op_6290_cast_fp16")];
            tensor<int32, [1]> var_6291_axes_0 = const()[name = string("op_6291_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_6291_cast_fp16 = expand_dims(axes = var_6291_axes_0, x = var_6290_cast_fp16)[name = string("op_6291_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_217_cast_fp16 = mul(x = state_215_cast_fp16, y = var_6291_cast_fp16)[name = string("state_217_cast_fp16")];
            tensor<int32, [4]> key_token_109_begin_0 = const()[name = string("key_token_109_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> key_token_109_end_0 = const()[name = string("key_token_109_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> key_token_109_end_mask_0 = const()[name = string("key_token_109_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_109_squeeze_mask_0 = const()[name = string("key_token_109_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_109_cast_fp16 = slice_by_index(begin = key_token_109_begin_0, end = key_token_109_end_0, end_mask = key_token_109_end_mask_0, squeeze_mask = key_token_109_squeeze_mask_0, x = key_71_cast_fp16)[name = string("key_token_109_cast_fp16")];
            tensor<int32, [4]> value_token_109_begin_0 = const()[name = string("value_token_109_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> value_token_109_end_0 = const()[name = string("value_token_109_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> value_token_109_end_mask_0 = const()[name = string("value_token_109_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_109_squeeze_mask_0 = const()[name = string("value_token_109_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_109_cast_fp16 = slice_by_index(begin = value_token_109_begin_0, end = value_token_109_end_0, end_mask = value_token_109_end_mask_0, squeeze_mask = value_token_109_squeeze_mask_0, x = value_257_cast_fp16)[name = string("value_token_109_cast_fp16")];
            tensor<int32, [1]> var_6299_axes_0 = const()[name = string("op_6299_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_6299_cast_fp16 = expand_dims(axes = var_6299_axes_0, x = key_token_109_cast_fp16)[name = string("op_6299_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6300_cast_fp16 = mul(x = state_217_cast_fp16, y = var_6299_cast_fp16)[name = string("op_6300_cast_fp16")];
            tensor<int32, [1]> memory_109_axes_0 = const()[name = string("memory_109_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_109_keep_dims_0 = const()[name = string("memory_109_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_109_cast_fp16 = reduce_sum(axes = memory_109_axes_0, keep_dims = memory_109_keep_dims_0, x = var_6300_cast_fp16)[name = string("memory_109_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_6303_cast_fp16 = sub(x = value_token_109_cast_fp16, y = memory_109_cast_fp16)[name = string("op_6303_cast_fp16")];
            tensor<int32, [3]> var_6306_begin_0 = const()[name = string("op_6306_begin_0"), val = tensor<int32, [3]>([0, 0, 2])];
            tensor<int32, [3]> var_6306_end_0 = const()[name = string("op_6306_end_0"), val = tensor<int32, [3]>([1, 16, 3])];
            tensor<bool, [3]> var_6306_end_mask_0 = const()[name = string("op_6306_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_6306_squeeze_mask_0 = const()[name = string("op_6306_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_6306_cast_fp16 = slice_by_index(begin = var_6306_begin_0, end = var_6306_end_0, end_mask = var_6306_end_mask_0, squeeze_mask = var_6306_squeeze_mask_0, x = beta_27_cast_fp16)[name = string("op_6306_cast_fp16")];
            tensor<int32, [1]> var_6307_axes_0 = const()[name = string("op_6307_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_6307_cast_fp16 = expand_dims(axes = var_6307_axes_0, x = var_6306_cast_fp16)[name = string("op_6307_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_109_cast_fp16 = mul(x = var_6303_cast_fp16, y = var_6307_cast_fp16)[name = string("delta_109_cast_fp16")];
            tensor<int32, [1]> var_6310_axes_0 = const()[name = string("op_6310_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_6310_cast_fp16 = expand_dims(axes = var_6310_axes_0, x = delta_109_cast_fp16)[name = string("op_6310_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6311_cast_fp16 = mul(x = var_6299_cast_fp16, y = var_6310_cast_fp16)[name = string("op_6311_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_219_cast_fp16 = add(x = state_217_cast_fp16, y = var_6311_cast_fp16)[name = string("state_219_cast_fp16")];
            tensor<int32, [4]> var_6315_begin_0 = const()[name = string("op_6315_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> var_6315_end_0 = const()[name = string("op_6315_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> var_6315_end_mask_0 = const()[name = string("op_6315_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_6315_squeeze_mask_0 = const()[name = string("op_6315_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_6315_cast_fp16 = slice_by_index(begin = var_6315_begin_0, end = var_6315_end_0, end_mask = var_6315_end_mask_0, squeeze_mask = var_6315_squeeze_mask_0, x = _inversed_query_71_cast_fp16)[name = string("op_6315_cast_fp16")];
            tensor<int32, [1]> var_6316_axes_0 = const()[name = string("op_6316_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_6316_cast_fp16 = expand_dims(axes = var_6316_axes_0, x = var_6315_cast_fp16)[name = string("op_6316_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6317_cast_fp16 = mul(x = state_219_cast_fp16, y = var_6316_cast_fp16)[name = string("op_6317_cast_fp16")];
            tensor<int32, [1]> var_6319_axes_0 = const()[name = string("op_6319_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_6319_keep_dims_0 = const()[name = string("op_6319_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_6319_cast_fp16 = reduce_sum(axes = var_6319_axes_0, keep_dims = var_6319_keep_dims_0, x = var_6317_cast_fp16)[name = string("op_6319_cast_fp16")];
            tensor<int32, [3]> var_6322_begin_0 = const()[name = string("op_6322_begin_0"), val = tensor<int32, [3]>([0, 0, 3])];
            tensor<int32, [3]> var_6322_end_0 = const()[name = string("op_6322_end_0"), val = tensor<int32, [3]>([1, 16, 4])];
            tensor<bool, [3]> var_6322_end_mask_0 = const()[name = string("op_6322_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_6322_squeeze_mask_0 = const()[name = string("op_6322_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_6322_cast_fp16 = slice_by_index(begin = var_6322_begin_0, end = var_6322_end_0, end_mask = var_6322_end_mask_0, squeeze_mask = var_6322_squeeze_mask_0, x = decay_27_cast_fp16)[name = string("op_6322_cast_fp16")];
            tensor<fp16, [1, 16]> var_6323_cast_fp16 = exp(x = var_6322_cast_fp16)[name = string("op_6323_cast_fp16")];
            tensor<int32, [1]> var_6324_axes_0 = const()[name = string("op_6324_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_6324_cast_fp16 = expand_dims(axes = var_6324_axes_0, x = var_6323_cast_fp16)[name = string("op_6324_cast_fp16")];
            tensor<int32, [1]> var_6325_axes_0 = const()[name = string("op_6325_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_6325_cast_fp16 = expand_dims(axes = var_6325_axes_0, x = var_6324_cast_fp16)[name = string("op_6325_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_221_cast_fp16 = mul(x = state_219_cast_fp16, y = var_6325_cast_fp16)[name = string("state_221_cast_fp16")];
            tensor<int32, [4]> key_token_111_begin_0 = const()[name = string("key_token_111_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> key_token_111_end_0 = const()[name = string("key_token_111_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> key_token_111_end_mask_0 = const()[name = string("key_token_111_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_111_squeeze_mask_0 = const()[name = string("key_token_111_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_111_cast_fp16 = slice_by_index(begin = key_token_111_begin_0, end = key_token_111_end_0, end_mask = key_token_111_end_mask_0, squeeze_mask = key_token_111_squeeze_mask_0, x = key_71_cast_fp16)[name = string("key_token_111_cast_fp16")];
            tensor<int32, [4]> value_token_111_begin_0 = const()[name = string("value_token_111_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> value_token_111_end_0 = const()[name = string("value_token_111_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> value_token_111_end_mask_0 = const()[name = string("value_token_111_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_111_squeeze_mask_0 = const()[name = string("value_token_111_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_111_cast_fp16 = slice_by_index(begin = value_token_111_begin_0, end = value_token_111_end_0, end_mask = value_token_111_end_mask_0, squeeze_mask = value_token_111_squeeze_mask_0, x = value_257_cast_fp16)[name = string("value_token_111_cast_fp16")];
            tensor<int32, [1]> var_6333_axes_0 = const()[name = string("op_6333_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_6333_cast_fp16 = expand_dims(axes = var_6333_axes_0, x = key_token_111_cast_fp16)[name = string("op_6333_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6334_cast_fp16 = mul(x = state_221_cast_fp16, y = var_6333_cast_fp16)[name = string("op_6334_cast_fp16")];
            tensor<int32, [1]> memory_111_axes_0 = const()[name = string("memory_111_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_111_keep_dims_0 = const()[name = string("memory_111_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_111_cast_fp16 = reduce_sum(axes = memory_111_axes_0, keep_dims = memory_111_keep_dims_0, x = var_6334_cast_fp16)[name = string("memory_111_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_6337_cast_fp16 = sub(x = value_token_111_cast_fp16, y = memory_111_cast_fp16)[name = string("op_6337_cast_fp16")];
            tensor<int32, [3]> var_6340_begin_0 = const()[name = string("op_6340_begin_0"), val = tensor<int32, [3]>([0, 0, 3])];
            tensor<int32, [3]> var_6340_end_0 = const()[name = string("op_6340_end_0"), val = tensor<int32, [3]>([1, 16, 4])];
            tensor<bool, [3]> var_6340_end_mask_0 = const()[name = string("op_6340_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_6340_squeeze_mask_0 = const()[name = string("op_6340_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_6340_cast_fp16 = slice_by_index(begin = var_6340_begin_0, end = var_6340_end_0, end_mask = var_6340_end_mask_0, squeeze_mask = var_6340_squeeze_mask_0, x = beta_27_cast_fp16)[name = string("op_6340_cast_fp16")];
            tensor<int32, [1]> var_6341_axes_0 = const()[name = string("op_6341_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_6341_cast_fp16 = expand_dims(axes = var_6341_axes_0, x = var_6340_cast_fp16)[name = string("op_6341_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_111_cast_fp16 = mul(x = var_6337_cast_fp16, y = var_6341_cast_fp16)[name = string("delta_111_cast_fp16")];
            tensor<int32, [1]> var_6344_axes_0 = const()[name = string("op_6344_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_6344_cast_fp16 = expand_dims(axes = var_6344_axes_0, x = delta_111_cast_fp16)[name = string("op_6344_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6345_cast_fp16 = mul(x = var_6333_cast_fp16, y = var_6344_cast_fp16)[name = string("op_6345_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> rec17_out = add(x = state_221_cast_fp16, y = var_6345_cast_fp16)[name = string("state_223_cast_fp16")];
            tensor<int32, [4]> var_6349_begin_0 = const()[name = string("op_6349_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> var_6349_end_0 = const()[name = string("op_6349_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> var_6349_end_mask_0 = const()[name = string("op_6349_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_6349_squeeze_mask_0 = const()[name = string("op_6349_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_6349_cast_fp16 = slice_by_index(begin = var_6349_begin_0, end = var_6349_end_0, end_mask = var_6349_end_mask_0, squeeze_mask = var_6349_squeeze_mask_0, x = _inversed_query_71_cast_fp16)[name = string("op_6349_cast_fp16")];
            tensor<int32, [1]> var_6350_axes_0 = const()[name = string("op_6350_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_6350_cast_fp16 = expand_dims(axes = var_6350_axes_0, x = var_6349_cast_fp16)[name = string("op_6350_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6351_cast_fp16 = mul(x = rec17_out, y = var_6350_cast_fp16)[name = string("op_6351_cast_fp16")];
            tensor<int32, [1]> var_6353_axes_0 = const()[name = string("op_6353_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_6353_keep_dims_0 = const()[name = string("op_6353_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_6353_cast_fp16 = reduce_sum(axes = var_6353_axes_0, keep_dims = var_6353_keep_dims_0, x = var_6351_cast_fp16)[name = string("op_6353_cast_fp16")];
            int32 attention_131_axis_0 = const()[name = string("attention_131_axis_0"), val = int32(1)];
            tensor<fp16, [1, 4, 16, 128]> attention_131_cast_fp16 = stack(axis = attention_131_axis_0, values = (var_6251_cast_fp16, var_6285_cast_fp16, var_6319_cast_fp16, var_6353_cast_fp16))[name = string("attention_131_cast_fp16")];
            tensor<int32, [2]> var_6356 = const()[name = string("op_6356"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [64, 128]> attention_133_cast_fp16 = reshape(shape = var_6356, x = attention_131_cast_fp16)[name = string("attention_133_cast_fp16")];
            tensor<int32, [2]> var_6358 = const()[name = string("op_6358"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [64, 128]> input_227_cast_fp16 = reshape(shape = var_6358, x = linear_135_cast_fp16)[name = string("input_227_cast_fp16")];
            tensor<fp16, [64, 128]> var_6360_cast_fp16 = mul(x = attention_133_cast_fp16, y = attention_133_cast_fp16)[name = string("op_6360_cast_fp16")];
            tensor<int32, [1]> variance_113_axes_0 = const()[name = string("variance_113_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_113_keep_dims_0 = const()[name = string("variance_113_keep_dims_0"), val = bool(true)];
            tensor<fp16, [64, 1]> variance_113_cast_fp16 = reduce_mean(axes = variance_113_axes_0, keep_dims = variance_113_keep_dims_0, x = var_6360_cast_fp16)[name = string("variance_113_cast_fp16")];
            fp16 var_6363_to_fp16 = const()[name = string("op_6363_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [64, 1]> var_6364_cast_fp16 = add(x = variance_113_cast_fp16, y = var_6363_to_fp16)[name = string("op_6364_cast_fp16")];
            fp32 var_6365_epsilon_0 = const()[name = string("op_6365_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [64, 1]> var_6365_cast_fp16 = rsqrt(epsilon = var_6365_epsilon_0, x = var_6364_cast_fp16)[name = string("op_6365_cast_fp16")];
            tensor<fp16, [64, 128]> attention_135_cast_fp16 = mul(x = attention_133_cast_fp16, y = var_6365_cast_fp16)[name = string("attention_135_cast_fp16")];
            tensor<fp16, [128]> final_group_1_gated_norm_promoted_to_fp16 = const()[name = string("final_group_1_gated_norm_promoted_to_fp16"), val = tensor<fp16, [128]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(274695232)))];
            tensor<fp16, [64, 128]> attention_137_cast_fp16 = mul(x = attention_135_cast_fp16, y = final_group_1_gated_norm_promoted_to_fp16)[name = string("attention_137_cast_fp16")];
            tensor<fp16, [64, 128]> var_6368_cast_fp16 = silu(x = input_227_cast_fp16)[name = string("op_6368_cast_fp16")];
            tensor<fp16, [64, 128]> attention_139_cast_fp16 = mul(x = attention_137_cast_fp16, y = var_6368_cast_fp16)[name = string("attention_139_cast_fp16")];
            tensor<int32, [2]> var_6370 = const()[name = string("op_6370"), val = tensor<int32, [2]>([4, 2048])];
            tensor<fp16, [4, 2048]> input_229_cast_fp16 = reshape(shape = var_6370, x = attention_139_cast_fp16)[name = string("input_229_cast_fp16")];
            tensor<fp16, [1024, 2048]> final_group_1_out_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 2048]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(274695552))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(276268480))))[name = string("final_group_1_out_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 1024]> linear_136_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = final_group_1_out_proj_weight_promoted_to_fp16_palettized, x = input_229_cast_fp16)[name = string("linear_136_cast_fp16")];
            tensor<fp16, [4, 1024]> value_259_cast_fp16 = add(x = value_247_cast_fp16, y = linear_136_cast_fp16)[name = string("value_259_cast_fp16")];
            tensor<fp16, [4, 1024]> var_6375_cast_fp16 = mul(x = value_259_cast_fp16, y = value_259_cast_fp16)[name = string("op_6375_cast_fp16")];
            tensor<int32, [1]> variance_115_axes_0 = const()[name = string("variance_115_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_115_keep_dims_0 = const()[name = string("variance_115_keep_dims_0"), val = bool(true)];
            tensor<fp16, [4, 1]> variance_115_cast_fp16 = reduce_mean(axes = variance_115_axes_0, keep_dims = variance_115_keep_dims_0, x = var_6375_cast_fp16)[name = string("variance_115_cast_fp16")];
            fp16 var_6378_to_fp16 = const()[name = string("op_6378_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [4, 1]> var_6379_cast_fp16 = add(x = variance_115_cast_fp16, y = var_6378_to_fp16)[name = string("op_6379_cast_fp16")];
            fp32 var_6380_epsilon_0 = const()[name = string("op_6380_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [4, 1]> var_6380_cast_fp16 = rsqrt(epsilon = var_6380_epsilon_0, x = var_6379_cast_fp16)[name = string("op_6380_cast_fp16")];
            tensor<fp16, [4, 1024]> var_6381_cast_fp16 = mul(x = value_259_cast_fp16, y = var_6380_cast_fp16)[name = string("op_6381_cast_fp16")];
            tensor<fp16, [1024]> var_6383_to_fp16 = const()[name = string("op_6383_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(276276736)))];
            tensor<fp16, [4, 1024]> input_231_cast_fp16 = mul(x = var_6381_cast_fp16, y = var_6383_to_fp16)[name = string("input_231_cast_fp16")];
            tensor<fp16, [3584, 1024]> final_group_1_gate_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(276278848))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(279031424))))[name = string("final_group_1_gate_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 3584]> linear_137_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = final_group_1_gate_proj_weight_promoted_to_fp16_palettized, x = input_231_cast_fp16)[name = string("linear_137_cast_fp16")];
            tensor<fp16, [4, 3584]> var_6387_cast_fp16 = silu(x = linear_137_cast_fp16)[name = string("op_6387_cast_fp16")];
            tensor<fp16, [3584, 1024]> final_group_1_up_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(279060160))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(281812736))))[name = string("final_group_1_up_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 3584]> linear_138_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = final_group_1_up_proj_weight_promoted_to_fp16_palettized, x = input_231_cast_fp16)[name = string("linear_138_cast_fp16")];
            tensor<fp16, [4, 3584]> input_235_cast_fp16 = mul(x = var_6387_cast_fp16, y = linear_138_cast_fp16)[name = string("input_235_cast_fp16")];
            tensor<fp16, [1024, 3584]> final_group_1_down_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 3584]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(281841472))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(284594048))))[name = string("final_group_1_down_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 1024]> linear_139_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = final_group_1_down_proj_weight_promoted_to_fp16_palettized, x = input_235_cast_fp16)[name = string("linear_139_cast_fp16")];
            tensor<fp16, [4, 1024]> value_261_cast_fp16 = add(x = value_259_cast_fp16, y = linear_139_cast_fp16)[name = string("value_261_cast_fp16")];
            int32 var_6414 = const()[name = string("op_6414"), val = int32(-1)];
            tensor<fp16, [4, 1024]> var_6429_cast_fp16 = mul(x = value_261_cast_fp16, y = value_261_cast_fp16)[name = string("op_6429_cast_fp16")];
            tensor<int32, [1]> variance_117_axes_0 = const()[name = string("variance_117_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_117_keep_dims_0 = const()[name = string("variance_117_keep_dims_0"), val = bool(true)];
            tensor<fp16, [4, 1]> variance_117_cast_fp16 = reduce_mean(axes = variance_117_axes_0, keep_dims = variance_117_keep_dims_0, x = var_6429_cast_fp16)[name = string("variance_117_cast_fp16")];
            fp16 var_6432_to_fp16 = const()[name = string("op_6432_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [4, 1]> var_6433_cast_fp16 = add(x = variance_117_cast_fp16, y = var_6432_to_fp16)[name = string("op_6433_cast_fp16")];
            fp32 var_6434_epsilon_0 = const()[name = string("op_6434_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [4, 1]> var_6434_cast_fp16 = rsqrt(epsilon = var_6434_epsilon_0, x = var_6433_cast_fp16)[name = string("op_6434_cast_fp16")];
            tensor<fp16, [4, 1024]> var_6435_cast_fp16 = mul(x = value_261_cast_fp16, y = var_6434_cast_fp16)[name = string("op_6435_cast_fp16")];
            tensor<fp16, [1024]> var_6437_to_fp16 = const()[name = string("op_6437_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(284602304)))];
            tensor<fp16, [4, 1024]> input_237_cast_fp16 = mul(x = var_6435_cast_fp16, y = var_6437_to_fp16)[name = string("input_237_cast_fp16")];
            tensor<fp16, [6144, 1024]> final_group_2_in_proj_qkv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(284604416))), lut = tensor<fp16, [384, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(289323072))))[name = string("final_group_2_in_proj_qkv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 6144]> linear_140_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = final_group_2_in_proj_qkv_weight_promoted_to_fp16_palettized, x = input_237_cast_fp16)[name = string("linear_140_cast_fp16")];
            tensor<int32, [2]> var_6441_perm_0 = const()[name = string("op_6441_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> mixed_axes_0 = const()[name = string("mixed_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [6144, 4]> var_6441_cast_fp16 = transpose(perm = var_6441_perm_0, x = linear_140_cast_fp16)[name = string("transpose_10")];
            tensor<fp16, [1, 6144, 4]> mixed_cast_fp16 = expand_dims(axes = mixed_axes_0, x = var_6441_cast_fp16)[name = string("mixed_cast_fp16")];
            bool convolution_input_interleave_0 = const()[name = string("convolution_input_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 6144, 7]> convolution_input_cast_fp16 = concat(axis = var_6414, interleave = convolution_input_interleave_0, values = (conv18, mixed_cast_fp16))[name = string("convolution_input_cast_fp16")];
            string input_239_pad_type_0 = const()[name = string("input_239_pad_type_0"), val = string("valid")];
            int32 input_239_groups_0 = const()[name = string("input_239_groups_0"), val = int32(6144)];
            tensor<int32, [1]> input_239_strides_0 = const()[name = string("input_239_strides_0"), val = tensor<int32, [1]>([1])];
            tensor<int32, [2]> input_239_pad_0 = const()[name = string("input_239_pad_0"), val = tensor<int32, [2]>([0, 0])];
            tensor<int32, [1]> input_239_dilations_0 = const()[name = string("input_239_dilations_0"), val = tensor<int32, [1]>([1])];
            tensor<fp16, [6144, 1, 4]> final_group_2_conv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1, 4]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(289372288))), lut = tensor<fp16, [384, 1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(289390784))))[name = string("final_group_2_conv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 6144, 4]> input_239_cast_fp16 = conv(dilations = input_239_dilations_0, groups = input_239_groups_0, pad = input_239_pad_0, pad_type = input_239_pad_type_0, strides = input_239_strides_0, weight = final_group_2_conv_weight_promoted_to_fp16_palettized, x = convolution_input_cast_fp16)[name = string("input_239_cast_fp16")];
            tensor<fp16, [1, 6144, 4]> var_6450_cast_fp16 = silu(x = input_239_cast_fp16)[name = string("op_6450_cast_fp16")];
            tensor<int32, [3]> var_6451_perm_0 = const()[name = string("op_6451_perm_0"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [3]> var_6454_begin_0 = const()[name = string("op_6454_begin_0"), val = tensor<int32, [3]>([0, 0, 4])];
            tensor<int32, [3]> var_6454_end_0 = const()[name = string("op_6454_end_0"), val = tensor<int32, [3]>([1, 6144, 7])];
            tensor<bool, [3]> var_6454_end_mask_0 = const()[name = string("op_6454_end_mask_0"), val = tensor<bool, [3]>([true, true, true])];
            tensor<fp16, [1, 6144, 3]> conv18_out = slice_by_index(begin = var_6454_begin_0, end = var_6454_end_0, end_mask = var_6454_end_mask_0, x = convolution_input_cast_fp16)[name = string("op_6454_cast_fp16")];
            tensor<int32, [3]> var_6455 = const()[name = string("op_6455"), val = tensor<int32, [3]>([2048, 2048, 2048])];
            int32 var_6456_axis_0 = const()[name = string("op_6456_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 4, 6144]> var_6451_cast_fp16 = transpose(perm = var_6451_perm_0, x = var_6450_cast_fp16)[name = string("transpose_9")];
            tensor<fp16, [1, 4, 2048]> var_6456_cast_fp16_0, tensor<fp16, [1, 4, 2048]> var_6456_cast_fp16_1, tensor<fp16, [1, 4, 2048]> var_6456_cast_fp16_2 = split(axis = var_6456_axis_0, split_sizes = var_6455, x = var_6451_cast_fp16)[name = string("op_6456_cast_fp16")];
            tensor<int32, [4]> var_6460 = const()[name = string("op_6460"), val = tensor<int32, [4]>([1, 4, 16, 128])];
            tensor<fp16, [1, 4, 16, 128]> value_265_cast_fp16 = reshape(shape = var_6460, x = var_6456_cast_fp16_0)[name = string("value_265_cast_fp16")];
            tensor<int32, [4]> var_6462 = const()[name = string("op_6462"), val = tensor<int32, [4]>([1, 4, 16, 128])];
            tensor<fp16, [1, 4, 16, 128]> value_267_cast_fp16 = reshape(shape = var_6462, x = var_6456_cast_fp16_1)[name = string("value_267_cast_fp16")];
            tensor<int32, [4]> var_6464 = const()[name = string("op_6464"), val = tensor<int32, [4]>([1, 4, 16, 128])];
            tensor<fp16, [1, 4, 16, 128]> value_269_cast_fp16 = reshape(shape = var_6464, x = var_6456_cast_fp16_2)[name = string("value_269_cast_fp16")];
            tensor<fp16, [1, 4, 16, 128]> var_6466_cast_fp16 = mul(x = value_265_cast_fp16, y = value_265_cast_fp16)[name = string("op_6466_cast_fp16")];
            tensor<int32, [1]> var_6468_axes_0 = const()[name = string("op_6468_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_6468_keep_dims_0 = const()[name = string("op_6468_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 4, 16, 1]> var_6468_cast_fp16 = reduce_sum(axes = var_6468_axes_0, keep_dims = var_6468_keep_dims_0, x = var_6466_cast_fp16)[name = string("op_6468_cast_fp16")];
            fp16 var_6469_to_fp16 = const()[name = string("op_6469_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 4, 16, 1]> var_6470_cast_fp16 = add(x = var_6468_cast_fp16, y = var_6469_to_fp16)[name = string("op_6470_cast_fp16")];
            fp32 var_6471_epsilon_0 = const()[name = string("op_6471_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 4, 16, 1]> var_6471_cast_fp16 = rsqrt(epsilon = var_6471_epsilon_0, x = var_6470_cast_fp16)[name = string("op_6471_cast_fp16")];
            tensor<fp16, [1, 4, 16, 128]> var_6472_cast_fp16 = mul(x = value_265_cast_fp16, y = var_6471_cast_fp16)[name = string("op_6472_cast_fp16")];
            tensor<int32, [4]> var_6473_perm_0 = const()[name = string("op_6473_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            fp16 _inversed_query_75_y_0_to_fp16 = const()[name = string("_inversed_query_75_y_0_to_fp16"), val = fp16(0x1.6ap-4)];
            tensor<fp16, [1, 16, 4, 128]> var_6473_cast_fp16 = transpose(perm = var_6473_perm_0, x = var_6472_cast_fp16)[name = string("transpose_8")];
            tensor<fp16, [1, 16, 4, 128]> _inversed_query_75_cast_fp16 = mul(x = var_6473_cast_fp16, y = _inversed_query_75_y_0_to_fp16)[name = string("_inversed_query_75_cast_fp16")];
            tensor<fp16, [1, 4, 16, 128]> var_6476_cast_fp16 = mul(x = value_267_cast_fp16, y = value_267_cast_fp16)[name = string("op_6476_cast_fp16")];
            tensor<int32, [1]> var_6478_axes_0 = const()[name = string("op_6478_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_6478_keep_dims_0 = const()[name = string("op_6478_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 4, 16, 1]> var_6478_cast_fp16 = reduce_sum(axes = var_6478_axes_0, keep_dims = var_6478_keep_dims_0, x = var_6476_cast_fp16)[name = string("op_6478_cast_fp16")];
            fp16 var_6479_to_fp16 = const()[name = string("op_6479_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 4, 16, 1]> var_6480_cast_fp16 = add(x = var_6478_cast_fp16, y = var_6479_to_fp16)[name = string("op_6480_cast_fp16")];
            fp32 var_6481_epsilon_0 = const()[name = string("op_6481_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 4, 16, 1]> var_6481_cast_fp16 = rsqrt(epsilon = var_6481_epsilon_0, x = var_6480_cast_fp16)[name = string("op_6481_cast_fp16")];
            tensor<fp16, [1, 4, 16, 128]> var_6482_cast_fp16 = mul(x = value_267_cast_fp16, y = var_6481_cast_fp16)[name = string("op_6482_cast_fp16")];
            tensor<int32, [4]> key_75_perm_0 = const()[name = string("key_75_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<int32, [4]> value_271_perm_0 = const()[name = string("value_271_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<fp16, [16, 1024]> final_group_2_in_proj_b_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(289440000))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(289452352))))[name = string("final_group_2_in_proj_b_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16]> linear_141_bias_0_to_fp16 = const()[name = string("linear_141_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(4862656)))];
            tensor<fp16, [4, 16]> linear_141_cast_fp16 = linear(bias = linear_141_bias_0_to_fp16, weight = final_group_2_in_proj_b_weight_promoted_to_fp16_palettized, x = input_237_cast_fp16)[name = string("linear_141_cast_fp16")];
            tensor<fp16, [4, 16]> var_6487_cast_fp16 = sigmoid(x = linear_141_cast_fp16)[name = string("op_6487_cast_fp16")];
            tensor<int32, [2]> var_6488_perm_0 = const()[name = string("op_6488_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> beta_axes_0 = const()[name = string("beta_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 4]> var_6488_cast_fp16 = transpose(perm = var_6488_perm_0, x = var_6487_cast_fp16)[name = string("transpose_7")];
            tensor<fp16, [1, 16, 4]> beta_cast_fp16 = expand_dims(axes = beta_axes_0, x = var_6488_cast_fp16)[name = string("beta_cast_fp16")];
            tensor<fp16, [16, 1024]> op_6494_weight_0_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(289452544))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(289464896))))[name = string("op_6494_weight_0_to_fp16_palettized")];
            tensor<fp16, [16]> var_6494_bias_0_to_fp16 = const()[name = string("op_6494_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(289465088)))];
            tensor<fp16, [4, 16]> var_6494_cast_fp16 = linear(bias = var_6494_bias_0_to_fp16, weight = op_6494_weight_0_to_fp16_palettized, x = input_237_cast_fp16)[name = string("op_6494_cast_fp16")];
            tensor<fp16, [4, 16]> var_6495_cast_fp16 = softplus(x = var_6494_cast_fp16)[name = string("op_6495_cast_fp16")];
            tensor<fp16, [16]> var_6491_promoted_to_fp16 = const()[name = string("op_6491_promoted_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(289465216)))];
            tensor<fp16, [4, 16]> var_6496_cast_fp16 = mul(x = var_6491_promoted_to_fp16, y = var_6495_cast_fp16)[name = string("op_6496_cast_fp16")];
            tensor<int32, [2]> var_6497_perm_0 = const()[name = string("op_6497_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> decay_axes_0 = const()[name = string("decay_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 4]> var_6497_cast_fp16 = transpose(perm = var_6497_perm_0, x = var_6496_cast_fp16)[name = string("transpose_6")];
            tensor<fp16, [1, 16, 4]> decay_cast_fp16 = expand_dims(axes = decay_axes_0, x = var_6497_cast_fp16)[name = string("decay_cast_fp16")];
            tensor<fp16, [2048, 1024]> final_group_2_in_proj_z_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [2048, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(289465344))), lut = tensor<fp16, [128, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(291038272))))[name = string("final_group_2_in_proj_z_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 2048]> linear_143_cast_fp16 = linear(bias = linear_3_bias_0_to_fp16, weight = final_group_2_in_proj_z_weight_promoted_to_fp16_palettized, x = input_237_cast_fp16)[name = string("linear_143_cast_fp16")];
            tensor<int32, [3]> var_6505_begin_0 = const()[name = string("op_6505_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_6505_end_0 = const()[name = string("op_6505_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_6505_end_mask_0 = const()[name = string("op_6505_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_6505_squeeze_mask_0 = const()[name = string("op_6505_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_6505_cast_fp16 = slice_by_index(begin = var_6505_begin_0, end = var_6505_end_0, end_mask = var_6505_end_mask_0, squeeze_mask = var_6505_squeeze_mask_0, x = decay_cast_fp16)[name = string("op_6505_cast_fp16")];
            tensor<fp16, [1, 16]> var_6506_cast_fp16 = exp(x = var_6505_cast_fp16)[name = string("op_6506_cast_fp16")];
            tensor<int32, [1]> var_6507_axes_0 = const()[name = string("op_6507_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_6507_cast_fp16 = expand_dims(axes = var_6507_axes_0, x = var_6506_cast_fp16)[name = string("op_6507_cast_fp16")];
            tensor<int32, [1]> var_6508_axes_0 = const()[name = string("op_6508_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_6508_cast_fp16 = expand_dims(axes = var_6508_axes_0, x = var_6507_cast_fp16)[name = string("op_6508_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_225_cast_fp16 = mul(x = rec18, y = var_6508_cast_fp16)[name = string("state_225_cast_fp16")];
            tensor<int32, [4]> key_token_113_begin_0 = const()[name = string("key_token_113_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> key_token_113_end_0 = const()[name = string("key_token_113_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> key_token_113_end_mask_0 = const()[name = string("key_token_113_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_113_squeeze_mask_0 = const()[name = string("key_token_113_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 4, 128]> key_75_cast_fp16 = transpose(perm = key_75_perm_0, x = var_6482_cast_fp16)[name = string("transpose_5")];
            tensor<fp16, [1, 16, 128]> key_token_113_cast_fp16 = slice_by_index(begin = key_token_113_begin_0, end = key_token_113_end_0, end_mask = key_token_113_end_mask_0, squeeze_mask = key_token_113_squeeze_mask_0, x = key_75_cast_fp16)[name = string("key_token_113_cast_fp16")];
            tensor<int32, [4]> value_token_113_begin_0 = const()[name = string("value_token_113_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> value_token_113_end_0 = const()[name = string("value_token_113_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> value_token_113_end_mask_0 = const()[name = string("value_token_113_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_113_squeeze_mask_0 = const()[name = string("value_token_113_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 4, 128]> value_271_cast_fp16 = transpose(perm = value_271_perm_0, x = value_269_cast_fp16)[name = string("transpose_4")];
            tensor<fp16, [1, 16, 128]> value_token_113_cast_fp16 = slice_by_index(begin = value_token_113_begin_0, end = value_token_113_end_0, end_mask = value_token_113_end_mask_0, squeeze_mask = value_token_113_squeeze_mask_0, x = value_271_cast_fp16)[name = string("value_token_113_cast_fp16")];
            tensor<int32, [1]> var_6516_axes_0 = const()[name = string("op_6516_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_6516_cast_fp16 = expand_dims(axes = var_6516_axes_0, x = key_token_113_cast_fp16)[name = string("op_6516_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6517_cast_fp16 = mul(x = state_225_cast_fp16, y = var_6516_cast_fp16)[name = string("op_6517_cast_fp16")];
            tensor<int32, [1]> memory_113_axes_0 = const()[name = string("memory_113_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_113_keep_dims_0 = const()[name = string("memory_113_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_113_cast_fp16 = reduce_sum(axes = memory_113_axes_0, keep_dims = memory_113_keep_dims_0, x = var_6517_cast_fp16)[name = string("memory_113_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_6520_cast_fp16 = sub(x = value_token_113_cast_fp16, y = memory_113_cast_fp16)[name = string("op_6520_cast_fp16")];
            tensor<int32, [3]> var_6523_begin_0 = const()[name = string("op_6523_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_6523_end_0 = const()[name = string("op_6523_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_6523_end_mask_0 = const()[name = string("op_6523_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_6523_squeeze_mask_0 = const()[name = string("op_6523_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_6523_cast_fp16 = slice_by_index(begin = var_6523_begin_0, end = var_6523_end_0, end_mask = var_6523_end_mask_0, squeeze_mask = var_6523_squeeze_mask_0, x = beta_cast_fp16)[name = string("op_6523_cast_fp16")];
            tensor<int32, [1]> var_6524_axes_0 = const()[name = string("op_6524_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_6524_cast_fp16 = expand_dims(axes = var_6524_axes_0, x = var_6523_cast_fp16)[name = string("op_6524_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_113_cast_fp16 = mul(x = var_6520_cast_fp16, y = var_6524_cast_fp16)[name = string("delta_113_cast_fp16")];
            tensor<int32, [1]> var_6527_axes_0 = const()[name = string("op_6527_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_6527_cast_fp16 = expand_dims(axes = var_6527_axes_0, x = delta_113_cast_fp16)[name = string("op_6527_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6528_cast_fp16 = mul(x = var_6516_cast_fp16, y = var_6527_cast_fp16)[name = string("op_6528_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_227_cast_fp16 = add(x = state_225_cast_fp16, y = var_6528_cast_fp16)[name = string("state_227_cast_fp16")];
            tensor<int32, [4]> var_6532_begin_0 = const()[name = string("op_6532_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> var_6532_end_0 = const()[name = string("op_6532_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> var_6532_end_mask_0 = const()[name = string("op_6532_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_6532_squeeze_mask_0 = const()[name = string("op_6532_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_6532_cast_fp16 = slice_by_index(begin = var_6532_begin_0, end = var_6532_end_0, end_mask = var_6532_end_mask_0, squeeze_mask = var_6532_squeeze_mask_0, x = _inversed_query_75_cast_fp16)[name = string("op_6532_cast_fp16")];
            tensor<int32, [1]> var_6533_axes_0 = const()[name = string("op_6533_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_6533_cast_fp16 = expand_dims(axes = var_6533_axes_0, x = var_6532_cast_fp16)[name = string("op_6533_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6534_cast_fp16 = mul(x = state_227_cast_fp16, y = var_6533_cast_fp16)[name = string("op_6534_cast_fp16")];
            tensor<int32, [1]> var_6536_axes_0 = const()[name = string("op_6536_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_6536_keep_dims_0 = const()[name = string("op_6536_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_6536_cast_fp16 = reduce_sum(axes = var_6536_axes_0, keep_dims = var_6536_keep_dims_0, x = var_6534_cast_fp16)[name = string("op_6536_cast_fp16")];
            tensor<int32, [3]> var_6539_begin_0 = const()[name = string("op_6539_begin_0"), val = tensor<int32, [3]>([0, 0, 1])];
            tensor<int32, [3]> var_6539_end_0 = const()[name = string("op_6539_end_0"), val = tensor<int32, [3]>([1, 16, 2])];
            tensor<bool, [3]> var_6539_end_mask_0 = const()[name = string("op_6539_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_6539_squeeze_mask_0 = const()[name = string("op_6539_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_6539_cast_fp16 = slice_by_index(begin = var_6539_begin_0, end = var_6539_end_0, end_mask = var_6539_end_mask_0, squeeze_mask = var_6539_squeeze_mask_0, x = decay_cast_fp16)[name = string("op_6539_cast_fp16")];
            tensor<fp16, [1, 16]> var_6540_cast_fp16 = exp(x = var_6539_cast_fp16)[name = string("op_6540_cast_fp16")];
            tensor<int32, [1]> var_6541_axes_0 = const()[name = string("op_6541_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_6541_cast_fp16 = expand_dims(axes = var_6541_axes_0, x = var_6540_cast_fp16)[name = string("op_6541_cast_fp16")];
            tensor<int32, [1]> var_6542_axes_0 = const()[name = string("op_6542_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_6542_cast_fp16 = expand_dims(axes = var_6542_axes_0, x = var_6541_cast_fp16)[name = string("op_6542_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_229_cast_fp16 = mul(x = state_227_cast_fp16, y = var_6542_cast_fp16)[name = string("state_229_cast_fp16")];
            tensor<int32, [4]> key_token_115_begin_0 = const()[name = string("key_token_115_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> key_token_115_end_0 = const()[name = string("key_token_115_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> key_token_115_end_mask_0 = const()[name = string("key_token_115_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_115_squeeze_mask_0 = const()[name = string("key_token_115_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_115_cast_fp16 = slice_by_index(begin = key_token_115_begin_0, end = key_token_115_end_0, end_mask = key_token_115_end_mask_0, squeeze_mask = key_token_115_squeeze_mask_0, x = key_75_cast_fp16)[name = string("key_token_115_cast_fp16")];
            tensor<int32, [4]> value_token_115_begin_0 = const()[name = string("value_token_115_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> value_token_115_end_0 = const()[name = string("value_token_115_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> value_token_115_end_mask_0 = const()[name = string("value_token_115_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_115_squeeze_mask_0 = const()[name = string("value_token_115_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_115_cast_fp16 = slice_by_index(begin = value_token_115_begin_0, end = value_token_115_end_0, end_mask = value_token_115_end_mask_0, squeeze_mask = value_token_115_squeeze_mask_0, x = value_271_cast_fp16)[name = string("value_token_115_cast_fp16")];
            tensor<int32, [1]> var_6550_axes_0 = const()[name = string("op_6550_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_6550_cast_fp16 = expand_dims(axes = var_6550_axes_0, x = key_token_115_cast_fp16)[name = string("op_6550_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6551_cast_fp16 = mul(x = state_229_cast_fp16, y = var_6550_cast_fp16)[name = string("op_6551_cast_fp16")];
            tensor<int32, [1]> memory_115_axes_0 = const()[name = string("memory_115_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_115_keep_dims_0 = const()[name = string("memory_115_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_115_cast_fp16 = reduce_sum(axes = memory_115_axes_0, keep_dims = memory_115_keep_dims_0, x = var_6551_cast_fp16)[name = string("memory_115_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_6554_cast_fp16 = sub(x = value_token_115_cast_fp16, y = memory_115_cast_fp16)[name = string("op_6554_cast_fp16")];
            tensor<int32, [3]> var_6557_begin_0 = const()[name = string("op_6557_begin_0"), val = tensor<int32, [3]>([0, 0, 1])];
            tensor<int32, [3]> var_6557_end_0 = const()[name = string("op_6557_end_0"), val = tensor<int32, [3]>([1, 16, 2])];
            tensor<bool, [3]> var_6557_end_mask_0 = const()[name = string("op_6557_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_6557_squeeze_mask_0 = const()[name = string("op_6557_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_6557_cast_fp16 = slice_by_index(begin = var_6557_begin_0, end = var_6557_end_0, end_mask = var_6557_end_mask_0, squeeze_mask = var_6557_squeeze_mask_0, x = beta_cast_fp16)[name = string("op_6557_cast_fp16")];
            tensor<int32, [1]> var_6558_axes_0 = const()[name = string("op_6558_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_6558_cast_fp16 = expand_dims(axes = var_6558_axes_0, x = var_6557_cast_fp16)[name = string("op_6558_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_115_cast_fp16 = mul(x = var_6554_cast_fp16, y = var_6558_cast_fp16)[name = string("delta_115_cast_fp16")];
            tensor<int32, [1]> var_6561_axes_0 = const()[name = string("op_6561_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_6561_cast_fp16 = expand_dims(axes = var_6561_axes_0, x = delta_115_cast_fp16)[name = string("op_6561_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6562_cast_fp16 = mul(x = var_6550_cast_fp16, y = var_6561_cast_fp16)[name = string("op_6562_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_231_cast_fp16 = add(x = state_229_cast_fp16, y = var_6562_cast_fp16)[name = string("state_231_cast_fp16")];
            tensor<int32, [4]> var_6566_begin_0 = const()[name = string("op_6566_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> var_6566_end_0 = const()[name = string("op_6566_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> var_6566_end_mask_0 = const()[name = string("op_6566_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_6566_squeeze_mask_0 = const()[name = string("op_6566_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_6566_cast_fp16 = slice_by_index(begin = var_6566_begin_0, end = var_6566_end_0, end_mask = var_6566_end_mask_0, squeeze_mask = var_6566_squeeze_mask_0, x = _inversed_query_75_cast_fp16)[name = string("op_6566_cast_fp16")];
            tensor<int32, [1]> var_6567_axes_0 = const()[name = string("op_6567_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_6567_cast_fp16 = expand_dims(axes = var_6567_axes_0, x = var_6566_cast_fp16)[name = string("op_6567_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6568_cast_fp16 = mul(x = state_231_cast_fp16, y = var_6567_cast_fp16)[name = string("op_6568_cast_fp16")];
            tensor<int32, [1]> var_6570_axes_0 = const()[name = string("op_6570_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_6570_keep_dims_0 = const()[name = string("op_6570_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_6570_cast_fp16 = reduce_sum(axes = var_6570_axes_0, keep_dims = var_6570_keep_dims_0, x = var_6568_cast_fp16)[name = string("op_6570_cast_fp16")];
            tensor<int32, [3]> var_6573_begin_0 = const()[name = string("op_6573_begin_0"), val = tensor<int32, [3]>([0, 0, 2])];
            tensor<int32, [3]> var_6573_end_0 = const()[name = string("op_6573_end_0"), val = tensor<int32, [3]>([1, 16, 3])];
            tensor<bool, [3]> var_6573_end_mask_0 = const()[name = string("op_6573_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_6573_squeeze_mask_0 = const()[name = string("op_6573_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_6573_cast_fp16 = slice_by_index(begin = var_6573_begin_0, end = var_6573_end_0, end_mask = var_6573_end_mask_0, squeeze_mask = var_6573_squeeze_mask_0, x = decay_cast_fp16)[name = string("op_6573_cast_fp16")];
            tensor<fp16, [1, 16]> var_6574_cast_fp16 = exp(x = var_6573_cast_fp16)[name = string("op_6574_cast_fp16")];
            tensor<int32, [1]> var_6575_axes_0 = const()[name = string("op_6575_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_6575_cast_fp16 = expand_dims(axes = var_6575_axes_0, x = var_6574_cast_fp16)[name = string("op_6575_cast_fp16")];
            tensor<int32, [1]> var_6576_axes_0 = const()[name = string("op_6576_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_6576_cast_fp16 = expand_dims(axes = var_6576_axes_0, x = var_6575_cast_fp16)[name = string("op_6576_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_233_cast_fp16 = mul(x = state_231_cast_fp16, y = var_6576_cast_fp16)[name = string("state_233_cast_fp16")];
            tensor<int32, [4]> key_token_117_begin_0 = const()[name = string("key_token_117_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> key_token_117_end_0 = const()[name = string("key_token_117_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> key_token_117_end_mask_0 = const()[name = string("key_token_117_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_117_squeeze_mask_0 = const()[name = string("key_token_117_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_117_cast_fp16 = slice_by_index(begin = key_token_117_begin_0, end = key_token_117_end_0, end_mask = key_token_117_end_mask_0, squeeze_mask = key_token_117_squeeze_mask_0, x = key_75_cast_fp16)[name = string("key_token_117_cast_fp16")];
            tensor<int32, [4]> value_token_117_begin_0 = const()[name = string("value_token_117_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> value_token_117_end_0 = const()[name = string("value_token_117_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> value_token_117_end_mask_0 = const()[name = string("value_token_117_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_117_squeeze_mask_0 = const()[name = string("value_token_117_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_117_cast_fp16 = slice_by_index(begin = value_token_117_begin_0, end = value_token_117_end_0, end_mask = value_token_117_end_mask_0, squeeze_mask = value_token_117_squeeze_mask_0, x = value_271_cast_fp16)[name = string("value_token_117_cast_fp16")];
            tensor<int32, [1]> var_6584_axes_0 = const()[name = string("op_6584_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_6584_cast_fp16 = expand_dims(axes = var_6584_axes_0, x = key_token_117_cast_fp16)[name = string("op_6584_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6585_cast_fp16 = mul(x = state_233_cast_fp16, y = var_6584_cast_fp16)[name = string("op_6585_cast_fp16")];
            tensor<int32, [1]> memory_117_axes_0 = const()[name = string("memory_117_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_117_keep_dims_0 = const()[name = string("memory_117_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_117_cast_fp16 = reduce_sum(axes = memory_117_axes_0, keep_dims = memory_117_keep_dims_0, x = var_6585_cast_fp16)[name = string("memory_117_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_6588_cast_fp16 = sub(x = value_token_117_cast_fp16, y = memory_117_cast_fp16)[name = string("op_6588_cast_fp16")];
            tensor<int32, [3]> var_6591_begin_0 = const()[name = string("op_6591_begin_0"), val = tensor<int32, [3]>([0, 0, 2])];
            tensor<int32, [3]> var_6591_end_0 = const()[name = string("op_6591_end_0"), val = tensor<int32, [3]>([1, 16, 3])];
            tensor<bool, [3]> var_6591_end_mask_0 = const()[name = string("op_6591_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_6591_squeeze_mask_0 = const()[name = string("op_6591_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_6591_cast_fp16 = slice_by_index(begin = var_6591_begin_0, end = var_6591_end_0, end_mask = var_6591_end_mask_0, squeeze_mask = var_6591_squeeze_mask_0, x = beta_cast_fp16)[name = string("op_6591_cast_fp16")];
            tensor<int32, [1]> var_6592_axes_0 = const()[name = string("op_6592_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_6592_cast_fp16 = expand_dims(axes = var_6592_axes_0, x = var_6591_cast_fp16)[name = string("op_6592_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_117_cast_fp16 = mul(x = var_6588_cast_fp16, y = var_6592_cast_fp16)[name = string("delta_117_cast_fp16")];
            tensor<int32, [1]> var_6595_axes_0 = const()[name = string("op_6595_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_6595_cast_fp16 = expand_dims(axes = var_6595_axes_0, x = delta_117_cast_fp16)[name = string("op_6595_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6596_cast_fp16 = mul(x = var_6584_cast_fp16, y = var_6595_cast_fp16)[name = string("op_6596_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_235_cast_fp16 = add(x = state_233_cast_fp16, y = var_6596_cast_fp16)[name = string("state_235_cast_fp16")];
            tensor<int32, [4]> var_6600_begin_0 = const()[name = string("op_6600_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> var_6600_end_0 = const()[name = string("op_6600_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> var_6600_end_mask_0 = const()[name = string("op_6600_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_6600_squeeze_mask_0 = const()[name = string("op_6600_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_6600_cast_fp16 = slice_by_index(begin = var_6600_begin_0, end = var_6600_end_0, end_mask = var_6600_end_mask_0, squeeze_mask = var_6600_squeeze_mask_0, x = _inversed_query_75_cast_fp16)[name = string("op_6600_cast_fp16")];
            tensor<int32, [1]> var_6601_axes_0 = const()[name = string("op_6601_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_6601_cast_fp16 = expand_dims(axes = var_6601_axes_0, x = var_6600_cast_fp16)[name = string("op_6601_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6602_cast_fp16 = mul(x = state_235_cast_fp16, y = var_6601_cast_fp16)[name = string("op_6602_cast_fp16")];
            tensor<int32, [1]> var_6604_axes_0 = const()[name = string("op_6604_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_6604_keep_dims_0 = const()[name = string("op_6604_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_6604_cast_fp16 = reduce_sum(axes = var_6604_axes_0, keep_dims = var_6604_keep_dims_0, x = var_6602_cast_fp16)[name = string("op_6604_cast_fp16")];
            tensor<int32, [3]> var_6607_begin_0 = const()[name = string("op_6607_begin_0"), val = tensor<int32, [3]>([0, 0, 3])];
            tensor<int32, [3]> var_6607_end_0 = const()[name = string("op_6607_end_0"), val = tensor<int32, [3]>([1, 16, 4])];
            tensor<bool, [3]> var_6607_end_mask_0 = const()[name = string("op_6607_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_6607_squeeze_mask_0 = const()[name = string("op_6607_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_6607_cast_fp16 = slice_by_index(begin = var_6607_begin_0, end = var_6607_end_0, end_mask = var_6607_end_mask_0, squeeze_mask = var_6607_squeeze_mask_0, x = decay_cast_fp16)[name = string("op_6607_cast_fp16")];
            tensor<fp16, [1, 16]> var_6608_cast_fp16 = exp(x = var_6607_cast_fp16)[name = string("op_6608_cast_fp16")];
            tensor<int32, [1]> var_6609_axes_0 = const()[name = string("op_6609_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_6609_cast_fp16 = expand_dims(axes = var_6609_axes_0, x = var_6608_cast_fp16)[name = string("op_6609_cast_fp16")];
            tensor<int32, [1]> var_6610_axes_0 = const()[name = string("op_6610_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_6610_cast_fp16 = expand_dims(axes = var_6610_axes_0, x = var_6609_cast_fp16)[name = string("op_6610_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_237_cast_fp16 = mul(x = state_235_cast_fp16, y = var_6610_cast_fp16)[name = string("state_237_cast_fp16")];
            tensor<int32, [4]> key_token_begin_0 = const()[name = string("key_token_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> key_token_end_0 = const()[name = string("key_token_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> key_token_end_mask_0 = const()[name = string("key_token_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_squeeze_mask_0 = const()[name = string("key_token_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_cast_fp16 = slice_by_index(begin = key_token_begin_0, end = key_token_end_0, end_mask = key_token_end_mask_0, squeeze_mask = key_token_squeeze_mask_0, x = key_75_cast_fp16)[name = string("key_token_cast_fp16")];
            tensor<int32, [4]> value_token_begin_0 = const()[name = string("value_token_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> value_token_end_0 = const()[name = string("value_token_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> value_token_end_mask_0 = const()[name = string("value_token_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_squeeze_mask_0 = const()[name = string("value_token_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_cast_fp16 = slice_by_index(begin = value_token_begin_0, end = value_token_end_0, end_mask = value_token_end_mask_0, squeeze_mask = value_token_squeeze_mask_0, x = value_271_cast_fp16)[name = string("value_token_cast_fp16")];
            tensor<int32, [1]> var_6618_axes_0 = const()[name = string("op_6618_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_6618_cast_fp16 = expand_dims(axes = var_6618_axes_0, x = key_token_cast_fp16)[name = string("op_6618_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6619_cast_fp16 = mul(x = state_237_cast_fp16, y = var_6618_cast_fp16)[name = string("op_6619_cast_fp16")];
            tensor<int32, [1]> memory_axes_0 = const()[name = string("memory_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_keep_dims_0 = const()[name = string("memory_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_cast_fp16 = reduce_sum(axes = memory_axes_0, keep_dims = memory_keep_dims_0, x = var_6619_cast_fp16)[name = string("memory_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_6622_cast_fp16 = sub(x = value_token_cast_fp16, y = memory_cast_fp16)[name = string("op_6622_cast_fp16")];
            tensor<int32, [3]> var_6625_begin_0 = const()[name = string("op_6625_begin_0"), val = tensor<int32, [3]>([0, 0, 3])];
            tensor<int32, [3]> var_6625_end_0 = const()[name = string("op_6625_end_0"), val = tensor<int32, [3]>([1, 16, 4])];
            tensor<bool, [3]> var_6625_end_mask_0 = const()[name = string("op_6625_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_6625_squeeze_mask_0 = const()[name = string("op_6625_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_6625_cast_fp16 = slice_by_index(begin = var_6625_begin_0, end = var_6625_end_0, end_mask = var_6625_end_mask_0, squeeze_mask = var_6625_squeeze_mask_0, x = beta_cast_fp16)[name = string("op_6625_cast_fp16")];
            tensor<int32, [1]> var_6626_axes_0 = const()[name = string("op_6626_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_6626_cast_fp16 = expand_dims(axes = var_6626_axes_0, x = var_6625_cast_fp16)[name = string("op_6626_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_cast_fp16 = mul(x = var_6622_cast_fp16, y = var_6626_cast_fp16)[name = string("delta_cast_fp16")];
            tensor<int32, [1]> var_6629_axes_0 = const()[name = string("op_6629_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_6629_cast_fp16 = expand_dims(axes = var_6629_axes_0, x = delta_cast_fp16)[name = string("op_6629_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6630_cast_fp16 = mul(x = var_6618_cast_fp16, y = var_6629_cast_fp16)[name = string("op_6630_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> rec18_out = add(x = state_237_cast_fp16, y = var_6630_cast_fp16)[name = string("state_cast_fp16")];
            tensor<int32, [4]> var_6634_begin_0 = const()[name = string("op_6634_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> var_6634_end_0 = const()[name = string("op_6634_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> var_6634_end_mask_0 = const()[name = string("op_6634_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_6634_squeeze_mask_0 = const()[name = string("op_6634_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_6634_cast_fp16 = slice_by_index(begin = var_6634_begin_0, end = var_6634_end_0, end_mask = var_6634_end_mask_0, squeeze_mask = var_6634_squeeze_mask_0, x = _inversed_query_75_cast_fp16)[name = string("op_6634_cast_fp16")];
            tensor<int32, [1]> var_6635_axes_0 = const()[name = string("op_6635_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_6635_cast_fp16 = expand_dims(axes = var_6635_axes_0, x = var_6634_cast_fp16)[name = string("op_6635_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_6636_cast_fp16 = mul(x = rec18_out, y = var_6635_cast_fp16)[name = string("op_6636_cast_fp16")];
            tensor<int32, [1]> var_6638_axes_0 = const()[name = string("op_6638_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_6638_keep_dims_0 = const()[name = string("op_6638_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_6638_cast_fp16 = reduce_sum(axes = var_6638_axes_0, keep_dims = var_6638_keep_dims_0, x = var_6636_cast_fp16)[name = string("op_6638_cast_fp16")];
            int32 attention_141_axis_0 = const()[name = string("attention_141_axis_0"), val = int32(1)];
            tensor<fp16, [1, 4, 16, 128]> attention_141_cast_fp16 = stack(axis = attention_141_axis_0, values = (var_6536_cast_fp16, var_6570_cast_fp16, var_6604_cast_fp16, var_6638_cast_fp16))[name = string("attention_141_cast_fp16")];
            tensor<int32, [2]> var_6641 = const()[name = string("op_6641"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [64, 128]> attention_143_cast_fp16 = reshape(shape = var_6641, x = attention_141_cast_fp16)[name = string("attention_143_cast_fp16")];
            tensor<int32, [2]> var_6643 = const()[name = string("op_6643"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [64, 128]> input_241_cast_fp16 = reshape(shape = var_6643, x = linear_143_cast_fp16)[name = string("input_241_cast_fp16")];
            tensor<fp16, [64, 128]> var_6645_cast_fp16 = mul(x = attention_143_cast_fp16, y = attention_143_cast_fp16)[name = string("op_6645_cast_fp16")];
            tensor<int32, [1]> variance_119_axes_0 = const()[name = string("variance_119_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_119_keep_dims_0 = const()[name = string("variance_119_keep_dims_0"), val = bool(true)];
            tensor<fp16, [64, 1]> variance_119_cast_fp16 = reduce_mean(axes = variance_119_axes_0, keep_dims = variance_119_keep_dims_0, x = var_6645_cast_fp16)[name = string("variance_119_cast_fp16")];
            fp16 var_6648_to_fp16 = const()[name = string("op_6648_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [64, 1]> var_6649_cast_fp16 = add(x = variance_119_cast_fp16, y = var_6648_to_fp16)[name = string("op_6649_cast_fp16")];
            fp32 var_6650_epsilon_0 = const()[name = string("op_6650_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [64, 1]> var_6650_cast_fp16 = rsqrt(epsilon = var_6650_epsilon_0, x = var_6649_cast_fp16)[name = string("op_6650_cast_fp16")];
            tensor<fp16, [64, 128]> attention_145_cast_fp16 = mul(x = attention_143_cast_fp16, y = var_6650_cast_fp16)[name = string("attention_145_cast_fp16")];
            tensor<fp16, [128]> final_group_2_gated_norm_promoted_to_fp16 = const()[name = string("final_group_2_gated_norm_promoted_to_fp16"), val = tensor<fp16, [128]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(291054720)))];
            tensor<fp16, [64, 128]> attention_147_cast_fp16 = mul(x = attention_145_cast_fp16, y = final_group_2_gated_norm_promoted_to_fp16)[name = string("attention_147_cast_fp16")];
            tensor<fp16, [64, 128]> var_6653_cast_fp16 = silu(x = input_241_cast_fp16)[name = string("op_6653_cast_fp16")];
            tensor<fp16, [64, 128]> attention_149_cast_fp16 = mul(x = attention_147_cast_fp16, y = var_6653_cast_fp16)[name = string("attention_149_cast_fp16")];
            tensor<int32, [2]> var_6655 = const()[name = string("op_6655"), val = tensor<int32, [2]>([4, 2048])];
            tensor<fp16, [4, 2048]> input_243_cast_fp16 = reshape(shape = var_6655, x = attention_149_cast_fp16)[name = string("input_243_cast_fp16")];
            tensor<fp16, [1024, 2048]> final_group_2_out_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 2048]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(291055040))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(292627968))))[name = string("final_group_2_out_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 1024]> linear_144_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = final_group_2_out_proj_weight_promoted_to_fp16_palettized, x = input_243_cast_fp16)[name = string("linear_144_cast_fp16")];
            tensor<fp16, [4, 1024]> value_273_cast_fp16 = add(x = value_261_cast_fp16, y = linear_144_cast_fp16)[name = string("value_273_cast_fp16")];
            tensor<fp16, [4, 1024]> var_6660_cast_fp16 = mul(x = value_273_cast_fp16, y = value_273_cast_fp16)[name = string("op_6660_cast_fp16")];
            tensor<int32, [1]> variance_121_axes_0 = const()[name = string("variance_121_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_121_keep_dims_0 = const()[name = string("variance_121_keep_dims_0"), val = bool(true)];
            tensor<fp16, [4, 1]> variance_121_cast_fp16 = reduce_mean(axes = variance_121_axes_0, keep_dims = variance_121_keep_dims_0, x = var_6660_cast_fp16)[name = string("variance_121_cast_fp16")];
            fp16 var_6663_to_fp16 = const()[name = string("op_6663_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [4, 1]> var_6664_cast_fp16 = add(x = variance_121_cast_fp16, y = var_6663_to_fp16)[name = string("op_6664_cast_fp16")];
            fp32 var_6665_epsilon_0 = const()[name = string("op_6665_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [4, 1]> var_6665_cast_fp16 = rsqrt(epsilon = var_6665_epsilon_0, x = var_6664_cast_fp16)[name = string("op_6665_cast_fp16")];
            tensor<fp16, [4, 1024]> var_6666_cast_fp16 = mul(x = value_273_cast_fp16, y = var_6665_cast_fp16)[name = string("op_6666_cast_fp16")];
            tensor<fp16, [1024]> var_6668_to_fp16 = const()[name = string("op_6668_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(292636224)))];
            tensor<fp16, [4, 1024]> input_245_cast_fp16 = mul(x = var_6666_cast_fp16, y = var_6668_to_fp16)[name = string("input_245_cast_fp16")];
            tensor<fp16, [3584, 1024]> final_group_2_gate_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(292638336))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(295390912))))[name = string("final_group_2_gate_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 3584]> linear_145_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = final_group_2_gate_proj_weight_promoted_to_fp16_palettized, x = input_245_cast_fp16)[name = string("linear_145_cast_fp16")];
            tensor<fp16, [4, 3584]> var_6672_cast_fp16 = silu(x = linear_145_cast_fp16)[name = string("op_6672_cast_fp16")];
            tensor<fp16, [3584, 1024]> final_group_2_up_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(295419648))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(298172224))))[name = string("final_group_2_up_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 3584]> linear_146_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = final_group_2_up_proj_weight_promoted_to_fp16_palettized, x = input_245_cast_fp16)[name = string("linear_146_cast_fp16")];
            tensor<fp16, [4, 3584]> input_249_cast_fp16 = mul(x = var_6672_cast_fp16, y = linear_146_cast_fp16)[name = string("input_249_cast_fp16")];
            tensor<fp16, [1024, 3584]> final_group_2_down_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 3584]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(298200960))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(300953536))))[name = string("final_group_2_down_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 1024]> linear_147_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = final_group_2_down_proj_weight_promoted_to_fp16_palettized, x = input_249_cast_fp16)[name = string("linear_147_cast_fp16")];
            tensor<fp16, [4, 1024]> hidden_out = add(x = value_273_cast_fp16, y = linear_147_cast_fp16)[name = string("value_275_cast_fp16")];
            int32 var_6696 = const()[name = string("op_6696"), val = int32(-1)];
            tensor<fp16, [4, 1024]> var_6703_cast_fp16 = mul(x = hidden_out, y = hidden_out)[name = string("op_6703_cast_fp16")];
            tensor<int32, [1]> variance_123_axes_0 = const()[name = string("variance_123_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_123_keep_dims_0 = const()[name = string("variance_123_keep_dims_0"), val = bool(true)];
            tensor<fp16, [4, 1]> variance_123_cast_fp16 = reduce_mean(axes = variance_123_axes_0, keep_dims = variance_123_keep_dims_0, x = var_6703_cast_fp16)[name = string("variance_123_cast_fp16")];
            fp16 var_6706_to_fp16 = const()[name = string("op_6706_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [4, 1]> var_6707_cast_fp16 = add(x = variance_123_cast_fp16, y = var_6706_to_fp16)[name = string("op_6707_cast_fp16")];
            fp32 var_6708_epsilon_0 = const()[name = string("op_6708_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [4, 1]> var_6708_cast_fp16 = rsqrt(epsilon = var_6708_epsilon_0, x = var_6707_cast_fp16)[name = string("op_6708_cast_fp16")];
            tensor<fp16, [4, 1024]> var_6709_cast_fp16 = mul(x = hidden_out, y = var_6708_cast_fp16)[name = string("op_6709_cast_fp16")];
            tensor<fp16, [1024]> var_6711_to_fp16 = const()[name = string("op_6711_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(300961792)))];
            tensor<fp16, [4, 1024]> input_cast_fp16 = mul(x = var_6709_cast_fp16, y = var_6711_to_fp16)[name = string("input_cast_fp16")];
            tensor<fp16, [4096, 1024]> projection19_q_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [4096, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(300963904))), lut = tensor<fp16, [256, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(304109696))))[name = string("projection19_q_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 4096]> linear_148_cast_fp16 = linear(bias = linear_24_bias_0_to_fp16, weight = projection19_q_proj_weight_promoted_to_fp16_palettized, x = input_cast_fp16)[name = string("linear_148_cast_fp16")];
            tensor<int32, [3]> var_6715 = const()[name = string("op_6715"), val = tensor<int32, [3]>([4, 8, 512])];
            tensor<fp16, [4, 8, 512]> query_and_gate_cast_fp16 = reshape(shape = var_6715, x = linear_148_cast_fp16)[name = string("query_and_gate_cast_fp16")];
            tensor<int32, [2]> var_6717_split_sizes_0 = const()[name = string("op_6717_split_sizes_0"), val = tensor<int32, [2]>([256, 256])];
            int32 var_6717_axis_0 = const()[name = string("op_6717_axis_0"), val = int32(-1)];
            tensor<fp16, [4, 8, 256]> var_6717_cast_fp16_0, tensor<fp16, [4, 8, 256]> var_6717_cast_fp16_1 = split(axis = var_6717_axis_0, split_sizes = var_6717_split_sizes_0, x = query_and_gate_cast_fp16)[name = string("op_6717_cast_fp16")];
            tensor<fp16, [512, 1024]> projection19_k_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [512, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(304142528))), lut = tensor<fp16, [32, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(304535808))))[name = string("projection19_k_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 512]> linear_149_cast_fp16 = linear(bias = linear_25_bias_0_to_fp16, weight = projection19_k_proj_weight_promoted_to_fp16_palettized, x = input_cast_fp16)[name = string("linear_149_cast_fp16")];
            tensor<int32, [3]> var_6721 = const()[name = string("op_6721"), val = tensor<int32, [3]>([4, 2, 256])];
            tensor<fp16, [4, 2, 256]> value_279_cast_fp16 = reshape(shape = var_6721, x = linear_149_cast_fp16)[name = string("value_279_cast_fp16")];
            tensor<fp16, [512, 1024]> projection19_v_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [512, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(304539968))), lut = tensor<fp16, [32, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(304933248))))[name = string("projection19_v_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 512]> linear_150_cast_fp16 = linear(bias = linear_25_bias_0_to_fp16, weight = projection19_v_proj_weight_promoted_to_fp16_palettized, x = input_cast_fp16)[name = string("linear_150_cast_fp16")];
            tensor<int32, [3]> var_6725 = const()[name = string("op_6725"), val = tensor<int32, [3]>([4, 2, 256])];
            tensor<fp16, [4, 2, 256]> value_cast_fp16 = reshape(shape = var_6725, x = linear_150_cast_fp16)[name = string("value_cast_fp16")];
            tensor<fp16, [4, 8, 256]> var_6727_cast_fp16 = mul(x = var_6717_cast_fp16_0, y = var_6717_cast_fp16_0)[name = string("op_6727_cast_fp16")];
            tensor<int32, [1]> variance_125_axes_0 = const()[name = string("variance_125_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_125_keep_dims_0 = const()[name = string("variance_125_keep_dims_0"), val = bool(true)];
            tensor<fp16, [4, 8, 1]> variance_125_cast_fp16 = reduce_mean(axes = variance_125_axes_0, keep_dims = variance_125_keep_dims_0, x = var_6727_cast_fp16)[name = string("variance_125_cast_fp16")];
            fp16 var_6730_to_fp16 = const()[name = string("op_6730_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [4, 8, 1]> var_6731_cast_fp16 = add(x = variance_125_cast_fp16, y = var_6730_to_fp16)[name = string("op_6731_cast_fp16")];
            fp32 var_6732_epsilon_0 = const()[name = string("op_6732_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [4, 8, 1]> var_6732_cast_fp16 = rsqrt(epsilon = var_6732_epsilon_0, x = var_6731_cast_fp16)[name = string("op_6732_cast_fp16")];
            tensor<fp16, [4, 8, 256]> var_6733_cast_fp16 = mul(x = var_6717_cast_fp16_0, y = var_6732_cast_fp16)[name = string("op_6733_cast_fp16")];
            tensor<fp16, [256]> var_6735_to_fp16 = const()[name = string("op_6735_to_fp16"), val = tensor<fp16, [256]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(304937408)))];
            tensor<fp16, [4, 8, 256]> var_6736_cast_fp16 = mul(x = var_6733_cast_fp16, y = var_6735_to_fp16)[name = string("op_6736_cast_fp16")];
            tensor<int32, [3]> var_6737 = const()[name = string("op_6737"), val = tensor<int32, [3]>([1, 0, 2])];
            tensor<int32, [1]> value_281_axes_0 = const()[name = string("value_281_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [8, 4, 256]> var_6738_cast_fp16 = transpose(perm = var_6737, x = var_6736_cast_fp16)[name = string("transpose_3")];
            tensor<fp16, [1, 8, 4, 256]> value_281_cast_fp16 = expand_dims(axes = value_281_axes_0, x = var_6738_cast_fp16)[name = string("value_281_cast_fp16")];
            tensor<fp16, [4, 2, 256]> var_6740_cast_fp16 = mul(x = value_279_cast_fp16, y = value_279_cast_fp16)[name = string("op_6740_cast_fp16")];
            tensor<int32, [1]> variance_axes_0 = const()[name = string("variance_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_keep_dims_0 = const()[name = string("variance_keep_dims_0"), val = bool(true)];
            tensor<fp16, [4, 2, 1]> variance_cast_fp16 = reduce_mean(axes = variance_axes_0, keep_dims = variance_keep_dims_0, x = var_6740_cast_fp16)[name = string("variance_cast_fp16")];
            fp16 var_6743_to_fp16 = const()[name = string("op_6743_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [4, 2, 1]> var_6744_cast_fp16 = add(x = variance_cast_fp16, y = var_6743_to_fp16)[name = string("op_6744_cast_fp16")];
            fp32 var_6745_epsilon_0 = const()[name = string("op_6745_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [4, 2, 1]> var_6745_cast_fp16 = rsqrt(epsilon = var_6745_epsilon_0, x = var_6744_cast_fp16)[name = string("op_6745_cast_fp16")];
            tensor<fp16, [4, 2, 256]> var_6746_cast_fp16 = mul(x = value_279_cast_fp16, y = var_6745_cast_fp16)[name = string("op_6746_cast_fp16")];
            tensor<fp16, [256]> var_6748_to_fp16 = const()[name = string("op_6748_to_fp16"), val = tensor<fp16, [256]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(304937984)))];
            tensor<fp16, [4, 2, 256]> var_6749_cast_fp16 = mul(x = var_6746_cast_fp16, y = var_6748_to_fp16)[name = string("op_6749_cast_fp16")];
            tensor<int32, [3]> var_6750 = const()[name = string("op_6750"), val = tensor<int32, [3]>([1, 0, 2])];
            tensor<int32, [1]> value_283_axes_0 = const()[name = string("value_283_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [2, 4, 256]> var_6751_cast_fp16 = transpose(perm = var_6750, x = var_6749_cast_fp16)[name = string("transpose_2")];
            tensor<fp16, [1, 2, 4, 256]> value_283_cast_fp16 = expand_dims(axes = value_283_axes_0, x = var_6751_cast_fp16)[name = string("value_283_cast_fp16")];
            tensor<int32, [4]> rotated_17_begin_0 = const()[name = string("rotated_17_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> rotated_17_end_0 = const()[name = string("rotated_17_end_0"), val = tensor<int32, [4]>([1, 8, 4, 64])];
            tensor<bool, [4]> rotated_17_end_mask_0 = const()[name = string("rotated_17_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
            tensor<fp16, [1, 8, 4, 64]> rotated_17_cast_fp16 = slice_by_index(begin = rotated_17_begin_0, end = rotated_17_end_0, end_mask = rotated_17_end_mask_0, x = value_281_cast_fp16)[name = string("rotated_17_cast_fp16")];
            tensor<int32, [4]> passthrough_17_begin_0 = const()[name = string("passthrough_17_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 64])];
            tensor<int32, [4]> passthrough_17_end_0 = const()[name = string("passthrough_17_end_0"), val = tensor<int32, [4]>([1, 8, 4, 256])];
            tensor<bool, [4]> passthrough_17_end_mask_0 = const()[name = string("passthrough_17_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
            tensor<fp16, [1, 8, 4, 192]> passthrough_17_cast_fp16 = slice_by_index(begin = passthrough_17_begin_0, end = passthrough_17_end_0, end_mask = passthrough_17_end_mask_0, x = value_281_cast_fp16)[name = string("passthrough_17_cast_fp16")];
            tensor<int32, [2]> var_6755_split_sizes_0 = const()[name = string("op_6755_split_sizes_0"), val = tensor<int32, [2]>([32, 32])];
            int32 var_6755_axis_0 = const()[name = string("op_6755_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 8, 4, 32]> var_6755_cast_fp16_0, tensor<fp16, [1, 8, 4, 32]> var_6755_cast_fp16_1 = split(axis = var_6755_axis_0, split_sizes = var_6755_split_sizes_0, x = rotated_17_cast_fp16)[name = string("op_6755_cast_fp16")];
            fp16 const_175_promoted_to_fp16 = const()[name = string("const_175_promoted_to_fp16"), val = fp16(-0x1p+0)];
            tensor<fp16, [1, 8, 4, 32]> var_6757_cast_fp16 = mul(x = var_6755_cast_fp16_1, y = const_175_promoted_to_fp16)[name = string("op_6757_cast_fp16")];
            bool rotated_half_17_interleave_0 = const()[name = string("rotated_half_17_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 8, 4, 64]> rotated_half_17_cast_fp16 = concat(axis = var_6696, interleave = rotated_half_17_interleave_0, values = (var_6757_cast_fp16, var_6755_cast_fp16_0))[name = string("rotated_half_17_cast_fp16")];
            tensor<fp16, [1, 8, 4, 64]> var_6760_cast_fp16 = mul(x = rotated_17_cast_fp16, y = cos)[name = string("op_6760_cast_fp16")];
            tensor<fp16, [1, 8, 4, 64]> var_6761_cast_fp16 = mul(x = rotated_half_17_cast_fp16, y = sin)[name = string("op_6761_cast_fp16")];
            tensor<fp16, [1, 8, 4, 64]> var_6762_cast_fp16 = add(x = var_6760_cast_fp16, y = var_6761_cast_fp16)[name = string("op_6762_cast_fp16")];
            bool query_interleave_0 = const()[name = string("query_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 8, 4, 256]> query_cast_fp16 = concat(axis = var_6696, interleave = query_interleave_0, values = (var_6762_cast_fp16, passthrough_17_cast_fp16))[name = string("query_cast_fp16")];
            tensor<int32, [4]> rotated_begin_0 = const()[name = string("rotated_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> rotated_end_0 = const()[name = string("rotated_end_0"), val = tensor<int32, [4]>([1, 2, 4, 64])];
            tensor<bool, [4]> rotated_end_mask_0 = const()[name = string("rotated_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
            tensor<fp16, [1, 2, 4, 64]> rotated_cast_fp16 = slice_by_index(begin = rotated_begin_0, end = rotated_end_0, end_mask = rotated_end_mask_0, x = value_283_cast_fp16)[name = string("rotated_cast_fp16")];
            tensor<int32, [4]> passthrough_begin_0 = const()[name = string("passthrough_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 64])];
            tensor<int32, [4]> passthrough_end_0 = const()[name = string("passthrough_end_0"), val = tensor<int32, [4]>([1, 2, 4, 256])];
            tensor<bool, [4]> passthrough_end_mask_0 = const()[name = string("passthrough_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
            tensor<fp16, [1, 2, 4, 192]> passthrough_cast_fp16 = slice_by_index(begin = passthrough_begin_0, end = passthrough_end_0, end_mask = passthrough_end_mask_0, x = value_283_cast_fp16)[name = string("passthrough_cast_fp16")];
            tensor<int32, [2]> var_6767_split_sizes_0 = const()[name = string("op_6767_split_sizes_0"), val = tensor<int32, [2]>([32, 32])];
            int32 var_6767_axis_0 = const()[name = string("op_6767_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 2, 4, 32]> var_6767_cast_fp16_0, tensor<fp16, [1, 2, 4, 32]> var_6767_cast_fp16_1 = split(axis = var_6767_axis_0, split_sizes = var_6767_split_sizes_0, x = rotated_cast_fp16)[name = string("op_6767_cast_fp16")];
            fp16 const_176_promoted_to_fp16 = const()[name = string("const_176_promoted_to_fp16"), val = fp16(-0x1p+0)];
            tensor<fp16, [1, 2, 4, 32]> var_6769_cast_fp16 = mul(x = var_6767_cast_fp16_1, y = const_176_promoted_to_fp16)[name = string("op_6769_cast_fp16")];
            bool rotated_half_interleave_0 = const()[name = string("rotated_half_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 2, 4, 64]> rotated_half_cast_fp16 = concat(axis = var_6696, interleave = rotated_half_interleave_0, values = (var_6769_cast_fp16, var_6767_cast_fp16_0))[name = string("rotated_half_cast_fp16")];
            tensor<fp16, [1, 2, 4, 64]> var_6772_cast_fp16 = mul(x = rotated_cast_fp16, y = cos)[name = string("op_6772_cast_fp16")];
            tensor<fp16, [1, 2, 4, 64]> var_6773_cast_fp16 = mul(x = rotated_half_cast_fp16, y = sin)[name = string("op_6773_cast_fp16")];
            tensor<fp16, [1, 2, 4, 64]> var_6774_cast_fp16 = add(x = var_6772_cast_fp16, y = var_6773_cast_fp16)[name = string("op_6774_cast_fp16")];
            bool key_interleave_0 = const()[name = string("key_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 2, 4, 256]> key_cast_fp16 = concat(axis = var_6696, interleave = key_interleave_0, values = (var_6774_cast_fp16, passthrough_cast_fp16))[name = string("key_cast_fp16")];
            tensor<int32, [4]> var_6777 = const()[name = string("op_6777"), val = tensor<int32, [4]>([2, 4, 4, 256])];
            tensor<fp16, [2, 4, 4, 256]> var_6778_cast_fp16 = reshape(shape = var_6777, x = query_cast_fp16)[name = string("op_6778_cast_fp16")];
            tensor<int32, [4]> var_6779 = const()[name = string("op_6779"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<int32, [3]> var_6781 = const()[name = string("op_6781"), val = tensor<int32, [3]>([2, 4, 256])];
            tensor<fp16, [2, 4, 256]> k19 = reshape(shape = var_6781, x = key_cast_fp16)[name = string("op_6782_cast_fp16")];
            tensor<int32, [3]> var_6783 = const()[name = string("op_6783"), val = tensor<int32, [3]>([1, 0, 2])];
            tensor<int32, [2]> var_6785 = const()[name = string("op_6785"), val = tensor<int32, [2]>([4, 2048])];
            tensor<fp16, [4, 2048]> gate19 = reshape(shape = var_6785, x = var_6717_cast_fp16_1)[name = string("op_6786_cast_fp16")];
            tensor<fp16, [2, 4, 4, 256]> q19 = transpose(perm = var_6779, x = var_6778_cast_fp16)[name = string("transpose_0")];
            tensor<fp16, [2, 4, 256]> v19 = transpose(perm = var_6783, x = value_cast_fp16)[name = string("transpose_1")];
        } -> (hidden_out, conv0_out, rec0_out, conv1_out, rec1_out, conv2_out, rec2_out, conv4_out, rec4_out, conv5_out, rec5_out, conv6_out, rec6_out, conv8_out, rec8_out, conv9_out, rec9_out, conv10_out, rec10_out, conv12_out, rec12_out, conv13_out, rec13_out, conv14_out, rec14_out, conv16_out, rec16_out, conv17_out, rec17_out, conv18_out, rec18_out, q19, k19, v19, gate19);
}