program(1.3)
[buildInfo = dict<string, string>({{"coremlc-component-MIL", "3600.16.1"}, {"coremlc-version", "3600.22.1"}})]
{
    func embed<ios18>(tensor<int32, [1, 1]> token_id) {
            int32 var_11_batch_dims_0 = const()[name = string("op_11_batch_dims_0"), val = int32(0)];
            bool var_11_validate_indices_0 = const()[name = string("op_11_validate_indices_0"), val = bool(false)];
            tensor<fp16, [248320, 1024]> weight_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [248320, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(64))), lut = tensor<fp16, [15520, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(190709888))))[name = string("weight_to_fp16_palettized")];
            int32 greater_equal_0_y_0 = const()[name = string("greater_equal_0_y_0"), val = int32(0)];
            tensor<bool, [1, 1]> greater_equal_0 = greater_equal(x = token_id, y = greater_equal_0_y_0)[name = string("greater_equal_0")];
            int32 slice_by_index_0 = const()[name = string("slice_by_index_0"), val = int32(248320)];
            tensor<int32, [1, 1]> add_0 = add(x = token_id, y = slice_by_index_0)[name = string("add_0")];
            tensor<int32, [1, 1]> select_0 = select(a = token_id, b = add_0, cond = greater_equal_0)[name = string("select_0")];
            int32 greater_equal_0_y_0_1 = const()[name = string("greater_equal_0_y_0_1"), val = int32(0)];
            tensor<bool, [1, 1]> greater_equal_0_1 = greater_equal(x = select_0, y = greater_equal_0_y_0_1)[name = string("greater_equal_0_1")];
            int32 slice_by_index_0_1 = const()[name = string("slice_by_index_0_1"), val = int32(248320)];
            tensor<int32, [1, 1]> add_0_1 = add(x = select_0, y = slice_by_index_0_1)[name = string("add_0_1")];
            tensor<int32, [1, 1]> select_0_1 = select(a = select_0, b = add_0_1, cond = greater_equal_0_1)[name = string("select_0_1")];
            int32 op_11_cast_fp16_axis_0 = const()[name = string("op_11_cast_fp16_axis_0"), val = int32(0)];
            tensor<fp16, [1, 1, 1024]> hidden_states = gather(axis = op_11_cast_fp16_axis_0, batch_dims = var_11_batch_dims_0, indices = select_0_1, validate_indices = var_11_validate_indices_0, x = weight_to_fp16_palettized)[name = string("op_11_cast_fp16")];
        } -> (hidden_states);
    func lm_head_argmax<ios18>(tensor<fp16, [1, 1, 1024]> hidden_states) {
            tensor<fp16, [248320, 1024]> weight_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [248320, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(64))), lut = tensor<fp16, [15520, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(190709888))))[name = string("weight_to_fp16_palettized")];
            tensor<fp16, [248320]> linear_0_bias_0_to_fp16 = const()[name = string("linear_0_bias_0_to_fp16"), val = tensor<fp16, [248320]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(192696512)))];
            tensor<fp16, [1, 1, 248320]> linear_0_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = weight_to_fp16_palettized, x = hidden_states)[name = string("linear_0_cast_fp16")];
            int32 var_7_axis_0 = const()[name = string("op_7_axis_0"), val = int32(-1)];
            bool var_7_keep_dims_0 = const()[name = string("op_7_keep_dims_0"), val = bool(false)];
            string var_7_output_dtype_0 = const()[name = string("op_7_output_dtype_0"), val = string("int32")];
            tensor<int32, [1, 1]> next_token = reduce_argmax(axis = var_7_axis_0, keep_dims = var_7_keep_dims_0, output_dtype = var_7_output_dtype_0, x = linear_0_cast_fp16)[name = string("op_7_cast_fp16")];
        } -> (next_token);
    func prefill16<ios18>(tensor<fp16, [1, 6144, 3]> conv20, tensor<fp16, [1, 6144, 3]> conv21, tensor<fp16, [1, 6144, 3]> conv22, tensor<fp16, [1, 1, 16, 64]> cos, tensor<int32, [1]> current_pos, tensor<fp16, [16, 2048]> gate19, tensor<fp16, [16, 1024]> hidden, tensor<fp16, [2, 16, 256]> k19, state<tensor<fp16, [2, 2, 2, 2048, 256]>> kv_cache, tensor<fp16, [1, 16, 2048]> mask19, tensor<fp16, [2, 16, 4, 256]> q19, tensor<fp16, [1, 16, 128, 128]> rec20, tensor<fp16, [1, 16, 128, 128]> rec21, tensor<fp16, [1, 16, 128, 128]> rec22, tensor<fp16, [1, 1, 16, 64]> sin, tensor<fp16, [2, 16, 256]> v19) {
            tensor<int32, [1]> var_33_axes_0 = const()[name = string("op_33_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [1, 2, 16, 256]> var_33_cast_fp16 = expand_dims(axes = var_33_axes_0, x = k19)[name = string("op_33_cast_fp16")];
            tensor<int32, [1]> var_35_axes_0 = const()[name = string("op_35_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [1, 1, 2, 16, 256]> var_35_cast_fp16 = expand_dims(axes = var_35_axes_0, x = var_33_cast_fp16)[name = string("op_35_cast_fp16")];
            int32 var_37 = const()[name = string("op_37"), val = int32(16)];
            tensor<int32, [1]> var_38 = add(x = current_pos, y = var_37)[name = string("op_38")];
            tensor<fp16, [2, 2, 2, 2048, 256]> read_state_0 = read_state(input = kv_cache)[name = string("read_state_0")];
            tensor<int32, [1]> expand_dims_0 = const()[name = string("expand_dims_0"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_1 = const()[name = string("expand_dims_1"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_2 = const()[name = string("expand_dims_2"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_4 = const()[name = string("expand_dims_4"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_5 = const()[name = string("expand_dims_5"), val = tensor<int32, [1]>([1])];
            tensor<int32, [1]> expand_dims_6 = const()[name = string("expand_dims_6"), val = tensor<int32, [1]>([1])];
            int32 concat_2_axis_0 = const()[name = string("concat_2_axis_0"), val = int32(0)];
            bool concat_2_interleave_0 = const()[name = string("concat_2_interleave_0"), val = bool(false)];
            tensor<int32, [5]> concat_2 = concat(axis = concat_2_axis_0, interleave = concat_2_interleave_0, values = (expand_dims_0, expand_dims_1, expand_dims_2, current_pos, expand_dims_4))[name = string("concat_2")];
            tensor<int32, [1]> concat_3_values2_0 = const()[name = string("concat_3_values2_0"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> concat_3_values4_0 = const()[name = string("concat_3_values4_0"), val = tensor<int32, [1]>([0])];
            int32 concat_3_axis_0 = const()[name = string("concat_3_axis_0"), val = int32(0)];
            bool concat_3_interleave_0 = const()[name = string("concat_3_interleave_0"), val = bool(false)];
            tensor<int32, [5]> concat_3 = concat(axis = concat_3_axis_0, interleave = concat_3_interleave_0, values = (expand_dims_5, expand_dims_6, concat_3_values2_0, var_38, concat_3_values4_0))[name = string("concat_3")];
            tensor<int32, [5]> kv_cache_internal_tensor_assign_1_stride_0 = const()[name = string("kv_cache_internal_tensor_assign_1_stride_0"), val = tensor<int32, [5]>([1, 1, 1, 1, 1])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_1_begin_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_1_begin_mask_0"), val = tensor<bool, [5]>([false, false, false, false, false])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_1_end_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_1_end_mask_0"), val = tensor<bool, [5]>([false, false, true, false, true])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_1_squeeze_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_1_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, false, false, false])];
            tensor<fp16, [2, 2, 2, 2048, 256]> kv_cache_internal_tensor_assign_1_cast_fp16 = slice_update(begin = concat_2, begin_mask = kv_cache_internal_tensor_assign_1_begin_mask_0, end = concat_3, end_mask = kv_cache_internal_tensor_assign_1_end_mask_0, squeeze_mask = kv_cache_internal_tensor_assign_1_squeeze_mask_0, stride = kv_cache_internal_tensor_assign_1_stride_0, update = var_35_cast_fp16, x = read_state_0)[name = string("kv_cache_internal_tensor_assign_1_cast_fp16")];
            write_state(data = kv_cache_internal_tensor_assign_1_cast_fp16, input = kv_cache)[name = string("coreml_update_state_4_write_state")];
            tensor<fp16, [2, 2, 2, 2048, 256]> coreml_update_state_4 = read_state(input = kv_cache)[name = string("coreml_update_state_4")];
            tensor<int32, [1]> var_66_axes_0 = const()[name = string("op_66_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [1, 2, 16, 256]> var_66_cast_fp16 = expand_dims(axes = var_66_axes_0, x = v19)[name = string("op_66_cast_fp16")];
            tensor<int32, [1]> var_68_axes_0 = const()[name = string("op_68_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [1, 1, 2, 16, 256]> var_68_cast_fp16 = expand_dims(axes = var_68_axes_0, x = var_66_cast_fp16)[name = string("op_68_cast_fp16")];
            tensor<int32, [1]> expand_dims_8 = const()[name = string("expand_dims_8"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_9 = const()[name = string("expand_dims_9"), val = tensor<int32, [1]>([1])];
            tensor<int32, [1]> expand_dims_10 = const()[name = string("expand_dims_10"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_12 = const()[name = string("expand_dims_12"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_13 = const()[name = string("expand_dims_13"), val = tensor<int32, [1]>([1])];
            tensor<int32, [1]> expand_dims_14 = const()[name = string("expand_dims_14"), val = tensor<int32, [1]>([2])];
            int32 concat_6_axis_0 = const()[name = string("concat_6_axis_0"), val = int32(0)];
            bool concat_6_interleave_0 = const()[name = string("concat_6_interleave_0"), val = bool(false)];
            tensor<int32, [5]> concat_6 = concat(axis = concat_6_axis_0, interleave = concat_6_interleave_0, values = (expand_dims_8, expand_dims_9, expand_dims_10, current_pos, expand_dims_12))[name = string("concat_6")];
            tensor<int32, [1]> concat_7_values2_0 = const()[name = string("concat_7_values2_0"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> concat_7_values4_0 = const()[name = string("concat_7_values4_0"), val = tensor<int32, [1]>([0])];
            int32 concat_7_axis_0 = const()[name = string("concat_7_axis_0"), val = int32(0)];
            bool concat_7_interleave_0 = const()[name = string("concat_7_interleave_0"), val = bool(false)];
            tensor<int32, [5]> concat_7 = concat(axis = concat_7_axis_0, interleave = concat_7_interleave_0, values = (expand_dims_13, expand_dims_14, concat_7_values2_0, var_38, concat_7_values4_0))[name = string("concat_7")];
            tensor<int32, [5]> kv_cache_internal_tensor_assign_2_stride_0 = const()[name = string("kv_cache_internal_tensor_assign_2_stride_0"), val = tensor<int32, [5]>([1, 1, 1, 1, 1])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_2_begin_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_2_begin_mask_0"), val = tensor<bool, [5]>([false, false, false, false, false])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_2_end_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_2_end_mask_0"), val = tensor<bool, [5]>([false, false, true, false, true])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_2_squeeze_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_2_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, false, false, false])];
            tensor<fp16, [2, 2, 2, 2048, 256]> kv_cache_internal_tensor_assign_2_cast_fp16 = slice_update(begin = concat_6, begin_mask = kv_cache_internal_tensor_assign_2_begin_mask_0, end = concat_7, end_mask = kv_cache_internal_tensor_assign_2_end_mask_0, squeeze_mask = kv_cache_internal_tensor_assign_2_squeeze_mask_0, stride = kv_cache_internal_tensor_assign_2_stride_0, update = var_68_cast_fp16, x = coreml_update_state_4)[name = string("kv_cache_internal_tensor_assign_2_cast_fp16")];
            write_state(data = kv_cache_internal_tensor_assign_2_cast_fp16, input = kv_cache)[name = string("coreml_update_state_5_write_state")];
            tensor<fp16, [2, 2, 2, 2048, 256]> coreml_update_state_5 = read_state(input = kv_cache)[name = string("coreml_update_state_5")];
            tensor<int32, [5]> var_100_begin_0 = const()[name = string("op_100_begin_0"), val = tensor<int32, [5]>([0, 0, 0, 0, 0])];
            tensor<int32, [5]> var_100_end_0 = const()[name = string("op_100_end_0"), val = tensor<int32, [5]>([1, 2, 2, 2048, 256])];
            tensor<bool, [5]> var_100_end_mask_0 = const()[name = string("op_100_end_mask_0"), val = tensor<bool, [5]>([false, true, true, true, true])];
            tensor<bool, [5]> var_100_squeeze_mask_0 = const()[name = string("op_100_squeeze_mask_0"), val = tensor<bool, [5]>([true, false, false, false, false])];
            tensor<fp16, [2, 2, 2048, 256]> var_100_cast_fp16 = slice_by_index(begin = var_100_begin_0, end = var_100_end_0, end_mask = var_100_end_mask_0, squeeze_mask = var_100_squeeze_mask_0, x = coreml_update_state_5)[name = string("op_100_cast_fp16")];
            tensor<int32, [4]> keys_1_begin_0 = const()[name = string("keys_1_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> keys_1_end_0 = const()[name = string("keys_1_end_0"), val = tensor<int32, [4]>([1, 2, 2048, 256])];
            tensor<bool, [4]> keys_1_end_mask_0 = const()[name = string("keys_1_end_mask_0"), val = tensor<bool, [4]>([false, true, true, true])];
            tensor<bool, [4]> keys_1_squeeze_mask_0 = const()[name = string("keys_1_squeeze_mask_0"), val = tensor<bool, [4]>([true, false, false, false])];
            tensor<fp16, [2, 2048, 256]> keys_1_cast_fp16 = slice_by_index(begin = keys_1_begin_0, end = keys_1_end_0, end_mask = keys_1_end_mask_0, squeeze_mask = keys_1_squeeze_mask_0, x = var_100_cast_fp16)[name = string("keys_1_cast_fp16")];
            tensor<int32, [4]> values_1_begin_0 = const()[name = string("values_1_begin_0"), val = tensor<int32, [4]>([1, 0, 0, 0])];
            tensor<int32, [4]> values_1_end_0 = const()[name = string("values_1_end_0"), val = tensor<int32, [4]>([2, 2, 2048, 256])];
            tensor<bool, [4]> values_1_end_mask_0 = const()[name = string("values_1_end_mask_0"), val = tensor<bool, [4]>([false, true, true, true])];
            tensor<bool, [4]> values_1_squeeze_mask_0 = const()[name = string("values_1_squeeze_mask_0"), val = tensor<bool, [4]>([true, false, false, false])];
            tensor<fp16, [2, 2048, 256]> values_1_cast_fp16 = slice_by_index(begin = values_1_begin_0, end = values_1_end_0, end_mask = values_1_end_mask_0, squeeze_mask = values_1_squeeze_mask_0, x = var_100_cast_fp16)[name = string("values_1_cast_fp16")];
            int32 var_118 = const()[name = string("op_118"), val = int32(1)];
            tensor<int32, [4]> var_122 = const()[name = string("op_122"), val = tensor<int32, [4]>([0, 2, 3, 1])];
            tensor<int32, [4]> var_125 = const()[name = string("op_125"), val = tensor<int32, [4]>([1, 2048, 1, 16])];
            tensor<fp16, [2, 4, 256, 16]> var_123_cast_fp16 = transpose(perm = var_122, x = q19)[name = string("transpose_34")];
            tensor<fp16, [1, 2048, 1, 16]> var_126_cast_fp16 = reshape(shape = var_125, x = var_123_cast_fp16)[name = string("op_126_cast_fp16")];
            tensor<int32, [3]> var_127 = const()[name = string("op_127"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [4]> var_130 = const()[name = string("op_130"), val = tensor<int32, [4]>([1, 512, 1, 2048])];
            tensor<fp16, [2, 256, 2048]> var_128_cast_fp16 = transpose(perm = var_127, x = keys_1_cast_fp16)[name = string("transpose_33")];
            tensor<fp16, [1, 512, 1, 2048]> key_native_1_cast_fp16 = reshape(shape = var_130, x = var_128_cast_fp16)[name = string("key_native_1_cast_fp16")];
            tensor<int32, [3]> var_132 = const()[name = string("op_132"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [4]> var_135 = const()[name = string("op_135"), val = tensor<int32, [4]>([1, 512, 1, 2048])];
            tensor<fp16, [2, 256, 2048]> var_133_cast_fp16 = transpose(perm = var_132, x = values_1_cast_fp16)[name = string("transpose_32")];
            tensor<fp16, [1, 512, 1, 2048]> var_136_cast_fp16 = reshape(shape = var_135, x = var_133_cast_fp16)[name = string("op_136_cast_fp16")];
            tensor<int32, [3]> var_137 = const()[name = string("op_137"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [1]> mask_native_1_axes_0 = const()[name = string("mask_native_1_axes_0"), val = tensor<int32, [1]>([2])];
            tensor<fp16, [1, 2048, 16]> var_138_cast_fp16 = transpose(perm = var_137, x = mask19)[name = string("transpose_31")];
            tensor<fp16, [1, 2048, 1, 16]> mask_native_1_cast_fp16 = expand_dims(axes = mask_native_1_axes_0, x = var_138_cast_fp16)[name = string("mask_native_1_cast_fp16")];
            tensor<int32, [8]> tile_0 = const()[name = string("tile_0"), val = tensor<int32, [8]>([256, 256, 256, 256, 256, 256, 256, 256])];
            int32 var_140_axis_0 = const()[name = string("op_140_axis_0"), val = int32(1)];
            tensor<fp16, [1, 256, 1, 16]> var_140_cast_fp16_0, tensor<fp16, [1, 256, 1, 16]> var_140_cast_fp16_1, tensor<fp16, [1, 256, 1, 16]> var_140_cast_fp16_2, tensor<fp16, [1, 256, 1, 16]> var_140_cast_fp16_3, tensor<fp16, [1, 256, 1, 16]> var_140_cast_fp16_4, tensor<fp16, [1, 256, 1, 16]> var_140_cast_fp16_5, tensor<fp16, [1, 256, 1, 16]> var_140_cast_fp16_6, tensor<fp16, [1, 256, 1, 16]> var_140_cast_fp16_7 = split(axis = var_140_axis_0, split_sizes = tile_0, x = var_126_cast_fp16)[name = string("op_140_cast_fp16")];
            tensor<int32, [4]> var_149_perm_0 = const()[name = string("op_149_perm_0"), val = tensor<int32, [4]>([0, 3, 2, 1])];
            tensor<int32, [2]> tile_1 = const()[name = string("tile_1"), val = tensor<int32, [2]>([256, 256])];
            int32 var_150_axis_0 = const()[name = string("op_150_axis_0"), val = int32(3)];
            tensor<fp16, [1, 2048, 1, 512]> var_149_cast_fp16 = transpose(perm = var_149_perm_0, x = key_native_1_cast_fp16)[name = string("transpose_30")];
            tensor<fp16, [1, 2048, 1, 256]> var_150_cast_fp16_0, tensor<fp16, [1, 2048, 1, 256]> var_150_cast_fp16_1 = split(axis = var_150_axis_0, split_sizes = tile_1, x = var_149_cast_fp16)[name = string("op_150_cast_fp16")];
            tensor<int32, [2]> tile_2 = const()[name = string("tile_2"), val = tensor<int32, [2]>([256, 256])];
            int32 var_153_axis_0 = const()[name = string("op_153_axis_0"), val = int32(1)];
            tensor<fp16, [1, 256, 1, 2048]> var_153_cast_fp16_0, tensor<fp16, [1, 256, 1, 2048]> var_153_cast_fp16_1 = split(axis = var_153_axis_0, split_sizes = tile_2, x = var_136_cast_fp16)[name = string("op_153_cast_fp16")];
            string scores_1_equation_0 = const()[name = string("scores_1_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 16]> scores_1_cast_fp16 = einsum(equation = scores_1_equation_0, values = (var_150_cast_fp16_0, var_140_cast_fp16_0))[name = string("scores_1_cast_fp16")];
            fp16 var_158_to_fp16 = const()[name = string("op_158_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 16]> var_159_cast_fp16 = mul(x = scores_1_cast_fp16, y = var_158_to_fp16)[name = string("op_159_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_160_cast_fp16 = add(x = var_159_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_160_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_161_cast_fp16 = softmax(axis = var_118, x = var_160_cast_fp16)[name = string("op_161_cast_fp16")];
            tensor<int32, [4]> tile_3 = const()[name = string("tile_3"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_162_axis_0 = const()[name = string("op_162_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 16]> var_162_cast_fp16_0, tensor<fp16, [1, 512, 1, 16]> var_162_cast_fp16_1, tensor<fp16, [1, 512, 1, 16]> var_162_cast_fp16_2, tensor<fp16, [1, 512, 1, 16]> var_162_cast_fp16_3 = split(axis = var_162_axis_0, split_sizes = tile_3, x = var_161_cast_fp16)[name = string("op_162_cast_fp16")];
            tensor<int32, [4]> tile_4 = const()[name = string("tile_4"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_167_axis_0 = const()[name = string("op_167_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_167_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_167_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_167_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_167_cast_fp16_3 = split(axis = var_167_axis_0, split_sizes = tile_4, x = var_153_cast_fp16_0)[name = string("op_167_cast_fp16")];
            fp16 var_172_to_fp16 = const()[name = string("op_172_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_173_cast_fp16 = mul(x = var_162_cast_fp16_0, y = var_172_to_fp16)[name = string("op_173_cast_fp16")];
            string var_175_equation_0 = const()[name = string("op_175_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_175_cast_fp16 = einsum(equation = var_175_equation_0, values = (var_167_cast_fp16_0, var_173_cast_fp16))[name = string("op_175_cast_fp16")];
            fp16 var_176_to_fp16 = const()[name = string("op_176_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_177_cast_fp16 = mul(x = var_162_cast_fp16_1, y = var_176_to_fp16)[name = string("op_177_cast_fp16")];
            string var_179_equation_0 = const()[name = string("op_179_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_179_cast_fp16 = einsum(equation = var_179_equation_0, values = (var_167_cast_fp16_1, var_177_cast_fp16))[name = string("op_179_cast_fp16")];
            fp16 var_180_to_fp16 = const()[name = string("op_180_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_181_cast_fp16 = mul(x = var_162_cast_fp16_2, y = var_180_to_fp16)[name = string("op_181_cast_fp16")];
            string var_183_equation_0 = const()[name = string("op_183_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_183_cast_fp16 = einsum(equation = var_183_equation_0, values = (var_167_cast_fp16_2, var_181_cast_fp16))[name = string("op_183_cast_fp16")];
            fp16 var_184_to_fp16 = const()[name = string("op_184_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_185_cast_fp16 = mul(x = var_162_cast_fp16_3, y = var_184_to_fp16)[name = string("op_185_cast_fp16")];
            string var_187_equation_0 = const()[name = string("op_187_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_187_cast_fp16 = einsum(equation = var_187_equation_0, values = (var_167_cast_fp16_3, var_185_cast_fp16))[name = string("op_187_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_188_cast_fp16 = add(x = var_175_cast_fp16, y = var_179_cast_fp16)[name = string("op_188_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_189_cast_fp16 = add(x = var_183_cast_fp16, y = var_187_cast_fp16)[name = string("op_189_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_190_cast_fp16 = add(x = var_188_cast_fp16, y = var_189_cast_fp16)[name = string("op_190_cast_fp16")];
            fp16 _inversed_192_y_0_to_fp16 = const()[name = string("_inversed_192_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 16]> _inversed_192_cast_fp16 = mul(x = var_190_cast_fp16, y = _inversed_192_y_0_to_fp16)[name = string("_inversed_192_cast_fp16")];
            string scores_3_equation_0 = const()[name = string("scores_3_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 16]> scores_3_cast_fp16 = einsum(equation = scores_3_equation_0, values = (var_150_cast_fp16_0, var_140_cast_fp16_1))[name = string("scores_3_cast_fp16")];
            fp16 var_195_to_fp16 = const()[name = string("op_195_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 16]> var_196_cast_fp16 = mul(x = scores_3_cast_fp16, y = var_195_to_fp16)[name = string("op_196_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_197_cast_fp16 = add(x = var_196_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_197_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_198_cast_fp16 = softmax(axis = var_118, x = var_197_cast_fp16)[name = string("op_198_cast_fp16")];
            tensor<int32, [4]> tile_5 = const()[name = string("tile_5"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_199_axis_0 = const()[name = string("op_199_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 16]> var_199_cast_fp16_0, tensor<fp16, [1, 512, 1, 16]> var_199_cast_fp16_1, tensor<fp16, [1, 512, 1, 16]> var_199_cast_fp16_2, tensor<fp16, [1, 512, 1, 16]> var_199_cast_fp16_3 = split(axis = var_199_axis_0, split_sizes = tile_5, x = var_198_cast_fp16)[name = string("op_199_cast_fp16")];
            tensor<int32, [4]> tile_6 = const()[name = string("tile_6"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_204_axis_0 = const()[name = string("op_204_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_204_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_204_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_204_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_204_cast_fp16_3 = split(axis = var_204_axis_0, split_sizes = tile_6, x = var_153_cast_fp16_0)[name = string("op_204_cast_fp16")];
            fp16 var_209_to_fp16 = const()[name = string("op_209_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_210_cast_fp16 = mul(x = var_199_cast_fp16_0, y = var_209_to_fp16)[name = string("op_210_cast_fp16")];
            string var_212_equation_0 = const()[name = string("op_212_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_212_cast_fp16 = einsum(equation = var_212_equation_0, values = (var_204_cast_fp16_0, var_210_cast_fp16))[name = string("op_212_cast_fp16")];
            fp16 var_213_to_fp16 = const()[name = string("op_213_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_214_cast_fp16 = mul(x = var_199_cast_fp16_1, y = var_213_to_fp16)[name = string("op_214_cast_fp16")];
            string var_216_equation_0 = const()[name = string("op_216_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_216_cast_fp16 = einsum(equation = var_216_equation_0, values = (var_204_cast_fp16_1, var_214_cast_fp16))[name = string("op_216_cast_fp16")];
            fp16 var_217_to_fp16 = const()[name = string("op_217_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_218_cast_fp16 = mul(x = var_199_cast_fp16_2, y = var_217_to_fp16)[name = string("op_218_cast_fp16")];
            string var_220_equation_0 = const()[name = string("op_220_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_220_cast_fp16 = einsum(equation = var_220_equation_0, values = (var_204_cast_fp16_2, var_218_cast_fp16))[name = string("op_220_cast_fp16")];
            fp16 var_221_to_fp16 = const()[name = string("op_221_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_222_cast_fp16 = mul(x = var_199_cast_fp16_3, y = var_221_to_fp16)[name = string("op_222_cast_fp16")];
            string var_224_equation_0 = const()[name = string("op_224_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_224_cast_fp16 = einsum(equation = var_224_equation_0, values = (var_204_cast_fp16_3, var_222_cast_fp16))[name = string("op_224_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_225_cast_fp16 = add(x = var_212_cast_fp16, y = var_216_cast_fp16)[name = string("op_225_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_226_cast_fp16 = add(x = var_220_cast_fp16, y = var_224_cast_fp16)[name = string("op_226_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_227_cast_fp16 = add(x = var_225_cast_fp16, y = var_226_cast_fp16)[name = string("op_227_cast_fp16")];
            fp16 _inversed_229_y_0_to_fp16 = const()[name = string("_inversed_229_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 16]> _inversed_229_cast_fp16 = mul(x = var_227_cast_fp16, y = _inversed_229_y_0_to_fp16)[name = string("_inversed_229_cast_fp16")];
            string scores_5_equation_0 = const()[name = string("scores_5_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 16]> scores_5_cast_fp16 = einsum(equation = scores_5_equation_0, values = (var_150_cast_fp16_0, var_140_cast_fp16_2))[name = string("scores_5_cast_fp16")];
            fp16 var_232_to_fp16 = const()[name = string("op_232_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 16]> var_233_cast_fp16 = mul(x = scores_5_cast_fp16, y = var_232_to_fp16)[name = string("op_233_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_234_cast_fp16 = add(x = var_233_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_234_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_235_cast_fp16 = softmax(axis = var_118, x = var_234_cast_fp16)[name = string("op_235_cast_fp16")];
            tensor<int32, [4]> tile_7 = const()[name = string("tile_7"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_236_axis_0 = const()[name = string("op_236_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 16]> var_236_cast_fp16_0, tensor<fp16, [1, 512, 1, 16]> var_236_cast_fp16_1, tensor<fp16, [1, 512, 1, 16]> var_236_cast_fp16_2, tensor<fp16, [1, 512, 1, 16]> var_236_cast_fp16_3 = split(axis = var_236_axis_0, split_sizes = tile_7, x = var_235_cast_fp16)[name = string("op_236_cast_fp16")];
            tensor<int32, [4]> tile_8 = const()[name = string("tile_8"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_241_axis_0 = const()[name = string("op_241_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_241_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_241_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_241_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_241_cast_fp16_3 = split(axis = var_241_axis_0, split_sizes = tile_8, x = var_153_cast_fp16_0)[name = string("op_241_cast_fp16")];
            fp16 var_246_to_fp16 = const()[name = string("op_246_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_247_cast_fp16 = mul(x = var_236_cast_fp16_0, y = var_246_to_fp16)[name = string("op_247_cast_fp16")];
            string var_249_equation_0 = const()[name = string("op_249_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_249_cast_fp16 = einsum(equation = var_249_equation_0, values = (var_241_cast_fp16_0, var_247_cast_fp16))[name = string("op_249_cast_fp16")];
            fp16 var_250_to_fp16 = const()[name = string("op_250_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_251_cast_fp16 = mul(x = var_236_cast_fp16_1, y = var_250_to_fp16)[name = string("op_251_cast_fp16")];
            string var_253_equation_0 = const()[name = string("op_253_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_253_cast_fp16 = einsum(equation = var_253_equation_0, values = (var_241_cast_fp16_1, var_251_cast_fp16))[name = string("op_253_cast_fp16")];
            fp16 var_254_to_fp16 = const()[name = string("op_254_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_255_cast_fp16 = mul(x = var_236_cast_fp16_2, y = var_254_to_fp16)[name = string("op_255_cast_fp16")];
            string var_257_equation_0 = const()[name = string("op_257_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_257_cast_fp16 = einsum(equation = var_257_equation_0, values = (var_241_cast_fp16_2, var_255_cast_fp16))[name = string("op_257_cast_fp16")];
            fp16 var_258_to_fp16 = const()[name = string("op_258_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_259_cast_fp16 = mul(x = var_236_cast_fp16_3, y = var_258_to_fp16)[name = string("op_259_cast_fp16")];
            string var_261_equation_0 = const()[name = string("op_261_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_261_cast_fp16 = einsum(equation = var_261_equation_0, values = (var_241_cast_fp16_3, var_259_cast_fp16))[name = string("op_261_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_262_cast_fp16 = add(x = var_249_cast_fp16, y = var_253_cast_fp16)[name = string("op_262_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_263_cast_fp16 = add(x = var_257_cast_fp16, y = var_261_cast_fp16)[name = string("op_263_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_264_cast_fp16 = add(x = var_262_cast_fp16, y = var_263_cast_fp16)[name = string("op_264_cast_fp16")];
            fp16 _inversed_266_y_0_to_fp16 = const()[name = string("_inversed_266_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 16]> _inversed_266_cast_fp16 = mul(x = var_264_cast_fp16, y = _inversed_266_y_0_to_fp16)[name = string("_inversed_266_cast_fp16")];
            string scores_7_equation_0 = const()[name = string("scores_7_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 16]> scores_7_cast_fp16 = einsum(equation = scores_7_equation_0, values = (var_150_cast_fp16_0, var_140_cast_fp16_3))[name = string("scores_7_cast_fp16")];
            fp16 var_269_to_fp16 = const()[name = string("op_269_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 16]> var_270_cast_fp16 = mul(x = scores_7_cast_fp16, y = var_269_to_fp16)[name = string("op_270_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_271_cast_fp16 = add(x = var_270_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_271_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_272_cast_fp16 = softmax(axis = var_118, x = var_271_cast_fp16)[name = string("op_272_cast_fp16")];
            tensor<int32, [4]> tile_9 = const()[name = string("tile_9"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_273_axis_0 = const()[name = string("op_273_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 16]> var_273_cast_fp16_0, tensor<fp16, [1, 512, 1, 16]> var_273_cast_fp16_1, tensor<fp16, [1, 512, 1, 16]> var_273_cast_fp16_2, tensor<fp16, [1, 512, 1, 16]> var_273_cast_fp16_3 = split(axis = var_273_axis_0, split_sizes = tile_9, x = var_272_cast_fp16)[name = string("op_273_cast_fp16")];
            tensor<int32, [4]> tile_10 = const()[name = string("tile_10"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_278_axis_0 = const()[name = string("op_278_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_278_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_278_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_278_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_278_cast_fp16_3 = split(axis = var_278_axis_0, split_sizes = tile_10, x = var_153_cast_fp16_0)[name = string("op_278_cast_fp16")];
            fp16 var_283_to_fp16 = const()[name = string("op_283_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_284_cast_fp16 = mul(x = var_273_cast_fp16_0, y = var_283_to_fp16)[name = string("op_284_cast_fp16")];
            string var_286_equation_0 = const()[name = string("op_286_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_286_cast_fp16 = einsum(equation = var_286_equation_0, values = (var_278_cast_fp16_0, var_284_cast_fp16))[name = string("op_286_cast_fp16")];
            fp16 var_287_to_fp16 = const()[name = string("op_287_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_288_cast_fp16 = mul(x = var_273_cast_fp16_1, y = var_287_to_fp16)[name = string("op_288_cast_fp16")];
            string var_290_equation_0 = const()[name = string("op_290_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_290_cast_fp16 = einsum(equation = var_290_equation_0, values = (var_278_cast_fp16_1, var_288_cast_fp16))[name = string("op_290_cast_fp16")];
            fp16 var_291_to_fp16 = const()[name = string("op_291_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_292_cast_fp16 = mul(x = var_273_cast_fp16_2, y = var_291_to_fp16)[name = string("op_292_cast_fp16")];
            string var_294_equation_0 = const()[name = string("op_294_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_294_cast_fp16 = einsum(equation = var_294_equation_0, values = (var_278_cast_fp16_2, var_292_cast_fp16))[name = string("op_294_cast_fp16")];
            fp16 var_295_to_fp16 = const()[name = string("op_295_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_296_cast_fp16 = mul(x = var_273_cast_fp16_3, y = var_295_to_fp16)[name = string("op_296_cast_fp16")];
            string var_298_equation_0 = const()[name = string("op_298_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_298_cast_fp16 = einsum(equation = var_298_equation_0, values = (var_278_cast_fp16_3, var_296_cast_fp16))[name = string("op_298_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_299_cast_fp16 = add(x = var_286_cast_fp16, y = var_290_cast_fp16)[name = string("op_299_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_300_cast_fp16 = add(x = var_294_cast_fp16, y = var_298_cast_fp16)[name = string("op_300_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_301_cast_fp16 = add(x = var_299_cast_fp16, y = var_300_cast_fp16)[name = string("op_301_cast_fp16")];
            fp16 _inversed_303_y_0_to_fp16 = const()[name = string("_inversed_303_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 16]> _inversed_303_cast_fp16 = mul(x = var_301_cast_fp16, y = _inversed_303_y_0_to_fp16)[name = string("_inversed_303_cast_fp16")];
            string scores_9_equation_0 = const()[name = string("scores_9_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 16]> scores_9_cast_fp16 = einsum(equation = scores_9_equation_0, values = (var_150_cast_fp16_1, var_140_cast_fp16_4))[name = string("scores_9_cast_fp16")];
            fp16 var_306_to_fp16 = const()[name = string("op_306_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 16]> var_307_cast_fp16 = mul(x = scores_9_cast_fp16, y = var_306_to_fp16)[name = string("op_307_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_308_cast_fp16 = add(x = var_307_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_308_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_309_cast_fp16 = softmax(axis = var_118, x = var_308_cast_fp16)[name = string("op_309_cast_fp16")];
            tensor<int32, [4]> tile_11 = const()[name = string("tile_11"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_310_axis_0 = const()[name = string("op_310_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 16]> var_310_cast_fp16_0, tensor<fp16, [1, 512, 1, 16]> var_310_cast_fp16_1, tensor<fp16, [1, 512, 1, 16]> var_310_cast_fp16_2, tensor<fp16, [1, 512, 1, 16]> var_310_cast_fp16_3 = split(axis = var_310_axis_0, split_sizes = tile_11, x = var_309_cast_fp16)[name = string("op_310_cast_fp16")];
            tensor<int32, [4]> tile_12 = const()[name = string("tile_12"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_315_axis_0 = const()[name = string("op_315_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_315_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_315_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_315_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_315_cast_fp16_3 = split(axis = var_315_axis_0, split_sizes = tile_12, x = var_153_cast_fp16_1)[name = string("op_315_cast_fp16")];
            fp16 var_320_to_fp16 = const()[name = string("op_320_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_321_cast_fp16 = mul(x = var_310_cast_fp16_0, y = var_320_to_fp16)[name = string("op_321_cast_fp16")];
            string var_323_equation_0 = const()[name = string("op_323_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_323_cast_fp16 = einsum(equation = var_323_equation_0, values = (var_315_cast_fp16_0, var_321_cast_fp16))[name = string("op_323_cast_fp16")];
            fp16 var_324_to_fp16 = const()[name = string("op_324_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_325_cast_fp16 = mul(x = var_310_cast_fp16_1, y = var_324_to_fp16)[name = string("op_325_cast_fp16")];
            string var_327_equation_0 = const()[name = string("op_327_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_327_cast_fp16 = einsum(equation = var_327_equation_0, values = (var_315_cast_fp16_1, var_325_cast_fp16))[name = string("op_327_cast_fp16")];
            fp16 var_328_to_fp16 = const()[name = string("op_328_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_329_cast_fp16 = mul(x = var_310_cast_fp16_2, y = var_328_to_fp16)[name = string("op_329_cast_fp16")];
            string var_331_equation_0 = const()[name = string("op_331_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_331_cast_fp16 = einsum(equation = var_331_equation_0, values = (var_315_cast_fp16_2, var_329_cast_fp16))[name = string("op_331_cast_fp16")];
            fp16 var_332_to_fp16 = const()[name = string("op_332_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_333_cast_fp16 = mul(x = var_310_cast_fp16_3, y = var_332_to_fp16)[name = string("op_333_cast_fp16")];
            string var_335_equation_0 = const()[name = string("op_335_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_335_cast_fp16 = einsum(equation = var_335_equation_0, values = (var_315_cast_fp16_3, var_333_cast_fp16))[name = string("op_335_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_336_cast_fp16 = add(x = var_323_cast_fp16, y = var_327_cast_fp16)[name = string("op_336_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_337_cast_fp16 = add(x = var_331_cast_fp16, y = var_335_cast_fp16)[name = string("op_337_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_338_cast_fp16 = add(x = var_336_cast_fp16, y = var_337_cast_fp16)[name = string("op_338_cast_fp16")];
            fp16 _inversed_340_y_0_to_fp16 = const()[name = string("_inversed_340_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 16]> _inversed_340_cast_fp16 = mul(x = var_338_cast_fp16, y = _inversed_340_y_0_to_fp16)[name = string("_inversed_340_cast_fp16")];
            string scores_11_equation_0 = const()[name = string("scores_11_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 16]> scores_11_cast_fp16 = einsum(equation = scores_11_equation_0, values = (var_150_cast_fp16_1, var_140_cast_fp16_5))[name = string("scores_11_cast_fp16")];
            fp16 var_343_to_fp16 = const()[name = string("op_343_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 16]> var_344_cast_fp16 = mul(x = scores_11_cast_fp16, y = var_343_to_fp16)[name = string("op_344_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_345_cast_fp16 = add(x = var_344_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_345_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_346_cast_fp16 = softmax(axis = var_118, x = var_345_cast_fp16)[name = string("op_346_cast_fp16")];
            tensor<int32, [4]> tile_13 = const()[name = string("tile_13"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_347_axis_0 = const()[name = string("op_347_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 16]> var_347_cast_fp16_0, tensor<fp16, [1, 512, 1, 16]> var_347_cast_fp16_1, tensor<fp16, [1, 512, 1, 16]> var_347_cast_fp16_2, tensor<fp16, [1, 512, 1, 16]> var_347_cast_fp16_3 = split(axis = var_347_axis_0, split_sizes = tile_13, x = var_346_cast_fp16)[name = string("op_347_cast_fp16")];
            tensor<int32, [4]> tile_14 = const()[name = string("tile_14"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_352_axis_0 = const()[name = string("op_352_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_352_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_352_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_352_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_352_cast_fp16_3 = split(axis = var_352_axis_0, split_sizes = tile_14, x = var_153_cast_fp16_1)[name = string("op_352_cast_fp16")];
            fp16 var_357_to_fp16 = const()[name = string("op_357_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_358_cast_fp16 = mul(x = var_347_cast_fp16_0, y = var_357_to_fp16)[name = string("op_358_cast_fp16")];
            string var_360_equation_0 = const()[name = string("op_360_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_360_cast_fp16 = einsum(equation = var_360_equation_0, values = (var_352_cast_fp16_0, var_358_cast_fp16))[name = string("op_360_cast_fp16")];
            fp16 var_361_to_fp16 = const()[name = string("op_361_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_362_cast_fp16 = mul(x = var_347_cast_fp16_1, y = var_361_to_fp16)[name = string("op_362_cast_fp16")];
            string var_364_equation_0 = const()[name = string("op_364_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_364_cast_fp16 = einsum(equation = var_364_equation_0, values = (var_352_cast_fp16_1, var_362_cast_fp16))[name = string("op_364_cast_fp16")];
            fp16 var_365_to_fp16 = const()[name = string("op_365_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_366_cast_fp16 = mul(x = var_347_cast_fp16_2, y = var_365_to_fp16)[name = string("op_366_cast_fp16")];
            string var_368_equation_0 = const()[name = string("op_368_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_368_cast_fp16 = einsum(equation = var_368_equation_0, values = (var_352_cast_fp16_2, var_366_cast_fp16))[name = string("op_368_cast_fp16")];
            fp16 var_369_to_fp16 = const()[name = string("op_369_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_370_cast_fp16 = mul(x = var_347_cast_fp16_3, y = var_369_to_fp16)[name = string("op_370_cast_fp16")];
            string var_372_equation_0 = const()[name = string("op_372_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_372_cast_fp16 = einsum(equation = var_372_equation_0, values = (var_352_cast_fp16_3, var_370_cast_fp16))[name = string("op_372_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_373_cast_fp16 = add(x = var_360_cast_fp16, y = var_364_cast_fp16)[name = string("op_373_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_374_cast_fp16 = add(x = var_368_cast_fp16, y = var_372_cast_fp16)[name = string("op_374_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_375_cast_fp16 = add(x = var_373_cast_fp16, y = var_374_cast_fp16)[name = string("op_375_cast_fp16")];
            fp16 _inversed_377_y_0_to_fp16 = const()[name = string("_inversed_377_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 16]> _inversed_377_cast_fp16 = mul(x = var_375_cast_fp16, y = _inversed_377_y_0_to_fp16)[name = string("_inversed_377_cast_fp16")];
            string scores_13_equation_0 = const()[name = string("scores_13_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 16]> scores_13_cast_fp16 = einsum(equation = scores_13_equation_0, values = (var_150_cast_fp16_1, var_140_cast_fp16_6))[name = string("scores_13_cast_fp16")];
            fp16 var_380_to_fp16 = const()[name = string("op_380_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 16]> var_381_cast_fp16 = mul(x = scores_13_cast_fp16, y = var_380_to_fp16)[name = string("op_381_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_382_cast_fp16 = add(x = var_381_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_382_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_383_cast_fp16 = softmax(axis = var_118, x = var_382_cast_fp16)[name = string("op_383_cast_fp16")];
            tensor<int32, [4]> tile_15 = const()[name = string("tile_15"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_384_axis_0 = const()[name = string("op_384_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 16]> var_384_cast_fp16_0, tensor<fp16, [1, 512, 1, 16]> var_384_cast_fp16_1, tensor<fp16, [1, 512, 1, 16]> var_384_cast_fp16_2, tensor<fp16, [1, 512, 1, 16]> var_384_cast_fp16_3 = split(axis = var_384_axis_0, split_sizes = tile_15, x = var_383_cast_fp16)[name = string("op_384_cast_fp16")];
            tensor<int32, [4]> tile_16 = const()[name = string("tile_16"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_389_axis_0 = const()[name = string("op_389_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_389_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_389_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_389_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_389_cast_fp16_3 = split(axis = var_389_axis_0, split_sizes = tile_16, x = var_153_cast_fp16_1)[name = string("op_389_cast_fp16")];
            fp16 var_394_to_fp16 = const()[name = string("op_394_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_395_cast_fp16 = mul(x = var_384_cast_fp16_0, y = var_394_to_fp16)[name = string("op_395_cast_fp16")];
            string var_397_equation_0 = const()[name = string("op_397_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_397_cast_fp16 = einsum(equation = var_397_equation_0, values = (var_389_cast_fp16_0, var_395_cast_fp16))[name = string("op_397_cast_fp16")];
            fp16 var_398_to_fp16 = const()[name = string("op_398_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_399_cast_fp16 = mul(x = var_384_cast_fp16_1, y = var_398_to_fp16)[name = string("op_399_cast_fp16")];
            string var_401_equation_0 = const()[name = string("op_401_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_401_cast_fp16 = einsum(equation = var_401_equation_0, values = (var_389_cast_fp16_1, var_399_cast_fp16))[name = string("op_401_cast_fp16")];
            fp16 var_402_to_fp16 = const()[name = string("op_402_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_403_cast_fp16 = mul(x = var_384_cast_fp16_2, y = var_402_to_fp16)[name = string("op_403_cast_fp16")];
            string var_405_equation_0 = const()[name = string("op_405_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_405_cast_fp16 = einsum(equation = var_405_equation_0, values = (var_389_cast_fp16_2, var_403_cast_fp16))[name = string("op_405_cast_fp16")];
            fp16 var_406_to_fp16 = const()[name = string("op_406_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_407_cast_fp16 = mul(x = var_384_cast_fp16_3, y = var_406_to_fp16)[name = string("op_407_cast_fp16")];
            string var_409_equation_0 = const()[name = string("op_409_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_409_cast_fp16 = einsum(equation = var_409_equation_0, values = (var_389_cast_fp16_3, var_407_cast_fp16))[name = string("op_409_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_410_cast_fp16 = add(x = var_397_cast_fp16, y = var_401_cast_fp16)[name = string("op_410_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_411_cast_fp16 = add(x = var_405_cast_fp16, y = var_409_cast_fp16)[name = string("op_411_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_412_cast_fp16 = add(x = var_410_cast_fp16, y = var_411_cast_fp16)[name = string("op_412_cast_fp16")];
            fp16 _inversed_414_y_0_to_fp16 = const()[name = string("_inversed_414_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 16]> _inversed_414_cast_fp16 = mul(x = var_412_cast_fp16, y = _inversed_414_y_0_to_fp16)[name = string("_inversed_414_cast_fp16")];
            string scores_15_equation_0 = const()[name = string("scores_15_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 16]> scores_15_cast_fp16 = einsum(equation = scores_15_equation_0, values = (var_150_cast_fp16_1, var_140_cast_fp16_7))[name = string("scores_15_cast_fp16")];
            fp16 var_417_to_fp16 = const()[name = string("op_417_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 16]> var_418_cast_fp16 = mul(x = scores_15_cast_fp16, y = var_417_to_fp16)[name = string("op_418_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_419_cast_fp16 = add(x = var_418_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_419_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_420_cast_fp16 = softmax(axis = var_118, x = var_419_cast_fp16)[name = string("op_420_cast_fp16")];
            tensor<int32, [4]> tile_17 = const()[name = string("tile_17"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_421_axis_0 = const()[name = string("op_421_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 16]> var_421_cast_fp16_0, tensor<fp16, [1, 512, 1, 16]> var_421_cast_fp16_1, tensor<fp16, [1, 512, 1, 16]> var_421_cast_fp16_2, tensor<fp16, [1, 512, 1, 16]> var_421_cast_fp16_3 = split(axis = var_421_axis_0, split_sizes = tile_17, x = var_420_cast_fp16)[name = string("op_421_cast_fp16")];
            tensor<int32, [4]> tile_18 = const()[name = string("tile_18"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_426_axis_0 = const()[name = string("op_426_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_426_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_426_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_426_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_426_cast_fp16_3 = split(axis = var_426_axis_0, split_sizes = tile_18, x = var_153_cast_fp16_1)[name = string("op_426_cast_fp16")];
            fp16 var_431_to_fp16 = const()[name = string("op_431_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_432_cast_fp16 = mul(x = var_421_cast_fp16_0, y = var_431_to_fp16)[name = string("op_432_cast_fp16")];
            string var_434_equation_0 = const()[name = string("op_434_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_434_cast_fp16 = einsum(equation = var_434_equation_0, values = (var_426_cast_fp16_0, var_432_cast_fp16))[name = string("op_434_cast_fp16")];
            fp16 var_435_to_fp16 = const()[name = string("op_435_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_436_cast_fp16 = mul(x = var_421_cast_fp16_1, y = var_435_to_fp16)[name = string("op_436_cast_fp16")];
            string var_438_equation_0 = const()[name = string("op_438_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_438_cast_fp16 = einsum(equation = var_438_equation_0, values = (var_426_cast_fp16_1, var_436_cast_fp16))[name = string("op_438_cast_fp16")];
            fp16 var_439_to_fp16 = const()[name = string("op_439_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_440_cast_fp16 = mul(x = var_421_cast_fp16_2, y = var_439_to_fp16)[name = string("op_440_cast_fp16")];
            string var_442_equation_0 = const()[name = string("op_442_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_442_cast_fp16 = einsum(equation = var_442_equation_0, values = (var_426_cast_fp16_2, var_440_cast_fp16))[name = string("op_442_cast_fp16")];
            fp16 var_443_to_fp16 = const()[name = string("op_443_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_444_cast_fp16 = mul(x = var_421_cast_fp16_3, y = var_443_to_fp16)[name = string("op_444_cast_fp16")];
            string var_446_equation_0 = const()[name = string("op_446_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_446_cast_fp16 = einsum(equation = var_446_equation_0, values = (var_426_cast_fp16_3, var_444_cast_fp16))[name = string("op_446_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_447_cast_fp16 = add(x = var_434_cast_fp16, y = var_438_cast_fp16)[name = string("op_447_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_448_cast_fp16 = add(x = var_442_cast_fp16, y = var_446_cast_fp16)[name = string("op_448_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_449_cast_fp16 = add(x = var_447_cast_fp16, y = var_448_cast_fp16)[name = string("op_449_cast_fp16")];
            fp16 _inversed_451_y_0_to_fp16 = const()[name = string("_inversed_451_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 16]> _inversed_451_cast_fp16 = mul(x = var_449_cast_fp16, y = _inversed_451_y_0_to_fp16)[name = string("_inversed_451_cast_fp16")];
            bool var_453_interleave_0 = const()[name = string("op_453_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 2048, 1, 16]> var_453_cast_fp16 = concat(axis = var_118, interleave = var_453_interleave_0, values = (_inversed_192_cast_fp16, _inversed_229_cast_fp16, _inversed_266_cast_fp16, _inversed_303_cast_fp16, _inversed_340_cast_fp16, _inversed_377_cast_fp16, _inversed_414_cast_fp16, _inversed_451_cast_fp16))[name = string("op_453_cast_fp16")];
            tensor<int32, [4]> var_454 = const()[name = string("op_454"), val = tensor<int32, [4]>([2, 4, 256, 16])];
            tensor<fp16, [2, 4, 256, 16]> var_455_cast_fp16 = reshape(shape = var_454, x = var_453_cast_fp16)[name = string("op_455_cast_fp16")];
            tensor<int32, [4]> transpose_0_perm_0 = const()[name = string("transpose_0_perm_0"), val = tensor<int32, [4]>([3, 0, 1, 2])];
            tensor<int32, [2]> var_474 = const()[name = string("op_474"), val = tensor<int32, [2]>([16, 2048])];
            tensor<fp16, [16, 2, 4, 256]> transpose_0_cast_fp16 = transpose(perm = transpose_0_perm_0, x = var_455_cast_fp16)[name = string("transpose_29")];
            tensor<fp16, [16, 2048]> attention_output_3_cast_fp16 = reshape(shape = var_474, x = transpose_0_cast_fp16)[name = string("attention_output_3_cast_fp16")];
            tensor<fp16, [16, 2048]> var_476_cast_fp16 = sigmoid(x = gate19)[name = string("op_476_cast_fp16")];
            tensor<fp16, [16, 2048]> input_1_cast_fp16 = mul(x = attention_output_3_cast_fp16, y = var_476_cast_fp16)[name = string("input_1_cast_fp16")];
            tensor<fp16, [1024, 2048]> completion19_o_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 2048]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(193193216))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(194766144))))[name = string("completion19_o_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1024]> linear_0_bias_0_to_fp16 = const()[name = string("linear_0_bias_0_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(194774400)))];
            tensor<fp16, [16, 1024]> linear_0_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = completion19_o_proj_weight_promoted_to_fp16_palettized, x = input_1_cast_fp16)[name = string("linear_0_cast_fp16")];
            tensor<fp16, [16, 1024]> value_1_cast_fp16 = add(x = hidden, y = linear_0_cast_fp16)[name = string("value_1_cast_fp16")];
            tensor<fp16, [16, 1024]> var_481_cast_fp16 = mul(x = value_1_cast_fp16, y = value_1_cast_fp16)[name = string("op_481_cast_fp16")];
            tensor<int32, [1]> variance_1_axes_0 = const()[name = string("variance_1_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_1_keep_dims_0 = const()[name = string("variance_1_keep_dims_0"), val = bool(true)];
            tensor<fp16, [16, 1]> variance_1_cast_fp16 = reduce_mean(axes = variance_1_axes_0, keep_dims = variance_1_keep_dims_0, x = var_481_cast_fp16)[name = string("variance_1_cast_fp16")];
            fp16 var_484_to_fp16 = const()[name = string("op_484_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [16, 1]> var_485_cast_fp16 = add(x = variance_1_cast_fp16, y = var_484_to_fp16)[name = string("op_485_cast_fp16")];
            fp32 var_486_epsilon_0 = const()[name = string("op_486_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [16, 1]> var_486_cast_fp16 = rsqrt(epsilon = var_486_epsilon_0, x = var_485_cast_fp16)[name = string("op_486_cast_fp16")];
            tensor<fp16, [16, 1024]> var_487_cast_fp16 = mul(x = value_1_cast_fp16, y = var_486_cast_fp16)[name = string("op_487_cast_fp16")];
            tensor<fp16, [1024]> var_489_to_fp16 = const()[name = string("op_489_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(194776512)))];
            tensor<fp16, [16, 1024]> input_3_cast_fp16 = mul(x = var_487_cast_fp16, y = var_489_to_fp16)[name = string("input_3_cast_fp16")];
            tensor<fp16, [3584, 1024]> completion19_gate_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(194778624))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(197531200))))[name = string("completion19_gate_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [3584]> linear_1_bias_0_to_fp16 = const()[name = string("linear_1_bias_0_to_fp16"), val = tensor<fp16, [3584]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(197559936)))];
            tensor<fp16, [16, 3584]> linear_1_cast_fp16 = linear(bias = linear_1_bias_0_to_fp16, weight = completion19_gate_proj_weight_promoted_to_fp16_palettized, x = input_3_cast_fp16)[name = string("linear_1_cast_fp16")];
            tensor<fp16, [16, 3584]> var_493_cast_fp16 = silu(x = linear_1_cast_fp16)[name = string("op_493_cast_fp16")];
            tensor<fp16, [3584, 1024]> completion19_up_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(197567168))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(200319744))))[name = string("completion19_up_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 3584]> linear_2_cast_fp16 = linear(bias = linear_1_bias_0_to_fp16, weight = completion19_up_proj_weight_promoted_to_fp16_palettized, x = input_3_cast_fp16)[name = string("linear_2_cast_fp16")];
            tensor<fp16, [16, 3584]> input_7_cast_fp16 = mul(x = var_493_cast_fp16, y = linear_2_cast_fp16)[name = string("input_7_cast_fp16")];
            tensor<fp16, [1024, 3584]> completion19_down_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 3584]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(200348480))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(203101056))))[name = string("completion19_down_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 1024]> linear_3_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = completion19_down_proj_weight_promoted_to_fp16_palettized, x = input_7_cast_fp16)[name = string("linear_3_cast_fp16")];
            tensor<fp16, [16, 1024]> value_3_cast_fp16 = add(x = value_1_cast_fp16, y = linear_3_cast_fp16)[name = string("value_3_cast_fp16")];
            int32 var_527 = const()[name = string("op_527"), val = int32(-1)];
            tensor<fp16, [16, 1024]> var_542_cast_fp16 = mul(x = value_3_cast_fp16, y = value_3_cast_fp16)[name = string("op_542_cast_fp16")];
            tensor<int32, [1]> variance_3_axes_0 = const()[name = string("variance_3_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_3_keep_dims_0 = const()[name = string("variance_3_keep_dims_0"), val = bool(true)];
            tensor<fp16, [16, 1]> variance_3_cast_fp16 = reduce_mean(axes = variance_3_axes_0, keep_dims = variance_3_keep_dims_0, x = var_542_cast_fp16)[name = string("variance_3_cast_fp16")];
            fp16 var_545_to_fp16 = const()[name = string("op_545_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [16, 1]> var_546_cast_fp16 = add(x = variance_3_cast_fp16, y = var_545_to_fp16)[name = string("op_546_cast_fp16")];
            fp32 var_547_epsilon_0 = const()[name = string("op_547_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [16, 1]> var_547_cast_fp16 = rsqrt(epsilon = var_547_epsilon_0, x = var_546_cast_fp16)[name = string("op_547_cast_fp16")];
            tensor<fp16, [16, 1024]> var_548_cast_fp16 = mul(x = value_3_cast_fp16, y = var_547_cast_fp16)[name = string("op_548_cast_fp16")];
            tensor<fp16, [1024]> var_550_to_fp16 = const()[name = string("op_550_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(203109312)))];
            tensor<fp16, [16, 1024]> input_9_cast_fp16 = mul(x = var_548_cast_fp16, y = var_550_to_fp16)[name = string("input_9_cast_fp16")];
            tensor<fp16, [6144, 1024]> layers20_0_in_proj_qkv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(203111424))), lut = tensor<fp16, [384, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(207830080))))[name = string("layers20_0_in_proj_qkv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [6144]> linear_4_bias_0_to_fp16 = const()[name = string("linear_4_bias_0_to_fp16"), val = tensor<fp16, [6144]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(207879296)))];
            tensor<fp16, [16, 6144]> linear_4_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = layers20_0_in_proj_qkv_weight_promoted_to_fp16_palettized, x = input_9_cast_fp16)[name = string("linear_4_cast_fp16")];
            tensor<int32, [2]> var_554_perm_0 = const()[name = string("op_554_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> mixed_1_axes_0 = const()[name = string("mixed_1_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [6144, 16]> var_554_cast_fp16 = transpose(perm = var_554_perm_0, x = linear_4_cast_fp16)[name = string("transpose_28")];
            tensor<fp16, [1, 6144, 16]> mixed_1_cast_fp16 = expand_dims(axes = mixed_1_axes_0, x = var_554_cast_fp16)[name = string("mixed_1_cast_fp16")];
            bool convolution_input_1_interleave_0 = const()[name = string("convolution_input_1_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 6144, 19]> convolution_input_1_cast_fp16 = concat(axis = var_527, interleave = convolution_input_1_interleave_0, values = (conv20, mixed_1_cast_fp16))[name = string("convolution_input_1_cast_fp16")];
            string input_11_pad_type_0 = const()[name = string("input_11_pad_type_0"), val = string("valid")];
            int32 input_11_groups_0 = const()[name = string("input_11_groups_0"), val = int32(6144)];
            tensor<int32, [1]> input_11_strides_0 = const()[name = string("input_11_strides_0"), val = tensor<int32, [1]>([1])];
            tensor<int32, [2]> input_11_pad_0 = const()[name = string("input_11_pad_0"), val = tensor<int32, [2]>([0, 0])];
            tensor<int32, [1]> input_11_dilations_0 = const()[name = string("input_11_dilations_0"), val = tensor<int32, [1]>([1])];
            tensor<fp16, [6144, 1, 4]> layers20_0_conv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1, 4]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(207891648))), lut = tensor<fp16, [384, 1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(207910144))))[name = string("layers20_0_conv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 6144, 16]> input_11_cast_fp16 = conv(dilations = input_11_dilations_0, groups = input_11_groups_0, pad = input_11_pad_0, pad_type = input_11_pad_type_0, strides = input_11_strides_0, weight = layers20_0_conv_weight_promoted_to_fp16_palettized, x = convolution_input_1_cast_fp16)[name = string("input_11_cast_fp16")];
            tensor<fp16, [1, 6144, 16]> var_563_cast_fp16 = silu(x = input_11_cast_fp16)[name = string("op_563_cast_fp16")];
            tensor<int32, [3]> var_564_perm_0 = const()[name = string("op_564_perm_0"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [3]> var_567_begin_0 = const()[name = string("op_567_begin_0"), val = tensor<int32, [3]>([0, 0, 16])];
            tensor<int32, [3]> var_567_end_0 = const()[name = string("op_567_end_0"), val = tensor<int32, [3]>([1, 6144, 19])];
            tensor<bool, [3]> var_567_end_mask_0 = const()[name = string("op_567_end_mask_0"), val = tensor<bool, [3]>([true, true, true])];
            tensor<fp16, [1, 6144, 3]> conv20_out = slice_by_index(begin = var_567_begin_0, end = var_567_end_0, end_mask = var_567_end_mask_0, x = convolution_input_1_cast_fp16)[name = string("op_567_cast_fp16")];
            tensor<int32, [3]> var_568 = const()[name = string("op_568"), val = tensor<int32, [3]>([2048, 2048, 2048])];
            int32 var_569_axis_0 = const()[name = string("op_569_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 16, 6144]> var_564_cast_fp16 = transpose(perm = var_564_perm_0, x = var_563_cast_fp16)[name = string("transpose_27")];
            tensor<fp16, [1, 16, 2048]> var_569_cast_fp16_0, tensor<fp16, [1, 16, 2048]> var_569_cast_fp16_1, tensor<fp16, [1, 16, 2048]> var_569_cast_fp16_2 = split(axis = var_569_axis_0, split_sizes = var_568, x = var_564_cast_fp16)[name = string("op_569_cast_fp16")];
            tensor<int32, [4]> var_573 = const()[name = string("op_573"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<fp16, [1, 16, 16, 128]> value_7_cast_fp16 = reshape(shape = var_573, x = var_569_cast_fp16_0)[name = string("value_7_cast_fp16")];
            tensor<int32, [4]> var_575 = const()[name = string("op_575"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<fp16, [1, 16, 16, 128]> value_9_cast_fp16 = reshape(shape = var_575, x = var_569_cast_fp16_1)[name = string("value_9_cast_fp16")];
            tensor<int32, [4]> var_577 = const()[name = string("op_577"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<fp16, [1, 16, 16, 128]> value_11_cast_fp16 = reshape(shape = var_577, x = var_569_cast_fp16_2)[name = string("value_11_cast_fp16")];
            tensor<fp16, [1, 16, 16, 128]> var_579_cast_fp16 = mul(x = value_7_cast_fp16, y = value_7_cast_fp16)[name = string("op_579_cast_fp16")];
            tensor<int32, [1]> var_581_axes_0 = const()[name = string("op_581_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_581_keep_dims_0 = const()[name = string("op_581_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 16, 16, 1]> var_581_cast_fp16 = reduce_sum(axes = var_581_axes_0, keep_dims = var_581_keep_dims_0, x = var_579_cast_fp16)[name = string("op_581_cast_fp16")];
            fp16 var_582_to_fp16 = const()[name = string("op_582_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 16, 16, 1]> var_583_cast_fp16 = add(x = var_581_cast_fp16, y = var_582_to_fp16)[name = string("op_583_cast_fp16")];
            fp32 var_584_epsilon_0 = const()[name = string("op_584_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 16, 16, 1]> var_584_cast_fp16 = rsqrt(epsilon = var_584_epsilon_0, x = var_583_cast_fp16)[name = string("op_584_cast_fp16")];
            tensor<fp16, [1, 16, 16, 128]> var_585_cast_fp16 = mul(x = value_7_cast_fp16, y = var_584_cast_fp16)[name = string("op_585_cast_fp16")];
            tensor<int32, [4]> var_586_perm_0 = const()[name = string("op_586_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            fp16 _inversed_query_3_y_0_to_fp16 = const()[name = string("_inversed_query_3_y_0_to_fp16"), val = fp16(0x1.6ap-4)];
            tensor<fp16, [1, 16, 16, 128]> var_586_cast_fp16 = transpose(perm = var_586_perm_0, x = var_585_cast_fp16)[name = string("transpose_26")];
            tensor<fp16, [1, 16, 16, 128]> _inversed_query_3_cast_fp16 = mul(x = var_586_cast_fp16, y = _inversed_query_3_y_0_to_fp16)[name = string("_inversed_query_3_cast_fp16")];
            tensor<fp16, [1, 16, 16, 128]> var_589_cast_fp16 = mul(x = value_9_cast_fp16, y = value_9_cast_fp16)[name = string("op_589_cast_fp16")];
            tensor<int32, [1]> var_591_axes_0 = const()[name = string("op_591_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_591_keep_dims_0 = const()[name = string("op_591_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 16, 16, 1]> var_591_cast_fp16 = reduce_sum(axes = var_591_axes_0, keep_dims = var_591_keep_dims_0, x = var_589_cast_fp16)[name = string("op_591_cast_fp16")];
            fp16 var_592_to_fp16 = const()[name = string("op_592_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 16, 16, 1]> var_593_cast_fp16 = add(x = var_591_cast_fp16, y = var_592_to_fp16)[name = string("op_593_cast_fp16")];
            fp32 var_594_epsilon_0 = const()[name = string("op_594_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 16, 16, 1]> var_594_cast_fp16 = rsqrt(epsilon = var_594_epsilon_0, x = var_593_cast_fp16)[name = string("op_594_cast_fp16")];
            tensor<fp16, [1, 16, 16, 128]> var_595_cast_fp16 = mul(x = value_9_cast_fp16, y = var_594_cast_fp16)[name = string("op_595_cast_fp16")];
            tensor<int32, [4]> key_3_perm_0 = const()[name = string("key_3_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<int32, [4]> value_13_perm_0 = const()[name = string("value_13_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<fp16, [16, 1024]> layers20_0_in_proj_b_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(207959360))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(207971712))))[name = string("layers20_0_in_proj_b_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16]> linear_5_bias_0_to_fp16 = const()[name = string("linear_5_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(207971904)))];
            tensor<fp16, [16, 16]> linear_5_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = layers20_0_in_proj_b_weight_promoted_to_fp16_palettized, x = input_9_cast_fp16)[name = string("linear_5_cast_fp16")];
            tensor<fp16, [16, 16]> var_600_cast_fp16 = sigmoid(x = linear_5_cast_fp16)[name = string("op_600_cast_fp16")];
            tensor<int32, [2]> var_601_perm_0 = const()[name = string("op_601_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> beta_1_axes_0 = const()[name = string("beta_1_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 16]> var_601_cast_fp16 = transpose(perm = var_601_perm_0, x = var_600_cast_fp16)[name = string("transpose_25")];
            tensor<fp16, [1, 16, 16]> beta_1_cast_fp16 = expand_dims(axes = beta_1_axes_0, x = var_601_cast_fp16)[name = string("beta_1_cast_fp16")];
            tensor<fp16, [16, 1024]> op_607_weight_0_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(207972032))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(207984384))))[name = string("op_607_weight_0_to_fp16_palettized")];
            tensor<fp16, [16]> var_607_bias_0_to_fp16 = const()[name = string("op_607_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(207984576)))];
            tensor<fp16, [16, 16]> var_607_cast_fp16 = linear(bias = var_607_bias_0_to_fp16, weight = op_607_weight_0_to_fp16_palettized, x = input_9_cast_fp16)[name = string("op_607_cast_fp16")];
            tensor<fp16, [16, 16]> var_608_cast_fp16 = softplus(x = var_607_cast_fp16)[name = string("op_608_cast_fp16")];
            tensor<fp16, [16]> var_604_promoted_to_fp16 = const()[name = string("op_604_promoted_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(207984704)))];
            tensor<fp16, [16, 16]> var_609_cast_fp16 = mul(x = var_604_promoted_to_fp16, y = var_608_cast_fp16)[name = string("op_609_cast_fp16")];
            tensor<int32, [2]> var_610_perm_0 = const()[name = string("op_610_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> decay_1_axes_0 = const()[name = string("decay_1_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 16]> var_610_cast_fp16 = transpose(perm = var_610_perm_0, x = var_609_cast_fp16)[name = string("transpose_24")];
            tensor<fp16, [1, 16, 16]> decay_1_cast_fp16 = expand_dims(axes = decay_1_axes_0, x = var_610_cast_fp16)[name = string("decay_1_cast_fp16")];
            tensor<fp16, [2048, 1024]> layers20_0_in_proj_z_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [2048, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(207984832))), lut = tensor<fp16, [128, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(209557760))))[name = string("layers20_0_in_proj_z_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [2048]> linear_7_bias_0_to_fp16 = const()[name = string("linear_7_bias_0_to_fp16"), val = tensor<fp16, [2048]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(209574208)))];
            tensor<fp16, [16, 2048]> linear_7_cast_fp16 = linear(bias = linear_7_bias_0_to_fp16, weight = layers20_0_in_proj_z_weight_promoted_to_fp16_palettized, x = input_9_cast_fp16)[name = string("linear_7_cast_fp16")];
            tensor<int32, [3]> var_618_begin_0 = const()[name = string("op_618_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_618_end_0 = const()[name = string("op_618_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_618_end_mask_0 = const()[name = string("op_618_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_618_squeeze_mask_0 = const()[name = string("op_618_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_618_cast_fp16 = slice_by_index(begin = var_618_begin_0, end = var_618_end_0, end_mask = var_618_end_mask_0, squeeze_mask = var_618_squeeze_mask_0, x = decay_1_cast_fp16)[name = string("op_618_cast_fp16")];
            tensor<fp16, [1, 16]> var_619_cast_fp16 = exp(x = var_618_cast_fp16)[name = string("op_619_cast_fp16")];
            tensor<int32, [1]> var_620_axes_0 = const()[name = string("op_620_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_620_cast_fp16 = expand_dims(axes = var_620_axes_0, x = var_619_cast_fp16)[name = string("op_620_cast_fp16")];
            tensor<int32, [1]> var_621_axes_0 = const()[name = string("op_621_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_621_cast_fp16 = expand_dims(axes = var_621_axes_0, x = var_620_cast_fp16)[name = string("op_621_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_1_cast_fp16 = mul(x = rec20, y = var_621_cast_fp16)[name = string("state_1_cast_fp16")];
            tensor<int32, [4]> key_token_1_begin_0 = const()[name = string("key_token_1_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> key_token_1_end_0 = const()[name = string("key_token_1_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> key_token_1_end_mask_0 = const()[name = string("key_token_1_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_1_squeeze_mask_0 = const()[name = string("key_token_1_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 16, 128]> key_3_cast_fp16 = transpose(perm = key_3_perm_0, x = var_595_cast_fp16)[name = string("transpose_23")];
            tensor<fp16, [1, 16, 128]> key_token_1_cast_fp16 = slice_by_index(begin = key_token_1_begin_0, end = key_token_1_end_0, end_mask = key_token_1_end_mask_0, squeeze_mask = key_token_1_squeeze_mask_0, x = key_3_cast_fp16)[name = string("key_token_1_cast_fp16")];
            tensor<int32, [4]> value_token_1_begin_0 = const()[name = string("value_token_1_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> value_token_1_end_0 = const()[name = string("value_token_1_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> value_token_1_end_mask_0 = const()[name = string("value_token_1_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_1_squeeze_mask_0 = const()[name = string("value_token_1_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 16, 128]> value_13_cast_fp16 = transpose(perm = value_13_perm_0, x = value_11_cast_fp16)[name = string("transpose_22")];
            tensor<fp16, [1, 16, 128]> value_token_1_cast_fp16 = slice_by_index(begin = value_token_1_begin_0, end = value_token_1_end_0, end_mask = value_token_1_end_mask_0, squeeze_mask = value_token_1_squeeze_mask_0, x = value_13_cast_fp16)[name = string("value_token_1_cast_fp16")];
            tensor<int32, [1]> var_629_axes_0 = const()[name = string("op_629_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_629_cast_fp16 = expand_dims(axes = var_629_axes_0, x = key_token_1_cast_fp16)[name = string("op_629_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_630_cast_fp16 = mul(x = state_1_cast_fp16, y = var_629_cast_fp16)[name = string("op_630_cast_fp16")];
            tensor<int32, [1]> memory_1_axes_0 = const()[name = string("memory_1_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_1_keep_dims_0 = const()[name = string("memory_1_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_1_cast_fp16 = reduce_sum(axes = memory_1_axes_0, keep_dims = memory_1_keep_dims_0, x = var_630_cast_fp16)[name = string("memory_1_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_633_cast_fp16 = sub(x = value_token_1_cast_fp16, y = memory_1_cast_fp16)[name = string("op_633_cast_fp16")];
            tensor<int32, [3]> var_636_begin_0 = const()[name = string("op_636_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_636_end_0 = const()[name = string("op_636_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_636_end_mask_0 = const()[name = string("op_636_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_636_squeeze_mask_0 = const()[name = string("op_636_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_636_cast_fp16 = slice_by_index(begin = var_636_begin_0, end = var_636_end_0, end_mask = var_636_end_mask_0, squeeze_mask = var_636_squeeze_mask_0, x = beta_1_cast_fp16)[name = string("op_636_cast_fp16")];
            tensor<int32, [1]> var_637_axes_0 = const()[name = string("op_637_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_637_cast_fp16 = expand_dims(axes = var_637_axes_0, x = var_636_cast_fp16)[name = string("op_637_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_1_cast_fp16 = mul(x = var_633_cast_fp16, y = var_637_cast_fp16)[name = string("delta_1_cast_fp16")];
            tensor<int32, [1]> var_640_axes_0 = const()[name = string("op_640_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_640_cast_fp16 = expand_dims(axes = var_640_axes_0, x = delta_1_cast_fp16)[name = string("op_640_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_641_cast_fp16 = mul(x = var_629_cast_fp16, y = var_640_cast_fp16)[name = string("op_641_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_3_cast_fp16 = add(x = state_1_cast_fp16, y = var_641_cast_fp16)[name = string("state_3_cast_fp16")];
            tensor<int32, [4]> var_645_begin_0 = const()[name = string("op_645_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> var_645_end_0 = const()[name = string("op_645_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> var_645_end_mask_0 = const()[name = string("op_645_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_645_squeeze_mask_0 = const()[name = string("op_645_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_645_cast_fp16 = slice_by_index(begin = var_645_begin_0, end = var_645_end_0, end_mask = var_645_end_mask_0, squeeze_mask = var_645_squeeze_mask_0, x = _inversed_query_3_cast_fp16)[name = string("op_645_cast_fp16")];
            tensor<int32, [1]> var_646_axes_0 = const()[name = string("op_646_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_646_cast_fp16 = expand_dims(axes = var_646_axes_0, x = var_645_cast_fp16)[name = string("op_646_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_647_cast_fp16 = mul(x = state_3_cast_fp16, y = var_646_cast_fp16)[name = string("op_647_cast_fp16")];
            tensor<int32, [1]> var_649_axes_0 = const()[name = string("op_649_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_649_keep_dims_0 = const()[name = string("op_649_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_649_cast_fp16 = reduce_sum(axes = var_649_axes_0, keep_dims = var_649_keep_dims_0, x = var_647_cast_fp16)[name = string("op_649_cast_fp16")];
            tensor<int32, [3]> var_652_begin_0 = const()[name = string("op_652_begin_0"), val = tensor<int32, [3]>([0, 0, 1])];
            tensor<int32, [3]> var_652_end_0 = const()[name = string("op_652_end_0"), val = tensor<int32, [3]>([1, 16, 2])];
            tensor<bool, [3]> var_652_end_mask_0 = const()[name = string("op_652_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_652_squeeze_mask_0 = const()[name = string("op_652_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_652_cast_fp16 = slice_by_index(begin = var_652_begin_0, end = var_652_end_0, end_mask = var_652_end_mask_0, squeeze_mask = var_652_squeeze_mask_0, x = decay_1_cast_fp16)[name = string("op_652_cast_fp16")];
            tensor<fp16, [1, 16]> var_653_cast_fp16 = exp(x = var_652_cast_fp16)[name = string("op_653_cast_fp16")];
            tensor<int32, [1]> var_654_axes_0 = const()[name = string("op_654_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_654_cast_fp16 = expand_dims(axes = var_654_axes_0, x = var_653_cast_fp16)[name = string("op_654_cast_fp16")];
            tensor<int32, [1]> var_655_axes_0 = const()[name = string("op_655_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_655_cast_fp16 = expand_dims(axes = var_655_axes_0, x = var_654_cast_fp16)[name = string("op_655_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_5_cast_fp16 = mul(x = state_3_cast_fp16, y = var_655_cast_fp16)[name = string("state_5_cast_fp16")];
            tensor<int32, [4]> key_token_3_begin_0 = const()[name = string("key_token_3_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> key_token_3_end_0 = const()[name = string("key_token_3_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> key_token_3_end_mask_0 = const()[name = string("key_token_3_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_3_squeeze_mask_0 = const()[name = string("key_token_3_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_3_cast_fp16 = slice_by_index(begin = key_token_3_begin_0, end = key_token_3_end_0, end_mask = key_token_3_end_mask_0, squeeze_mask = key_token_3_squeeze_mask_0, x = key_3_cast_fp16)[name = string("key_token_3_cast_fp16")];
            tensor<int32, [4]> value_token_3_begin_0 = const()[name = string("value_token_3_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> value_token_3_end_0 = const()[name = string("value_token_3_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> value_token_3_end_mask_0 = const()[name = string("value_token_3_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_3_squeeze_mask_0 = const()[name = string("value_token_3_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_3_cast_fp16 = slice_by_index(begin = value_token_3_begin_0, end = value_token_3_end_0, end_mask = value_token_3_end_mask_0, squeeze_mask = value_token_3_squeeze_mask_0, x = value_13_cast_fp16)[name = string("value_token_3_cast_fp16")];
            tensor<int32, [1]> var_663_axes_0 = const()[name = string("op_663_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_663_cast_fp16 = expand_dims(axes = var_663_axes_0, x = key_token_3_cast_fp16)[name = string("op_663_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_664_cast_fp16 = mul(x = state_5_cast_fp16, y = var_663_cast_fp16)[name = string("op_664_cast_fp16")];
            tensor<int32, [1]> memory_3_axes_0 = const()[name = string("memory_3_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_3_keep_dims_0 = const()[name = string("memory_3_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_3_cast_fp16 = reduce_sum(axes = memory_3_axes_0, keep_dims = memory_3_keep_dims_0, x = var_664_cast_fp16)[name = string("memory_3_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_667_cast_fp16 = sub(x = value_token_3_cast_fp16, y = memory_3_cast_fp16)[name = string("op_667_cast_fp16")];
            tensor<int32, [3]> var_670_begin_0 = const()[name = string("op_670_begin_0"), val = tensor<int32, [3]>([0, 0, 1])];
            tensor<int32, [3]> var_670_end_0 = const()[name = string("op_670_end_0"), val = tensor<int32, [3]>([1, 16, 2])];
            tensor<bool, [3]> var_670_end_mask_0 = const()[name = string("op_670_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_670_squeeze_mask_0 = const()[name = string("op_670_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_670_cast_fp16 = slice_by_index(begin = var_670_begin_0, end = var_670_end_0, end_mask = var_670_end_mask_0, squeeze_mask = var_670_squeeze_mask_0, x = beta_1_cast_fp16)[name = string("op_670_cast_fp16")];
            tensor<int32, [1]> var_671_axes_0 = const()[name = string("op_671_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_671_cast_fp16 = expand_dims(axes = var_671_axes_0, x = var_670_cast_fp16)[name = string("op_671_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_3_cast_fp16 = mul(x = var_667_cast_fp16, y = var_671_cast_fp16)[name = string("delta_3_cast_fp16")];
            tensor<int32, [1]> var_674_axes_0 = const()[name = string("op_674_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_674_cast_fp16 = expand_dims(axes = var_674_axes_0, x = delta_3_cast_fp16)[name = string("op_674_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_675_cast_fp16 = mul(x = var_663_cast_fp16, y = var_674_cast_fp16)[name = string("op_675_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_7_cast_fp16 = add(x = state_5_cast_fp16, y = var_675_cast_fp16)[name = string("state_7_cast_fp16")];
            tensor<int32, [4]> var_679_begin_0 = const()[name = string("op_679_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> var_679_end_0 = const()[name = string("op_679_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> var_679_end_mask_0 = const()[name = string("op_679_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_679_squeeze_mask_0 = const()[name = string("op_679_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_679_cast_fp16 = slice_by_index(begin = var_679_begin_0, end = var_679_end_0, end_mask = var_679_end_mask_0, squeeze_mask = var_679_squeeze_mask_0, x = _inversed_query_3_cast_fp16)[name = string("op_679_cast_fp16")];
            tensor<int32, [1]> var_680_axes_0 = const()[name = string("op_680_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_680_cast_fp16 = expand_dims(axes = var_680_axes_0, x = var_679_cast_fp16)[name = string("op_680_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_681_cast_fp16 = mul(x = state_7_cast_fp16, y = var_680_cast_fp16)[name = string("op_681_cast_fp16")];
            tensor<int32, [1]> var_683_axes_0 = const()[name = string("op_683_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_683_keep_dims_0 = const()[name = string("op_683_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_683_cast_fp16 = reduce_sum(axes = var_683_axes_0, keep_dims = var_683_keep_dims_0, x = var_681_cast_fp16)[name = string("op_683_cast_fp16")];
            tensor<int32, [3]> var_686_begin_0 = const()[name = string("op_686_begin_0"), val = tensor<int32, [3]>([0, 0, 2])];
            tensor<int32, [3]> var_686_end_0 = const()[name = string("op_686_end_0"), val = tensor<int32, [3]>([1, 16, 3])];
            tensor<bool, [3]> var_686_end_mask_0 = const()[name = string("op_686_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_686_squeeze_mask_0 = const()[name = string("op_686_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_686_cast_fp16 = slice_by_index(begin = var_686_begin_0, end = var_686_end_0, end_mask = var_686_end_mask_0, squeeze_mask = var_686_squeeze_mask_0, x = decay_1_cast_fp16)[name = string("op_686_cast_fp16")];
            tensor<fp16, [1, 16]> var_687_cast_fp16 = exp(x = var_686_cast_fp16)[name = string("op_687_cast_fp16")];
            tensor<int32, [1]> var_688_axes_0 = const()[name = string("op_688_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_688_cast_fp16 = expand_dims(axes = var_688_axes_0, x = var_687_cast_fp16)[name = string("op_688_cast_fp16")];
            tensor<int32, [1]> var_689_axes_0 = const()[name = string("op_689_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_689_cast_fp16 = expand_dims(axes = var_689_axes_0, x = var_688_cast_fp16)[name = string("op_689_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_9_cast_fp16 = mul(x = state_7_cast_fp16, y = var_689_cast_fp16)[name = string("state_9_cast_fp16")];
            tensor<int32, [4]> key_token_5_begin_0 = const()[name = string("key_token_5_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> key_token_5_end_0 = const()[name = string("key_token_5_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> key_token_5_end_mask_0 = const()[name = string("key_token_5_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_5_squeeze_mask_0 = const()[name = string("key_token_5_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_5_cast_fp16 = slice_by_index(begin = key_token_5_begin_0, end = key_token_5_end_0, end_mask = key_token_5_end_mask_0, squeeze_mask = key_token_5_squeeze_mask_0, x = key_3_cast_fp16)[name = string("key_token_5_cast_fp16")];
            tensor<int32, [4]> value_token_5_begin_0 = const()[name = string("value_token_5_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> value_token_5_end_0 = const()[name = string("value_token_5_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> value_token_5_end_mask_0 = const()[name = string("value_token_5_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_5_squeeze_mask_0 = const()[name = string("value_token_5_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_5_cast_fp16 = slice_by_index(begin = value_token_5_begin_0, end = value_token_5_end_0, end_mask = value_token_5_end_mask_0, squeeze_mask = value_token_5_squeeze_mask_0, x = value_13_cast_fp16)[name = string("value_token_5_cast_fp16")];
            tensor<int32, [1]> var_697_axes_0 = const()[name = string("op_697_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_697_cast_fp16 = expand_dims(axes = var_697_axes_0, x = key_token_5_cast_fp16)[name = string("op_697_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_698_cast_fp16 = mul(x = state_9_cast_fp16, y = var_697_cast_fp16)[name = string("op_698_cast_fp16")];
            tensor<int32, [1]> memory_5_axes_0 = const()[name = string("memory_5_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_5_keep_dims_0 = const()[name = string("memory_5_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_5_cast_fp16 = reduce_sum(axes = memory_5_axes_0, keep_dims = memory_5_keep_dims_0, x = var_698_cast_fp16)[name = string("memory_5_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_701_cast_fp16 = sub(x = value_token_5_cast_fp16, y = memory_5_cast_fp16)[name = string("op_701_cast_fp16")];
            tensor<int32, [3]> var_704_begin_0 = const()[name = string("op_704_begin_0"), val = tensor<int32, [3]>([0, 0, 2])];
            tensor<int32, [3]> var_704_end_0 = const()[name = string("op_704_end_0"), val = tensor<int32, [3]>([1, 16, 3])];
            tensor<bool, [3]> var_704_end_mask_0 = const()[name = string("op_704_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_704_squeeze_mask_0 = const()[name = string("op_704_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_704_cast_fp16 = slice_by_index(begin = var_704_begin_0, end = var_704_end_0, end_mask = var_704_end_mask_0, squeeze_mask = var_704_squeeze_mask_0, x = beta_1_cast_fp16)[name = string("op_704_cast_fp16")];
            tensor<int32, [1]> var_705_axes_0 = const()[name = string("op_705_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_705_cast_fp16 = expand_dims(axes = var_705_axes_0, x = var_704_cast_fp16)[name = string("op_705_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_5_cast_fp16 = mul(x = var_701_cast_fp16, y = var_705_cast_fp16)[name = string("delta_5_cast_fp16")];
            tensor<int32, [1]> var_708_axes_0 = const()[name = string("op_708_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_708_cast_fp16 = expand_dims(axes = var_708_axes_0, x = delta_5_cast_fp16)[name = string("op_708_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_709_cast_fp16 = mul(x = var_697_cast_fp16, y = var_708_cast_fp16)[name = string("op_709_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_11_cast_fp16 = add(x = state_9_cast_fp16, y = var_709_cast_fp16)[name = string("state_11_cast_fp16")];
            tensor<int32, [4]> var_713_begin_0 = const()[name = string("op_713_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> var_713_end_0 = const()[name = string("op_713_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> var_713_end_mask_0 = const()[name = string("op_713_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_713_squeeze_mask_0 = const()[name = string("op_713_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_713_cast_fp16 = slice_by_index(begin = var_713_begin_0, end = var_713_end_0, end_mask = var_713_end_mask_0, squeeze_mask = var_713_squeeze_mask_0, x = _inversed_query_3_cast_fp16)[name = string("op_713_cast_fp16")];
            tensor<int32, [1]> var_714_axes_0 = const()[name = string("op_714_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_714_cast_fp16 = expand_dims(axes = var_714_axes_0, x = var_713_cast_fp16)[name = string("op_714_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_715_cast_fp16 = mul(x = state_11_cast_fp16, y = var_714_cast_fp16)[name = string("op_715_cast_fp16")];
            tensor<int32, [1]> var_717_axes_0 = const()[name = string("op_717_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_717_keep_dims_0 = const()[name = string("op_717_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_717_cast_fp16 = reduce_sum(axes = var_717_axes_0, keep_dims = var_717_keep_dims_0, x = var_715_cast_fp16)[name = string("op_717_cast_fp16")];
            tensor<int32, [3]> var_720_begin_0 = const()[name = string("op_720_begin_0"), val = tensor<int32, [3]>([0, 0, 3])];
            tensor<int32, [3]> var_720_end_0 = const()[name = string("op_720_end_0"), val = tensor<int32, [3]>([1, 16, 4])];
            tensor<bool, [3]> var_720_end_mask_0 = const()[name = string("op_720_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_720_squeeze_mask_0 = const()[name = string("op_720_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_720_cast_fp16 = slice_by_index(begin = var_720_begin_0, end = var_720_end_0, end_mask = var_720_end_mask_0, squeeze_mask = var_720_squeeze_mask_0, x = decay_1_cast_fp16)[name = string("op_720_cast_fp16")];
            tensor<fp16, [1, 16]> var_721_cast_fp16 = exp(x = var_720_cast_fp16)[name = string("op_721_cast_fp16")];
            tensor<int32, [1]> var_722_axes_0 = const()[name = string("op_722_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_722_cast_fp16 = expand_dims(axes = var_722_axes_0, x = var_721_cast_fp16)[name = string("op_722_cast_fp16")];
            tensor<int32, [1]> var_723_axes_0 = const()[name = string("op_723_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_723_cast_fp16 = expand_dims(axes = var_723_axes_0, x = var_722_cast_fp16)[name = string("op_723_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_13_cast_fp16 = mul(x = state_11_cast_fp16, y = var_723_cast_fp16)[name = string("state_13_cast_fp16")];
            tensor<int32, [4]> key_token_7_begin_0 = const()[name = string("key_token_7_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> key_token_7_end_0 = const()[name = string("key_token_7_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> key_token_7_end_mask_0 = const()[name = string("key_token_7_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_7_squeeze_mask_0 = const()[name = string("key_token_7_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_7_cast_fp16 = slice_by_index(begin = key_token_7_begin_0, end = key_token_7_end_0, end_mask = key_token_7_end_mask_0, squeeze_mask = key_token_7_squeeze_mask_0, x = key_3_cast_fp16)[name = string("key_token_7_cast_fp16")];
            tensor<int32, [4]> value_token_7_begin_0 = const()[name = string("value_token_7_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> value_token_7_end_0 = const()[name = string("value_token_7_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> value_token_7_end_mask_0 = const()[name = string("value_token_7_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_7_squeeze_mask_0 = const()[name = string("value_token_7_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_7_cast_fp16 = slice_by_index(begin = value_token_7_begin_0, end = value_token_7_end_0, end_mask = value_token_7_end_mask_0, squeeze_mask = value_token_7_squeeze_mask_0, x = value_13_cast_fp16)[name = string("value_token_7_cast_fp16")];
            tensor<int32, [1]> var_731_axes_0 = const()[name = string("op_731_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_731_cast_fp16 = expand_dims(axes = var_731_axes_0, x = key_token_7_cast_fp16)[name = string("op_731_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_732_cast_fp16 = mul(x = state_13_cast_fp16, y = var_731_cast_fp16)[name = string("op_732_cast_fp16")];
            tensor<int32, [1]> memory_7_axes_0 = const()[name = string("memory_7_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_7_keep_dims_0 = const()[name = string("memory_7_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_7_cast_fp16 = reduce_sum(axes = memory_7_axes_0, keep_dims = memory_7_keep_dims_0, x = var_732_cast_fp16)[name = string("memory_7_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_735_cast_fp16 = sub(x = value_token_7_cast_fp16, y = memory_7_cast_fp16)[name = string("op_735_cast_fp16")];
            tensor<int32, [3]> var_738_begin_0 = const()[name = string("op_738_begin_0"), val = tensor<int32, [3]>([0, 0, 3])];
            tensor<int32, [3]> var_738_end_0 = const()[name = string("op_738_end_0"), val = tensor<int32, [3]>([1, 16, 4])];
            tensor<bool, [3]> var_738_end_mask_0 = const()[name = string("op_738_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_738_squeeze_mask_0 = const()[name = string("op_738_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_738_cast_fp16 = slice_by_index(begin = var_738_begin_0, end = var_738_end_0, end_mask = var_738_end_mask_0, squeeze_mask = var_738_squeeze_mask_0, x = beta_1_cast_fp16)[name = string("op_738_cast_fp16")];
            tensor<int32, [1]> var_739_axes_0 = const()[name = string("op_739_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_739_cast_fp16 = expand_dims(axes = var_739_axes_0, x = var_738_cast_fp16)[name = string("op_739_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_7_cast_fp16 = mul(x = var_735_cast_fp16, y = var_739_cast_fp16)[name = string("delta_7_cast_fp16")];
            tensor<int32, [1]> var_742_axes_0 = const()[name = string("op_742_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_742_cast_fp16 = expand_dims(axes = var_742_axes_0, x = delta_7_cast_fp16)[name = string("op_742_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_743_cast_fp16 = mul(x = var_731_cast_fp16, y = var_742_cast_fp16)[name = string("op_743_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_15_cast_fp16 = add(x = state_13_cast_fp16, y = var_743_cast_fp16)[name = string("state_15_cast_fp16")];
            tensor<int32, [4]> var_747_begin_0 = const()[name = string("op_747_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> var_747_end_0 = const()[name = string("op_747_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> var_747_end_mask_0 = const()[name = string("op_747_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_747_squeeze_mask_0 = const()[name = string("op_747_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_747_cast_fp16 = slice_by_index(begin = var_747_begin_0, end = var_747_end_0, end_mask = var_747_end_mask_0, squeeze_mask = var_747_squeeze_mask_0, x = _inversed_query_3_cast_fp16)[name = string("op_747_cast_fp16")];
            tensor<int32, [1]> var_748_axes_0 = const()[name = string("op_748_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_748_cast_fp16 = expand_dims(axes = var_748_axes_0, x = var_747_cast_fp16)[name = string("op_748_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_749_cast_fp16 = mul(x = state_15_cast_fp16, y = var_748_cast_fp16)[name = string("op_749_cast_fp16")];
            tensor<int32, [1]> var_751_axes_0 = const()[name = string("op_751_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_751_keep_dims_0 = const()[name = string("op_751_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_751_cast_fp16 = reduce_sum(axes = var_751_axes_0, keep_dims = var_751_keep_dims_0, x = var_749_cast_fp16)[name = string("op_751_cast_fp16")];
            tensor<int32, [3]> var_754_begin_0 = const()[name = string("op_754_begin_0"), val = tensor<int32, [3]>([0, 0, 4])];
            tensor<int32, [3]> var_754_end_0 = const()[name = string("op_754_end_0"), val = tensor<int32, [3]>([1, 16, 5])];
            tensor<bool, [3]> var_754_end_mask_0 = const()[name = string("op_754_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_754_squeeze_mask_0 = const()[name = string("op_754_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_754_cast_fp16 = slice_by_index(begin = var_754_begin_0, end = var_754_end_0, end_mask = var_754_end_mask_0, squeeze_mask = var_754_squeeze_mask_0, x = decay_1_cast_fp16)[name = string("op_754_cast_fp16")];
            tensor<fp16, [1, 16]> var_755_cast_fp16 = exp(x = var_754_cast_fp16)[name = string("op_755_cast_fp16")];
            tensor<int32, [1]> var_756_axes_0 = const()[name = string("op_756_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_756_cast_fp16 = expand_dims(axes = var_756_axes_0, x = var_755_cast_fp16)[name = string("op_756_cast_fp16")];
            tensor<int32, [1]> var_757_axes_0 = const()[name = string("op_757_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_757_cast_fp16 = expand_dims(axes = var_757_axes_0, x = var_756_cast_fp16)[name = string("op_757_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_17_cast_fp16 = mul(x = state_15_cast_fp16, y = var_757_cast_fp16)[name = string("state_17_cast_fp16")];
            tensor<int32, [4]> key_token_9_begin_0 = const()[name = string("key_token_9_begin_0"), val = tensor<int32, [4]>([0, 0, 4, 0])];
            tensor<int32, [4]> key_token_9_end_0 = const()[name = string("key_token_9_end_0"), val = tensor<int32, [4]>([1, 16, 5, 128])];
            tensor<bool, [4]> key_token_9_end_mask_0 = const()[name = string("key_token_9_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_9_squeeze_mask_0 = const()[name = string("key_token_9_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_9_cast_fp16 = slice_by_index(begin = key_token_9_begin_0, end = key_token_9_end_0, end_mask = key_token_9_end_mask_0, squeeze_mask = key_token_9_squeeze_mask_0, x = key_3_cast_fp16)[name = string("key_token_9_cast_fp16")];
            tensor<int32, [4]> value_token_9_begin_0 = const()[name = string("value_token_9_begin_0"), val = tensor<int32, [4]>([0, 0, 4, 0])];
            tensor<int32, [4]> value_token_9_end_0 = const()[name = string("value_token_9_end_0"), val = tensor<int32, [4]>([1, 16, 5, 128])];
            tensor<bool, [4]> value_token_9_end_mask_0 = const()[name = string("value_token_9_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_9_squeeze_mask_0 = const()[name = string("value_token_9_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_9_cast_fp16 = slice_by_index(begin = value_token_9_begin_0, end = value_token_9_end_0, end_mask = value_token_9_end_mask_0, squeeze_mask = value_token_9_squeeze_mask_0, x = value_13_cast_fp16)[name = string("value_token_9_cast_fp16")];
            tensor<int32, [1]> var_765_axes_0 = const()[name = string("op_765_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_765_cast_fp16 = expand_dims(axes = var_765_axes_0, x = key_token_9_cast_fp16)[name = string("op_765_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_766_cast_fp16 = mul(x = state_17_cast_fp16, y = var_765_cast_fp16)[name = string("op_766_cast_fp16")];
            tensor<int32, [1]> memory_9_axes_0 = const()[name = string("memory_9_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_9_keep_dims_0 = const()[name = string("memory_9_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_9_cast_fp16 = reduce_sum(axes = memory_9_axes_0, keep_dims = memory_9_keep_dims_0, x = var_766_cast_fp16)[name = string("memory_9_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_769_cast_fp16 = sub(x = value_token_9_cast_fp16, y = memory_9_cast_fp16)[name = string("op_769_cast_fp16")];
            tensor<int32, [3]> var_772_begin_0 = const()[name = string("op_772_begin_0"), val = tensor<int32, [3]>([0, 0, 4])];
            tensor<int32, [3]> var_772_end_0 = const()[name = string("op_772_end_0"), val = tensor<int32, [3]>([1, 16, 5])];
            tensor<bool, [3]> var_772_end_mask_0 = const()[name = string("op_772_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_772_squeeze_mask_0 = const()[name = string("op_772_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_772_cast_fp16 = slice_by_index(begin = var_772_begin_0, end = var_772_end_0, end_mask = var_772_end_mask_0, squeeze_mask = var_772_squeeze_mask_0, x = beta_1_cast_fp16)[name = string("op_772_cast_fp16")];
            tensor<int32, [1]> var_773_axes_0 = const()[name = string("op_773_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_773_cast_fp16 = expand_dims(axes = var_773_axes_0, x = var_772_cast_fp16)[name = string("op_773_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_9_cast_fp16 = mul(x = var_769_cast_fp16, y = var_773_cast_fp16)[name = string("delta_9_cast_fp16")];
            tensor<int32, [1]> var_776_axes_0 = const()[name = string("op_776_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_776_cast_fp16 = expand_dims(axes = var_776_axes_0, x = delta_9_cast_fp16)[name = string("op_776_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_777_cast_fp16 = mul(x = var_765_cast_fp16, y = var_776_cast_fp16)[name = string("op_777_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_19_cast_fp16 = add(x = state_17_cast_fp16, y = var_777_cast_fp16)[name = string("state_19_cast_fp16")];
            tensor<int32, [4]> var_781_begin_0 = const()[name = string("op_781_begin_0"), val = tensor<int32, [4]>([0, 0, 4, 0])];
            tensor<int32, [4]> var_781_end_0 = const()[name = string("op_781_end_0"), val = tensor<int32, [4]>([1, 16, 5, 128])];
            tensor<bool, [4]> var_781_end_mask_0 = const()[name = string("op_781_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_781_squeeze_mask_0 = const()[name = string("op_781_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_781_cast_fp16 = slice_by_index(begin = var_781_begin_0, end = var_781_end_0, end_mask = var_781_end_mask_0, squeeze_mask = var_781_squeeze_mask_0, x = _inversed_query_3_cast_fp16)[name = string("op_781_cast_fp16")];
            tensor<int32, [1]> var_782_axes_0 = const()[name = string("op_782_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_782_cast_fp16 = expand_dims(axes = var_782_axes_0, x = var_781_cast_fp16)[name = string("op_782_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_783_cast_fp16 = mul(x = state_19_cast_fp16, y = var_782_cast_fp16)[name = string("op_783_cast_fp16")];
            tensor<int32, [1]> var_785_axes_0 = const()[name = string("op_785_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_785_keep_dims_0 = const()[name = string("op_785_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_785_cast_fp16 = reduce_sum(axes = var_785_axes_0, keep_dims = var_785_keep_dims_0, x = var_783_cast_fp16)[name = string("op_785_cast_fp16")];
            tensor<int32, [3]> var_788_begin_0 = const()[name = string("op_788_begin_0"), val = tensor<int32, [3]>([0, 0, 5])];
            tensor<int32, [3]> var_788_end_0 = const()[name = string("op_788_end_0"), val = tensor<int32, [3]>([1, 16, 6])];
            tensor<bool, [3]> var_788_end_mask_0 = const()[name = string("op_788_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_788_squeeze_mask_0 = const()[name = string("op_788_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_788_cast_fp16 = slice_by_index(begin = var_788_begin_0, end = var_788_end_0, end_mask = var_788_end_mask_0, squeeze_mask = var_788_squeeze_mask_0, x = decay_1_cast_fp16)[name = string("op_788_cast_fp16")];
            tensor<fp16, [1, 16]> var_789_cast_fp16 = exp(x = var_788_cast_fp16)[name = string("op_789_cast_fp16")];
            tensor<int32, [1]> var_790_axes_0 = const()[name = string("op_790_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_790_cast_fp16 = expand_dims(axes = var_790_axes_0, x = var_789_cast_fp16)[name = string("op_790_cast_fp16")];
            tensor<int32, [1]> var_791_axes_0 = const()[name = string("op_791_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_791_cast_fp16 = expand_dims(axes = var_791_axes_0, x = var_790_cast_fp16)[name = string("op_791_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_21_cast_fp16 = mul(x = state_19_cast_fp16, y = var_791_cast_fp16)[name = string("state_21_cast_fp16")];
            tensor<int32, [4]> key_token_11_begin_0 = const()[name = string("key_token_11_begin_0"), val = tensor<int32, [4]>([0, 0, 5, 0])];
            tensor<int32, [4]> key_token_11_end_0 = const()[name = string("key_token_11_end_0"), val = tensor<int32, [4]>([1, 16, 6, 128])];
            tensor<bool, [4]> key_token_11_end_mask_0 = const()[name = string("key_token_11_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_11_squeeze_mask_0 = const()[name = string("key_token_11_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_11_cast_fp16 = slice_by_index(begin = key_token_11_begin_0, end = key_token_11_end_0, end_mask = key_token_11_end_mask_0, squeeze_mask = key_token_11_squeeze_mask_0, x = key_3_cast_fp16)[name = string("key_token_11_cast_fp16")];
            tensor<int32, [4]> value_token_11_begin_0 = const()[name = string("value_token_11_begin_0"), val = tensor<int32, [4]>([0, 0, 5, 0])];
            tensor<int32, [4]> value_token_11_end_0 = const()[name = string("value_token_11_end_0"), val = tensor<int32, [4]>([1, 16, 6, 128])];
            tensor<bool, [4]> value_token_11_end_mask_0 = const()[name = string("value_token_11_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_11_squeeze_mask_0 = const()[name = string("value_token_11_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_11_cast_fp16 = slice_by_index(begin = value_token_11_begin_0, end = value_token_11_end_0, end_mask = value_token_11_end_mask_0, squeeze_mask = value_token_11_squeeze_mask_0, x = value_13_cast_fp16)[name = string("value_token_11_cast_fp16")];
            tensor<int32, [1]> var_799_axes_0 = const()[name = string("op_799_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_799_cast_fp16 = expand_dims(axes = var_799_axes_0, x = key_token_11_cast_fp16)[name = string("op_799_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_800_cast_fp16 = mul(x = state_21_cast_fp16, y = var_799_cast_fp16)[name = string("op_800_cast_fp16")];
            tensor<int32, [1]> memory_11_axes_0 = const()[name = string("memory_11_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_11_keep_dims_0 = const()[name = string("memory_11_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_11_cast_fp16 = reduce_sum(axes = memory_11_axes_0, keep_dims = memory_11_keep_dims_0, x = var_800_cast_fp16)[name = string("memory_11_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_803_cast_fp16 = sub(x = value_token_11_cast_fp16, y = memory_11_cast_fp16)[name = string("op_803_cast_fp16")];
            tensor<int32, [3]> var_806_begin_0 = const()[name = string("op_806_begin_0"), val = tensor<int32, [3]>([0, 0, 5])];
            tensor<int32, [3]> var_806_end_0 = const()[name = string("op_806_end_0"), val = tensor<int32, [3]>([1, 16, 6])];
            tensor<bool, [3]> var_806_end_mask_0 = const()[name = string("op_806_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_806_squeeze_mask_0 = const()[name = string("op_806_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_806_cast_fp16 = slice_by_index(begin = var_806_begin_0, end = var_806_end_0, end_mask = var_806_end_mask_0, squeeze_mask = var_806_squeeze_mask_0, x = beta_1_cast_fp16)[name = string("op_806_cast_fp16")];
            tensor<int32, [1]> var_807_axes_0 = const()[name = string("op_807_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_807_cast_fp16 = expand_dims(axes = var_807_axes_0, x = var_806_cast_fp16)[name = string("op_807_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_11_cast_fp16 = mul(x = var_803_cast_fp16, y = var_807_cast_fp16)[name = string("delta_11_cast_fp16")];
            tensor<int32, [1]> var_810_axes_0 = const()[name = string("op_810_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_810_cast_fp16 = expand_dims(axes = var_810_axes_0, x = delta_11_cast_fp16)[name = string("op_810_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_811_cast_fp16 = mul(x = var_799_cast_fp16, y = var_810_cast_fp16)[name = string("op_811_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_23_cast_fp16 = add(x = state_21_cast_fp16, y = var_811_cast_fp16)[name = string("state_23_cast_fp16")];
            tensor<int32, [4]> var_815_begin_0 = const()[name = string("op_815_begin_0"), val = tensor<int32, [4]>([0, 0, 5, 0])];
            tensor<int32, [4]> var_815_end_0 = const()[name = string("op_815_end_0"), val = tensor<int32, [4]>([1, 16, 6, 128])];
            tensor<bool, [4]> var_815_end_mask_0 = const()[name = string("op_815_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_815_squeeze_mask_0 = const()[name = string("op_815_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_815_cast_fp16 = slice_by_index(begin = var_815_begin_0, end = var_815_end_0, end_mask = var_815_end_mask_0, squeeze_mask = var_815_squeeze_mask_0, x = _inversed_query_3_cast_fp16)[name = string("op_815_cast_fp16")];
            tensor<int32, [1]> var_816_axes_0 = const()[name = string("op_816_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_816_cast_fp16 = expand_dims(axes = var_816_axes_0, x = var_815_cast_fp16)[name = string("op_816_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_817_cast_fp16 = mul(x = state_23_cast_fp16, y = var_816_cast_fp16)[name = string("op_817_cast_fp16")];
            tensor<int32, [1]> var_819_axes_0 = const()[name = string("op_819_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_819_keep_dims_0 = const()[name = string("op_819_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_819_cast_fp16 = reduce_sum(axes = var_819_axes_0, keep_dims = var_819_keep_dims_0, x = var_817_cast_fp16)[name = string("op_819_cast_fp16")];
            tensor<int32, [3]> var_822_begin_0 = const()[name = string("op_822_begin_0"), val = tensor<int32, [3]>([0, 0, 6])];
            tensor<int32, [3]> var_822_end_0 = const()[name = string("op_822_end_0"), val = tensor<int32, [3]>([1, 16, 7])];
            tensor<bool, [3]> var_822_end_mask_0 = const()[name = string("op_822_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_822_squeeze_mask_0 = const()[name = string("op_822_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_822_cast_fp16 = slice_by_index(begin = var_822_begin_0, end = var_822_end_0, end_mask = var_822_end_mask_0, squeeze_mask = var_822_squeeze_mask_0, x = decay_1_cast_fp16)[name = string("op_822_cast_fp16")];
            tensor<fp16, [1, 16]> var_823_cast_fp16 = exp(x = var_822_cast_fp16)[name = string("op_823_cast_fp16")];
            tensor<int32, [1]> var_824_axes_0 = const()[name = string("op_824_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_824_cast_fp16 = expand_dims(axes = var_824_axes_0, x = var_823_cast_fp16)[name = string("op_824_cast_fp16")];
            tensor<int32, [1]> var_825_axes_0 = const()[name = string("op_825_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_825_cast_fp16 = expand_dims(axes = var_825_axes_0, x = var_824_cast_fp16)[name = string("op_825_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_25_cast_fp16 = mul(x = state_23_cast_fp16, y = var_825_cast_fp16)[name = string("state_25_cast_fp16")];
            tensor<int32, [4]> key_token_13_begin_0 = const()[name = string("key_token_13_begin_0"), val = tensor<int32, [4]>([0, 0, 6, 0])];
            tensor<int32, [4]> key_token_13_end_0 = const()[name = string("key_token_13_end_0"), val = tensor<int32, [4]>([1, 16, 7, 128])];
            tensor<bool, [4]> key_token_13_end_mask_0 = const()[name = string("key_token_13_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_13_squeeze_mask_0 = const()[name = string("key_token_13_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_13_cast_fp16 = slice_by_index(begin = key_token_13_begin_0, end = key_token_13_end_0, end_mask = key_token_13_end_mask_0, squeeze_mask = key_token_13_squeeze_mask_0, x = key_3_cast_fp16)[name = string("key_token_13_cast_fp16")];
            tensor<int32, [4]> value_token_13_begin_0 = const()[name = string("value_token_13_begin_0"), val = tensor<int32, [4]>([0, 0, 6, 0])];
            tensor<int32, [4]> value_token_13_end_0 = const()[name = string("value_token_13_end_0"), val = tensor<int32, [4]>([1, 16, 7, 128])];
            tensor<bool, [4]> value_token_13_end_mask_0 = const()[name = string("value_token_13_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_13_squeeze_mask_0 = const()[name = string("value_token_13_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_13_cast_fp16 = slice_by_index(begin = value_token_13_begin_0, end = value_token_13_end_0, end_mask = value_token_13_end_mask_0, squeeze_mask = value_token_13_squeeze_mask_0, x = value_13_cast_fp16)[name = string("value_token_13_cast_fp16")];
            tensor<int32, [1]> var_833_axes_0 = const()[name = string("op_833_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_833_cast_fp16 = expand_dims(axes = var_833_axes_0, x = key_token_13_cast_fp16)[name = string("op_833_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_834_cast_fp16 = mul(x = state_25_cast_fp16, y = var_833_cast_fp16)[name = string("op_834_cast_fp16")];
            tensor<int32, [1]> memory_13_axes_0 = const()[name = string("memory_13_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_13_keep_dims_0 = const()[name = string("memory_13_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_13_cast_fp16 = reduce_sum(axes = memory_13_axes_0, keep_dims = memory_13_keep_dims_0, x = var_834_cast_fp16)[name = string("memory_13_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_837_cast_fp16 = sub(x = value_token_13_cast_fp16, y = memory_13_cast_fp16)[name = string("op_837_cast_fp16")];
            tensor<int32, [3]> var_840_begin_0 = const()[name = string("op_840_begin_0"), val = tensor<int32, [3]>([0, 0, 6])];
            tensor<int32, [3]> var_840_end_0 = const()[name = string("op_840_end_0"), val = tensor<int32, [3]>([1, 16, 7])];
            tensor<bool, [3]> var_840_end_mask_0 = const()[name = string("op_840_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_840_squeeze_mask_0 = const()[name = string("op_840_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_840_cast_fp16 = slice_by_index(begin = var_840_begin_0, end = var_840_end_0, end_mask = var_840_end_mask_0, squeeze_mask = var_840_squeeze_mask_0, x = beta_1_cast_fp16)[name = string("op_840_cast_fp16")];
            tensor<int32, [1]> var_841_axes_0 = const()[name = string("op_841_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_841_cast_fp16 = expand_dims(axes = var_841_axes_0, x = var_840_cast_fp16)[name = string("op_841_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_13_cast_fp16 = mul(x = var_837_cast_fp16, y = var_841_cast_fp16)[name = string("delta_13_cast_fp16")];
            tensor<int32, [1]> var_844_axes_0 = const()[name = string("op_844_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_844_cast_fp16 = expand_dims(axes = var_844_axes_0, x = delta_13_cast_fp16)[name = string("op_844_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_845_cast_fp16 = mul(x = var_833_cast_fp16, y = var_844_cast_fp16)[name = string("op_845_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_27_cast_fp16 = add(x = state_25_cast_fp16, y = var_845_cast_fp16)[name = string("state_27_cast_fp16")];
            tensor<int32, [4]> var_849_begin_0 = const()[name = string("op_849_begin_0"), val = tensor<int32, [4]>([0, 0, 6, 0])];
            tensor<int32, [4]> var_849_end_0 = const()[name = string("op_849_end_0"), val = tensor<int32, [4]>([1, 16, 7, 128])];
            tensor<bool, [4]> var_849_end_mask_0 = const()[name = string("op_849_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_849_squeeze_mask_0 = const()[name = string("op_849_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_849_cast_fp16 = slice_by_index(begin = var_849_begin_0, end = var_849_end_0, end_mask = var_849_end_mask_0, squeeze_mask = var_849_squeeze_mask_0, x = _inversed_query_3_cast_fp16)[name = string("op_849_cast_fp16")];
            tensor<int32, [1]> var_850_axes_0 = const()[name = string("op_850_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_850_cast_fp16 = expand_dims(axes = var_850_axes_0, x = var_849_cast_fp16)[name = string("op_850_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_851_cast_fp16 = mul(x = state_27_cast_fp16, y = var_850_cast_fp16)[name = string("op_851_cast_fp16")];
            tensor<int32, [1]> var_853_axes_0 = const()[name = string("op_853_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_853_keep_dims_0 = const()[name = string("op_853_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_853_cast_fp16 = reduce_sum(axes = var_853_axes_0, keep_dims = var_853_keep_dims_0, x = var_851_cast_fp16)[name = string("op_853_cast_fp16")];
            tensor<int32, [3]> var_856_begin_0 = const()[name = string("op_856_begin_0"), val = tensor<int32, [3]>([0, 0, 7])];
            tensor<int32, [3]> var_856_end_0 = const()[name = string("op_856_end_0"), val = tensor<int32, [3]>([1, 16, 8])];
            tensor<bool, [3]> var_856_end_mask_0 = const()[name = string("op_856_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_856_squeeze_mask_0 = const()[name = string("op_856_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_856_cast_fp16 = slice_by_index(begin = var_856_begin_0, end = var_856_end_0, end_mask = var_856_end_mask_0, squeeze_mask = var_856_squeeze_mask_0, x = decay_1_cast_fp16)[name = string("op_856_cast_fp16")];
            tensor<fp16, [1, 16]> var_857_cast_fp16 = exp(x = var_856_cast_fp16)[name = string("op_857_cast_fp16")];
            tensor<int32, [1]> var_858_axes_0 = const()[name = string("op_858_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_858_cast_fp16 = expand_dims(axes = var_858_axes_0, x = var_857_cast_fp16)[name = string("op_858_cast_fp16")];
            tensor<int32, [1]> var_859_axes_0 = const()[name = string("op_859_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_859_cast_fp16 = expand_dims(axes = var_859_axes_0, x = var_858_cast_fp16)[name = string("op_859_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_29_cast_fp16 = mul(x = state_27_cast_fp16, y = var_859_cast_fp16)[name = string("state_29_cast_fp16")];
            tensor<int32, [4]> key_token_15_begin_0 = const()[name = string("key_token_15_begin_0"), val = tensor<int32, [4]>([0, 0, 7, 0])];
            tensor<int32, [4]> key_token_15_end_0 = const()[name = string("key_token_15_end_0"), val = tensor<int32, [4]>([1, 16, 8, 128])];
            tensor<bool, [4]> key_token_15_end_mask_0 = const()[name = string("key_token_15_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_15_squeeze_mask_0 = const()[name = string("key_token_15_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_15_cast_fp16 = slice_by_index(begin = key_token_15_begin_0, end = key_token_15_end_0, end_mask = key_token_15_end_mask_0, squeeze_mask = key_token_15_squeeze_mask_0, x = key_3_cast_fp16)[name = string("key_token_15_cast_fp16")];
            tensor<int32, [4]> value_token_15_begin_0 = const()[name = string("value_token_15_begin_0"), val = tensor<int32, [4]>([0, 0, 7, 0])];
            tensor<int32, [4]> value_token_15_end_0 = const()[name = string("value_token_15_end_0"), val = tensor<int32, [4]>([1, 16, 8, 128])];
            tensor<bool, [4]> value_token_15_end_mask_0 = const()[name = string("value_token_15_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_15_squeeze_mask_0 = const()[name = string("value_token_15_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_15_cast_fp16 = slice_by_index(begin = value_token_15_begin_0, end = value_token_15_end_0, end_mask = value_token_15_end_mask_0, squeeze_mask = value_token_15_squeeze_mask_0, x = value_13_cast_fp16)[name = string("value_token_15_cast_fp16")];
            tensor<int32, [1]> var_867_axes_0 = const()[name = string("op_867_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_867_cast_fp16 = expand_dims(axes = var_867_axes_0, x = key_token_15_cast_fp16)[name = string("op_867_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_868_cast_fp16 = mul(x = state_29_cast_fp16, y = var_867_cast_fp16)[name = string("op_868_cast_fp16")];
            tensor<int32, [1]> memory_15_axes_0 = const()[name = string("memory_15_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_15_keep_dims_0 = const()[name = string("memory_15_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_15_cast_fp16 = reduce_sum(axes = memory_15_axes_0, keep_dims = memory_15_keep_dims_0, x = var_868_cast_fp16)[name = string("memory_15_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_871_cast_fp16 = sub(x = value_token_15_cast_fp16, y = memory_15_cast_fp16)[name = string("op_871_cast_fp16")];
            tensor<int32, [3]> var_874_begin_0 = const()[name = string("op_874_begin_0"), val = tensor<int32, [3]>([0, 0, 7])];
            tensor<int32, [3]> var_874_end_0 = const()[name = string("op_874_end_0"), val = tensor<int32, [3]>([1, 16, 8])];
            tensor<bool, [3]> var_874_end_mask_0 = const()[name = string("op_874_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_874_squeeze_mask_0 = const()[name = string("op_874_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_874_cast_fp16 = slice_by_index(begin = var_874_begin_0, end = var_874_end_0, end_mask = var_874_end_mask_0, squeeze_mask = var_874_squeeze_mask_0, x = beta_1_cast_fp16)[name = string("op_874_cast_fp16")];
            tensor<int32, [1]> var_875_axes_0 = const()[name = string("op_875_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_875_cast_fp16 = expand_dims(axes = var_875_axes_0, x = var_874_cast_fp16)[name = string("op_875_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_15_cast_fp16 = mul(x = var_871_cast_fp16, y = var_875_cast_fp16)[name = string("delta_15_cast_fp16")];
            tensor<int32, [1]> var_878_axes_0 = const()[name = string("op_878_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_878_cast_fp16 = expand_dims(axes = var_878_axes_0, x = delta_15_cast_fp16)[name = string("op_878_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_879_cast_fp16 = mul(x = var_867_cast_fp16, y = var_878_cast_fp16)[name = string("op_879_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_31_cast_fp16 = add(x = state_29_cast_fp16, y = var_879_cast_fp16)[name = string("state_31_cast_fp16")];
            tensor<int32, [4]> var_883_begin_0 = const()[name = string("op_883_begin_0"), val = tensor<int32, [4]>([0, 0, 7, 0])];
            tensor<int32, [4]> var_883_end_0 = const()[name = string("op_883_end_0"), val = tensor<int32, [4]>([1, 16, 8, 128])];
            tensor<bool, [4]> var_883_end_mask_0 = const()[name = string("op_883_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_883_squeeze_mask_0 = const()[name = string("op_883_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_883_cast_fp16 = slice_by_index(begin = var_883_begin_0, end = var_883_end_0, end_mask = var_883_end_mask_0, squeeze_mask = var_883_squeeze_mask_0, x = _inversed_query_3_cast_fp16)[name = string("op_883_cast_fp16")];
            tensor<int32, [1]> var_884_axes_0 = const()[name = string("op_884_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_884_cast_fp16 = expand_dims(axes = var_884_axes_0, x = var_883_cast_fp16)[name = string("op_884_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_885_cast_fp16 = mul(x = state_31_cast_fp16, y = var_884_cast_fp16)[name = string("op_885_cast_fp16")];
            tensor<int32, [1]> var_887_axes_0 = const()[name = string("op_887_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_887_keep_dims_0 = const()[name = string("op_887_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_887_cast_fp16 = reduce_sum(axes = var_887_axes_0, keep_dims = var_887_keep_dims_0, x = var_885_cast_fp16)[name = string("op_887_cast_fp16")];
            tensor<int32, [3]> var_890_begin_0 = const()[name = string("op_890_begin_0"), val = tensor<int32, [3]>([0, 0, 8])];
            tensor<int32, [3]> var_890_end_0 = const()[name = string("op_890_end_0"), val = tensor<int32, [3]>([1, 16, 9])];
            tensor<bool, [3]> var_890_end_mask_0 = const()[name = string("op_890_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_890_squeeze_mask_0 = const()[name = string("op_890_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_890_cast_fp16 = slice_by_index(begin = var_890_begin_0, end = var_890_end_0, end_mask = var_890_end_mask_0, squeeze_mask = var_890_squeeze_mask_0, x = decay_1_cast_fp16)[name = string("op_890_cast_fp16")];
            tensor<fp16, [1, 16]> var_891_cast_fp16 = exp(x = var_890_cast_fp16)[name = string("op_891_cast_fp16")];
            tensor<int32, [1]> var_892_axes_0 = const()[name = string("op_892_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_892_cast_fp16 = expand_dims(axes = var_892_axes_0, x = var_891_cast_fp16)[name = string("op_892_cast_fp16")];
            tensor<int32, [1]> var_893_axes_0 = const()[name = string("op_893_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_893_cast_fp16 = expand_dims(axes = var_893_axes_0, x = var_892_cast_fp16)[name = string("op_893_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_33_cast_fp16 = mul(x = state_31_cast_fp16, y = var_893_cast_fp16)[name = string("state_33_cast_fp16")];
            tensor<int32, [4]> key_token_17_begin_0 = const()[name = string("key_token_17_begin_0"), val = tensor<int32, [4]>([0, 0, 8, 0])];
            tensor<int32, [4]> key_token_17_end_0 = const()[name = string("key_token_17_end_0"), val = tensor<int32, [4]>([1, 16, 9, 128])];
            tensor<bool, [4]> key_token_17_end_mask_0 = const()[name = string("key_token_17_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_17_squeeze_mask_0 = const()[name = string("key_token_17_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_17_cast_fp16 = slice_by_index(begin = key_token_17_begin_0, end = key_token_17_end_0, end_mask = key_token_17_end_mask_0, squeeze_mask = key_token_17_squeeze_mask_0, x = key_3_cast_fp16)[name = string("key_token_17_cast_fp16")];
            tensor<int32, [4]> value_token_17_begin_0 = const()[name = string("value_token_17_begin_0"), val = tensor<int32, [4]>([0, 0, 8, 0])];
            tensor<int32, [4]> value_token_17_end_0 = const()[name = string("value_token_17_end_0"), val = tensor<int32, [4]>([1, 16, 9, 128])];
            tensor<bool, [4]> value_token_17_end_mask_0 = const()[name = string("value_token_17_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_17_squeeze_mask_0 = const()[name = string("value_token_17_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_17_cast_fp16 = slice_by_index(begin = value_token_17_begin_0, end = value_token_17_end_0, end_mask = value_token_17_end_mask_0, squeeze_mask = value_token_17_squeeze_mask_0, x = value_13_cast_fp16)[name = string("value_token_17_cast_fp16")];
            tensor<int32, [1]> var_901_axes_0 = const()[name = string("op_901_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_901_cast_fp16 = expand_dims(axes = var_901_axes_0, x = key_token_17_cast_fp16)[name = string("op_901_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_902_cast_fp16 = mul(x = state_33_cast_fp16, y = var_901_cast_fp16)[name = string("op_902_cast_fp16")];
            tensor<int32, [1]> memory_17_axes_0 = const()[name = string("memory_17_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_17_keep_dims_0 = const()[name = string("memory_17_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_17_cast_fp16 = reduce_sum(axes = memory_17_axes_0, keep_dims = memory_17_keep_dims_0, x = var_902_cast_fp16)[name = string("memory_17_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_905_cast_fp16 = sub(x = value_token_17_cast_fp16, y = memory_17_cast_fp16)[name = string("op_905_cast_fp16")];
            tensor<int32, [3]> var_908_begin_0 = const()[name = string("op_908_begin_0"), val = tensor<int32, [3]>([0, 0, 8])];
            tensor<int32, [3]> var_908_end_0 = const()[name = string("op_908_end_0"), val = tensor<int32, [3]>([1, 16, 9])];
            tensor<bool, [3]> var_908_end_mask_0 = const()[name = string("op_908_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_908_squeeze_mask_0 = const()[name = string("op_908_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_908_cast_fp16 = slice_by_index(begin = var_908_begin_0, end = var_908_end_0, end_mask = var_908_end_mask_0, squeeze_mask = var_908_squeeze_mask_0, x = beta_1_cast_fp16)[name = string("op_908_cast_fp16")];
            tensor<int32, [1]> var_909_axes_0 = const()[name = string("op_909_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_909_cast_fp16 = expand_dims(axes = var_909_axes_0, x = var_908_cast_fp16)[name = string("op_909_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_17_cast_fp16 = mul(x = var_905_cast_fp16, y = var_909_cast_fp16)[name = string("delta_17_cast_fp16")];
            tensor<int32, [1]> var_912_axes_0 = const()[name = string("op_912_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_912_cast_fp16 = expand_dims(axes = var_912_axes_0, x = delta_17_cast_fp16)[name = string("op_912_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_913_cast_fp16 = mul(x = var_901_cast_fp16, y = var_912_cast_fp16)[name = string("op_913_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_35_cast_fp16 = add(x = state_33_cast_fp16, y = var_913_cast_fp16)[name = string("state_35_cast_fp16")];
            tensor<int32, [4]> var_917_begin_0 = const()[name = string("op_917_begin_0"), val = tensor<int32, [4]>([0, 0, 8, 0])];
            tensor<int32, [4]> var_917_end_0 = const()[name = string("op_917_end_0"), val = tensor<int32, [4]>([1, 16, 9, 128])];
            tensor<bool, [4]> var_917_end_mask_0 = const()[name = string("op_917_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_917_squeeze_mask_0 = const()[name = string("op_917_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_917_cast_fp16 = slice_by_index(begin = var_917_begin_0, end = var_917_end_0, end_mask = var_917_end_mask_0, squeeze_mask = var_917_squeeze_mask_0, x = _inversed_query_3_cast_fp16)[name = string("op_917_cast_fp16")];
            tensor<int32, [1]> var_918_axes_0 = const()[name = string("op_918_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_918_cast_fp16 = expand_dims(axes = var_918_axes_0, x = var_917_cast_fp16)[name = string("op_918_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_919_cast_fp16 = mul(x = state_35_cast_fp16, y = var_918_cast_fp16)[name = string("op_919_cast_fp16")];
            tensor<int32, [1]> var_921_axes_0 = const()[name = string("op_921_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_921_keep_dims_0 = const()[name = string("op_921_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_921_cast_fp16 = reduce_sum(axes = var_921_axes_0, keep_dims = var_921_keep_dims_0, x = var_919_cast_fp16)[name = string("op_921_cast_fp16")];
            tensor<int32, [3]> var_924_begin_0 = const()[name = string("op_924_begin_0"), val = tensor<int32, [3]>([0, 0, 9])];
            tensor<int32, [3]> var_924_end_0 = const()[name = string("op_924_end_0"), val = tensor<int32, [3]>([1, 16, 10])];
            tensor<bool, [3]> var_924_end_mask_0 = const()[name = string("op_924_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_924_squeeze_mask_0 = const()[name = string("op_924_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_924_cast_fp16 = slice_by_index(begin = var_924_begin_0, end = var_924_end_0, end_mask = var_924_end_mask_0, squeeze_mask = var_924_squeeze_mask_0, x = decay_1_cast_fp16)[name = string("op_924_cast_fp16")];
            tensor<fp16, [1, 16]> var_925_cast_fp16 = exp(x = var_924_cast_fp16)[name = string("op_925_cast_fp16")];
            tensor<int32, [1]> var_926_axes_0 = const()[name = string("op_926_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_926_cast_fp16 = expand_dims(axes = var_926_axes_0, x = var_925_cast_fp16)[name = string("op_926_cast_fp16")];
            tensor<int32, [1]> var_927_axes_0 = const()[name = string("op_927_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_927_cast_fp16 = expand_dims(axes = var_927_axes_0, x = var_926_cast_fp16)[name = string("op_927_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_37_cast_fp16 = mul(x = state_35_cast_fp16, y = var_927_cast_fp16)[name = string("state_37_cast_fp16")];
            tensor<int32, [4]> key_token_19_begin_0 = const()[name = string("key_token_19_begin_0"), val = tensor<int32, [4]>([0, 0, 9, 0])];
            tensor<int32, [4]> key_token_19_end_0 = const()[name = string("key_token_19_end_0"), val = tensor<int32, [4]>([1, 16, 10, 128])];
            tensor<bool, [4]> key_token_19_end_mask_0 = const()[name = string("key_token_19_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_19_squeeze_mask_0 = const()[name = string("key_token_19_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_19_cast_fp16 = slice_by_index(begin = key_token_19_begin_0, end = key_token_19_end_0, end_mask = key_token_19_end_mask_0, squeeze_mask = key_token_19_squeeze_mask_0, x = key_3_cast_fp16)[name = string("key_token_19_cast_fp16")];
            tensor<int32, [4]> value_token_19_begin_0 = const()[name = string("value_token_19_begin_0"), val = tensor<int32, [4]>([0, 0, 9, 0])];
            tensor<int32, [4]> value_token_19_end_0 = const()[name = string("value_token_19_end_0"), val = tensor<int32, [4]>([1, 16, 10, 128])];
            tensor<bool, [4]> value_token_19_end_mask_0 = const()[name = string("value_token_19_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_19_squeeze_mask_0 = const()[name = string("value_token_19_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_19_cast_fp16 = slice_by_index(begin = value_token_19_begin_0, end = value_token_19_end_0, end_mask = value_token_19_end_mask_0, squeeze_mask = value_token_19_squeeze_mask_0, x = value_13_cast_fp16)[name = string("value_token_19_cast_fp16")];
            tensor<int32, [1]> var_935_axes_0 = const()[name = string("op_935_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_935_cast_fp16 = expand_dims(axes = var_935_axes_0, x = key_token_19_cast_fp16)[name = string("op_935_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_936_cast_fp16 = mul(x = state_37_cast_fp16, y = var_935_cast_fp16)[name = string("op_936_cast_fp16")];
            tensor<int32, [1]> memory_19_axes_0 = const()[name = string("memory_19_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_19_keep_dims_0 = const()[name = string("memory_19_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_19_cast_fp16 = reduce_sum(axes = memory_19_axes_0, keep_dims = memory_19_keep_dims_0, x = var_936_cast_fp16)[name = string("memory_19_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_939_cast_fp16 = sub(x = value_token_19_cast_fp16, y = memory_19_cast_fp16)[name = string("op_939_cast_fp16")];
            tensor<int32, [3]> var_942_begin_0 = const()[name = string("op_942_begin_0"), val = tensor<int32, [3]>([0, 0, 9])];
            tensor<int32, [3]> var_942_end_0 = const()[name = string("op_942_end_0"), val = tensor<int32, [3]>([1, 16, 10])];
            tensor<bool, [3]> var_942_end_mask_0 = const()[name = string("op_942_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_942_squeeze_mask_0 = const()[name = string("op_942_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_942_cast_fp16 = slice_by_index(begin = var_942_begin_0, end = var_942_end_0, end_mask = var_942_end_mask_0, squeeze_mask = var_942_squeeze_mask_0, x = beta_1_cast_fp16)[name = string("op_942_cast_fp16")];
            tensor<int32, [1]> var_943_axes_0 = const()[name = string("op_943_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_943_cast_fp16 = expand_dims(axes = var_943_axes_0, x = var_942_cast_fp16)[name = string("op_943_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_19_cast_fp16 = mul(x = var_939_cast_fp16, y = var_943_cast_fp16)[name = string("delta_19_cast_fp16")];
            tensor<int32, [1]> var_946_axes_0 = const()[name = string("op_946_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_946_cast_fp16 = expand_dims(axes = var_946_axes_0, x = delta_19_cast_fp16)[name = string("op_946_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_947_cast_fp16 = mul(x = var_935_cast_fp16, y = var_946_cast_fp16)[name = string("op_947_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_39_cast_fp16 = add(x = state_37_cast_fp16, y = var_947_cast_fp16)[name = string("state_39_cast_fp16")];
            tensor<int32, [4]> var_951_begin_0 = const()[name = string("op_951_begin_0"), val = tensor<int32, [4]>([0, 0, 9, 0])];
            tensor<int32, [4]> var_951_end_0 = const()[name = string("op_951_end_0"), val = tensor<int32, [4]>([1, 16, 10, 128])];
            tensor<bool, [4]> var_951_end_mask_0 = const()[name = string("op_951_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_951_squeeze_mask_0 = const()[name = string("op_951_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_951_cast_fp16 = slice_by_index(begin = var_951_begin_0, end = var_951_end_0, end_mask = var_951_end_mask_0, squeeze_mask = var_951_squeeze_mask_0, x = _inversed_query_3_cast_fp16)[name = string("op_951_cast_fp16")];
            tensor<int32, [1]> var_952_axes_0 = const()[name = string("op_952_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_952_cast_fp16 = expand_dims(axes = var_952_axes_0, x = var_951_cast_fp16)[name = string("op_952_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_953_cast_fp16 = mul(x = state_39_cast_fp16, y = var_952_cast_fp16)[name = string("op_953_cast_fp16")];
            tensor<int32, [1]> var_955_axes_0 = const()[name = string("op_955_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_955_keep_dims_0 = const()[name = string("op_955_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_955_cast_fp16 = reduce_sum(axes = var_955_axes_0, keep_dims = var_955_keep_dims_0, x = var_953_cast_fp16)[name = string("op_955_cast_fp16")];
            tensor<int32, [3]> var_958_begin_0 = const()[name = string("op_958_begin_0"), val = tensor<int32, [3]>([0, 0, 10])];
            tensor<int32, [3]> var_958_end_0 = const()[name = string("op_958_end_0"), val = tensor<int32, [3]>([1, 16, 11])];
            tensor<bool, [3]> var_958_end_mask_0 = const()[name = string("op_958_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_958_squeeze_mask_0 = const()[name = string("op_958_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_958_cast_fp16 = slice_by_index(begin = var_958_begin_0, end = var_958_end_0, end_mask = var_958_end_mask_0, squeeze_mask = var_958_squeeze_mask_0, x = decay_1_cast_fp16)[name = string("op_958_cast_fp16")];
            tensor<fp16, [1, 16]> var_959_cast_fp16 = exp(x = var_958_cast_fp16)[name = string("op_959_cast_fp16")];
            tensor<int32, [1]> var_960_axes_0 = const()[name = string("op_960_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_960_cast_fp16 = expand_dims(axes = var_960_axes_0, x = var_959_cast_fp16)[name = string("op_960_cast_fp16")];
            tensor<int32, [1]> var_961_axes_0 = const()[name = string("op_961_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_961_cast_fp16 = expand_dims(axes = var_961_axes_0, x = var_960_cast_fp16)[name = string("op_961_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_41_cast_fp16 = mul(x = state_39_cast_fp16, y = var_961_cast_fp16)[name = string("state_41_cast_fp16")];
            tensor<int32, [4]> key_token_21_begin_0 = const()[name = string("key_token_21_begin_0"), val = tensor<int32, [4]>([0, 0, 10, 0])];
            tensor<int32, [4]> key_token_21_end_0 = const()[name = string("key_token_21_end_0"), val = tensor<int32, [4]>([1, 16, 11, 128])];
            tensor<bool, [4]> key_token_21_end_mask_0 = const()[name = string("key_token_21_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_21_squeeze_mask_0 = const()[name = string("key_token_21_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_21_cast_fp16 = slice_by_index(begin = key_token_21_begin_0, end = key_token_21_end_0, end_mask = key_token_21_end_mask_0, squeeze_mask = key_token_21_squeeze_mask_0, x = key_3_cast_fp16)[name = string("key_token_21_cast_fp16")];
            tensor<int32, [4]> value_token_21_begin_0 = const()[name = string("value_token_21_begin_0"), val = tensor<int32, [4]>([0, 0, 10, 0])];
            tensor<int32, [4]> value_token_21_end_0 = const()[name = string("value_token_21_end_0"), val = tensor<int32, [4]>([1, 16, 11, 128])];
            tensor<bool, [4]> value_token_21_end_mask_0 = const()[name = string("value_token_21_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_21_squeeze_mask_0 = const()[name = string("value_token_21_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_21_cast_fp16 = slice_by_index(begin = value_token_21_begin_0, end = value_token_21_end_0, end_mask = value_token_21_end_mask_0, squeeze_mask = value_token_21_squeeze_mask_0, x = value_13_cast_fp16)[name = string("value_token_21_cast_fp16")];
            tensor<int32, [1]> var_969_axes_0 = const()[name = string("op_969_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_969_cast_fp16 = expand_dims(axes = var_969_axes_0, x = key_token_21_cast_fp16)[name = string("op_969_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_970_cast_fp16 = mul(x = state_41_cast_fp16, y = var_969_cast_fp16)[name = string("op_970_cast_fp16")];
            tensor<int32, [1]> memory_21_axes_0 = const()[name = string("memory_21_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_21_keep_dims_0 = const()[name = string("memory_21_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_21_cast_fp16 = reduce_sum(axes = memory_21_axes_0, keep_dims = memory_21_keep_dims_0, x = var_970_cast_fp16)[name = string("memory_21_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_973_cast_fp16 = sub(x = value_token_21_cast_fp16, y = memory_21_cast_fp16)[name = string("op_973_cast_fp16")];
            tensor<int32, [3]> var_976_begin_0 = const()[name = string("op_976_begin_0"), val = tensor<int32, [3]>([0, 0, 10])];
            tensor<int32, [3]> var_976_end_0 = const()[name = string("op_976_end_0"), val = tensor<int32, [3]>([1, 16, 11])];
            tensor<bool, [3]> var_976_end_mask_0 = const()[name = string("op_976_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_976_squeeze_mask_0 = const()[name = string("op_976_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_976_cast_fp16 = slice_by_index(begin = var_976_begin_0, end = var_976_end_0, end_mask = var_976_end_mask_0, squeeze_mask = var_976_squeeze_mask_0, x = beta_1_cast_fp16)[name = string("op_976_cast_fp16")];
            tensor<int32, [1]> var_977_axes_0 = const()[name = string("op_977_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_977_cast_fp16 = expand_dims(axes = var_977_axes_0, x = var_976_cast_fp16)[name = string("op_977_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_21_cast_fp16 = mul(x = var_973_cast_fp16, y = var_977_cast_fp16)[name = string("delta_21_cast_fp16")];
            tensor<int32, [1]> var_980_axes_0 = const()[name = string("op_980_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_980_cast_fp16 = expand_dims(axes = var_980_axes_0, x = delta_21_cast_fp16)[name = string("op_980_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_981_cast_fp16 = mul(x = var_969_cast_fp16, y = var_980_cast_fp16)[name = string("op_981_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_43_cast_fp16 = add(x = state_41_cast_fp16, y = var_981_cast_fp16)[name = string("state_43_cast_fp16")];
            tensor<int32, [4]> var_985_begin_0 = const()[name = string("op_985_begin_0"), val = tensor<int32, [4]>([0, 0, 10, 0])];
            tensor<int32, [4]> var_985_end_0 = const()[name = string("op_985_end_0"), val = tensor<int32, [4]>([1, 16, 11, 128])];
            tensor<bool, [4]> var_985_end_mask_0 = const()[name = string("op_985_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_985_squeeze_mask_0 = const()[name = string("op_985_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_985_cast_fp16 = slice_by_index(begin = var_985_begin_0, end = var_985_end_0, end_mask = var_985_end_mask_0, squeeze_mask = var_985_squeeze_mask_0, x = _inversed_query_3_cast_fp16)[name = string("op_985_cast_fp16")];
            tensor<int32, [1]> var_986_axes_0 = const()[name = string("op_986_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_986_cast_fp16 = expand_dims(axes = var_986_axes_0, x = var_985_cast_fp16)[name = string("op_986_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_987_cast_fp16 = mul(x = state_43_cast_fp16, y = var_986_cast_fp16)[name = string("op_987_cast_fp16")];
            tensor<int32, [1]> var_989_axes_0 = const()[name = string("op_989_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_989_keep_dims_0 = const()[name = string("op_989_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_989_cast_fp16 = reduce_sum(axes = var_989_axes_0, keep_dims = var_989_keep_dims_0, x = var_987_cast_fp16)[name = string("op_989_cast_fp16")];
            tensor<int32, [3]> var_992_begin_0 = const()[name = string("op_992_begin_0"), val = tensor<int32, [3]>([0, 0, 11])];
            tensor<int32, [3]> var_992_end_0 = const()[name = string("op_992_end_0"), val = tensor<int32, [3]>([1, 16, 12])];
            tensor<bool, [3]> var_992_end_mask_0 = const()[name = string("op_992_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_992_squeeze_mask_0 = const()[name = string("op_992_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_992_cast_fp16 = slice_by_index(begin = var_992_begin_0, end = var_992_end_0, end_mask = var_992_end_mask_0, squeeze_mask = var_992_squeeze_mask_0, x = decay_1_cast_fp16)[name = string("op_992_cast_fp16")];
            tensor<fp16, [1, 16]> var_993_cast_fp16 = exp(x = var_992_cast_fp16)[name = string("op_993_cast_fp16")];
            tensor<int32, [1]> var_994_axes_0 = const()[name = string("op_994_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_994_cast_fp16 = expand_dims(axes = var_994_axes_0, x = var_993_cast_fp16)[name = string("op_994_cast_fp16")];
            tensor<int32, [1]> var_995_axes_0 = const()[name = string("op_995_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_995_cast_fp16 = expand_dims(axes = var_995_axes_0, x = var_994_cast_fp16)[name = string("op_995_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_45_cast_fp16 = mul(x = state_43_cast_fp16, y = var_995_cast_fp16)[name = string("state_45_cast_fp16")];
            tensor<int32, [4]> key_token_23_begin_0 = const()[name = string("key_token_23_begin_0"), val = tensor<int32, [4]>([0, 0, 11, 0])];
            tensor<int32, [4]> key_token_23_end_0 = const()[name = string("key_token_23_end_0"), val = tensor<int32, [4]>([1, 16, 12, 128])];
            tensor<bool, [4]> key_token_23_end_mask_0 = const()[name = string("key_token_23_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_23_squeeze_mask_0 = const()[name = string("key_token_23_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_23_cast_fp16 = slice_by_index(begin = key_token_23_begin_0, end = key_token_23_end_0, end_mask = key_token_23_end_mask_0, squeeze_mask = key_token_23_squeeze_mask_0, x = key_3_cast_fp16)[name = string("key_token_23_cast_fp16")];
            tensor<int32, [4]> value_token_23_begin_0 = const()[name = string("value_token_23_begin_0"), val = tensor<int32, [4]>([0, 0, 11, 0])];
            tensor<int32, [4]> value_token_23_end_0 = const()[name = string("value_token_23_end_0"), val = tensor<int32, [4]>([1, 16, 12, 128])];
            tensor<bool, [4]> value_token_23_end_mask_0 = const()[name = string("value_token_23_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_23_squeeze_mask_0 = const()[name = string("value_token_23_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_23_cast_fp16 = slice_by_index(begin = value_token_23_begin_0, end = value_token_23_end_0, end_mask = value_token_23_end_mask_0, squeeze_mask = value_token_23_squeeze_mask_0, x = value_13_cast_fp16)[name = string("value_token_23_cast_fp16")];
            tensor<int32, [1]> var_1003_axes_0 = const()[name = string("op_1003_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1003_cast_fp16 = expand_dims(axes = var_1003_axes_0, x = key_token_23_cast_fp16)[name = string("op_1003_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1004_cast_fp16 = mul(x = state_45_cast_fp16, y = var_1003_cast_fp16)[name = string("op_1004_cast_fp16")];
            tensor<int32, [1]> memory_23_axes_0 = const()[name = string("memory_23_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_23_keep_dims_0 = const()[name = string("memory_23_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_23_cast_fp16 = reduce_sum(axes = memory_23_axes_0, keep_dims = memory_23_keep_dims_0, x = var_1004_cast_fp16)[name = string("memory_23_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_1007_cast_fp16 = sub(x = value_token_23_cast_fp16, y = memory_23_cast_fp16)[name = string("op_1007_cast_fp16")];
            tensor<int32, [3]> var_1010_begin_0 = const()[name = string("op_1010_begin_0"), val = tensor<int32, [3]>([0, 0, 11])];
            tensor<int32, [3]> var_1010_end_0 = const()[name = string("op_1010_end_0"), val = tensor<int32, [3]>([1, 16, 12])];
            tensor<bool, [3]> var_1010_end_mask_0 = const()[name = string("op_1010_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1010_squeeze_mask_0 = const()[name = string("op_1010_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1010_cast_fp16 = slice_by_index(begin = var_1010_begin_0, end = var_1010_end_0, end_mask = var_1010_end_mask_0, squeeze_mask = var_1010_squeeze_mask_0, x = beta_1_cast_fp16)[name = string("op_1010_cast_fp16")];
            tensor<int32, [1]> var_1011_axes_0 = const()[name = string("op_1011_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1011_cast_fp16 = expand_dims(axes = var_1011_axes_0, x = var_1010_cast_fp16)[name = string("op_1011_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_23_cast_fp16 = mul(x = var_1007_cast_fp16, y = var_1011_cast_fp16)[name = string("delta_23_cast_fp16")];
            tensor<int32, [1]> var_1014_axes_0 = const()[name = string("op_1014_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_1014_cast_fp16 = expand_dims(axes = var_1014_axes_0, x = delta_23_cast_fp16)[name = string("op_1014_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1015_cast_fp16 = mul(x = var_1003_cast_fp16, y = var_1014_cast_fp16)[name = string("op_1015_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_47_cast_fp16 = add(x = state_45_cast_fp16, y = var_1015_cast_fp16)[name = string("state_47_cast_fp16")];
            tensor<int32, [4]> var_1019_begin_0 = const()[name = string("op_1019_begin_0"), val = tensor<int32, [4]>([0, 0, 11, 0])];
            tensor<int32, [4]> var_1019_end_0 = const()[name = string("op_1019_end_0"), val = tensor<int32, [4]>([1, 16, 12, 128])];
            tensor<bool, [4]> var_1019_end_mask_0 = const()[name = string("op_1019_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_1019_squeeze_mask_0 = const()[name = string("op_1019_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_1019_cast_fp16 = slice_by_index(begin = var_1019_begin_0, end = var_1019_end_0, end_mask = var_1019_end_mask_0, squeeze_mask = var_1019_squeeze_mask_0, x = _inversed_query_3_cast_fp16)[name = string("op_1019_cast_fp16")];
            tensor<int32, [1]> var_1020_axes_0 = const()[name = string("op_1020_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1020_cast_fp16 = expand_dims(axes = var_1020_axes_0, x = var_1019_cast_fp16)[name = string("op_1020_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1021_cast_fp16 = mul(x = state_47_cast_fp16, y = var_1020_cast_fp16)[name = string("op_1021_cast_fp16")];
            tensor<int32, [1]> var_1023_axes_0 = const()[name = string("op_1023_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_1023_keep_dims_0 = const()[name = string("op_1023_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_1023_cast_fp16 = reduce_sum(axes = var_1023_axes_0, keep_dims = var_1023_keep_dims_0, x = var_1021_cast_fp16)[name = string("op_1023_cast_fp16")];
            tensor<int32, [3]> var_1026_begin_0 = const()[name = string("op_1026_begin_0"), val = tensor<int32, [3]>([0, 0, 12])];
            tensor<int32, [3]> var_1026_end_0 = const()[name = string("op_1026_end_0"), val = tensor<int32, [3]>([1, 16, 13])];
            tensor<bool, [3]> var_1026_end_mask_0 = const()[name = string("op_1026_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1026_squeeze_mask_0 = const()[name = string("op_1026_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1026_cast_fp16 = slice_by_index(begin = var_1026_begin_0, end = var_1026_end_0, end_mask = var_1026_end_mask_0, squeeze_mask = var_1026_squeeze_mask_0, x = decay_1_cast_fp16)[name = string("op_1026_cast_fp16")];
            tensor<fp16, [1, 16]> var_1027_cast_fp16 = exp(x = var_1026_cast_fp16)[name = string("op_1027_cast_fp16")];
            tensor<int32, [1]> var_1028_axes_0 = const()[name = string("op_1028_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1028_cast_fp16 = expand_dims(axes = var_1028_axes_0, x = var_1027_cast_fp16)[name = string("op_1028_cast_fp16")];
            tensor<int32, [1]> var_1029_axes_0 = const()[name = string("op_1029_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_1029_cast_fp16 = expand_dims(axes = var_1029_axes_0, x = var_1028_cast_fp16)[name = string("op_1029_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_49_cast_fp16 = mul(x = state_47_cast_fp16, y = var_1029_cast_fp16)[name = string("state_49_cast_fp16")];
            tensor<int32, [4]> key_token_25_begin_0 = const()[name = string("key_token_25_begin_0"), val = tensor<int32, [4]>([0, 0, 12, 0])];
            tensor<int32, [4]> key_token_25_end_0 = const()[name = string("key_token_25_end_0"), val = tensor<int32, [4]>([1, 16, 13, 128])];
            tensor<bool, [4]> key_token_25_end_mask_0 = const()[name = string("key_token_25_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_25_squeeze_mask_0 = const()[name = string("key_token_25_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_25_cast_fp16 = slice_by_index(begin = key_token_25_begin_0, end = key_token_25_end_0, end_mask = key_token_25_end_mask_0, squeeze_mask = key_token_25_squeeze_mask_0, x = key_3_cast_fp16)[name = string("key_token_25_cast_fp16")];
            tensor<int32, [4]> value_token_25_begin_0 = const()[name = string("value_token_25_begin_0"), val = tensor<int32, [4]>([0, 0, 12, 0])];
            tensor<int32, [4]> value_token_25_end_0 = const()[name = string("value_token_25_end_0"), val = tensor<int32, [4]>([1, 16, 13, 128])];
            tensor<bool, [4]> value_token_25_end_mask_0 = const()[name = string("value_token_25_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_25_squeeze_mask_0 = const()[name = string("value_token_25_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_25_cast_fp16 = slice_by_index(begin = value_token_25_begin_0, end = value_token_25_end_0, end_mask = value_token_25_end_mask_0, squeeze_mask = value_token_25_squeeze_mask_0, x = value_13_cast_fp16)[name = string("value_token_25_cast_fp16")];
            tensor<int32, [1]> var_1037_axes_0 = const()[name = string("op_1037_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1037_cast_fp16 = expand_dims(axes = var_1037_axes_0, x = key_token_25_cast_fp16)[name = string("op_1037_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1038_cast_fp16 = mul(x = state_49_cast_fp16, y = var_1037_cast_fp16)[name = string("op_1038_cast_fp16")];
            tensor<int32, [1]> memory_25_axes_0 = const()[name = string("memory_25_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_25_keep_dims_0 = const()[name = string("memory_25_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_25_cast_fp16 = reduce_sum(axes = memory_25_axes_0, keep_dims = memory_25_keep_dims_0, x = var_1038_cast_fp16)[name = string("memory_25_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_1041_cast_fp16 = sub(x = value_token_25_cast_fp16, y = memory_25_cast_fp16)[name = string("op_1041_cast_fp16")];
            tensor<int32, [3]> var_1044_begin_0 = const()[name = string("op_1044_begin_0"), val = tensor<int32, [3]>([0, 0, 12])];
            tensor<int32, [3]> var_1044_end_0 = const()[name = string("op_1044_end_0"), val = tensor<int32, [3]>([1, 16, 13])];
            tensor<bool, [3]> var_1044_end_mask_0 = const()[name = string("op_1044_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1044_squeeze_mask_0 = const()[name = string("op_1044_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1044_cast_fp16 = slice_by_index(begin = var_1044_begin_0, end = var_1044_end_0, end_mask = var_1044_end_mask_0, squeeze_mask = var_1044_squeeze_mask_0, x = beta_1_cast_fp16)[name = string("op_1044_cast_fp16")];
            tensor<int32, [1]> var_1045_axes_0 = const()[name = string("op_1045_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1045_cast_fp16 = expand_dims(axes = var_1045_axes_0, x = var_1044_cast_fp16)[name = string("op_1045_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_25_cast_fp16 = mul(x = var_1041_cast_fp16, y = var_1045_cast_fp16)[name = string("delta_25_cast_fp16")];
            tensor<int32, [1]> var_1048_axes_0 = const()[name = string("op_1048_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_1048_cast_fp16 = expand_dims(axes = var_1048_axes_0, x = delta_25_cast_fp16)[name = string("op_1048_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1049_cast_fp16 = mul(x = var_1037_cast_fp16, y = var_1048_cast_fp16)[name = string("op_1049_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_51_cast_fp16 = add(x = state_49_cast_fp16, y = var_1049_cast_fp16)[name = string("state_51_cast_fp16")];
            tensor<int32, [4]> var_1053_begin_0 = const()[name = string("op_1053_begin_0"), val = tensor<int32, [4]>([0, 0, 12, 0])];
            tensor<int32, [4]> var_1053_end_0 = const()[name = string("op_1053_end_0"), val = tensor<int32, [4]>([1, 16, 13, 128])];
            tensor<bool, [4]> var_1053_end_mask_0 = const()[name = string("op_1053_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_1053_squeeze_mask_0 = const()[name = string("op_1053_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_1053_cast_fp16 = slice_by_index(begin = var_1053_begin_0, end = var_1053_end_0, end_mask = var_1053_end_mask_0, squeeze_mask = var_1053_squeeze_mask_0, x = _inversed_query_3_cast_fp16)[name = string("op_1053_cast_fp16")];
            tensor<int32, [1]> var_1054_axes_0 = const()[name = string("op_1054_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1054_cast_fp16 = expand_dims(axes = var_1054_axes_0, x = var_1053_cast_fp16)[name = string("op_1054_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1055_cast_fp16 = mul(x = state_51_cast_fp16, y = var_1054_cast_fp16)[name = string("op_1055_cast_fp16")];
            tensor<int32, [1]> var_1057_axes_0 = const()[name = string("op_1057_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_1057_keep_dims_0 = const()[name = string("op_1057_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_1057_cast_fp16 = reduce_sum(axes = var_1057_axes_0, keep_dims = var_1057_keep_dims_0, x = var_1055_cast_fp16)[name = string("op_1057_cast_fp16")];
            tensor<int32, [3]> var_1060_begin_0 = const()[name = string("op_1060_begin_0"), val = tensor<int32, [3]>([0, 0, 13])];
            tensor<int32, [3]> var_1060_end_0 = const()[name = string("op_1060_end_0"), val = tensor<int32, [3]>([1, 16, 14])];
            tensor<bool, [3]> var_1060_end_mask_0 = const()[name = string("op_1060_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1060_squeeze_mask_0 = const()[name = string("op_1060_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1060_cast_fp16 = slice_by_index(begin = var_1060_begin_0, end = var_1060_end_0, end_mask = var_1060_end_mask_0, squeeze_mask = var_1060_squeeze_mask_0, x = decay_1_cast_fp16)[name = string("op_1060_cast_fp16")];
            tensor<fp16, [1, 16]> var_1061_cast_fp16 = exp(x = var_1060_cast_fp16)[name = string("op_1061_cast_fp16")];
            tensor<int32, [1]> var_1062_axes_0 = const()[name = string("op_1062_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1062_cast_fp16 = expand_dims(axes = var_1062_axes_0, x = var_1061_cast_fp16)[name = string("op_1062_cast_fp16")];
            tensor<int32, [1]> var_1063_axes_0 = const()[name = string("op_1063_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_1063_cast_fp16 = expand_dims(axes = var_1063_axes_0, x = var_1062_cast_fp16)[name = string("op_1063_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_53_cast_fp16 = mul(x = state_51_cast_fp16, y = var_1063_cast_fp16)[name = string("state_53_cast_fp16")];
            tensor<int32, [4]> key_token_27_begin_0 = const()[name = string("key_token_27_begin_0"), val = tensor<int32, [4]>([0, 0, 13, 0])];
            tensor<int32, [4]> key_token_27_end_0 = const()[name = string("key_token_27_end_0"), val = tensor<int32, [4]>([1, 16, 14, 128])];
            tensor<bool, [4]> key_token_27_end_mask_0 = const()[name = string("key_token_27_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_27_squeeze_mask_0 = const()[name = string("key_token_27_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_27_cast_fp16 = slice_by_index(begin = key_token_27_begin_0, end = key_token_27_end_0, end_mask = key_token_27_end_mask_0, squeeze_mask = key_token_27_squeeze_mask_0, x = key_3_cast_fp16)[name = string("key_token_27_cast_fp16")];
            tensor<int32, [4]> value_token_27_begin_0 = const()[name = string("value_token_27_begin_0"), val = tensor<int32, [4]>([0, 0, 13, 0])];
            tensor<int32, [4]> value_token_27_end_0 = const()[name = string("value_token_27_end_0"), val = tensor<int32, [4]>([1, 16, 14, 128])];
            tensor<bool, [4]> value_token_27_end_mask_0 = const()[name = string("value_token_27_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_27_squeeze_mask_0 = const()[name = string("value_token_27_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_27_cast_fp16 = slice_by_index(begin = value_token_27_begin_0, end = value_token_27_end_0, end_mask = value_token_27_end_mask_0, squeeze_mask = value_token_27_squeeze_mask_0, x = value_13_cast_fp16)[name = string("value_token_27_cast_fp16")];
            tensor<int32, [1]> var_1071_axes_0 = const()[name = string("op_1071_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1071_cast_fp16 = expand_dims(axes = var_1071_axes_0, x = key_token_27_cast_fp16)[name = string("op_1071_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1072_cast_fp16 = mul(x = state_53_cast_fp16, y = var_1071_cast_fp16)[name = string("op_1072_cast_fp16")];
            tensor<int32, [1]> memory_27_axes_0 = const()[name = string("memory_27_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_27_keep_dims_0 = const()[name = string("memory_27_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_27_cast_fp16 = reduce_sum(axes = memory_27_axes_0, keep_dims = memory_27_keep_dims_0, x = var_1072_cast_fp16)[name = string("memory_27_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_1075_cast_fp16 = sub(x = value_token_27_cast_fp16, y = memory_27_cast_fp16)[name = string("op_1075_cast_fp16")];
            tensor<int32, [3]> var_1078_begin_0 = const()[name = string("op_1078_begin_0"), val = tensor<int32, [3]>([0, 0, 13])];
            tensor<int32, [3]> var_1078_end_0 = const()[name = string("op_1078_end_0"), val = tensor<int32, [3]>([1, 16, 14])];
            tensor<bool, [3]> var_1078_end_mask_0 = const()[name = string("op_1078_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1078_squeeze_mask_0 = const()[name = string("op_1078_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1078_cast_fp16 = slice_by_index(begin = var_1078_begin_0, end = var_1078_end_0, end_mask = var_1078_end_mask_0, squeeze_mask = var_1078_squeeze_mask_0, x = beta_1_cast_fp16)[name = string("op_1078_cast_fp16")];
            tensor<int32, [1]> var_1079_axes_0 = const()[name = string("op_1079_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1079_cast_fp16 = expand_dims(axes = var_1079_axes_0, x = var_1078_cast_fp16)[name = string("op_1079_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_27_cast_fp16 = mul(x = var_1075_cast_fp16, y = var_1079_cast_fp16)[name = string("delta_27_cast_fp16")];
            tensor<int32, [1]> var_1082_axes_0 = const()[name = string("op_1082_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_1082_cast_fp16 = expand_dims(axes = var_1082_axes_0, x = delta_27_cast_fp16)[name = string("op_1082_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1083_cast_fp16 = mul(x = var_1071_cast_fp16, y = var_1082_cast_fp16)[name = string("op_1083_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_55_cast_fp16 = add(x = state_53_cast_fp16, y = var_1083_cast_fp16)[name = string("state_55_cast_fp16")];
            tensor<int32, [4]> var_1087_begin_0 = const()[name = string("op_1087_begin_0"), val = tensor<int32, [4]>([0, 0, 13, 0])];
            tensor<int32, [4]> var_1087_end_0 = const()[name = string("op_1087_end_0"), val = tensor<int32, [4]>([1, 16, 14, 128])];
            tensor<bool, [4]> var_1087_end_mask_0 = const()[name = string("op_1087_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_1087_squeeze_mask_0 = const()[name = string("op_1087_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_1087_cast_fp16 = slice_by_index(begin = var_1087_begin_0, end = var_1087_end_0, end_mask = var_1087_end_mask_0, squeeze_mask = var_1087_squeeze_mask_0, x = _inversed_query_3_cast_fp16)[name = string("op_1087_cast_fp16")];
            tensor<int32, [1]> var_1088_axes_0 = const()[name = string("op_1088_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1088_cast_fp16 = expand_dims(axes = var_1088_axes_0, x = var_1087_cast_fp16)[name = string("op_1088_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1089_cast_fp16 = mul(x = state_55_cast_fp16, y = var_1088_cast_fp16)[name = string("op_1089_cast_fp16")];
            tensor<int32, [1]> var_1091_axes_0 = const()[name = string("op_1091_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_1091_keep_dims_0 = const()[name = string("op_1091_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_1091_cast_fp16 = reduce_sum(axes = var_1091_axes_0, keep_dims = var_1091_keep_dims_0, x = var_1089_cast_fp16)[name = string("op_1091_cast_fp16")];
            tensor<int32, [3]> var_1094_begin_0 = const()[name = string("op_1094_begin_0"), val = tensor<int32, [3]>([0, 0, 14])];
            tensor<int32, [3]> var_1094_end_0 = const()[name = string("op_1094_end_0"), val = tensor<int32, [3]>([1, 16, 15])];
            tensor<bool, [3]> var_1094_end_mask_0 = const()[name = string("op_1094_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1094_squeeze_mask_0 = const()[name = string("op_1094_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1094_cast_fp16 = slice_by_index(begin = var_1094_begin_0, end = var_1094_end_0, end_mask = var_1094_end_mask_0, squeeze_mask = var_1094_squeeze_mask_0, x = decay_1_cast_fp16)[name = string("op_1094_cast_fp16")];
            tensor<fp16, [1, 16]> var_1095_cast_fp16 = exp(x = var_1094_cast_fp16)[name = string("op_1095_cast_fp16")];
            tensor<int32, [1]> var_1096_axes_0 = const()[name = string("op_1096_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1096_cast_fp16 = expand_dims(axes = var_1096_axes_0, x = var_1095_cast_fp16)[name = string("op_1096_cast_fp16")];
            tensor<int32, [1]> var_1097_axes_0 = const()[name = string("op_1097_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_1097_cast_fp16 = expand_dims(axes = var_1097_axes_0, x = var_1096_cast_fp16)[name = string("op_1097_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_57_cast_fp16 = mul(x = state_55_cast_fp16, y = var_1097_cast_fp16)[name = string("state_57_cast_fp16")];
            tensor<int32, [4]> key_token_29_begin_0 = const()[name = string("key_token_29_begin_0"), val = tensor<int32, [4]>([0, 0, 14, 0])];
            tensor<int32, [4]> key_token_29_end_0 = const()[name = string("key_token_29_end_0"), val = tensor<int32, [4]>([1, 16, 15, 128])];
            tensor<bool, [4]> key_token_29_end_mask_0 = const()[name = string("key_token_29_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_29_squeeze_mask_0 = const()[name = string("key_token_29_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_29_cast_fp16 = slice_by_index(begin = key_token_29_begin_0, end = key_token_29_end_0, end_mask = key_token_29_end_mask_0, squeeze_mask = key_token_29_squeeze_mask_0, x = key_3_cast_fp16)[name = string("key_token_29_cast_fp16")];
            tensor<int32, [4]> value_token_29_begin_0 = const()[name = string("value_token_29_begin_0"), val = tensor<int32, [4]>([0, 0, 14, 0])];
            tensor<int32, [4]> value_token_29_end_0 = const()[name = string("value_token_29_end_0"), val = tensor<int32, [4]>([1, 16, 15, 128])];
            tensor<bool, [4]> value_token_29_end_mask_0 = const()[name = string("value_token_29_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_29_squeeze_mask_0 = const()[name = string("value_token_29_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_29_cast_fp16 = slice_by_index(begin = value_token_29_begin_0, end = value_token_29_end_0, end_mask = value_token_29_end_mask_0, squeeze_mask = value_token_29_squeeze_mask_0, x = value_13_cast_fp16)[name = string("value_token_29_cast_fp16")];
            tensor<int32, [1]> var_1105_axes_0 = const()[name = string("op_1105_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1105_cast_fp16 = expand_dims(axes = var_1105_axes_0, x = key_token_29_cast_fp16)[name = string("op_1105_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1106_cast_fp16 = mul(x = state_57_cast_fp16, y = var_1105_cast_fp16)[name = string("op_1106_cast_fp16")];
            tensor<int32, [1]> memory_29_axes_0 = const()[name = string("memory_29_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_29_keep_dims_0 = const()[name = string("memory_29_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_29_cast_fp16 = reduce_sum(axes = memory_29_axes_0, keep_dims = memory_29_keep_dims_0, x = var_1106_cast_fp16)[name = string("memory_29_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_1109_cast_fp16 = sub(x = value_token_29_cast_fp16, y = memory_29_cast_fp16)[name = string("op_1109_cast_fp16")];
            tensor<int32, [3]> var_1112_begin_0 = const()[name = string("op_1112_begin_0"), val = tensor<int32, [3]>([0, 0, 14])];
            tensor<int32, [3]> var_1112_end_0 = const()[name = string("op_1112_end_0"), val = tensor<int32, [3]>([1, 16, 15])];
            tensor<bool, [3]> var_1112_end_mask_0 = const()[name = string("op_1112_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1112_squeeze_mask_0 = const()[name = string("op_1112_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1112_cast_fp16 = slice_by_index(begin = var_1112_begin_0, end = var_1112_end_0, end_mask = var_1112_end_mask_0, squeeze_mask = var_1112_squeeze_mask_0, x = beta_1_cast_fp16)[name = string("op_1112_cast_fp16")];
            tensor<int32, [1]> var_1113_axes_0 = const()[name = string("op_1113_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1113_cast_fp16 = expand_dims(axes = var_1113_axes_0, x = var_1112_cast_fp16)[name = string("op_1113_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_29_cast_fp16 = mul(x = var_1109_cast_fp16, y = var_1113_cast_fp16)[name = string("delta_29_cast_fp16")];
            tensor<int32, [1]> var_1116_axes_0 = const()[name = string("op_1116_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_1116_cast_fp16 = expand_dims(axes = var_1116_axes_0, x = delta_29_cast_fp16)[name = string("op_1116_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1117_cast_fp16 = mul(x = var_1105_cast_fp16, y = var_1116_cast_fp16)[name = string("op_1117_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_59_cast_fp16 = add(x = state_57_cast_fp16, y = var_1117_cast_fp16)[name = string("state_59_cast_fp16")];
            tensor<int32, [4]> var_1121_begin_0 = const()[name = string("op_1121_begin_0"), val = tensor<int32, [4]>([0, 0, 14, 0])];
            tensor<int32, [4]> var_1121_end_0 = const()[name = string("op_1121_end_0"), val = tensor<int32, [4]>([1, 16, 15, 128])];
            tensor<bool, [4]> var_1121_end_mask_0 = const()[name = string("op_1121_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_1121_squeeze_mask_0 = const()[name = string("op_1121_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_1121_cast_fp16 = slice_by_index(begin = var_1121_begin_0, end = var_1121_end_0, end_mask = var_1121_end_mask_0, squeeze_mask = var_1121_squeeze_mask_0, x = _inversed_query_3_cast_fp16)[name = string("op_1121_cast_fp16")];
            tensor<int32, [1]> var_1122_axes_0 = const()[name = string("op_1122_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1122_cast_fp16 = expand_dims(axes = var_1122_axes_0, x = var_1121_cast_fp16)[name = string("op_1122_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1123_cast_fp16 = mul(x = state_59_cast_fp16, y = var_1122_cast_fp16)[name = string("op_1123_cast_fp16")];
            tensor<int32, [1]> var_1125_axes_0 = const()[name = string("op_1125_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_1125_keep_dims_0 = const()[name = string("op_1125_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_1125_cast_fp16 = reduce_sum(axes = var_1125_axes_0, keep_dims = var_1125_keep_dims_0, x = var_1123_cast_fp16)[name = string("op_1125_cast_fp16")];
            tensor<int32, [3]> var_1128_begin_0 = const()[name = string("op_1128_begin_0"), val = tensor<int32, [3]>([0, 0, 15])];
            tensor<int32, [3]> var_1128_end_0 = const()[name = string("op_1128_end_0"), val = tensor<int32, [3]>([1, 16, 16])];
            tensor<bool, [3]> var_1128_end_mask_0 = const()[name = string("op_1128_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1128_squeeze_mask_0 = const()[name = string("op_1128_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1128_cast_fp16 = slice_by_index(begin = var_1128_begin_0, end = var_1128_end_0, end_mask = var_1128_end_mask_0, squeeze_mask = var_1128_squeeze_mask_0, x = decay_1_cast_fp16)[name = string("op_1128_cast_fp16")];
            tensor<fp16, [1, 16]> var_1129_cast_fp16 = exp(x = var_1128_cast_fp16)[name = string("op_1129_cast_fp16")];
            tensor<int32, [1]> var_1130_axes_0 = const()[name = string("op_1130_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1130_cast_fp16 = expand_dims(axes = var_1130_axes_0, x = var_1129_cast_fp16)[name = string("op_1130_cast_fp16")];
            tensor<int32, [1]> var_1131_axes_0 = const()[name = string("op_1131_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_1131_cast_fp16 = expand_dims(axes = var_1131_axes_0, x = var_1130_cast_fp16)[name = string("op_1131_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_61_cast_fp16 = mul(x = state_59_cast_fp16, y = var_1131_cast_fp16)[name = string("state_61_cast_fp16")];
            tensor<int32, [4]> key_token_31_begin_0 = const()[name = string("key_token_31_begin_0"), val = tensor<int32, [4]>([0, 0, 15, 0])];
            tensor<int32, [4]> key_token_31_end_0 = const()[name = string("key_token_31_end_0"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<bool, [4]> key_token_31_end_mask_0 = const()[name = string("key_token_31_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_31_squeeze_mask_0 = const()[name = string("key_token_31_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_31_cast_fp16 = slice_by_index(begin = key_token_31_begin_0, end = key_token_31_end_0, end_mask = key_token_31_end_mask_0, squeeze_mask = key_token_31_squeeze_mask_0, x = key_3_cast_fp16)[name = string("key_token_31_cast_fp16")];
            tensor<int32, [4]> value_token_31_begin_0 = const()[name = string("value_token_31_begin_0"), val = tensor<int32, [4]>([0, 0, 15, 0])];
            tensor<int32, [4]> value_token_31_end_0 = const()[name = string("value_token_31_end_0"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<bool, [4]> value_token_31_end_mask_0 = const()[name = string("value_token_31_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_31_squeeze_mask_0 = const()[name = string("value_token_31_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_31_cast_fp16 = slice_by_index(begin = value_token_31_begin_0, end = value_token_31_end_0, end_mask = value_token_31_end_mask_0, squeeze_mask = value_token_31_squeeze_mask_0, x = value_13_cast_fp16)[name = string("value_token_31_cast_fp16")];
            tensor<int32, [1]> var_1139_axes_0 = const()[name = string("op_1139_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1139_cast_fp16 = expand_dims(axes = var_1139_axes_0, x = key_token_31_cast_fp16)[name = string("op_1139_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1140_cast_fp16 = mul(x = state_61_cast_fp16, y = var_1139_cast_fp16)[name = string("op_1140_cast_fp16")];
            tensor<int32, [1]> memory_31_axes_0 = const()[name = string("memory_31_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_31_keep_dims_0 = const()[name = string("memory_31_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_31_cast_fp16 = reduce_sum(axes = memory_31_axes_0, keep_dims = memory_31_keep_dims_0, x = var_1140_cast_fp16)[name = string("memory_31_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_1143_cast_fp16 = sub(x = value_token_31_cast_fp16, y = memory_31_cast_fp16)[name = string("op_1143_cast_fp16")];
            tensor<int32, [3]> var_1146_begin_0 = const()[name = string("op_1146_begin_0"), val = tensor<int32, [3]>([0, 0, 15])];
            tensor<int32, [3]> var_1146_end_0 = const()[name = string("op_1146_end_0"), val = tensor<int32, [3]>([1, 16, 16])];
            tensor<bool, [3]> var_1146_end_mask_0 = const()[name = string("op_1146_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1146_squeeze_mask_0 = const()[name = string("op_1146_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1146_cast_fp16 = slice_by_index(begin = var_1146_begin_0, end = var_1146_end_0, end_mask = var_1146_end_mask_0, squeeze_mask = var_1146_squeeze_mask_0, x = beta_1_cast_fp16)[name = string("op_1146_cast_fp16")];
            tensor<int32, [1]> var_1147_axes_0 = const()[name = string("op_1147_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1147_cast_fp16 = expand_dims(axes = var_1147_axes_0, x = var_1146_cast_fp16)[name = string("op_1147_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_31_cast_fp16 = mul(x = var_1143_cast_fp16, y = var_1147_cast_fp16)[name = string("delta_31_cast_fp16")];
            tensor<int32, [1]> var_1150_axes_0 = const()[name = string("op_1150_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_1150_cast_fp16 = expand_dims(axes = var_1150_axes_0, x = delta_31_cast_fp16)[name = string("op_1150_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1151_cast_fp16 = mul(x = var_1139_cast_fp16, y = var_1150_cast_fp16)[name = string("op_1151_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> rec20_out = add(x = state_61_cast_fp16, y = var_1151_cast_fp16)[name = string("state_63_cast_fp16")];
            tensor<int32, [4]> var_1155_begin_0 = const()[name = string("op_1155_begin_0"), val = tensor<int32, [4]>([0, 0, 15, 0])];
            tensor<int32, [4]> var_1155_end_0 = const()[name = string("op_1155_end_0"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<bool, [4]> var_1155_end_mask_0 = const()[name = string("op_1155_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_1155_squeeze_mask_0 = const()[name = string("op_1155_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_1155_cast_fp16 = slice_by_index(begin = var_1155_begin_0, end = var_1155_end_0, end_mask = var_1155_end_mask_0, squeeze_mask = var_1155_squeeze_mask_0, x = _inversed_query_3_cast_fp16)[name = string("op_1155_cast_fp16")];
            tensor<int32, [1]> var_1156_axes_0 = const()[name = string("op_1156_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1156_cast_fp16 = expand_dims(axes = var_1156_axes_0, x = var_1155_cast_fp16)[name = string("op_1156_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1157_cast_fp16 = mul(x = rec20_out, y = var_1156_cast_fp16)[name = string("op_1157_cast_fp16")];
            tensor<int32, [1]> var_1159_axes_0 = const()[name = string("op_1159_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_1159_keep_dims_0 = const()[name = string("op_1159_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_1159_cast_fp16 = reduce_sum(axes = var_1159_axes_0, keep_dims = var_1159_keep_dims_0, x = var_1157_cast_fp16)[name = string("op_1159_cast_fp16")];
            int32 attention_1_axis_0 = const()[name = string("attention_1_axis_0"), val = int32(1)];
            tensor<fp16, [1, 16, 16, 128]> attention_1_cast_fp16 = stack(axis = attention_1_axis_0, values = (var_649_cast_fp16, var_683_cast_fp16, var_717_cast_fp16, var_751_cast_fp16, var_785_cast_fp16, var_819_cast_fp16, var_853_cast_fp16, var_887_cast_fp16, var_921_cast_fp16, var_955_cast_fp16, var_989_cast_fp16, var_1023_cast_fp16, var_1057_cast_fp16, var_1091_cast_fp16, var_1125_cast_fp16, var_1159_cast_fp16))[name = string("attention_1_cast_fp16")];
            tensor<int32, [2]> var_1162 = const()[name = string("op_1162"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [256, 128]> attention_3_cast_fp16 = reshape(shape = var_1162, x = attention_1_cast_fp16)[name = string("attention_3_cast_fp16")];
            tensor<int32, [2]> var_1164 = const()[name = string("op_1164"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [256, 128]> input_13_cast_fp16 = reshape(shape = var_1164, x = linear_7_cast_fp16)[name = string("input_13_cast_fp16")];
            tensor<fp16, [256, 128]> var_1166_cast_fp16 = mul(x = attention_3_cast_fp16, y = attention_3_cast_fp16)[name = string("op_1166_cast_fp16")];
            tensor<int32, [1]> variance_5_axes_0 = const()[name = string("variance_5_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_5_keep_dims_0 = const()[name = string("variance_5_keep_dims_0"), val = bool(true)];
            tensor<fp16, [256, 1]> variance_5_cast_fp16 = reduce_mean(axes = variance_5_axes_0, keep_dims = variance_5_keep_dims_0, x = var_1166_cast_fp16)[name = string("variance_5_cast_fp16")];
            fp16 var_1169_to_fp16 = const()[name = string("op_1169_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [256, 1]> var_1170_cast_fp16 = add(x = variance_5_cast_fp16, y = var_1169_to_fp16)[name = string("op_1170_cast_fp16")];
            fp32 var_1171_epsilon_0 = const()[name = string("op_1171_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [256, 1]> var_1171_cast_fp16 = rsqrt(epsilon = var_1171_epsilon_0, x = var_1170_cast_fp16)[name = string("op_1171_cast_fp16")];
            tensor<fp16, [256, 128]> attention_5_cast_fp16 = mul(x = attention_3_cast_fp16, y = var_1171_cast_fp16)[name = string("attention_5_cast_fp16")];
            tensor<fp16, [128]> layers20_0_gated_norm_promoted_to_fp16 = const()[name = string("layers20_0_gated_norm_promoted_to_fp16"), val = tensor<fp16, [128]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(209578368)))];
            tensor<fp16, [256, 128]> attention_7_cast_fp16 = mul(x = attention_5_cast_fp16, y = layers20_0_gated_norm_promoted_to_fp16)[name = string("attention_7_cast_fp16")];
            tensor<fp16, [256, 128]> var_1174_cast_fp16 = silu(x = input_13_cast_fp16)[name = string("op_1174_cast_fp16")];
            tensor<fp16, [256, 128]> attention_9_cast_fp16 = mul(x = attention_7_cast_fp16, y = var_1174_cast_fp16)[name = string("attention_9_cast_fp16")];
            tensor<int32, [2]> var_1176 = const()[name = string("op_1176"), val = tensor<int32, [2]>([16, 2048])];
            tensor<fp16, [16, 2048]> input_15_cast_fp16 = reshape(shape = var_1176, x = attention_9_cast_fp16)[name = string("input_15_cast_fp16")];
            tensor<fp16, [1024, 2048]> layers20_0_out_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 2048]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(209578688))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(211151616))))[name = string("layers20_0_out_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 1024]> linear_8_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = layers20_0_out_proj_weight_promoted_to_fp16_palettized, x = input_15_cast_fp16)[name = string("linear_8_cast_fp16")];
            tensor<fp16, [16, 1024]> value_15_cast_fp16 = add(x = value_3_cast_fp16, y = linear_8_cast_fp16)[name = string("value_15_cast_fp16")];
            tensor<fp16, [16, 1024]> var_1181_cast_fp16 = mul(x = value_15_cast_fp16, y = value_15_cast_fp16)[name = string("op_1181_cast_fp16")];
            tensor<int32, [1]> variance_7_axes_0 = const()[name = string("variance_7_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_7_keep_dims_0 = const()[name = string("variance_7_keep_dims_0"), val = bool(true)];
            tensor<fp16, [16, 1]> variance_7_cast_fp16 = reduce_mean(axes = variance_7_axes_0, keep_dims = variance_7_keep_dims_0, x = var_1181_cast_fp16)[name = string("variance_7_cast_fp16")];
            fp16 var_1184_to_fp16 = const()[name = string("op_1184_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [16, 1]> var_1185_cast_fp16 = add(x = variance_7_cast_fp16, y = var_1184_to_fp16)[name = string("op_1185_cast_fp16")];
            fp32 var_1186_epsilon_0 = const()[name = string("op_1186_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [16, 1]> var_1186_cast_fp16 = rsqrt(epsilon = var_1186_epsilon_0, x = var_1185_cast_fp16)[name = string("op_1186_cast_fp16")];
            tensor<fp16, [16, 1024]> var_1187_cast_fp16 = mul(x = value_15_cast_fp16, y = var_1186_cast_fp16)[name = string("op_1187_cast_fp16")];
            tensor<fp16, [1024]> var_1189_to_fp16 = const()[name = string("op_1189_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(211159872)))];
            tensor<fp16, [16, 1024]> input_17_cast_fp16 = mul(x = var_1187_cast_fp16, y = var_1189_to_fp16)[name = string("input_17_cast_fp16")];
            tensor<fp16, [3584, 1024]> layers20_0_gate_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(211161984))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(213914560))))[name = string("layers20_0_gate_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 3584]> linear_9_cast_fp16 = linear(bias = linear_1_bias_0_to_fp16, weight = layers20_0_gate_proj_weight_promoted_to_fp16_palettized, x = input_17_cast_fp16)[name = string("linear_9_cast_fp16")];
            tensor<fp16, [16, 3584]> var_1193_cast_fp16 = silu(x = linear_9_cast_fp16)[name = string("op_1193_cast_fp16")];
            tensor<fp16, [3584, 1024]> layers20_0_up_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(213943296))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(216695872))))[name = string("layers20_0_up_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 3584]> linear_10_cast_fp16 = linear(bias = linear_1_bias_0_to_fp16, weight = layers20_0_up_proj_weight_promoted_to_fp16_palettized, x = input_17_cast_fp16)[name = string("linear_10_cast_fp16")];
            tensor<fp16, [16, 3584]> input_21_cast_fp16 = mul(x = var_1193_cast_fp16, y = linear_10_cast_fp16)[name = string("input_21_cast_fp16")];
            tensor<fp16, [1024, 3584]> layers20_0_down_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 3584]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(216724608))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(219477184))))[name = string("layers20_0_down_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 1024]> linear_11_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = layers20_0_down_proj_weight_promoted_to_fp16_palettized, x = input_21_cast_fp16)[name = string("linear_11_cast_fp16")];
            tensor<fp16, [16, 1024]> value_17_cast_fp16 = add(x = value_15_cast_fp16, y = linear_11_cast_fp16)[name = string("value_17_cast_fp16")];
            int32 var_1231 = const()[name = string("op_1231"), val = int32(-1)];
            tensor<fp16, [16, 1024]> var_1246_cast_fp16 = mul(x = value_17_cast_fp16, y = value_17_cast_fp16)[name = string("op_1246_cast_fp16")];
            tensor<int32, [1]> variance_9_axes_0 = const()[name = string("variance_9_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_9_keep_dims_0 = const()[name = string("variance_9_keep_dims_0"), val = bool(true)];
            tensor<fp16, [16, 1]> variance_9_cast_fp16 = reduce_mean(axes = variance_9_axes_0, keep_dims = variance_9_keep_dims_0, x = var_1246_cast_fp16)[name = string("variance_9_cast_fp16")];
            fp16 var_1249_to_fp16 = const()[name = string("op_1249_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [16, 1]> var_1250_cast_fp16 = add(x = variance_9_cast_fp16, y = var_1249_to_fp16)[name = string("op_1250_cast_fp16")];
            fp32 var_1251_epsilon_0 = const()[name = string("op_1251_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [16, 1]> var_1251_cast_fp16 = rsqrt(epsilon = var_1251_epsilon_0, x = var_1250_cast_fp16)[name = string("op_1251_cast_fp16")];
            tensor<fp16, [16, 1024]> var_1252_cast_fp16 = mul(x = value_17_cast_fp16, y = var_1251_cast_fp16)[name = string("op_1252_cast_fp16")];
            tensor<fp16, [1024]> var_1254_to_fp16 = const()[name = string("op_1254_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(219485440)))];
            tensor<fp16, [16, 1024]> input_23_cast_fp16 = mul(x = var_1252_cast_fp16, y = var_1254_to_fp16)[name = string("input_23_cast_fp16")];
            tensor<fp16, [6144, 1024]> layers20_1_in_proj_qkv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(219487552))), lut = tensor<fp16, [384, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(224206208))))[name = string("layers20_1_in_proj_qkv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 6144]> linear_12_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = layers20_1_in_proj_qkv_weight_promoted_to_fp16_palettized, x = input_23_cast_fp16)[name = string("linear_12_cast_fp16")];
            tensor<int32, [2]> var_1258_perm_0 = const()[name = string("op_1258_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> mixed_3_axes_0 = const()[name = string("mixed_3_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [6144, 16]> var_1258_cast_fp16 = transpose(perm = var_1258_perm_0, x = linear_12_cast_fp16)[name = string("transpose_21")];
            tensor<fp16, [1, 6144, 16]> mixed_3_cast_fp16 = expand_dims(axes = mixed_3_axes_0, x = var_1258_cast_fp16)[name = string("mixed_3_cast_fp16")];
            bool convolution_input_3_interleave_0 = const()[name = string("convolution_input_3_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 6144, 19]> convolution_input_3_cast_fp16 = concat(axis = var_1231, interleave = convolution_input_3_interleave_0, values = (conv21, mixed_3_cast_fp16))[name = string("convolution_input_3_cast_fp16")];
            string input_25_pad_type_0 = const()[name = string("input_25_pad_type_0"), val = string("valid")];
            int32 input_25_groups_0 = const()[name = string("input_25_groups_0"), val = int32(6144)];
            tensor<int32, [1]> input_25_strides_0 = const()[name = string("input_25_strides_0"), val = tensor<int32, [1]>([1])];
            tensor<int32, [2]> input_25_pad_0 = const()[name = string("input_25_pad_0"), val = tensor<int32, [2]>([0, 0])];
            tensor<int32, [1]> input_25_dilations_0 = const()[name = string("input_25_dilations_0"), val = tensor<int32, [1]>([1])];
            tensor<fp16, [6144, 1, 4]> layers20_1_conv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1, 4]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(224255424))), lut = tensor<fp16, [384, 1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(224273920))))[name = string("layers20_1_conv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 6144, 16]> input_25_cast_fp16 = conv(dilations = input_25_dilations_0, groups = input_25_groups_0, pad = input_25_pad_0, pad_type = input_25_pad_type_0, strides = input_25_strides_0, weight = layers20_1_conv_weight_promoted_to_fp16_palettized, x = convolution_input_3_cast_fp16)[name = string("input_25_cast_fp16")];
            tensor<fp16, [1, 6144, 16]> var_1267_cast_fp16 = silu(x = input_25_cast_fp16)[name = string("op_1267_cast_fp16")];
            tensor<int32, [3]> var_1268_perm_0 = const()[name = string("op_1268_perm_0"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [3]> var_1271_begin_0 = const()[name = string("op_1271_begin_0"), val = tensor<int32, [3]>([0, 0, 16])];
            tensor<int32, [3]> var_1271_end_0 = const()[name = string("op_1271_end_0"), val = tensor<int32, [3]>([1, 6144, 19])];
            tensor<bool, [3]> var_1271_end_mask_0 = const()[name = string("op_1271_end_mask_0"), val = tensor<bool, [3]>([true, true, true])];
            tensor<fp16, [1, 6144, 3]> conv21_out = slice_by_index(begin = var_1271_begin_0, end = var_1271_end_0, end_mask = var_1271_end_mask_0, x = convolution_input_3_cast_fp16)[name = string("op_1271_cast_fp16")];
            tensor<int32, [3]> var_1272 = const()[name = string("op_1272"), val = tensor<int32, [3]>([2048, 2048, 2048])];
            int32 var_1273_axis_0 = const()[name = string("op_1273_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 16, 6144]> var_1268_cast_fp16 = transpose(perm = var_1268_perm_0, x = var_1267_cast_fp16)[name = string("transpose_20")];
            tensor<fp16, [1, 16, 2048]> var_1273_cast_fp16_0, tensor<fp16, [1, 16, 2048]> var_1273_cast_fp16_1, tensor<fp16, [1, 16, 2048]> var_1273_cast_fp16_2 = split(axis = var_1273_axis_0, split_sizes = var_1272, x = var_1268_cast_fp16)[name = string("op_1273_cast_fp16")];
            tensor<int32, [4]> var_1277 = const()[name = string("op_1277"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<fp16, [1, 16, 16, 128]> value_21_cast_fp16 = reshape(shape = var_1277, x = var_1273_cast_fp16_0)[name = string("value_21_cast_fp16")];
            tensor<int32, [4]> var_1279 = const()[name = string("op_1279"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<fp16, [1, 16, 16, 128]> value_23_cast_fp16 = reshape(shape = var_1279, x = var_1273_cast_fp16_1)[name = string("value_23_cast_fp16")];
            tensor<int32, [4]> var_1281 = const()[name = string("op_1281"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<fp16, [1, 16, 16, 128]> value_25_cast_fp16 = reshape(shape = var_1281, x = var_1273_cast_fp16_2)[name = string("value_25_cast_fp16")];
            tensor<fp16, [1, 16, 16, 128]> var_1283_cast_fp16 = mul(x = value_21_cast_fp16, y = value_21_cast_fp16)[name = string("op_1283_cast_fp16")];
            tensor<int32, [1]> var_1285_axes_0 = const()[name = string("op_1285_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_1285_keep_dims_0 = const()[name = string("op_1285_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 16, 16, 1]> var_1285_cast_fp16 = reduce_sum(axes = var_1285_axes_0, keep_dims = var_1285_keep_dims_0, x = var_1283_cast_fp16)[name = string("op_1285_cast_fp16")];
            fp16 var_1286_to_fp16 = const()[name = string("op_1286_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 16, 16, 1]> var_1287_cast_fp16 = add(x = var_1285_cast_fp16, y = var_1286_to_fp16)[name = string("op_1287_cast_fp16")];
            fp32 var_1288_epsilon_0 = const()[name = string("op_1288_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 16, 16, 1]> var_1288_cast_fp16 = rsqrt(epsilon = var_1288_epsilon_0, x = var_1287_cast_fp16)[name = string("op_1288_cast_fp16")];
            tensor<fp16, [1, 16, 16, 128]> var_1289_cast_fp16 = mul(x = value_21_cast_fp16, y = var_1288_cast_fp16)[name = string("op_1289_cast_fp16")];
            tensor<int32, [4]> var_1290_perm_0 = const()[name = string("op_1290_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            fp16 _inversed_query_7_y_0_to_fp16 = const()[name = string("_inversed_query_7_y_0_to_fp16"), val = fp16(0x1.6ap-4)];
            tensor<fp16, [1, 16, 16, 128]> var_1290_cast_fp16 = transpose(perm = var_1290_perm_0, x = var_1289_cast_fp16)[name = string("transpose_19")];
            tensor<fp16, [1, 16, 16, 128]> _inversed_query_7_cast_fp16 = mul(x = var_1290_cast_fp16, y = _inversed_query_7_y_0_to_fp16)[name = string("_inversed_query_7_cast_fp16")];
            tensor<fp16, [1, 16, 16, 128]> var_1293_cast_fp16 = mul(x = value_23_cast_fp16, y = value_23_cast_fp16)[name = string("op_1293_cast_fp16")];
            tensor<int32, [1]> var_1295_axes_0 = const()[name = string("op_1295_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_1295_keep_dims_0 = const()[name = string("op_1295_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 16, 16, 1]> var_1295_cast_fp16 = reduce_sum(axes = var_1295_axes_0, keep_dims = var_1295_keep_dims_0, x = var_1293_cast_fp16)[name = string("op_1295_cast_fp16")];
            fp16 var_1296_to_fp16 = const()[name = string("op_1296_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 16, 16, 1]> var_1297_cast_fp16 = add(x = var_1295_cast_fp16, y = var_1296_to_fp16)[name = string("op_1297_cast_fp16")];
            fp32 var_1298_epsilon_0 = const()[name = string("op_1298_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 16, 16, 1]> var_1298_cast_fp16 = rsqrt(epsilon = var_1298_epsilon_0, x = var_1297_cast_fp16)[name = string("op_1298_cast_fp16")];
            tensor<fp16, [1, 16, 16, 128]> var_1299_cast_fp16 = mul(x = value_23_cast_fp16, y = var_1298_cast_fp16)[name = string("op_1299_cast_fp16")];
            tensor<int32, [4]> key_7_perm_0 = const()[name = string("key_7_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<int32, [4]> value_27_perm_0 = const()[name = string("value_27_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<fp16, [16, 1024]> layers20_1_in_proj_b_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(224323136))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(224335488))))[name = string("layers20_1_in_proj_b_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16]> linear_13_bias_0_to_fp16 = const()[name = string("linear_13_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(207971904)))];
            tensor<fp16, [16, 16]> linear_13_cast_fp16 = linear(bias = linear_13_bias_0_to_fp16, weight = layers20_1_in_proj_b_weight_promoted_to_fp16_palettized, x = input_23_cast_fp16)[name = string("linear_13_cast_fp16")];
            tensor<fp16, [16, 16]> var_1304_cast_fp16 = sigmoid(x = linear_13_cast_fp16)[name = string("op_1304_cast_fp16")];
            tensor<int32, [2]> var_1305_perm_0 = const()[name = string("op_1305_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> beta_3_axes_0 = const()[name = string("beta_3_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 16]> var_1305_cast_fp16 = transpose(perm = var_1305_perm_0, x = var_1304_cast_fp16)[name = string("transpose_18")];
            tensor<fp16, [1, 16, 16]> beta_3_cast_fp16 = expand_dims(axes = beta_3_axes_0, x = var_1305_cast_fp16)[name = string("beta_3_cast_fp16")];
            tensor<fp16, [16, 1024]> op_1311_weight_0_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(224335680))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(224348032))))[name = string("op_1311_weight_0_to_fp16_palettized")];
            tensor<fp16, [16]> var_1311_bias_0_to_fp16 = const()[name = string("op_1311_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(224348224)))];
            tensor<fp16, [16, 16]> var_1311_cast_fp16 = linear(bias = var_1311_bias_0_to_fp16, weight = op_1311_weight_0_to_fp16_palettized, x = input_23_cast_fp16)[name = string("op_1311_cast_fp16")];
            tensor<fp16, [16, 16]> var_1312_cast_fp16 = softplus(x = var_1311_cast_fp16)[name = string("op_1312_cast_fp16")];
            tensor<fp16, [16]> var_1308_promoted_to_fp16 = const()[name = string("op_1308_promoted_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(224348352)))];
            tensor<fp16, [16, 16]> var_1313_cast_fp16 = mul(x = var_1308_promoted_to_fp16, y = var_1312_cast_fp16)[name = string("op_1313_cast_fp16")];
            tensor<int32, [2]> var_1314_perm_0 = const()[name = string("op_1314_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> decay_3_axes_0 = const()[name = string("decay_3_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 16]> var_1314_cast_fp16 = transpose(perm = var_1314_perm_0, x = var_1313_cast_fp16)[name = string("transpose_17")];
            tensor<fp16, [1, 16, 16]> decay_3_cast_fp16 = expand_dims(axes = decay_3_axes_0, x = var_1314_cast_fp16)[name = string("decay_3_cast_fp16")];
            tensor<fp16, [2048, 1024]> layers20_1_in_proj_z_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [2048, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(224348480))), lut = tensor<fp16, [128, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(225921408))))[name = string("layers20_1_in_proj_z_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 2048]> linear_15_cast_fp16 = linear(bias = linear_7_bias_0_to_fp16, weight = layers20_1_in_proj_z_weight_promoted_to_fp16_palettized, x = input_23_cast_fp16)[name = string("linear_15_cast_fp16")];
            tensor<int32, [3]> var_1322_begin_0 = const()[name = string("op_1322_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_1322_end_0 = const()[name = string("op_1322_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_1322_end_mask_0 = const()[name = string("op_1322_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1322_squeeze_mask_0 = const()[name = string("op_1322_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1322_cast_fp16 = slice_by_index(begin = var_1322_begin_0, end = var_1322_end_0, end_mask = var_1322_end_mask_0, squeeze_mask = var_1322_squeeze_mask_0, x = decay_3_cast_fp16)[name = string("op_1322_cast_fp16")];
            tensor<fp16, [1, 16]> var_1323_cast_fp16 = exp(x = var_1322_cast_fp16)[name = string("op_1323_cast_fp16")];
            tensor<int32, [1]> var_1324_axes_0 = const()[name = string("op_1324_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1324_cast_fp16 = expand_dims(axes = var_1324_axes_0, x = var_1323_cast_fp16)[name = string("op_1324_cast_fp16")];
            tensor<int32, [1]> var_1325_axes_0 = const()[name = string("op_1325_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_1325_cast_fp16 = expand_dims(axes = var_1325_axes_0, x = var_1324_cast_fp16)[name = string("op_1325_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_65_cast_fp16 = mul(x = rec21, y = var_1325_cast_fp16)[name = string("state_65_cast_fp16")];
            tensor<int32, [4]> key_token_33_begin_0 = const()[name = string("key_token_33_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> key_token_33_end_0 = const()[name = string("key_token_33_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> key_token_33_end_mask_0 = const()[name = string("key_token_33_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_33_squeeze_mask_0 = const()[name = string("key_token_33_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 16, 128]> key_7_cast_fp16 = transpose(perm = key_7_perm_0, x = var_1299_cast_fp16)[name = string("transpose_16")];
            tensor<fp16, [1, 16, 128]> key_token_33_cast_fp16 = slice_by_index(begin = key_token_33_begin_0, end = key_token_33_end_0, end_mask = key_token_33_end_mask_0, squeeze_mask = key_token_33_squeeze_mask_0, x = key_7_cast_fp16)[name = string("key_token_33_cast_fp16")];
            tensor<int32, [4]> value_token_33_begin_0 = const()[name = string("value_token_33_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> value_token_33_end_0 = const()[name = string("value_token_33_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> value_token_33_end_mask_0 = const()[name = string("value_token_33_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_33_squeeze_mask_0 = const()[name = string("value_token_33_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 16, 128]> value_27_cast_fp16 = transpose(perm = value_27_perm_0, x = value_25_cast_fp16)[name = string("transpose_15")];
            tensor<fp16, [1, 16, 128]> value_token_33_cast_fp16 = slice_by_index(begin = value_token_33_begin_0, end = value_token_33_end_0, end_mask = value_token_33_end_mask_0, squeeze_mask = value_token_33_squeeze_mask_0, x = value_27_cast_fp16)[name = string("value_token_33_cast_fp16")];
            tensor<int32, [1]> var_1333_axes_0 = const()[name = string("op_1333_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1333_cast_fp16 = expand_dims(axes = var_1333_axes_0, x = key_token_33_cast_fp16)[name = string("op_1333_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1334_cast_fp16 = mul(x = state_65_cast_fp16, y = var_1333_cast_fp16)[name = string("op_1334_cast_fp16")];
            tensor<int32, [1]> memory_33_axes_0 = const()[name = string("memory_33_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_33_keep_dims_0 = const()[name = string("memory_33_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_33_cast_fp16 = reduce_sum(axes = memory_33_axes_0, keep_dims = memory_33_keep_dims_0, x = var_1334_cast_fp16)[name = string("memory_33_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_1337_cast_fp16 = sub(x = value_token_33_cast_fp16, y = memory_33_cast_fp16)[name = string("op_1337_cast_fp16")];
            tensor<int32, [3]> var_1340_begin_0 = const()[name = string("op_1340_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_1340_end_0 = const()[name = string("op_1340_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_1340_end_mask_0 = const()[name = string("op_1340_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1340_squeeze_mask_0 = const()[name = string("op_1340_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1340_cast_fp16 = slice_by_index(begin = var_1340_begin_0, end = var_1340_end_0, end_mask = var_1340_end_mask_0, squeeze_mask = var_1340_squeeze_mask_0, x = beta_3_cast_fp16)[name = string("op_1340_cast_fp16")];
            tensor<int32, [1]> var_1341_axes_0 = const()[name = string("op_1341_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1341_cast_fp16 = expand_dims(axes = var_1341_axes_0, x = var_1340_cast_fp16)[name = string("op_1341_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_33_cast_fp16 = mul(x = var_1337_cast_fp16, y = var_1341_cast_fp16)[name = string("delta_33_cast_fp16")];
            tensor<int32, [1]> var_1344_axes_0 = const()[name = string("op_1344_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_1344_cast_fp16 = expand_dims(axes = var_1344_axes_0, x = delta_33_cast_fp16)[name = string("op_1344_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1345_cast_fp16 = mul(x = var_1333_cast_fp16, y = var_1344_cast_fp16)[name = string("op_1345_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_67_cast_fp16 = add(x = state_65_cast_fp16, y = var_1345_cast_fp16)[name = string("state_67_cast_fp16")];
            tensor<int32, [4]> var_1349_begin_0 = const()[name = string("op_1349_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> var_1349_end_0 = const()[name = string("op_1349_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> var_1349_end_mask_0 = const()[name = string("op_1349_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_1349_squeeze_mask_0 = const()[name = string("op_1349_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_1349_cast_fp16 = slice_by_index(begin = var_1349_begin_0, end = var_1349_end_0, end_mask = var_1349_end_mask_0, squeeze_mask = var_1349_squeeze_mask_0, x = _inversed_query_7_cast_fp16)[name = string("op_1349_cast_fp16")];
            tensor<int32, [1]> var_1350_axes_0 = const()[name = string("op_1350_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1350_cast_fp16 = expand_dims(axes = var_1350_axes_0, x = var_1349_cast_fp16)[name = string("op_1350_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1351_cast_fp16 = mul(x = state_67_cast_fp16, y = var_1350_cast_fp16)[name = string("op_1351_cast_fp16")];
            tensor<int32, [1]> var_1353_axes_0 = const()[name = string("op_1353_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_1353_keep_dims_0 = const()[name = string("op_1353_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_1353_cast_fp16 = reduce_sum(axes = var_1353_axes_0, keep_dims = var_1353_keep_dims_0, x = var_1351_cast_fp16)[name = string("op_1353_cast_fp16")];
            tensor<int32, [3]> var_1356_begin_0 = const()[name = string("op_1356_begin_0"), val = tensor<int32, [3]>([0, 0, 1])];
            tensor<int32, [3]> var_1356_end_0 = const()[name = string("op_1356_end_0"), val = tensor<int32, [3]>([1, 16, 2])];
            tensor<bool, [3]> var_1356_end_mask_0 = const()[name = string("op_1356_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1356_squeeze_mask_0 = const()[name = string("op_1356_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1356_cast_fp16 = slice_by_index(begin = var_1356_begin_0, end = var_1356_end_0, end_mask = var_1356_end_mask_0, squeeze_mask = var_1356_squeeze_mask_0, x = decay_3_cast_fp16)[name = string("op_1356_cast_fp16")];
            tensor<fp16, [1, 16]> var_1357_cast_fp16 = exp(x = var_1356_cast_fp16)[name = string("op_1357_cast_fp16")];
            tensor<int32, [1]> var_1358_axes_0 = const()[name = string("op_1358_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1358_cast_fp16 = expand_dims(axes = var_1358_axes_0, x = var_1357_cast_fp16)[name = string("op_1358_cast_fp16")];
            tensor<int32, [1]> var_1359_axes_0 = const()[name = string("op_1359_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_1359_cast_fp16 = expand_dims(axes = var_1359_axes_0, x = var_1358_cast_fp16)[name = string("op_1359_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_69_cast_fp16 = mul(x = state_67_cast_fp16, y = var_1359_cast_fp16)[name = string("state_69_cast_fp16")];
            tensor<int32, [4]> key_token_35_begin_0 = const()[name = string("key_token_35_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> key_token_35_end_0 = const()[name = string("key_token_35_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> key_token_35_end_mask_0 = const()[name = string("key_token_35_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_35_squeeze_mask_0 = const()[name = string("key_token_35_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_35_cast_fp16 = slice_by_index(begin = key_token_35_begin_0, end = key_token_35_end_0, end_mask = key_token_35_end_mask_0, squeeze_mask = key_token_35_squeeze_mask_0, x = key_7_cast_fp16)[name = string("key_token_35_cast_fp16")];
            tensor<int32, [4]> value_token_35_begin_0 = const()[name = string("value_token_35_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> value_token_35_end_0 = const()[name = string("value_token_35_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> value_token_35_end_mask_0 = const()[name = string("value_token_35_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_35_squeeze_mask_0 = const()[name = string("value_token_35_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_35_cast_fp16 = slice_by_index(begin = value_token_35_begin_0, end = value_token_35_end_0, end_mask = value_token_35_end_mask_0, squeeze_mask = value_token_35_squeeze_mask_0, x = value_27_cast_fp16)[name = string("value_token_35_cast_fp16")];
            tensor<int32, [1]> var_1367_axes_0 = const()[name = string("op_1367_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1367_cast_fp16 = expand_dims(axes = var_1367_axes_0, x = key_token_35_cast_fp16)[name = string("op_1367_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1368_cast_fp16 = mul(x = state_69_cast_fp16, y = var_1367_cast_fp16)[name = string("op_1368_cast_fp16")];
            tensor<int32, [1]> memory_35_axes_0 = const()[name = string("memory_35_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_35_keep_dims_0 = const()[name = string("memory_35_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_35_cast_fp16 = reduce_sum(axes = memory_35_axes_0, keep_dims = memory_35_keep_dims_0, x = var_1368_cast_fp16)[name = string("memory_35_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_1371_cast_fp16 = sub(x = value_token_35_cast_fp16, y = memory_35_cast_fp16)[name = string("op_1371_cast_fp16")];
            tensor<int32, [3]> var_1374_begin_0 = const()[name = string("op_1374_begin_0"), val = tensor<int32, [3]>([0, 0, 1])];
            tensor<int32, [3]> var_1374_end_0 = const()[name = string("op_1374_end_0"), val = tensor<int32, [3]>([1, 16, 2])];
            tensor<bool, [3]> var_1374_end_mask_0 = const()[name = string("op_1374_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1374_squeeze_mask_0 = const()[name = string("op_1374_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1374_cast_fp16 = slice_by_index(begin = var_1374_begin_0, end = var_1374_end_0, end_mask = var_1374_end_mask_0, squeeze_mask = var_1374_squeeze_mask_0, x = beta_3_cast_fp16)[name = string("op_1374_cast_fp16")];
            tensor<int32, [1]> var_1375_axes_0 = const()[name = string("op_1375_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1375_cast_fp16 = expand_dims(axes = var_1375_axes_0, x = var_1374_cast_fp16)[name = string("op_1375_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_35_cast_fp16 = mul(x = var_1371_cast_fp16, y = var_1375_cast_fp16)[name = string("delta_35_cast_fp16")];
            tensor<int32, [1]> var_1378_axes_0 = const()[name = string("op_1378_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_1378_cast_fp16 = expand_dims(axes = var_1378_axes_0, x = delta_35_cast_fp16)[name = string("op_1378_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1379_cast_fp16 = mul(x = var_1367_cast_fp16, y = var_1378_cast_fp16)[name = string("op_1379_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_71_cast_fp16 = add(x = state_69_cast_fp16, y = var_1379_cast_fp16)[name = string("state_71_cast_fp16")];
            tensor<int32, [4]> var_1383_begin_0 = const()[name = string("op_1383_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> var_1383_end_0 = const()[name = string("op_1383_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> var_1383_end_mask_0 = const()[name = string("op_1383_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_1383_squeeze_mask_0 = const()[name = string("op_1383_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_1383_cast_fp16 = slice_by_index(begin = var_1383_begin_0, end = var_1383_end_0, end_mask = var_1383_end_mask_0, squeeze_mask = var_1383_squeeze_mask_0, x = _inversed_query_7_cast_fp16)[name = string("op_1383_cast_fp16")];
            tensor<int32, [1]> var_1384_axes_0 = const()[name = string("op_1384_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1384_cast_fp16 = expand_dims(axes = var_1384_axes_0, x = var_1383_cast_fp16)[name = string("op_1384_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1385_cast_fp16 = mul(x = state_71_cast_fp16, y = var_1384_cast_fp16)[name = string("op_1385_cast_fp16")];
            tensor<int32, [1]> var_1387_axes_0 = const()[name = string("op_1387_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_1387_keep_dims_0 = const()[name = string("op_1387_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_1387_cast_fp16 = reduce_sum(axes = var_1387_axes_0, keep_dims = var_1387_keep_dims_0, x = var_1385_cast_fp16)[name = string("op_1387_cast_fp16")];
            tensor<int32, [3]> var_1390_begin_0 = const()[name = string("op_1390_begin_0"), val = tensor<int32, [3]>([0, 0, 2])];
            tensor<int32, [3]> var_1390_end_0 = const()[name = string("op_1390_end_0"), val = tensor<int32, [3]>([1, 16, 3])];
            tensor<bool, [3]> var_1390_end_mask_0 = const()[name = string("op_1390_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1390_squeeze_mask_0 = const()[name = string("op_1390_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1390_cast_fp16 = slice_by_index(begin = var_1390_begin_0, end = var_1390_end_0, end_mask = var_1390_end_mask_0, squeeze_mask = var_1390_squeeze_mask_0, x = decay_3_cast_fp16)[name = string("op_1390_cast_fp16")];
            tensor<fp16, [1, 16]> var_1391_cast_fp16 = exp(x = var_1390_cast_fp16)[name = string("op_1391_cast_fp16")];
            tensor<int32, [1]> var_1392_axes_0 = const()[name = string("op_1392_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1392_cast_fp16 = expand_dims(axes = var_1392_axes_0, x = var_1391_cast_fp16)[name = string("op_1392_cast_fp16")];
            tensor<int32, [1]> var_1393_axes_0 = const()[name = string("op_1393_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_1393_cast_fp16 = expand_dims(axes = var_1393_axes_0, x = var_1392_cast_fp16)[name = string("op_1393_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_73_cast_fp16 = mul(x = state_71_cast_fp16, y = var_1393_cast_fp16)[name = string("state_73_cast_fp16")];
            tensor<int32, [4]> key_token_37_begin_0 = const()[name = string("key_token_37_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> key_token_37_end_0 = const()[name = string("key_token_37_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> key_token_37_end_mask_0 = const()[name = string("key_token_37_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_37_squeeze_mask_0 = const()[name = string("key_token_37_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_37_cast_fp16 = slice_by_index(begin = key_token_37_begin_0, end = key_token_37_end_0, end_mask = key_token_37_end_mask_0, squeeze_mask = key_token_37_squeeze_mask_0, x = key_7_cast_fp16)[name = string("key_token_37_cast_fp16")];
            tensor<int32, [4]> value_token_37_begin_0 = const()[name = string("value_token_37_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> value_token_37_end_0 = const()[name = string("value_token_37_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> value_token_37_end_mask_0 = const()[name = string("value_token_37_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_37_squeeze_mask_0 = const()[name = string("value_token_37_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_37_cast_fp16 = slice_by_index(begin = value_token_37_begin_0, end = value_token_37_end_0, end_mask = value_token_37_end_mask_0, squeeze_mask = value_token_37_squeeze_mask_0, x = value_27_cast_fp16)[name = string("value_token_37_cast_fp16")];
            tensor<int32, [1]> var_1401_axes_0 = const()[name = string("op_1401_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1401_cast_fp16 = expand_dims(axes = var_1401_axes_0, x = key_token_37_cast_fp16)[name = string("op_1401_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1402_cast_fp16 = mul(x = state_73_cast_fp16, y = var_1401_cast_fp16)[name = string("op_1402_cast_fp16")];
            tensor<int32, [1]> memory_37_axes_0 = const()[name = string("memory_37_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_37_keep_dims_0 = const()[name = string("memory_37_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_37_cast_fp16 = reduce_sum(axes = memory_37_axes_0, keep_dims = memory_37_keep_dims_0, x = var_1402_cast_fp16)[name = string("memory_37_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_1405_cast_fp16 = sub(x = value_token_37_cast_fp16, y = memory_37_cast_fp16)[name = string("op_1405_cast_fp16")];
            tensor<int32, [3]> var_1408_begin_0 = const()[name = string("op_1408_begin_0"), val = tensor<int32, [3]>([0, 0, 2])];
            tensor<int32, [3]> var_1408_end_0 = const()[name = string("op_1408_end_0"), val = tensor<int32, [3]>([1, 16, 3])];
            tensor<bool, [3]> var_1408_end_mask_0 = const()[name = string("op_1408_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1408_squeeze_mask_0 = const()[name = string("op_1408_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1408_cast_fp16 = slice_by_index(begin = var_1408_begin_0, end = var_1408_end_0, end_mask = var_1408_end_mask_0, squeeze_mask = var_1408_squeeze_mask_0, x = beta_3_cast_fp16)[name = string("op_1408_cast_fp16")];
            tensor<int32, [1]> var_1409_axes_0 = const()[name = string("op_1409_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1409_cast_fp16 = expand_dims(axes = var_1409_axes_0, x = var_1408_cast_fp16)[name = string("op_1409_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_37_cast_fp16 = mul(x = var_1405_cast_fp16, y = var_1409_cast_fp16)[name = string("delta_37_cast_fp16")];
            tensor<int32, [1]> var_1412_axes_0 = const()[name = string("op_1412_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_1412_cast_fp16 = expand_dims(axes = var_1412_axes_0, x = delta_37_cast_fp16)[name = string("op_1412_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1413_cast_fp16 = mul(x = var_1401_cast_fp16, y = var_1412_cast_fp16)[name = string("op_1413_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_75_cast_fp16 = add(x = state_73_cast_fp16, y = var_1413_cast_fp16)[name = string("state_75_cast_fp16")];
            tensor<int32, [4]> var_1417_begin_0 = const()[name = string("op_1417_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> var_1417_end_0 = const()[name = string("op_1417_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> var_1417_end_mask_0 = const()[name = string("op_1417_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_1417_squeeze_mask_0 = const()[name = string("op_1417_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_1417_cast_fp16 = slice_by_index(begin = var_1417_begin_0, end = var_1417_end_0, end_mask = var_1417_end_mask_0, squeeze_mask = var_1417_squeeze_mask_0, x = _inversed_query_7_cast_fp16)[name = string("op_1417_cast_fp16")];
            tensor<int32, [1]> var_1418_axes_0 = const()[name = string("op_1418_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1418_cast_fp16 = expand_dims(axes = var_1418_axes_0, x = var_1417_cast_fp16)[name = string("op_1418_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1419_cast_fp16 = mul(x = state_75_cast_fp16, y = var_1418_cast_fp16)[name = string("op_1419_cast_fp16")];
            tensor<int32, [1]> var_1421_axes_0 = const()[name = string("op_1421_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_1421_keep_dims_0 = const()[name = string("op_1421_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_1421_cast_fp16 = reduce_sum(axes = var_1421_axes_0, keep_dims = var_1421_keep_dims_0, x = var_1419_cast_fp16)[name = string("op_1421_cast_fp16")];
            tensor<int32, [3]> var_1424_begin_0 = const()[name = string("op_1424_begin_0"), val = tensor<int32, [3]>([0, 0, 3])];
            tensor<int32, [3]> var_1424_end_0 = const()[name = string("op_1424_end_0"), val = tensor<int32, [3]>([1, 16, 4])];
            tensor<bool, [3]> var_1424_end_mask_0 = const()[name = string("op_1424_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1424_squeeze_mask_0 = const()[name = string("op_1424_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1424_cast_fp16 = slice_by_index(begin = var_1424_begin_0, end = var_1424_end_0, end_mask = var_1424_end_mask_0, squeeze_mask = var_1424_squeeze_mask_0, x = decay_3_cast_fp16)[name = string("op_1424_cast_fp16")];
            tensor<fp16, [1, 16]> var_1425_cast_fp16 = exp(x = var_1424_cast_fp16)[name = string("op_1425_cast_fp16")];
            tensor<int32, [1]> var_1426_axes_0 = const()[name = string("op_1426_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1426_cast_fp16 = expand_dims(axes = var_1426_axes_0, x = var_1425_cast_fp16)[name = string("op_1426_cast_fp16")];
            tensor<int32, [1]> var_1427_axes_0 = const()[name = string("op_1427_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_1427_cast_fp16 = expand_dims(axes = var_1427_axes_0, x = var_1426_cast_fp16)[name = string("op_1427_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_77_cast_fp16 = mul(x = state_75_cast_fp16, y = var_1427_cast_fp16)[name = string("state_77_cast_fp16")];
            tensor<int32, [4]> key_token_39_begin_0 = const()[name = string("key_token_39_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> key_token_39_end_0 = const()[name = string("key_token_39_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> key_token_39_end_mask_0 = const()[name = string("key_token_39_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_39_squeeze_mask_0 = const()[name = string("key_token_39_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_39_cast_fp16 = slice_by_index(begin = key_token_39_begin_0, end = key_token_39_end_0, end_mask = key_token_39_end_mask_0, squeeze_mask = key_token_39_squeeze_mask_0, x = key_7_cast_fp16)[name = string("key_token_39_cast_fp16")];
            tensor<int32, [4]> value_token_39_begin_0 = const()[name = string("value_token_39_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> value_token_39_end_0 = const()[name = string("value_token_39_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> value_token_39_end_mask_0 = const()[name = string("value_token_39_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_39_squeeze_mask_0 = const()[name = string("value_token_39_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_39_cast_fp16 = slice_by_index(begin = value_token_39_begin_0, end = value_token_39_end_0, end_mask = value_token_39_end_mask_0, squeeze_mask = value_token_39_squeeze_mask_0, x = value_27_cast_fp16)[name = string("value_token_39_cast_fp16")];
            tensor<int32, [1]> var_1435_axes_0 = const()[name = string("op_1435_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1435_cast_fp16 = expand_dims(axes = var_1435_axes_0, x = key_token_39_cast_fp16)[name = string("op_1435_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1436_cast_fp16 = mul(x = state_77_cast_fp16, y = var_1435_cast_fp16)[name = string("op_1436_cast_fp16")];
            tensor<int32, [1]> memory_39_axes_0 = const()[name = string("memory_39_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_39_keep_dims_0 = const()[name = string("memory_39_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_39_cast_fp16 = reduce_sum(axes = memory_39_axes_0, keep_dims = memory_39_keep_dims_0, x = var_1436_cast_fp16)[name = string("memory_39_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_1439_cast_fp16 = sub(x = value_token_39_cast_fp16, y = memory_39_cast_fp16)[name = string("op_1439_cast_fp16")];
            tensor<int32, [3]> var_1442_begin_0 = const()[name = string("op_1442_begin_0"), val = tensor<int32, [3]>([0, 0, 3])];
            tensor<int32, [3]> var_1442_end_0 = const()[name = string("op_1442_end_0"), val = tensor<int32, [3]>([1, 16, 4])];
            tensor<bool, [3]> var_1442_end_mask_0 = const()[name = string("op_1442_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1442_squeeze_mask_0 = const()[name = string("op_1442_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1442_cast_fp16 = slice_by_index(begin = var_1442_begin_0, end = var_1442_end_0, end_mask = var_1442_end_mask_0, squeeze_mask = var_1442_squeeze_mask_0, x = beta_3_cast_fp16)[name = string("op_1442_cast_fp16")];
            tensor<int32, [1]> var_1443_axes_0 = const()[name = string("op_1443_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1443_cast_fp16 = expand_dims(axes = var_1443_axes_0, x = var_1442_cast_fp16)[name = string("op_1443_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_39_cast_fp16 = mul(x = var_1439_cast_fp16, y = var_1443_cast_fp16)[name = string("delta_39_cast_fp16")];
            tensor<int32, [1]> var_1446_axes_0 = const()[name = string("op_1446_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_1446_cast_fp16 = expand_dims(axes = var_1446_axes_0, x = delta_39_cast_fp16)[name = string("op_1446_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1447_cast_fp16 = mul(x = var_1435_cast_fp16, y = var_1446_cast_fp16)[name = string("op_1447_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_79_cast_fp16 = add(x = state_77_cast_fp16, y = var_1447_cast_fp16)[name = string("state_79_cast_fp16")];
            tensor<int32, [4]> var_1451_begin_0 = const()[name = string("op_1451_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> var_1451_end_0 = const()[name = string("op_1451_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> var_1451_end_mask_0 = const()[name = string("op_1451_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_1451_squeeze_mask_0 = const()[name = string("op_1451_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_1451_cast_fp16 = slice_by_index(begin = var_1451_begin_0, end = var_1451_end_0, end_mask = var_1451_end_mask_0, squeeze_mask = var_1451_squeeze_mask_0, x = _inversed_query_7_cast_fp16)[name = string("op_1451_cast_fp16")];
            tensor<int32, [1]> var_1452_axes_0 = const()[name = string("op_1452_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1452_cast_fp16 = expand_dims(axes = var_1452_axes_0, x = var_1451_cast_fp16)[name = string("op_1452_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1453_cast_fp16 = mul(x = state_79_cast_fp16, y = var_1452_cast_fp16)[name = string("op_1453_cast_fp16")];
            tensor<int32, [1]> var_1455_axes_0 = const()[name = string("op_1455_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_1455_keep_dims_0 = const()[name = string("op_1455_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_1455_cast_fp16 = reduce_sum(axes = var_1455_axes_0, keep_dims = var_1455_keep_dims_0, x = var_1453_cast_fp16)[name = string("op_1455_cast_fp16")];
            tensor<int32, [3]> var_1458_begin_0 = const()[name = string("op_1458_begin_0"), val = tensor<int32, [3]>([0, 0, 4])];
            tensor<int32, [3]> var_1458_end_0 = const()[name = string("op_1458_end_0"), val = tensor<int32, [3]>([1, 16, 5])];
            tensor<bool, [3]> var_1458_end_mask_0 = const()[name = string("op_1458_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1458_squeeze_mask_0 = const()[name = string("op_1458_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1458_cast_fp16 = slice_by_index(begin = var_1458_begin_0, end = var_1458_end_0, end_mask = var_1458_end_mask_0, squeeze_mask = var_1458_squeeze_mask_0, x = decay_3_cast_fp16)[name = string("op_1458_cast_fp16")];
            tensor<fp16, [1, 16]> var_1459_cast_fp16 = exp(x = var_1458_cast_fp16)[name = string("op_1459_cast_fp16")];
            tensor<int32, [1]> var_1460_axes_0 = const()[name = string("op_1460_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1460_cast_fp16 = expand_dims(axes = var_1460_axes_0, x = var_1459_cast_fp16)[name = string("op_1460_cast_fp16")];
            tensor<int32, [1]> var_1461_axes_0 = const()[name = string("op_1461_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_1461_cast_fp16 = expand_dims(axes = var_1461_axes_0, x = var_1460_cast_fp16)[name = string("op_1461_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_81_cast_fp16 = mul(x = state_79_cast_fp16, y = var_1461_cast_fp16)[name = string("state_81_cast_fp16")];
            tensor<int32, [4]> key_token_41_begin_0 = const()[name = string("key_token_41_begin_0"), val = tensor<int32, [4]>([0, 0, 4, 0])];
            tensor<int32, [4]> key_token_41_end_0 = const()[name = string("key_token_41_end_0"), val = tensor<int32, [4]>([1, 16, 5, 128])];
            tensor<bool, [4]> key_token_41_end_mask_0 = const()[name = string("key_token_41_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_41_squeeze_mask_0 = const()[name = string("key_token_41_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_41_cast_fp16 = slice_by_index(begin = key_token_41_begin_0, end = key_token_41_end_0, end_mask = key_token_41_end_mask_0, squeeze_mask = key_token_41_squeeze_mask_0, x = key_7_cast_fp16)[name = string("key_token_41_cast_fp16")];
            tensor<int32, [4]> value_token_41_begin_0 = const()[name = string("value_token_41_begin_0"), val = tensor<int32, [4]>([0, 0, 4, 0])];
            tensor<int32, [4]> value_token_41_end_0 = const()[name = string("value_token_41_end_0"), val = tensor<int32, [4]>([1, 16, 5, 128])];
            tensor<bool, [4]> value_token_41_end_mask_0 = const()[name = string("value_token_41_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_41_squeeze_mask_0 = const()[name = string("value_token_41_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_41_cast_fp16 = slice_by_index(begin = value_token_41_begin_0, end = value_token_41_end_0, end_mask = value_token_41_end_mask_0, squeeze_mask = value_token_41_squeeze_mask_0, x = value_27_cast_fp16)[name = string("value_token_41_cast_fp16")];
            tensor<int32, [1]> var_1469_axes_0 = const()[name = string("op_1469_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1469_cast_fp16 = expand_dims(axes = var_1469_axes_0, x = key_token_41_cast_fp16)[name = string("op_1469_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1470_cast_fp16 = mul(x = state_81_cast_fp16, y = var_1469_cast_fp16)[name = string("op_1470_cast_fp16")];
            tensor<int32, [1]> memory_41_axes_0 = const()[name = string("memory_41_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_41_keep_dims_0 = const()[name = string("memory_41_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_41_cast_fp16 = reduce_sum(axes = memory_41_axes_0, keep_dims = memory_41_keep_dims_0, x = var_1470_cast_fp16)[name = string("memory_41_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_1473_cast_fp16 = sub(x = value_token_41_cast_fp16, y = memory_41_cast_fp16)[name = string("op_1473_cast_fp16")];
            tensor<int32, [3]> var_1476_begin_0 = const()[name = string("op_1476_begin_0"), val = tensor<int32, [3]>([0, 0, 4])];
            tensor<int32, [3]> var_1476_end_0 = const()[name = string("op_1476_end_0"), val = tensor<int32, [3]>([1, 16, 5])];
            tensor<bool, [3]> var_1476_end_mask_0 = const()[name = string("op_1476_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1476_squeeze_mask_0 = const()[name = string("op_1476_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1476_cast_fp16 = slice_by_index(begin = var_1476_begin_0, end = var_1476_end_0, end_mask = var_1476_end_mask_0, squeeze_mask = var_1476_squeeze_mask_0, x = beta_3_cast_fp16)[name = string("op_1476_cast_fp16")];
            tensor<int32, [1]> var_1477_axes_0 = const()[name = string("op_1477_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1477_cast_fp16 = expand_dims(axes = var_1477_axes_0, x = var_1476_cast_fp16)[name = string("op_1477_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_41_cast_fp16 = mul(x = var_1473_cast_fp16, y = var_1477_cast_fp16)[name = string("delta_41_cast_fp16")];
            tensor<int32, [1]> var_1480_axes_0 = const()[name = string("op_1480_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_1480_cast_fp16 = expand_dims(axes = var_1480_axes_0, x = delta_41_cast_fp16)[name = string("op_1480_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1481_cast_fp16 = mul(x = var_1469_cast_fp16, y = var_1480_cast_fp16)[name = string("op_1481_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_83_cast_fp16 = add(x = state_81_cast_fp16, y = var_1481_cast_fp16)[name = string("state_83_cast_fp16")];
            tensor<int32, [4]> var_1485_begin_0 = const()[name = string("op_1485_begin_0"), val = tensor<int32, [4]>([0, 0, 4, 0])];
            tensor<int32, [4]> var_1485_end_0 = const()[name = string("op_1485_end_0"), val = tensor<int32, [4]>([1, 16, 5, 128])];
            tensor<bool, [4]> var_1485_end_mask_0 = const()[name = string("op_1485_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_1485_squeeze_mask_0 = const()[name = string("op_1485_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_1485_cast_fp16 = slice_by_index(begin = var_1485_begin_0, end = var_1485_end_0, end_mask = var_1485_end_mask_0, squeeze_mask = var_1485_squeeze_mask_0, x = _inversed_query_7_cast_fp16)[name = string("op_1485_cast_fp16")];
            tensor<int32, [1]> var_1486_axes_0 = const()[name = string("op_1486_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1486_cast_fp16 = expand_dims(axes = var_1486_axes_0, x = var_1485_cast_fp16)[name = string("op_1486_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1487_cast_fp16 = mul(x = state_83_cast_fp16, y = var_1486_cast_fp16)[name = string("op_1487_cast_fp16")];
            tensor<int32, [1]> var_1489_axes_0 = const()[name = string("op_1489_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_1489_keep_dims_0 = const()[name = string("op_1489_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_1489_cast_fp16 = reduce_sum(axes = var_1489_axes_0, keep_dims = var_1489_keep_dims_0, x = var_1487_cast_fp16)[name = string("op_1489_cast_fp16")];
            tensor<int32, [3]> var_1492_begin_0 = const()[name = string("op_1492_begin_0"), val = tensor<int32, [3]>([0, 0, 5])];
            tensor<int32, [3]> var_1492_end_0 = const()[name = string("op_1492_end_0"), val = tensor<int32, [3]>([1, 16, 6])];
            tensor<bool, [3]> var_1492_end_mask_0 = const()[name = string("op_1492_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1492_squeeze_mask_0 = const()[name = string("op_1492_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1492_cast_fp16 = slice_by_index(begin = var_1492_begin_0, end = var_1492_end_0, end_mask = var_1492_end_mask_0, squeeze_mask = var_1492_squeeze_mask_0, x = decay_3_cast_fp16)[name = string("op_1492_cast_fp16")];
            tensor<fp16, [1, 16]> var_1493_cast_fp16 = exp(x = var_1492_cast_fp16)[name = string("op_1493_cast_fp16")];
            tensor<int32, [1]> var_1494_axes_0 = const()[name = string("op_1494_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1494_cast_fp16 = expand_dims(axes = var_1494_axes_0, x = var_1493_cast_fp16)[name = string("op_1494_cast_fp16")];
            tensor<int32, [1]> var_1495_axes_0 = const()[name = string("op_1495_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_1495_cast_fp16 = expand_dims(axes = var_1495_axes_0, x = var_1494_cast_fp16)[name = string("op_1495_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_85_cast_fp16 = mul(x = state_83_cast_fp16, y = var_1495_cast_fp16)[name = string("state_85_cast_fp16")];
            tensor<int32, [4]> key_token_43_begin_0 = const()[name = string("key_token_43_begin_0"), val = tensor<int32, [4]>([0, 0, 5, 0])];
            tensor<int32, [4]> key_token_43_end_0 = const()[name = string("key_token_43_end_0"), val = tensor<int32, [4]>([1, 16, 6, 128])];
            tensor<bool, [4]> key_token_43_end_mask_0 = const()[name = string("key_token_43_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_43_squeeze_mask_0 = const()[name = string("key_token_43_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_43_cast_fp16 = slice_by_index(begin = key_token_43_begin_0, end = key_token_43_end_0, end_mask = key_token_43_end_mask_0, squeeze_mask = key_token_43_squeeze_mask_0, x = key_7_cast_fp16)[name = string("key_token_43_cast_fp16")];
            tensor<int32, [4]> value_token_43_begin_0 = const()[name = string("value_token_43_begin_0"), val = tensor<int32, [4]>([0, 0, 5, 0])];
            tensor<int32, [4]> value_token_43_end_0 = const()[name = string("value_token_43_end_0"), val = tensor<int32, [4]>([1, 16, 6, 128])];
            tensor<bool, [4]> value_token_43_end_mask_0 = const()[name = string("value_token_43_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_43_squeeze_mask_0 = const()[name = string("value_token_43_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_43_cast_fp16 = slice_by_index(begin = value_token_43_begin_0, end = value_token_43_end_0, end_mask = value_token_43_end_mask_0, squeeze_mask = value_token_43_squeeze_mask_0, x = value_27_cast_fp16)[name = string("value_token_43_cast_fp16")];
            tensor<int32, [1]> var_1503_axes_0 = const()[name = string("op_1503_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1503_cast_fp16 = expand_dims(axes = var_1503_axes_0, x = key_token_43_cast_fp16)[name = string("op_1503_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1504_cast_fp16 = mul(x = state_85_cast_fp16, y = var_1503_cast_fp16)[name = string("op_1504_cast_fp16")];
            tensor<int32, [1]> memory_43_axes_0 = const()[name = string("memory_43_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_43_keep_dims_0 = const()[name = string("memory_43_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_43_cast_fp16 = reduce_sum(axes = memory_43_axes_0, keep_dims = memory_43_keep_dims_0, x = var_1504_cast_fp16)[name = string("memory_43_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_1507_cast_fp16 = sub(x = value_token_43_cast_fp16, y = memory_43_cast_fp16)[name = string("op_1507_cast_fp16")];
            tensor<int32, [3]> var_1510_begin_0 = const()[name = string("op_1510_begin_0"), val = tensor<int32, [3]>([0, 0, 5])];
            tensor<int32, [3]> var_1510_end_0 = const()[name = string("op_1510_end_0"), val = tensor<int32, [3]>([1, 16, 6])];
            tensor<bool, [3]> var_1510_end_mask_0 = const()[name = string("op_1510_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1510_squeeze_mask_0 = const()[name = string("op_1510_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1510_cast_fp16 = slice_by_index(begin = var_1510_begin_0, end = var_1510_end_0, end_mask = var_1510_end_mask_0, squeeze_mask = var_1510_squeeze_mask_0, x = beta_3_cast_fp16)[name = string("op_1510_cast_fp16")];
            tensor<int32, [1]> var_1511_axes_0 = const()[name = string("op_1511_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1511_cast_fp16 = expand_dims(axes = var_1511_axes_0, x = var_1510_cast_fp16)[name = string("op_1511_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_43_cast_fp16 = mul(x = var_1507_cast_fp16, y = var_1511_cast_fp16)[name = string("delta_43_cast_fp16")];
            tensor<int32, [1]> var_1514_axes_0 = const()[name = string("op_1514_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_1514_cast_fp16 = expand_dims(axes = var_1514_axes_0, x = delta_43_cast_fp16)[name = string("op_1514_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1515_cast_fp16 = mul(x = var_1503_cast_fp16, y = var_1514_cast_fp16)[name = string("op_1515_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_87_cast_fp16 = add(x = state_85_cast_fp16, y = var_1515_cast_fp16)[name = string("state_87_cast_fp16")];
            tensor<int32, [4]> var_1519_begin_0 = const()[name = string("op_1519_begin_0"), val = tensor<int32, [4]>([0, 0, 5, 0])];
            tensor<int32, [4]> var_1519_end_0 = const()[name = string("op_1519_end_0"), val = tensor<int32, [4]>([1, 16, 6, 128])];
            tensor<bool, [4]> var_1519_end_mask_0 = const()[name = string("op_1519_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_1519_squeeze_mask_0 = const()[name = string("op_1519_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_1519_cast_fp16 = slice_by_index(begin = var_1519_begin_0, end = var_1519_end_0, end_mask = var_1519_end_mask_0, squeeze_mask = var_1519_squeeze_mask_0, x = _inversed_query_7_cast_fp16)[name = string("op_1519_cast_fp16")];
            tensor<int32, [1]> var_1520_axes_0 = const()[name = string("op_1520_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1520_cast_fp16 = expand_dims(axes = var_1520_axes_0, x = var_1519_cast_fp16)[name = string("op_1520_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1521_cast_fp16 = mul(x = state_87_cast_fp16, y = var_1520_cast_fp16)[name = string("op_1521_cast_fp16")];
            tensor<int32, [1]> var_1523_axes_0 = const()[name = string("op_1523_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_1523_keep_dims_0 = const()[name = string("op_1523_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_1523_cast_fp16 = reduce_sum(axes = var_1523_axes_0, keep_dims = var_1523_keep_dims_0, x = var_1521_cast_fp16)[name = string("op_1523_cast_fp16")];
            tensor<int32, [3]> var_1526_begin_0 = const()[name = string("op_1526_begin_0"), val = tensor<int32, [3]>([0, 0, 6])];
            tensor<int32, [3]> var_1526_end_0 = const()[name = string("op_1526_end_0"), val = tensor<int32, [3]>([1, 16, 7])];
            tensor<bool, [3]> var_1526_end_mask_0 = const()[name = string("op_1526_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1526_squeeze_mask_0 = const()[name = string("op_1526_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1526_cast_fp16 = slice_by_index(begin = var_1526_begin_0, end = var_1526_end_0, end_mask = var_1526_end_mask_0, squeeze_mask = var_1526_squeeze_mask_0, x = decay_3_cast_fp16)[name = string("op_1526_cast_fp16")];
            tensor<fp16, [1, 16]> var_1527_cast_fp16 = exp(x = var_1526_cast_fp16)[name = string("op_1527_cast_fp16")];
            tensor<int32, [1]> var_1528_axes_0 = const()[name = string("op_1528_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1528_cast_fp16 = expand_dims(axes = var_1528_axes_0, x = var_1527_cast_fp16)[name = string("op_1528_cast_fp16")];
            tensor<int32, [1]> var_1529_axes_0 = const()[name = string("op_1529_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_1529_cast_fp16 = expand_dims(axes = var_1529_axes_0, x = var_1528_cast_fp16)[name = string("op_1529_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_89_cast_fp16 = mul(x = state_87_cast_fp16, y = var_1529_cast_fp16)[name = string("state_89_cast_fp16")];
            tensor<int32, [4]> key_token_45_begin_0 = const()[name = string("key_token_45_begin_0"), val = tensor<int32, [4]>([0, 0, 6, 0])];
            tensor<int32, [4]> key_token_45_end_0 = const()[name = string("key_token_45_end_0"), val = tensor<int32, [4]>([1, 16, 7, 128])];
            tensor<bool, [4]> key_token_45_end_mask_0 = const()[name = string("key_token_45_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_45_squeeze_mask_0 = const()[name = string("key_token_45_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_45_cast_fp16 = slice_by_index(begin = key_token_45_begin_0, end = key_token_45_end_0, end_mask = key_token_45_end_mask_0, squeeze_mask = key_token_45_squeeze_mask_0, x = key_7_cast_fp16)[name = string("key_token_45_cast_fp16")];
            tensor<int32, [4]> value_token_45_begin_0 = const()[name = string("value_token_45_begin_0"), val = tensor<int32, [4]>([0, 0, 6, 0])];
            tensor<int32, [4]> value_token_45_end_0 = const()[name = string("value_token_45_end_0"), val = tensor<int32, [4]>([1, 16, 7, 128])];
            tensor<bool, [4]> value_token_45_end_mask_0 = const()[name = string("value_token_45_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_45_squeeze_mask_0 = const()[name = string("value_token_45_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_45_cast_fp16 = slice_by_index(begin = value_token_45_begin_0, end = value_token_45_end_0, end_mask = value_token_45_end_mask_0, squeeze_mask = value_token_45_squeeze_mask_0, x = value_27_cast_fp16)[name = string("value_token_45_cast_fp16")];
            tensor<int32, [1]> var_1537_axes_0 = const()[name = string("op_1537_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1537_cast_fp16 = expand_dims(axes = var_1537_axes_0, x = key_token_45_cast_fp16)[name = string("op_1537_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1538_cast_fp16 = mul(x = state_89_cast_fp16, y = var_1537_cast_fp16)[name = string("op_1538_cast_fp16")];
            tensor<int32, [1]> memory_45_axes_0 = const()[name = string("memory_45_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_45_keep_dims_0 = const()[name = string("memory_45_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_45_cast_fp16 = reduce_sum(axes = memory_45_axes_0, keep_dims = memory_45_keep_dims_0, x = var_1538_cast_fp16)[name = string("memory_45_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_1541_cast_fp16 = sub(x = value_token_45_cast_fp16, y = memory_45_cast_fp16)[name = string("op_1541_cast_fp16")];
            tensor<int32, [3]> var_1544_begin_0 = const()[name = string("op_1544_begin_0"), val = tensor<int32, [3]>([0, 0, 6])];
            tensor<int32, [3]> var_1544_end_0 = const()[name = string("op_1544_end_0"), val = tensor<int32, [3]>([1, 16, 7])];
            tensor<bool, [3]> var_1544_end_mask_0 = const()[name = string("op_1544_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1544_squeeze_mask_0 = const()[name = string("op_1544_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1544_cast_fp16 = slice_by_index(begin = var_1544_begin_0, end = var_1544_end_0, end_mask = var_1544_end_mask_0, squeeze_mask = var_1544_squeeze_mask_0, x = beta_3_cast_fp16)[name = string("op_1544_cast_fp16")];
            tensor<int32, [1]> var_1545_axes_0 = const()[name = string("op_1545_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1545_cast_fp16 = expand_dims(axes = var_1545_axes_0, x = var_1544_cast_fp16)[name = string("op_1545_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_45_cast_fp16 = mul(x = var_1541_cast_fp16, y = var_1545_cast_fp16)[name = string("delta_45_cast_fp16")];
            tensor<int32, [1]> var_1548_axes_0 = const()[name = string("op_1548_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_1548_cast_fp16 = expand_dims(axes = var_1548_axes_0, x = delta_45_cast_fp16)[name = string("op_1548_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1549_cast_fp16 = mul(x = var_1537_cast_fp16, y = var_1548_cast_fp16)[name = string("op_1549_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_91_cast_fp16 = add(x = state_89_cast_fp16, y = var_1549_cast_fp16)[name = string("state_91_cast_fp16")];
            tensor<int32, [4]> var_1553_begin_0 = const()[name = string("op_1553_begin_0"), val = tensor<int32, [4]>([0, 0, 6, 0])];
            tensor<int32, [4]> var_1553_end_0 = const()[name = string("op_1553_end_0"), val = tensor<int32, [4]>([1, 16, 7, 128])];
            tensor<bool, [4]> var_1553_end_mask_0 = const()[name = string("op_1553_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_1553_squeeze_mask_0 = const()[name = string("op_1553_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_1553_cast_fp16 = slice_by_index(begin = var_1553_begin_0, end = var_1553_end_0, end_mask = var_1553_end_mask_0, squeeze_mask = var_1553_squeeze_mask_0, x = _inversed_query_7_cast_fp16)[name = string("op_1553_cast_fp16")];
            tensor<int32, [1]> var_1554_axes_0 = const()[name = string("op_1554_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1554_cast_fp16 = expand_dims(axes = var_1554_axes_0, x = var_1553_cast_fp16)[name = string("op_1554_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1555_cast_fp16 = mul(x = state_91_cast_fp16, y = var_1554_cast_fp16)[name = string("op_1555_cast_fp16")];
            tensor<int32, [1]> var_1557_axes_0 = const()[name = string("op_1557_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_1557_keep_dims_0 = const()[name = string("op_1557_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_1557_cast_fp16 = reduce_sum(axes = var_1557_axes_0, keep_dims = var_1557_keep_dims_0, x = var_1555_cast_fp16)[name = string("op_1557_cast_fp16")];
            tensor<int32, [3]> var_1560_begin_0 = const()[name = string("op_1560_begin_0"), val = tensor<int32, [3]>([0, 0, 7])];
            tensor<int32, [3]> var_1560_end_0 = const()[name = string("op_1560_end_0"), val = tensor<int32, [3]>([1, 16, 8])];
            tensor<bool, [3]> var_1560_end_mask_0 = const()[name = string("op_1560_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1560_squeeze_mask_0 = const()[name = string("op_1560_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1560_cast_fp16 = slice_by_index(begin = var_1560_begin_0, end = var_1560_end_0, end_mask = var_1560_end_mask_0, squeeze_mask = var_1560_squeeze_mask_0, x = decay_3_cast_fp16)[name = string("op_1560_cast_fp16")];
            tensor<fp16, [1, 16]> var_1561_cast_fp16 = exp(x = var_1560_cast_fp16)[name = string("op_1561_cast_fp16")];
            tensor<int32, [1]> var_1562_axes_0 = const()[name = string("op_1562_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1562_cast_fp16 = expand_dims(axes = var_1562_axes_0, x = var_1561_cast_fp16)[name = string("op_1562_cast_fp16")];
            tensor<int32, [1]> var_1563_axes_0 = const()[name = string("op_1563_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_1563_cast_fp16 = expand_dims(axes = var_1563_axes_0, x = var_1562_cast_fp16)[name = string("op_1563_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_93_cast_fp16 = mul(x = state_91_cast_fp16, y = var_1563_cast_fp16)[name = string("state_93_cast_fp16")];
            tensor<int32, [4]> key_token_47_begin_0 = const()[name = string("key_token_47_begin_0"), val = tensor<int32, [4]>([0, 0, 7, 0])];
            tensor<int32, [4]> key_token_47_end_0 = const()[name = string("key_token_47_end_0"), val = tensor<int32, [4]>([1, 16, 8, 128])];
            tensor<bool, [4]> key_token_47_end_mask_0 = const()[name = string("key_token_47_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_47_squeeze_mask_0 = const()[name = string("key_token_47_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_47_cast_fp16 = slice_by_index(begin = key_token_47_begin_0, end = key_token_47_end_0, end_mask = key_token_47_end_mask_0, squeeze_mask = key_token_47_squeeze_mask_0, x = key_7_cast_fp16)[name = string("key_token_47_cast_fp16")];
            tensor<int32, [4]> value_token_47_begin_0 = const()[name = string("value_token_47_begin_0"), val = tensor<int32, [4]>([0, 0, 7, 0])];
            tensor<int32, [4]> value_token_47_end_0 = const()[name = string("value_token_47_end_0"), val = tensor<int32, [4]>([1, 16, 8, 128])];
            tensor<bool, [4]> value_token_47_end_mask_0 = const()[name = string("value_token_47_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_47_squeeze_mask_0 = const()[name = string("value_token_47_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_47_cast_fp16 = slice_by_index(begin = value_token_47_begin_0, end = value_token_47_end_0, end_mask = value_token_47_end_mask_0, squeeze_mask = value_token_47_squeeze_mask_0, x = value_27_cast_fp16)[name = string("value_token_47_cast_fp16")];
            tensor<int32, [1]> var_1571_axes_0 = const()[name = string("op_1571_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1571_cast_fp16 = expand_dims(axes = var_1571_axes_0, x = key_token_47_cast_fp16)[name = string("op_1571_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1572_cast_fp16 = mul(x = state_93_cast_fp16, y = var_1571_cast_fp16)[name = string("op_1572_cast_fp16")];
            tensor<int32, [1]> memory_47_axes_0 = const()[name = string("memory_47_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_47_keep_dims_0 = const()[name = string("memory_47_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_47_cast_fp16 = reduce_sum(axes = memory_47_axes_0, keep_dims = memory_47_keep_dims_0, x = var_1572_cast_fp16)[name = string("memory_47_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_1575_cast_fp16 = sub(x = value_token_47_cast_fp16, y = memory_47_cast_fp16)[name = string("op_1575_cast_fp16")];
            tensor<int32, [3]> var_1578_begin_0 = const()[name = string("op_1578_begin_0"), val = tensor<int32, [3]>([0, 0, 7])];
            tensor<int32, [3]> var_1578_end_0 = const()[name = string("op_1578_end_0"), val = tensor<int32, [3]>([1, 16, 8])];
            tensor<bool, [3]> var_1578_end_mask_0 = const()[name = string("op_1578_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1578_squeeze_mask_0 = const()[name = string("op_1578_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1578_cast_fp16 = slice_by_index(begin = var_1578_begin_0, end = var_1578_end_0, end_mask = var_1578_end_mask_0, squeeze_mask = var_1578_squeeze_mask_0, x = beta_3_cast_fp16)[name = string("op_1578_cast_fp16")];
            tensor<int32, [1]> var_1579_axes_0 = const()[name = string("op_1579_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1579_cast_fp16 = expand_dims(axes = var_1579_axes_0, x = var_1578_cast_fp16)[name = string("op_1579_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_47_cast_fp16 = mul(x = var_1575_cast_fp16, y = var_1579_cast_fp16)[name = string("delta_47_cast_fp16")];
            tensor<int32, [1]> var_1582_axes_0 = const()[name = string("op_1582_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_1582_cast_fp16 = expand_dims(axes = var_1582_axes_0, x = delta_47_cast_fp16)[name = string("op_1582_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1583_cast_fp16 = mul(x = var_1571_cast_fp16, y = var_1582_cast_fp16)[name = string("op_1583_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_95_cast_fp16 = add(x = state_93_cast_fp16, y = var_1583_cast_fp16)[name = string("state_95_cast_fp16")];
            tensor<int32, [4]> var_1587_begin_0 = const()[name = string("op_1587_begin_0"), val = tensor<int32, [4]>([0, 0, 7, 0])];
            tensor<int32, [4]> var_1587_end_0 = const()[name = string("op_1587_end_0"), val = tensor<int32, [4]>([1, 16, 8, 128])];
            tensor<bool, [4]> var_1587_end_mask_0 = const()[name = string("op_1587_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_1587_squeeze_mask_0 = const()[name = string("op_1587_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_1587_cast_fp16 = slice_by_index(begin = var_1587_begin_0, end = var_1587_end_0, end_mask = var_1587_end_mask_0, squeeze_mask = var_1587_squeeze_mask_0, x = _inversed_query_7_cast_fp16)[name = string("op_1587_cast_fp16")];
            tensor<int32, [1]> var_1588_axes_0 = const()[name = string("op_1588_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1588_cast_fp16 = expand_dims(axes = var_1588_axes_0, x = var_1587_cast_fp16)[name = string("op_1588_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1589_cast_fp16 = mul(x = state_95_cast_fp16, y = var_1588_cast_fp16)[name = string("op_1589_cast_fp16")];
            tensor<int32, [1]> var_1591_axes_0 = const()[name = string("op_1591_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_1591_keep_dims_0 = const()[name = string("op_1591_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_1591_cast_fp16 = reduce_sum(axes = var_1591_axes_0, keep_dims = var_1591_keep_dims_0, x = var_1589_cast_fp16)[name = string("op_1591_cast_fp16")];
            tensor<int32, [3]> var_1594_begin_0 = const()[name = string("op_1594_begin_0"), val = tensor<int32, [3]>([0, 0, 8])];
            tensor<int32, [3]> var_1594_end_0 = const()[name = string("op_1594_end_0"), val = tensor<int32, [3]>([1, 16, 9])];
            tensor<bool, [3]> var_1594_end_mask_0 = const()[name = string("op_1594_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1594_squeeze_mask_0 = const()[name = string("op_1594_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1594_cast_fp16 = slice_by_index(begin = var_1594_begin_0, end = var_1594_end_0, end_mask = var_1594_end_mask_0, squeeze_mask = var_1594_squeeze_mask_0, x = decay_3_cast_fp16)[name = string("op_1594_cast_fp16")];
            tensor<fp16, [1, 16]> var_1595_cast_fp16 = exp(x = var_1594_cast_fp16)[name = string("op_1595_cast_fp16")];
            tensor<int32, [1]> var_1596_axes_0 = const()[name = string("op_1596_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1596_cast_fp16 = expand_dims(axes = var_1596_axes_0, x = var_1595_cast_fp16)[name = string("op_1596_cast_fp16")];
            tensor<int32, [1]> var_1597_axes_0 = const()[name = string("op_1597_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_1597_cast_fp16 = expand_dims(axes = var_1597_axes_0, x = var_1596_cast_fp16)[name = string("op_1597_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_97_cast_fp16 = mul(x = state_95_cast_fp16, y = var_1597_cast_fp16)[name = string("state_97_cast_fp16")];
            tensor<int32, [4]> key_token_49_begin_0 = const()[name = string("key_token_49_begin_0"), val = tensor<int32, [4]>([0, 0, 8, 0])];
            tensor<int32, [4]> key_token_49_end_0 = const()[name = string("key_token_49_end_0"), val = tensor<int32, [4]>([1, 16, 9, 128])];
            tensor<bool, [4]> key_token_49_end_mask_0 = const()[name = string("key_token_49_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_49_squeeze_mask_0 = const()[name = string("key_token_49_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_49_cast_fp16 = slice_by_index(begin = key_token_49_begin_0, end = key_token_49_end_0, end_mask = key_token_49_end_mask_0, squeeze_mask = key_token_49_squeeze_mask_0, x = key_7_cast_fp16)[name = string("key_token_49_cast_fp16")];
            tensor<int32, [4]> value_token_49_begin_0 = const()[name = string("value_token_49_begin_0"), val = tensor<int32, [4]>([0, 0, 8, 0])];
            tensor<int32, [4]> value_token_49_end_0 = const()[name = string("value_token_49_end_0"), val = tensor<int32, [4]>([1, 16, 9, 128])];
            tensor<bool, [4]> value_token_49_end_mask_0 = const()[name = string("value_token_49_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_49_squeeze_mask_0 = const()[name = string("value_token_49_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_49_cast_fp16 = slice_by_index(begin = value_token_49_begin_0, end = value_token_49_end_0, end_mask = value_token_49_end_mask_0, squeeze_mask = value_token_49_squeeze_mask_0, x = value_27_cast_fp16)[name = string("value_token_49_cast_fp16")];
            tensor<int32, [1]> var_1605_axes_0 = const()[name = string("op_1605_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1605_cast_fp16 = expand_dims(axes = var_1605_axes_0, x = key_token_49_cast_fp16)[name = string("op_1605_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1606_cast_fp16 = mul(x = state_97_cast_fp16, y = var_1605_cast_fp16)[name = string("op_1606_cast_fp16")];
            tensor<int32, [1]> memory_49_axes_0 = const()[name = string("memory_49_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_49_keep_dims_0 = const()[name = string("memory_49_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_49_cast_fp16 = reduce_sum(axes = memory_49_axes_0, keep_dims = memory_49_keep_dims_0, x = var_1606_cast_fp16)[name = string("memory_49_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_1609_cast_fp16 = sub(x = value_token_49_cast_fp16, y = memory_49_cast_fp16)[name = string("op_1609_cast_fp16")];
            tensor<int32, [3]> var_1612_begin_0 = const()[name = string("op_1612_begin_0"), val = tensor<int32, [3]>([0, 0, 8])];
            tensor<int32, [3]> var_1612_end_0 = const()[name = string("op_1612_end_0"), val = tensor<int32, [3]>([1, 16, 9])];
            tensor<bool, [3]> var_1612_end_mask_0 = const()[name = string("op_1612_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1612_squeeze_mask_0 = const()[name = string("op_1612_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1612_cast_fp16 = slice_by_index(begin = var_1612_begin_0, end = var_1612_end_0, end_mask = var_1612_end_mask_0, squeeze_mask = var_1612_squeeze_mask_0, x = beta_3_cast_fp16)[name = string("op_1612_cast_fp16")];
            tensor<int32, [1]> var_1613_axes_0 = const()[name = string("op_1613_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1613_cast_fp16 = expand_dims(axes = var_1613_axes_0, x = var_1612_cast_fp16)[name = string("op_1613_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_49_cast_fp16 = mul(x = var_1609_cast_fp16, y = var_1613_cast_fp16)[name = string("delta_49_cast_fp16")];
            tensor<int32, [1]> var_1616_axes_0 = const()[name = string("op_1616_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_1616_cast_fp16 = expand_dims(axes = var_1616_axes_0, x = delta_49_cast_fp16)[name = string("op_1616_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1617_cast_fp16 = mul(x = var_1605_cast_fp16, y = var_1616_cast_fp16)[name = string("op_1617_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_99_cast_fp16 = add(x = state_97_cast_fp16, y = var_1617_cast_fp16)[name = string("state_99_cast_fp16")];
            tensor<int32, [4]> var_1621_begin_0 = const()[name = string("op_1621_begin_0"), val = tensor<int32, [4]>([0, 0, 8, 0])];
            tensor<int32, [4]> var_1621_end_0 = const()[name = string("op_1621_end_0"), val = tensor<int32, [4]>([1, 16, 9, 128])];
            tensor<bool, [4]> var_1621_end_mask_0 = const()[name = string("op_1621_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_1621_squeeze_mask_0 = const()[name = string("op_1621_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_1621_cast_fp16 = slice_by_index(begin = var_1621_begin_0, end = var_1621_end_0, end_mask = var_1621_end_mask_0, squeeze_mask = var_1621_squeeze_mask_0, x = _inversed_query_7_cast_fp16)[name = string("op_1621_cast_fp16")];
            tensor<int32, [1]> var_1622_axes_0 = const()[name = string("op_1622_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1622_cast_fp16 = expand_dims(axes = var_1622_axes_0, x = var_1621_cast_fp16)[name = string("op_1622_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1623_cast_fp16 = mul(x = state_99_cast_fp16, y = var_1622_cast_fp16)[name = string("op_1623_cast_fp16")];
            tensor<int32, [1]> var_1625_axes_0 = const()[name = string("op_1625_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_1625_keep_dims_0 = const()[name = string("op_1625_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_1625_cast_fp16 = reduce_sum(axes = var_1625_axes_0, keep_dims = var_1625_keep_dims_0, x = var_1623_cast_fp16)[name = string("op_1625_cast_fp16")];
            tensor<int32, [3]> var_1628_begin_0 = const()[name = string("op_1628_begin_0"), val = tensor<int32, [3]>([0, 0, 9])];
            tensor<int32, [3]> var_1628_end_0 = const()[name = string("op_1628_end_0"), val = tensor<int32, [3]>([1, 16, 10])];
            tensor<bool, [3]> var_1628_end_mask_0 = const()[name = string("op_1628_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1628_squeeze_mask_0 = const()[name = string("op_1628_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1628_cast_fp16 = slice_by_index(begin = var_1628_begin_0, end = var_1628_end_0, end_mask = var_1628_end_mask_0, squeeze_mask = var_1628_squeeze_mask_0, x = decay_3_cast_fp16)[name = string("op_1628_cast_fp16")];
            tensor<fp16, [1, 16]> var_1629_cast_fp16 = exp(x = var_1628_cast_fp16)[name = string("op_1629_cast_fp16")];
            tensor<int32, [1]> var_1630_axes_0 = const()[name = string("op_1630_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1630_cast_fp16 = expand_dims(axes = var_1630_axes_0, x = var_1629_cast_fp16)[name = string("op_1630_cast_fp16")];
            tensor<int32, [1]> var_1631_axes_0 = const()[name = string("op_1631_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_1631_cast_fp16 = expand_dims(axes = var_1631_axes_0, x = var_1630_cast_fp16)[name = string("op_1631_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_101_cast_fp16 = mul(x = state_99_cast_fp16, y = var_1631_cast_fp16)[name = string("state_101_cast_fp16")];
            tensor<int32, [4]> key_token_51_begin_0 = const()[name = string("key_token_51_begin_0"), val = tensor<int32, [4]>([0, 0, 9, 0])];
            tensor<int32, [4]> key_token_51_end_0 = const()[name = string("key_token_51_end_0"), val = tensor<int32, [4]>([1, 16, 10, 128])];
            tensor<bool, [4]> key_token_51_end_mask_0 = const()[name = string("key_token_51_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_51_squeeze_mask_0 = const()[name = string("key_token_51_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_51_cast_fp16 = slice_by_index(begin = key_token_51_begin_0, end = key_token_51_end_0, end_mask = key_token_51_end_mask_0, squeeze_mask = key_token_51_squeeze_mask_0, x = key_7_cast_fp16)[name = string("key_token_51_cast_fp16")];
            tensor<int32, [4]> value_token_51_begin_0 = const()[name = string("value_token_51_begin_0"), val = tensor<int32, [4]>([0, 0, 9, 0])];
            tensor<int32, [4]> value_token_51_end_0 = const()[name = string("value_token_51_end_0"), val = tensor<int32, [4]>([1, 16, 10, 128])];
            tensor<bool, [4]> value_token_51_end_mask_0 = const()[name = string("value_token_51_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_51_squeeze_mask_0 = const()[name = string("value_token_51_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_51_cast_fp16 = slice_by_index(begin = value_token_51_begin_0, end = value_token_51_end_0, end_mask = value_token_51_end_mask_0, squeeze_mask = value_token_51_squeeze_mask_0, x = value_27_cast_fp16)[name = string("value_token_51_cast_fp16")];
            tensor<int32, [1]> var_1639_axes_0 = const()[name = string("op_1639_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1639_cast_fp16 = expand_dims(axes = var_1639_axes_0, x = key_token_51_cast_fp16)[name = string("op_1639_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1640_cast_fp16 = mul(x = state_101_cast_fp16, y = var_1639_cast_fp16)[name = string("op_1640_cast_fp16")];
            tensor<int32, [1]> memory_51_axes_0 = const()[name = string("memory_51_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_51_keep_dims_0 = const()[name = string("memory_51_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_51_cast_fp16 = reduce_sum(axes = memory_51_axes_0, keep_dims = memory_51_keep_dims_0, x = var_1640_cast_fp16)[name = string("memory_51_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_1643_cast_fp16 = sub(x = value_token_51_cast_fp16, y = memory_51_cast_fp16)[name = string("op_1643_cast_fp16")];
            tensor<int32, [3]> var_1646_begin_0 = const()[name = string("op_1646_begin_0"), val = tensor<int32, [3]>([0, 0, 9])];
            tensor<int32, [3]> var_1646_end_0 = const()[name = string("op_1646_end_0"), val = tensor<int32, [3]>([1, 16, 10])];
            tensor<bool, [3]> var_1646_end_mask_0 = const()[name = string("op_1646_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1646_squeeze_mask_0 = const()[name = string("op_1646_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1646_cast_fp16 = slice_by_index(begin = var_1646_begin_0, end = var_1646_end_0, end_mask = var_1646_end_mask_0, squeeze_mask = var_1646_squeeze_mask_0, x = beta_3_cast_fp16)[name = string("op_1646_cast_fp16")];
            tensor<int32, [1]> var_1647_axes_0 = const()[name = string("op_1647_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1647_cast_fp16 = expand_dims(axes = var_1647_axes_0, x = var_1646_cast_fp16)[name = string("op_1647_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_51_cast_fp16 = mul(x = var_1643_cast_fp16, y = var_1647_cast_fp16)[name = string("delta_51_cast_fp16")];
            tensor<int32, [1]> var_1650_axes_0 = const()[name = string("op_1650_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_1650_cast_fp16 = expand_dims(axes = var_1650_axes_0, x = delta_51_cast_fp16)[name = string("op_1650_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1651_cast_fp16 = mul(x = var_1639_cast_fp16, y = var_1650_cast_fp16)[name = string("op_1651_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_103_cast_fp16 = add(x = state_101_cast_fp16, y = var_1651_cast_fp16)[name = string("state_103_cast_fp16")];
            tensor<int32, [4]> var_1655_begin_0 = const()[name = string("op_1655_begin_0"), val = tensor<int32, [4]>([0, 0, 9, 0])];
            tensor<int32, [4]> var_1655_end_0 = const()[name = string("op_1655_end_0"), val = tensor<int32, [4]>([1, 16, 10, 128])];
            tensor<bool, [4]> var_1655_end_mask_0 = const()[name = string("op_1655_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_1655_squeeze_mask_0 = const()[name = string("op_1655_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_1655_cast_fp16 = slice_by_index(begin = var_1655_begin_0, end = var_1655_end_0, end_mask = var_1655_end_mask_0, squeeze_mask = var_1655_squeeze_mask_0, x = _inversed_query_7_cast_fp16)[name = string("op_1655_cast_fp16")];
            tensor<int32, [1]> var_1656_axes_0 = const()[name = string("op_1656_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1656_cast_fp16 = expand_dims(axes = var_1656_axes_0, x = var_1655_cast_fp16)[name = string("op_1656_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1657_cast_fp16 = mul(x = state_103_cast_fp16, y = var_1656_cast_fp16)[name = string("op_1657_cast_fp16")];
            tensor<int32, [1]> var_1659_axes_0 = const()[name = string("op_1659_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_1659_keep_dims_0 = const()[name = string("op_1659_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_1659_cast_fp16 = reduce_sum(axes = var_1659_axes_0, keep_dims = var_1659_keep_dims_0, x = var_1657_cast_fp16)[name = string("op_1659_cast_fp16")];
            tensor<int32, [3]> var_1662_begin_0 = const()[name = string("op_1662_begin_0"), val = tensor<int32, [3]>([0, 0, 10])];
            tensor<int32, [3]> var_1662_end_0 = const()[name = string("op_1662_end_0"), val = tensor<int32, [3]>([1, 16, 11])];
            tensor<bool, [3]> var_1662_end_mask_0 = const()[name = string("op_1662_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1662_squeeze_mask_0 = const()[name = string("op_1662_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1662_cast_fp16 = slice_by_index(begin = var_1662_begin_0, end = var_1662_end_0, end_mask = var_1662_end_mask_0, squeeze_mask = var_1662_squeeze_mask_0, x = decay_3_cast_fp16)[name = string("op_1662_cast_fp16")];
            tensor<fp16, [1, 16]> var_1663_cast_fp16 = exp(x = var_1662_cast_fp16)[name = string("op_1663_cast_fp16")];
            tensor<int32, [1]> var_1664_axes_0 = const()[name = string("op_1664_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1664_cast_fp16 = expand_dims(axes = var_1664_axes_0, x = var_1663_cast_fp16)[name = string("op_1664_cast_fp16")];
            tensor<int32, [1]> var_1665_axes_0 = const()[name = string("op_1665_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_1665_cast_fp16 = expand_dims(axes = var_1665_axes_0, x = var_1664_cast_fp16)[name = string("op_1665_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_105_cast_fp16 = mul(x = state_103_cast_fp16, y = var_1665_cast_fp16)[name = string("state_105_cast_fp16")];
            tensor<int32, [4]> key_token_53_begin_0 = const()[name = string("key_token_53_begin_0"), val = tensor<int32, [4]>([0, 0, 10, 0])];
            tensor<int32, [4]> key_token_53_end_0 = const()[name = string("key_token_53_end_0"), val = tensor<int32, [4]>([1, 16, 11, 128])];
            tensor<bool, [4]> key_token_53_end_mask_0 = const()[name = string("key_token_53_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_53_squeeze_mask_0 = const()[name = string("key_token_53_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_53_cast_fp16 = slice_by_index(begin = key_token_53_begin_0, end = key_token_53_end_0, end_mask = key_token_53_end_mask_0, squeeze_mask = key_token_53_squeeze_mask_0, x = key_7_cast_fp16)[name = string("key_token_53_cast_fp16")];
            tensor<int32, [4]> value_token_53_begin_0 = const()[name = string("value_token_53_begin_0"), val = tensor<int32, [4]>([0, 0, 10, 0])];
            tensor<int32, [4]> value_token_53_end_0 = const()[name = string("value_token_53_end_0"), val = tensor<int32, [4]>([1, 16, 11, 128])];
            tensor<bool, [4]> value_token_53_end_mask_0 = const()[name = string("value_token_53_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_53_squeeze_mask_0 = const()[name = string("value_token_53_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_53_cast_fp16 = slice_by_index(begin = value_token_53_begin_0, end = value_token_53_end_0, end_mask = value_token_53_end_mask_0, squeeze_mask = value_token_53_squeeze_mask_0, x = value_27_cast_fp16)[name = string("value_token_53_cast_fp16")];
            tensor<int32, [1]> var_1673_axes_0 = const()[name = string("op_1673_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1673_cast_fp16 = expand_dims(axes = var_1673_axes_0, x = key_token_53_cast_fp16)[name = string("op_1673_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1674_cast_fp16 = mul(x = state_105_cast_fp16, y = var_1673_cast_fp16)[name = string("op_1674_cast_fp16")];
            tensor<int32, [1]> memory_53_axes_0 = const()[name = string("memory_53_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_53_keep_dims_0 = const()[name = string("memory_53_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_53_cast_fp16 = reduce_sum(axes = memory_53_axes_0, keep_dims = memory_53_keep_dims_0, x = var_1674_cast_fp16)[name = string("memory_53_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_1677_cast_fp16 = sub(x = value_token_53_cast_fp16, y = memory_53_cast_fp16)[name = string("op_1677_cast_fp16")];
            tensor<int32, [3]> var_1680_begin_0 = const()[name = string("op_1680_begin_0"), val = tensor<int32, [3]>([0, 0, 10])];
            tensor<int32, [3]> var_1680_end_0 = const()[name = string("op_1680_end_0"), val = tensor<int32, [3]>([1, 16, 11])];
            tensor<bool, [3]> var_1680_end_mask_0 = const()[name = string("op_1680_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1680_squeeze_mask_0 = const()[name = string("op_1680_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1680_cast_fp16 = slice_by_index(begin = var_1680_begin_0, end = var_1680_end_0, end_mask = var_1680_end_mask_0, squeeze_mask = var_1680_squeeze_mask_0, x = beta_3_cast_fp16)[name = string("op_1680_cast_fp16")];
            tensor<int32, [1]> var_1681_axes_0 = const()[name = string("op_1681_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1681_cast_fp16 = expand_dims(axes = var_1681_axes_0, x = var_1680_cast_fp16)[name = string("op_1681_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_53_cast_fp16 = mul(x = var_1677_cast_fp16, y = var_1681_cast_fp16)[name = string("delta_53_cast_fp16")];
            tensor<int32, [1]> var_1684_axes_0 = const()[name = string("op_1684_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_1684_cast_fp16 = expand_dims(axes = var_1684_axes_0, x = delta_53_cast_fp16)[name = string("op_1684_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1685_cast_fp16 = mul(x = var_1673_cast_fp16, y = var_1684_cast_fp16)[name = string("op_1685_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_107_cast_fp16 = add(x = state_105_cast_fp16, y = var_1685_cast_fp16)[name = string("state_107_cast_fp16")];
            tensor<int32, [4]> var_1689_begin_0 = const()[name = string("op_1689_begin_0"), val = tensor<int32, [4]>([0, 0, 10, 0])];
            tensor<int32, [4]> var_1689_end_0 = const()[name = string("op_1689_end_0"), val = tensor<int32, [4]>([1, 16, 11, 128])];
            tensor<bool, [4]> var_1689_end_mask_0 = const()[name = string("op_1689_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_1689_squeeze_mask_0 = const()[name = string("op_1689_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_1689_cast_fp16 = slice_by_index(begin = var_1689_begin_0, end = var_1689_end_0, end_mask = var_1689_end_mask_0, squeeze_mask = var_1689_squeeze_mask_0, x = _inversed_query_7_cast_fp16)[name = string("op_1689_cast_fp16")];
            tensor<int32, [1]> var_1690_axes_0 = const()[name = string("op_1690_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1690_cast_fp16 = expand_dims(axes = var_1690_axes_0, x = var_1689_cast_fp16)[name = string("op_1690_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1691_cast_fp16 = mul(x = state_107_cast_fp16, y = var_1690_cast_fp16)[name = string("op_1691_cast_fp16")];
            tensor<int32, [1]> var_1693_axes_0 = const()[name = string("op_1693_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_1693_keep_dims_0 = const()[name = string("op_1693_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_1693_cast_fp16 = reduce_sum(axes = var_1693_axes_0, keep_dims = var_1693_keep_dims_0, x = var_1691_cast_fp16)[name = string("op_1693_cast_fp16")];
            tensor<int32, [3]> var_1696_begin_0 = const()[name = string("op_1696_begin_0"), val = tensor<int32, [3]>([0, 0, 11])];
            tensor<int32, [3]> var_1696_end_0 = const()[name = string("op_1696_end_0"), val = tensor<int32, [3]>([1, 16, 12])];
            tensor<bool, [3]> var_1696_end_mask_0 = const()[name = string("op_1696_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1696_squeeze_mask_0 = const()[name = string("op_1696_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1696_cast_fp16 = slice_by_index(begin = var_1696_begin_0, end = var_1696_end_0, end_mask = var_1696_end_mask_0, squeeze_mask = var_1696_squeeze_mask_0, x = decay_3_cast_fp16)[name = string("op_1696_cast_fp16")];
            tensor<fp16, [1, 16]> var_1697_cast_fp16 = exp(x = var_1696_cast_fp16)[name = string("op_1697_cast_fp16")];
            tensor<int32, [1]> var_1698_axes_0 = const()[name = string("op_1698_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1698_cast_fp16 = expand_dims(axes = var_1698_axes_0, x = var_1697_cast_fp16)[name = string("op_1698_cast_fp16")];
            tensor<int32, [1]> var_1699_axes_0 = const()[name = string("op_1699_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_1699_cast_fp16 = expand_dims(axes = var_1699_axes_0, x = var_1698_cast_fp16)[name = string("op_1699_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_109_cast_fp16 = mul(x = state_107_cast_fp16, y = var_1699_cast_fp16)[name = string("state_109_cast_fp16")];
            tensor<int32, [4]> key_token_55_begin_0 = const()[name = string("key_token_55_begin_0"), val = tensor<int32, [4]>([0, 0, 11, 0])];
            tensor<int32, [4]> key_token_55_end_0 = const()[name = string("key_token_55_end_0"), val = tensor<int32, [4]>([1, 16, 12, 128])];
            tensor<bool, [4]> key_token_55_end_mask_0 = const()[name = string("key_token_55_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_55_squeeze_mask_0 = const()[name = string("key_token_55_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_55_cast_fp16 = slice_by_index(begin = key_token_55_begin_0, end = key_token_55_end_0, end_mask = key_token_55_end_mask_0, squeeze_mask = key_token_55_squeeze_mask_0, x = key_7_cast_fp16)[name = string("key_token_55_cast_fp16")];
            tensor<int32, [4]> value_token_55_begin_0 = const()[name = string("value_token_55_begin_0"), val = tensor<int32, [4]>([0, 0, 11, 0])];
            tensor<int32, [4]> value_token_55_end_0 = const()[name = string("value_token_55_end_0"), val = tensor<int32, [4]>([1, 16, 12, 128])];
            tensor<bool, [4]> value_token_55_end_mask_0 = const()[name = string("value_token_55_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_55_squeeze_mask_0 = const()[name = string("value_token_55_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_55_cast_fp16 = slice_by_index(begin = value_token_55_begin_0, end = value_token_55_end_0, end_mask = value_token_55_end_mask_0, squeeze_mask = value_token_55_squeeze_mask_0, x = value_27_cast_fp16)[name = string("value_token_55_cast_fp16")];
            tensor<int32, [1]> var_1707_axes_0 = const()[name = string("op_1707_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1707_cast_fp16 = expand_dims(axes = var_1707_axes_0, x = key_token_55_cast_fp16)[name = string("op_1707_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1708_cast_fp16 = mul(x = state_109_cast_fp16, y = var_1707_cast_fp16)[name = string("op_1708_cast_fp16")];
            tensor<int32, [1]> memory_55_axes_0 = const()[name = string("memory_55_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_55_keep_dims_0 = const()[name = string("memory_55_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_55_cast_fp16 = reduce_sum(axes = memory_55_axes_0, keep_dims = memory_55_keep_dims_0, x = var_1708_cast_fp16)[name = string("memory_55_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_1711_cast_fp16 = sub(x = value_token_55_cast_fp16, y = memory_55_cast_fp16)[name = string("op_1711_cast_fp16")];
            tensor<int32, [3]> var_1714_begin_0 = const()[name = string("op_1714_begin_0"), val = tensor<int32, [3]>([0, 0, 11])];
            tensor<int32, [3]> var_1714_end_0 = const()[name = string("op_1714_end_0"), val = tensor<int32, [3]>([1, 16, 12])];
            tensor<bool, [3]> var_1714_end_mask_0 = const()[name = string("op_1714_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1714_squeeze_mask_0 = const()[name = string("op_1714_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1714_cast_fp16 = slice_by_index(begin = var_1714_begin_0, end = var_1714_end_0, end_mask = var_1714_end_mask_0, squeeze_mask = var_1714_squeeze_mask_0, x = beta_3_cast_fp16)[name = string("op_1714_cast_fp16")];
            tensor<int32, [1]> var_1715_axes_0 = const()[name = string("op_1715_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1715_cast_fp16 = expand_dims(axes = var_1715_axes_0, x = var_1714_cast_fp16)[name = string("op_1715_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_55_cast_fp16 = mul(x = var_1711_cast_fp16, y = var_1715_cast_fp16)[name = string("delta_55_cast_fp16")];
            tensor<int32, [1]> var_1718_axes_0 = const()[name = string("op_1718_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_1718_cast_fp16 = expand_dims(axes = var_1718_axes_0, x = delta_55_cast_fp16)[name = string("op_1718_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1719_cast_fp16 = mul(x = var_1707_cast_fp16, y = var_1718_cast_fp16)[name = string("op_1719_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_111_cast_fp16 = add(x = state_109_cast_fp16, y = var_1719_cast_fp16)[name = string("state_111_cast_fp16")];
            tensor<int32, [4]> var_1723_begin_0 = const()[name = string("op_1723_begin_0"), val = tensor<int32, [4]>([0, 0, 11, 0])];
            tensor<int32, [4]> var_1723_end_0 = const()[name = string("op_1723_end_0"), val = tensor<int32, [4]>([1, 16, 12, 128])];
            tensor<bool, [4]> var_1723_end_mask_0 = const()[name = string("op_1723_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_1723_squeeze_mask_0 = const()[name = string("op_1723_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_1723_cast_fp16 = slice_by_index(begin = var_1723_begin_0, end = var_1723_end_0, end_mask = var_1723_end_mask_0, squeeze_mask = var_1723_squeeze_mask_0, x = _inversed_query_7_cast_fp16)[name = string("op_1723_cast_fp16")];
            tensor<int32, [1]> var_1724_axes_0 = const()[name = string("op_1724_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1724_cast_fp16 = expand_dims(axes = var_1724_axes_0, x = var_1723_cast_fp16)[name = string("op_1724_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1725_cast_fp16 = mul(x = state_111_cast_fp16, y = var_1724_cast_fp16)[name = string("op_1725_cast_fp16")];
            tensor<int32, [1]> var_1727_axes_0 = const()[name = string("op_1727_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_1727_keep_dims_0 = const()[name = string("op_1727_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_1727_cast_fp16 = reduce_sum(axes = var_1727_axes_0, keep_dims = var_1727_keep_dims_0, x = var_1725_cast_fp16)[name = string("op_1727_cast_fp16")];
            tensor<int32, [3]> var_1730_begin_0 = const()[name = string("op_1730_begin_0"), val = tensor<int32, [3]>([0, 0, 12])];
            tensor<int32, [3]> var_1730_end_0 = const()[name = string("op_1730_end_0"), val = tensor<int32, [3]>([1, 16, 13])];
            tensor<bool, [3]> var_1730_end_mask_0 = const()[name = string("op_1730_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1730_squeeze_mask_0 = const()[name = string("op_1730_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1730_cast_fp16 = slice_by_index(begin = var_1730_begin_0, end = var_1730_end_0, end_mask = var_1730_end_mask_0, squeeze_mask = var_1730_squeeze_mask_0, x = decay_3_cast_fp16)[name = string("op_1730_cast_fp16")];
            tensor<fp16, [1, 16]> var_1731_cast_fp16 = exp(x = var_1730_cast_fp16)[name = string("op_1731_cast_fp16")];
            tensor<int32, [1]> var_1732_axes_0 = const()[name = string("op_1732_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1732_cast_fp16 = expand_dims(axes = var_1732_axes_0, x = var_1731_cast_fp16)[name = string("op_1732_cast_fp16")];
            tensor<int32, [1]> var_1733_axes_0 = const()[name = string("op_1733_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_1733_cast_fp16 = expand_dims(axes = var_1733_axes_0, x = var_1732_cast_fp16)[name = string("op_1733_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_113_cast_fp16 = mul(x = state_111_cast_fp16, y = var_1733_cast_fp16)[name = string("state_113_cast_fp16")];
            tensor<int32, [4]> key_token_57_begin_0 = const()[name = string("key_token_57_begin_0"), val = tensor<int32, [4]>([0, 0, 12, 0])];
            tensor<int32, [4]> key_token_57_end_0 = const()[name = string("key_token_57_end_0"), val = tensor<int32, [4]>([1, 16, 13, 128])];
            tensor<bool, [4]> key_token_57_end_mask_0 = const()[name = string("key_token_57_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_57_squeeze_mask_0 = const()[name = string("key_token_57_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_57_cast_fp16 = slice_by_index(begin = key_token_57_begin_0, end = key_token_57_end_0, end_mask = key_token_57_end_mask_0, squeeze_mask = key_token_57_squeeze_mask_0, x = key_7_cast_fp16)[name = string("key_token_57_cast_fp16")];
            tensor<int32, [4]> value_token_57_begin_0 = const()[name = string("value_token_57_begin_0"), val = tensor<int32, [4]>([0, 0, 12, 0])];
            tensor<int32, [4]> value_token_57_end_0 = const()[name = string("value_token_57_end_0"), val = tensor<int32, [4]>([1, 16, 13, 128])];
            tensor<bool, [4]> value_token_57_end_mask_0 = const()[name = string("value_token_57_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_57_squeeze_mask_0 = const()[name = string("value_token_57_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_57_cast_fp16 = slice_by_index(begin = value_token_57_begin_0, end = value_token_57_end_0, end_mask = value_token_57_end_mask_0, squeeze_mask = value_token_57_squeeze_mask_0, x = value_27_cast_fp16)[name = string("value_token_57_cast_fp16")];
            tensor<int32, [1]> var_1741_axes_0 = const()[name = string("op_1741_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1741_cast_fp16 = expand_dims(axes = var_1741_axes_0, x = key_token_57_cast_fp16)[name = string("op_1741_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1742_cast_fp16 = mul(x = state_113_cast_fp16, y = var_1741_cast_fp16)[name = string("op_1742_cast_fp16")];
            tensor<int32, [1]> memory_57_axes_0 = const()[name = string("memory_57_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_57_keep_dims_0 = const()[name = string("memory_57_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_57_cast_fp16 = reduce_sum(axes = memory_57_axes_0, keep_dims = memory_57_keep_dims_0, x = var_1742_cast_fp16)[name = string("memory_57_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_1745_cast_fp16 = sub(x = value_token_57_cast_fp16, y = memory_57_cast_fp16)[name = string("op_1745_cast_fp16")];
            tensor<int32, [3]> var_1748_begin_0 = const()[name = string("op_1748_begin_0"), val = tensor<int32, [3]>([0, 0, 12])];
            tensor<int32, [3]> var_1748_end_0 = const()[name = string("op_1748_end_0"), val = tensor<int32, [3]>([1, 16, 13])];
            tensor<bool, [3]> var_1748_end_mask_0 = const()[name = string("op_1748_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1748_squeeze_mask_0 = const()[name = string("op_1748_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1748_cast_fp16 = slice_by_index(begin = var_1748_begin_0, end = var_1748_end_0, end_mask = var_1748_end_mask_0, squeeze_mask = var_1748_squeeze_mask_0, x = beta_3_cast_fp16)[name = string("op_1748_cast_fp16")];
            tensor<int32, [1]> var_1749_axes_0 = const()[name = string("op_1749_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1749_cast_fp16 = expand_dims(axes = var_1749_axes_0, x = var_1748_cast_fp16)[name = string("op_1749_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_57_cast_fp16 = mul(x = var_1745_cast_fp16, y = var_1749_cast_fp16)[name = string("delta_57_cast_fp16")];
            tensor<int32, [1]> var_1752_axes_0 = const()[name = string("op_1752_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_1752_cast_fp16 = expand_dims(axes = var_1752_axes_0, x = delta_57_cast_fp16)[name = string("op_1752_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1753_cast_fp16 = mul(x = var_1741_cast_fp16, y = var_1752_cast_fp16)[name = string("op_1753_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_115_cast_fp16 = add(x = state_113_cast_fp16, y = var_1753_cast_fp16)[name = string("state_115_cast_fp16")];
            tensor<int32, [4]> var_1757_begin_0 = const()[name = string("op_1757_begin_0"), val = tensor<int32, [4]>([0, 0, 12, 0])];
            tensor<int32, [4]> var_1757_end_0 = const()[name = string("op_1757_end_0"), val = tensor<int32, [4]>([1, 16, 13, 128])];
            tensor<bool, [4]> var_1757_end_mask_0 = const()[name = string("op_1757_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_1757_squeeze_mask_0 = const()[name = string("op_1757_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_1757_cast_fp16 = slice_by_index(begin = var_1757_begin_0, end = var_1757_end_0, end_mask = var_1757_end_mask_0, squeeze_mask = var_1757_squeeze_mask_0, x = _inversed_query_7_cast_fp16)[name = string("op_1757_cast_fp16")];
            tensor<int32, [1]> var_1758_axes_0 = const()[name = string("op_1758_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1758_cast_fp16 = expand_dims(axes = var_1758_axes_0, x = var_1757_cast_fp16)[name = string("op_1758_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1759_cast_fp16 = mul(x = state_115_cast_fp16, y = var_1758_cast_fp16)[name = string("op_1759_cast_fp16")];
            tensor<int32, [1]> var_1761_axes_0 = const()[name = string("op_1761_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_1761_keep_dims_0 = const()[name = string("op_1761_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_1761_cast_fp16 = reduce_sum(axes = var_1761_axes_0, keep_dims = var_1761_keep_dims_0, x = var_1759_cast_fp16)[name = string("op_1761_cast_fp16")];
            tensor<int32, [3]> var_1764_begin_0 = const()[name = string("op_1764_begin_0"), val = tensor<int32, [3]>([0, 0, 13])];
            tensor<int32, [3]> var_1764_end_0 = const()[name = string("op_1764_end_0"), val = tensor<int32, [3]>([1, 16, 14])];
            tensor<bool, [3]> var_1764_end_mask_0 = const()[name = string("op_1764_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1764_squeeze_mask_0 = const()[name = string("op_1764_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1764_cast_fp16 = slice_by_index(begin = var_1764_begin_0, end = var_1764_end_0, end_mask = var_1764_end_mask_0, squeeze_mask = var_1764_squeeze_mask_0, x = decay_3_cast_fp16)[name = string("op_1764_cast_fp16")];
            tensor<fp16, [1, 16]> var_1765_cast_fp16 = exp(x = var_1764_cast_fp16)[name = string("op_1765_cast_fp16")];
            tensor<int32, [1]> var_1766_axes_0 = const()[name = string("op_1766_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1766_cast_fp16 = expand_dims(axes = var_1766_axes_0, x = var_1765_cast_fp16)[name = string("op_1766_cast_fp16")];
            tensor<int32, [1]> var_1767_axes_0 = const()[name = string("op_1767_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_1767_cast_fp16 = expand_dims(axes = var_1767_axes_0, x = var_1766_cast_fp16)[name = string("op_1767_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_117_cast_fp16 = mul(x = state_115_cast_fp16, y = var_1767_cast_fp16)[name = string("state_117_cast_fp16")];
            tensor<int32, [4]> key_token_59_begin_0 = const()[name = string("key_token_59_begin_0"), val = tensor<int32, [4]>([0, 0, 13, 0])];
            tensor<int32, [4]> key_token_59_end_0 = const()[name = string("key_token_59_end_0"), val = tensor<int32, [4]>([1, 16, 14, 128])];
            tensor<bool, [4]> key_token_59_end_mask_0 = const()[name = string("key_token_59_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_59_squeeze_mask_0 = const()[name = string("key_token_59_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_59_cast_fp16 = slice_by_index(begin = key_token_59_begin_0, end = key_token_59_end_0, end_mask = key_token_59_end_mask_0, squeeze_mask = key_token_59_squeeze_mask_0, x = key_7_cast_fp16)[name = string("key_token_59_cast_fp16")];
            tensor<int32, [4]> value_token_59_begin_0 = const()[name = string("value_token_59_begin_0"), val = tensor<int32, [4]>([0, 0, 13, 0])];
            tensor<int32, [4]> value_token_59_end_0 = const()[name = string("value_token_59_end_0"), val = tensor<int32, [4]>([1, 16, 14, 128])];
            tensor<bool, [4]> value_token_59_end_mask_0 = const()[name = string("value_token_59_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_59_squeeze_mask_0 = const()[name = string("value_token_59_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_59_cast_fp16 = slice_by_index(begin = value_token_59_begin_0, end = value_token_59_end_0, end_mask = value_token_59_end_mask_0, squeeze_mask = value_token_59_squeeze_mask_0, x = value_27_cast_fp16)[name = string("value_token_59_cast_fp16")];
            tensor<int32, [1]> var_1775_axes_0 = const()[name = string("op_1775_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1775_cast_fp16 = expand_dims(axes = var_1775_axes_0, x = key_token_59_cast_fp16)[name = string("op_1775_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1776_cast_fp16 = mul(x = state_117_cast_fp16, y = var_1775_cast_fp16)[name = string("op_1776_cast_fp16")];
            tensor<int32, [1]> memory_59_axes_0 = const()[name = string("memory_59_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_59_keep_dims_0 = const()[name = string("memory_59_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_59_cast_fp16 = reduce_sum(axes = memory_59_axes_0, keep_dims = memory_59_keep_dims_0, x = var_1776_cast_fp16)[name = string("memory_59_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_1779_cast_fp16 = sub(x = value_token_59_cast_fp16, y = memory_59_cast_fp16)[name = string("op_1779_cast_fp16")];
            tensor<int32, [3]> var_1782_begin_0 = const()[name = string("op_1782_begin_0"), val = tensor<int32, [3]>([0, 0, 13])];
            tensor<int32, [3]> var_1782_end_0 = const()[name = string("op_1782_end_0"), val = tensor<int32, [3]>([1, 16, 14])];
            tensor<bool, [3]> var_1782_end_mask_0 = const()[name = string("op_1782_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1782_squeeze_mask_0 = const()[name = string("op_1782_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1782_cast_fp16 = slice_by_index(begin = var_1782_begin_0, end = var_1782_end_0, end_mask = var_1782_end_mask_0, squeeze_mask = var_1782_squeeze_mask_0, x = beta_3_cast_fp16)[name = string("op_1782_cast_fp16")];
            tensor<int32, [1]> var_1783_axes_0 = const()[name = string("op_1783_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1783_cast_fp16 = expand_dims(axes = var_1783_axes_0, x = var_1782_cast_fp16)[name = string("op_1783_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_59_cast_fp16 = mul(x = var_1779_cast_fp16, y = var_1783_cast_fp16)[name = string("delta_59_cast_fp16")];
            tensor<int32, [1]> var_1786_axes_0 = const()[name = string("op_1786_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_1786_cast_fp16 = expand_dims(axes = var_1786_axes_0, x = delta_59_cast_fp16)[name = string("op_1786_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1787_cast_fp16 = mul(x = var_1775_cast_fp16, y = var_1786_cast_fp16)[name = string("op_1787_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_119_cast_fp16 = add(x = state_117_cast_fp16, y = var_1787_cast_fp16)[name = string("state_119_cast_fp16")];
            tensor<int32, [4]> var_1791_begin_0 = const()[name = string("op_1791_begin_0"), val = tensor<int32, [4]>([0, 0, 13, 0])];
            tensor<int32, [4]> var_1791_end_0 = const()[name = string("op_1791_end_0"), val = tensor<int32, [4]>([1, 16, 14, 128])];
            tensor<bool, [4]> var_1791_end_mask_0 = const()[name = string("op_1791_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_1791_squeeze_mask_0 = const()[name = string("op_1791_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_1791_cast_fp16 = slice_by_index(begin = var_1791_begin_0, end = var_1791_end_0, end_mask = var_1791_end_mask_0, squeeze_mask = var_1791_squeeze_mask_0, x = _inversed_query_7_cast_fp16)[name = string("op_1791_cast_fp16")];
            tensor<int32, [1]> var_1792_axes_0 = const()[name = string("op_1792_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1792_cast_fp16 = expand_dims(axes = var_1792_axes_0, x = var_1791_cast_fp16)[name = string("op_1792_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1793_cast_fp16 = mul(x = state_119_cast_fp16, y = var_1792_cast_fp16)[name = string("op_1793_cast_fp16")];
            tensor<int32, [1]> var_1795_axes_0 = const()[name = string("op_1795_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_1795_keep_dims_0 = const()[name = string("op_1795_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_1795_cast_fp16 = reduce_sum(axes = var_1795_axes_0, keep_dims = var_1795_keep_dims_0, x = var_1793_cast_fp16)[name = string("op_1795_cast_fp16")];
            tensor<int32, [3]> var_1798_begin_0 = const()[name = string("op_1798_begin_0"), val = tensor<int32, [3]>([0, 0, 14])];
            tensor<int32, [3]> var_1798_end_0 = const()[name = string("op_1798_end_0"), val = tensor<int32, [3]>([1, 16, 15])];
            tensor<bool, [3]> var_1798_end_mask_0 = const()[name = string("op_1798_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1798_squeeze_mask_0 = const()[name = string("op_1798_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1798_cast_fp16 = slice_by_index(begin = var_1798_begin_0, end = var_1798_end_0, end_mask = var_1798_end_mask_0, squeeze_mask = var_1798_squeeze_mask_0, x = decay_3_cast_fp16)[name = string("op_1798_cast_fp16")];
            tensor<fp16, [1, 16]> var_1799_cast_fp16 = exp(x = var_1798_cast_fp16)[name = string("op_1799_cast_fp16")];
            tensor<int32, [1]> var_1800_axes_0 = const()[name = string("op_1800_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1800_cast_fp16 = expand_dims(axes = var_1800_axes_0, x = var_1799_cast_fp16)[name = string("op_1800_cast_fp16")];
            tensor<int32, [1]> var_1801_axes_0 = const()[name = string("op_1801_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_1801_cast_fp16 = expand_dims(axes = var_1801_axes_0, x = var_1800_cast_fp16)[name = string("op_1801_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_121_cast_fp16 = mul(x = state_119_cast_fp16, y = var_1801_cast_fp16)[name = string("state_121_cast_fp16")];
            tensor<int32, [4]> key_token_61_begin_0 = const()[name = string("key_token_61_begin_0"), val = tensor<int32, [4]>([0, 0, 14, 0])];
            tensor<int32, [4]> key_token_61_end_0 = const()[name = string("key_token_61_end_0"), val = tensor<int32, [4]>([1, 16, 15, 128])];
            tensor<bool, [4]> key_token_61_end_mask_0 = const()[name = string("key_token_61_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_61_squeeze_mask_0 = const()[name = string("key_token_61_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_61_cast_fp16 = slice_by_index(begin = key_token_61_begin_0, end = key_token_61_end_0, end_mask = key_token_61_end_mask_0, squeeze_mask = key_token_61_squeeze_mask_0, x = key_7_cast_fp16)[name = string("key_token_61_cast_fp16")];
            tensor<int32, [4]> value_token_61_begin_0 = const()[name = string("value_token_61_begin_0"), val = tensor<int32, [4]>([0, 0, 14, 0])];
            tensor<int32, [4]> value_token_61_end_0 = const()[name = string("value_token_61_end_0"), val = tensor<int32, [4]>([1, 16, 15, 128])];
            tensor<bool, [4]> value_token_61_end_mask_0 = const()[name = string("value_token_61_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_61_squeeze_mask_0 = const()[name = string("value_token_61_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_61_cast_fp16 = slice_by_index(begin = value_token_61_begin_0, end = value_token_61_end_0, end_mask = value_token_61_end_mask_0, squeeze_mask = value_token_61_squeeze_mask_0, x = value_27_cast_fp16)[name = string("value_token_61_cast_fp16")];
            tensor<int32, [1]> var_1809_axes_0 = const()[name = string("op_1809_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1809_cast_fp16 = expand_dims(axes = var_1809_axes_0, x = key_token_61_cast_fp16)[name = string("op_1809_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1810_cast_fp16 = mul(x = state_121_cast_fp16, y = var_1809_cast_fp16)[name = string("op_1810_cast_fp16")];
            tensor<int32, [1]> memory_61_axes_0 = const()[name = string("memory_61_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_61_keep_dims_0 = const()[name = string("memory_61_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_61_cast_fp16 = reduce_sum(axes = memory_61_axes_0, keep_dims = memory_61_keep_dims_0, x = var_1810_cast_fp16)[name = string("memory_61_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_1813_cast_fp16 = sub(x = value_token_61_cast_fp16, y = memory_61_cast_fp16)[name = string("op_1813_cast_fp16")];
            tensor<int32, [3]> var_1816_begin_0 = const()[name = string("op_1816_begin_0"), val = tensor<int32, [3]>([0, 0, 14])];
            tensor<int32, [3]> var_1816_end_0 = const()[name = string("op_1816_end_0"), val = tensor<int32, [3]>([1, 16, 15])];
            tensor<bool, [3]> var_1816_end_mask_0 = const()[name = string("op_1816_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1816_squeeze_mask_0 = const()[name = string("op_1816_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1816_cast_fp16 = slice_by_index(begin = var_1816_begin_0, end = var_1816_end_0, end_mask = var_1816_end_mask_0, squeeze_mask = var_1816_squeeze_mask_0, x = beta_3_cast_fp16)[name = string("op_1816_cast_fp16")];
            tensor<int32, [1]> var_1817_axes_0 = const()[name = string("op_1817_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1817_cast_fp16 = expand_dims(axes = var_1817_axes_0, x = var_1816_cast_fp16)[name = string("op_1817_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_61_cast_fp16 = mul(x = var_1813_cast_fp16, y = var_1817_cast_fp16)[name = string("delta_61_cast_fp16")];
            tensor<int32, [1]> var_1820_axes_0 = const()[name = string("op_1820_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_1820_cast_fp16 = expand_dims(axes = var_1820_axes_0, x = delta_61_cast_fp16)[name = string("op_1820_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1821_cast_fp16 = mul(x = var_1809_cast_fp16, y = var_1820_cast_fp16)[name = string("op_1821_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_123_cast_fp16 = add(x = state_121_cast_fp16, y = var_1821_cast_fp16)[name = string("state_123_cast_fp16")];
            tensor<int32, [4]> var_1825_begin_0 = const()[name = string("op_1825_begin_0"), val = tensor<int32, [4]>([0, 0, 14, 0])];
            tensor<int32, [4]> var_1825_end_0 = const()[name = string("op_1825_end_0"), val = tensor<int32, [4]>([1, 16, 15, 128])];
            tensor<bool, [4]> var_1825_end_mask_0 = const()[name = string("op_1825_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_1825_squeeze_mask_0 = const()[name = string("op_1825_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_1825_cast_fp16 = slice_by_index(begin = var_1825_begin_0, end = var_1825_end_0, end_mask = var_1825_end_mask_0, squeeze_mask = var_1825_squeeze_mask_0, x = _inversed_query_7_cast_fp16)[name = string("op_1825_cast_fp16")];
            tensor<int32, [1]> var_1826_axes_0 = const()[name = string("op_1826_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1826_cast_fp16 = expand_dims(axes = var_1826_axes_0, x = var_1825_cast_fp16)[name = string("op_1826_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1827_cast_fp16 = mul(x = state_123_cast_fp16, y = var_1826_cast_fp16)[name = string("op_1827_cast_fp16")];
            tensor<int32, [1]> var_1829_axes_0 = const()[name = string("op_1829_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_1829_keep_dims_0 = const()[name = string("op_1829_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_1829_cast_fp16 = reduce_sum(axes = var_1829_axes_0, keep_dims = var_1829_keep_dims_0, x = var_1827_cast_fp16)[name = string("op_1829_cast_fp16")];
            tensor<int32, [3]> var_1832_begin_0 = const()[name = string("op_1832_begin_0"), val = tensor<int32, [3]>([0, 0, 15])];
            tensor<int32, [3]> var_1832_end_0 = const()[name = string("op_1832_end_0"), val = tensor<int32, [3]>([1, 16, 16])];
            tensor<bool, [3]> var_1832_end_mask_0 = const()[name = string("op_1832_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1832_squeeze_mask_0 = const()[name = string("op_1832_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1832_cast_fp16 = slice_by_index(begin = var_1832_begin_0, end = var_1832_end_0, end_mask = var_1832_end_mask_0, squeeze_mask = var_1832_squeeze_mask_0, x = decay_3_cast_fp16)[name = string("op_1832_cast_fp16")];
            tensor<fp16, [1, 16]> var_1833_cast_fp16 = exp(x = var_1832_cast_fp16)[name = string("op_1833_cast_fp16")];
            tensor<int32, [1]> var_1834_axes_0 = const()[name = string("op_1834_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1834_cast_fp16 = expand_dims(axes = var_1834_axes_0, x = var_1833_cast_fp16)[name = string("op_1834_cast_fp16")];
            tensor<int32, [1]> var_1835_axes_0 = const()[name = string("op_1835_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_1835_cast_fp16 = expand_dims(axes = var_1835_axes_0, x = var_1834_cast_fp16)[name = string("op_1835_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_125_cast_fp16 = mul(x = state_123_cast_fp16, y = var_1835_cast_fp16)[name = string("state_125_cast_fp16")];
            tensor<int32, [4]> key_token_63_begin_0 = const()[name = string("key_token_63_begin_0"), val = tensor<int32, [4]>([0, 0, 15, 0])];
            tensor<int32, [4]> key_token_63_end_0 = const()[name = string("key_token_63_end_0"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<bool, [4]> key_token_63_end_mask_0 = const()[name = string("key_token_63_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_63_squeeze_mask_0 = const()[name = string("key_token_63_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_63_cast_fp16 = slice_by_index(begin = key_token_63_begin_0, end = key_token_63_end_0, end_mask = key_token_63_end_mask_0, squeeze_mask = key_token_63_squeeze_mask_0, x = key_7_cast_fp16)[name = string("key_token_63_cast_fp16")];
            tensor<int32, [4]> value_token_63_begin_0 = const()[name = string("value_token_63_begin_0"), val = tensor<int32, [4]>([0, 0, 15, 0])];
            tensor<int32, [4]> value_token_63_end_0 = const()[name = string("value_token_63_end_0"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<bool, [4]> value_token_63_end_mask_0 = const()[name = string("value_token_63_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_63_squeeze_mask_0 = const()[name = string("value_token_63_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_63_cast_fp16 = slice_by_index(begin = value_token_63_begin_0, end = value_token_63_end_0, end_mask = value_token_63_end_mask_0, squeeze_mask = value_token_63_squeeze_mask_0, x = value_27_cast_fp16)[name = string("value_token_63_cast_fp16")];
            tensor<int32, [1]> var_1843_axes_0 = const()[name = string("op_1843_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1843_cast_fp16 = expand_dims(axes = var_1843_axes_0, x = key_token_63_cast_fp16)[name = string("op_1843_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1844_cast_fp16 = mul(x = state_125_cast_fp16, y = var_1843_cast_fp16)[name = string("op_1844_cast_fp16")];
            tensor<int32, [1]> memory_63_axes_0 = const()[name = string("memory_63_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_63_keep_dims_0 = const()[name = string("memory_63_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_63_cast_fp16 = reduce_sum(axes = memory_63_axes_0, keep_dims = memory_63_keep_dims_0, x = var_1844_cast_fp16)[name = string("memory_63_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_1847_cast_fp16 = sub(x = value_token_63_cast_fp16, y = memory_63_cast_fp16)[name = string("op_1847_cast_fp16")];
            tensor<int32, [3]> var_1850_begin_0 = const()[name = string("op_1850_begin_0"), val = tensor<int32, [3]>([0, 0, 15])];
            tensor<int32, [3]> var_1850_end_0 = const()[name = string("op_1850_end_0"), val = tensor<int32, [3]>([1, 16, 16])];
            tensor<bool, [3]> var_1850_end_mask_0 = const()[name = string("op_1850_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1850_squeeze_mask_0 = const()[name = string("op_1850_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1850_cast_fp16 = slice_by_index(begin = var_1850_begin_0, end = var_1850_end_0, end_mask = var_1850_end_mask_0, squeeze_mask = var_1850_squeeze_mask_0, x = beta_3_cast_fp16)[name = string("op_1850_cast_fp16")];
            tensor<int32, [1]> var_1851_axes_0 = const()[name = string("op_1851_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1851_cast_fp16 = expand_dims(axes = var_1851_axes_0, x = var_1850_cast_fp16)[name = string("op_1851_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_63_cast_fp16 = mul(x = var_1847_cast_fp16, y = var_1851_cast_fp16)[name = string("delta_63_cast_fp16")];
            tensor<int32, [1]> var_1854_axes_0 = const()[name = string("op_1854_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_1854_cast_fp16 = expand_dims(axes = var_1854_axes_0, x = delta_63_cast_fp16)[name = string("op_1854_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1855_cast_fp16 = mul(x = var_1843_cast_fp16, y = var_1854_cast_fp16)[name = string("op_1855_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> rec21_out = add(x = state_125_cast_fp16, y = var_1855_cast_fp16)[name = string("state_127_cast_fp16")];
            tensor<int32, [4]> var_1859_begin_0 = const()[name = string("op_1859_begin_0"), val = tensor<int32, [4]>([0, 0, 15, 0])];
            tensor<int32, [4]> var_1859_end_0 = const()[name = string("op_1859_end_0"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<bool, [4]> var_1859_end_mask_0 = const()[name = string("op_1859_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_1859_squeeze_mask_0 = const()[name = string("op_1859_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_1859_cast_fp16 = slice_by_index(begin = var_1859_begin_0, end = var_1859_end_0, end_mask = var_1859_end_mask_0, squeeze_mask = var_1859_squeeze_mask_0, x = _inversed_query_7_cast_fp16)[name = string("op_1859_cast_fp16")];
            tensor<int32, [1]> var_1860_axes_0 = const()[name = string("op_1860_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1860_cast_fp16 = expand_dims(axes = var_1860_axes_0, x = var_1859_cast_fp16)[name = string("op_1860_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1861_cast_fp16 = mul(x = rec21_out, y = var_1860_cast_fp16)[name = string("op_1861_cast_fp16")];
            tensor<int32, [1]> var_1863_axes_0 = const()[name = string("op_1863_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_1863_keep_dims_0 = const()[name = string("op_1863_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_1863_cast_fp16 = reduce_sum(axes = var_1863_axes_0, keep_dims = var_1863_keep_dims_0, x = var_1861_cast_fp16)[name = string("op_1863_cast_fp16")];
            int32 attention_11_axis_0 = const()[name = string("attention_11_axis_0"), val = int32(1)];
            tensor<fp16, [1, 16, 16, 128]> attention_11_cast_fp16 = stack(axis = attention_11_axis_0, values = (var_1353_cast_fp16, var_1387_cast_fp16, var_1421_cast_fp16, var_1455_cast_fp16, var_1489_cast_fp16, var_1523_cast_fp16, var_1557_cast_fp16, var_1591_cast_fp16, var_1625_cast_fp16, var_1659_cast_fp16, var_1693_cast_fp16, var_1727_cast_fp16, var_1761_cast_fp16, var_1795_cast_fp16, var_1829_cast_fp16, var_1863_cast_fp16))[name = string("attention_11_cast_fp16")];
            tensor<int32, [2]> var_1866 = const()[name = string("op_1866"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [256, 128]> attention_13_cast_fp16 = reshape(shape = var_1866, x = attention_11_cast_fp16)[name = string("attention_13_cast_fp16")];
            tensor<int32, [2]> var_1868 = const()[name = string("op_1868"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [256, 128]> input_27_cast_fp16 = reshape(shape = var_1868, x = linear_15_cast_fp16)[name = string("input_27_cast_fp16")];
            tensor<fp16, [256, 128]> var_1870_cast_fp16 = mul(x = attention_13_cast_fp16, y = attention_13_cast_fp16)[name = string("op_1870_cast_fp16")];
            tensor<int32, [1]> variance_11_axes_0 = const()[name = string("variance_11_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_11_keep_dims_0 = const()[name = string("variance_11_keep_dims_0"), val = bool(true)];
            tensor<fp16, [256, 1]> variance_11_cast_fp16 = reduce_mean(axes = variance_11_axes_0, keep_dims = variance_11_keep_dims_0, x = var_1870_cast_fp16)[name = string("variance_11_cast_fp16")];
            fp16 var_1873_to_fp16 = const()[name = string("op_1873_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [256, 1]> var_1874_cast_fp16 = add(x = variance_11_cast_fp16, y = var_1873_to_fp16)[name = string("op_1874_cast_fp16")];
            fp32 var_1875_epsilon_0 = const()[name = string("op_1875_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [256, 1]> var_1875_cast_fp16 = rsqrt(epsilon = var_1875_epsilon_0, x = var_1874_cast_fp16)[name = string("op_1875_cast_fp16")];
            tensor<fp16, [256, 128]> attention_15_cast_fp16 = mul(x = attention_13_cast_fp16, y = var_1875_cast_fp16)[name = string("attention_15_cast_fp16")];
            tensor<fp16, [128]> layers20_1_gated_norm_promoted_to_fp16 = const()[name = string("layers20_1_gated_norm_promoted_to_fp16"), val = tensor<fp16, [128]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(225937856)))];
            tensor<fp16, [256, 128]> attention_17_cast_fp16 = mul(x = attention_15_cast_fp16, y = layers20_1_gated_norm_promoted_to_fp16)[name = string("attention_17_cast_fp16")];
            tensor<fp16, [256, 128]> var_1878_cast_fp16 = silu(x = input_27_cast_fp16)[name = string("op_1878_cast_fp16")];
            tensor<fp16, [256, 128]> attention_19_cast_fp16 = mul(x = attention_17_cast_fp16, y = var_1878_cast_fp16)[name = string("attention_19_cast_fp16")];
            tensor<int32, [2]> var_1880 = const()[name = string("op_1880"), val = tensor<int32, [2]>([16, 2048])];
            tensor<fp16, [16, 2048]> input_29_cast_fp16 = reshape(shape = var_1880, x = attention_19_cast_fp16)[name = string("input_29_cast_fp16")];
            tensor<fp16, [1024, 2048]> layers20_1_out_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 2048]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(225938176))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(227511104))))[name = string("layers20_1_out_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 1024]> linear_16_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = layers20_1_out_proj_weight_promoted_to_fp16_palettized, x = input_29_cast_fp16)[name = string("linear_16_cast_fp16")];
            tensor<fp16, [16, 1024]> value_29_cast_fp16 = add(x = value_17_cast_fp16, y = linear_16_cast_fp16)[name = string("value_29_cast_fp16")];
            tensor<fp16, [16, 1024]> var_1885_cast_fp16 = mul(x = value_29_cast_fp16, y = value_29_cast_fp16)[name = string("op_1885_cast_fp16")];
            tensor<int32, [1]> variance_13_axes_0 = const()[name = string("variance_13_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_13_keep_dims_0 = const()[name = string("variance_13_keep_dims_0"), val = bool(true)];
            tensor<fp16, [16, 1]> variance_13_cast_fp16 = reduce_mean(axes = variance_13_axes_0, keep_dims = variance_13_keep_dims_0, x = var_1885_cast_fp16)[name = string("variance_13_cast_fp16")];
            fp16 var_1888_to_fp16 = const()[name = string("op_1888_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [16, 1]> var_1889_cast_fp16 = add(x = variance_13_cast_fp16, y = var_1888_to_fp16)[name = string("op_1889_cast_fp16")];
            fp32 var_1890_epsilon_0 = const()[name = string("op_1890_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [16, 1]> var_1890_cast_fp16 = rsqrt(epsilon = var_1890_epsilon_0, x = var_1889_cast_fp16)[name = string("op_1890_cast_fp16")];
            tensor<fp16, [16, 1024]> var_1891_cast_fp16 = mul(x = value_29_cast_fp16, y = var_1890_cast_fp16)[name = string("op_1891_cast_fp16")];
            tensor<fp16, [1024]> var_1893_to_fp16 = const()[name = string("op_1893_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(227519360)))];
            tensor<fp16, [16, 1024]> input_31_cast_fp16 = mul(x = var_1891_cast_fp16, y = var_1893_to_fp16)[name = string("input_31_cast_fp16")];
            tensor<fp16, [3584, 1024]> layers20_1_gate_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(227521472))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(230274048))))[name = string("layers20_1_gate_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 3584]> linear_17_cast_fp16 = linear(bias = linear_1_bias_0_to_fp16, weight = layers20_1_gate_proj_weight_promoted_to_fp16_palettized, x = input_31_cast_fp16)[name = string("linear_17_cast_fp16")];
            tensor<fp16, [16, 3584]> var_1897_cast_fp16 = silu(x = linear_17_cast_fp16)[name = string("op_1897_cast_fp16")];
            tensor<fp16, [3584, 1024]> layers20_1_up_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(230302784))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(233055360))))[name = string("layers20_1_up_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 3584]> linear_18_cast_fp16 = linear(bias = linear_1_bias_0_to_fp16, weight = layers20_1_up_proj_weight_promoted_to_fp16_palettized, x = input_31_cast_fp16)[name = string("linear_18_cast_fp16")];
            tensor<fp16, [16, 3584]> input_35_cast_fp16 = mul(x = var_1897_cast_fp16, y = linear_18_cast_fp16)[name = string("input_35_cast_fp16")];
            tensor<fp16, [1024, 3584]> layers20_1_down_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 3584]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(233084096))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(235836672))))[name = string("layers20_1_down_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 1024]> linear_19_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = layers20_1_down_proj_weight_promoted_to_fp16_palettized, x = input_35_cast_fp16)[name = string("linear_19_cast_fp16")];
            tensor<fp16, [16, 1024]> value_31_cast_fp16 = add(x = value_29_cast_fp16, y = linear_19_cast_fp16)[name = string("value_31_cast_fp16")];
            int32 var_1935 = const()[name = string("op_1935"), val = int32(-1)];
            tensor<fp16, [16, 1024]> var_1950_cast_fp16 = mul(x = value_31_cast_fp16, y = value_31_cast_fp16)[name = string("op_1950_cast_fp16")];
            tensor<int32, [1]> variance_15_axes_0 = const()[name = string("variance_15_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_15_keep_dims_0 = const()[name = string("variance_15_keep_dims_0"), val = bool(true)];
            tensor<fp16, [16, 1]> variance_15_cast_fp16 = reduce_mean(axes = variance_15_axes_0, keep_dims = variance_15_keep_dims_0, x = var_1950_cast_fp16)[name = string("variance_15_cast_fp16")];
            fp16 var_1953_to_fp16 = const()[name = string("op_1953_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [16, 1]> var_1954_cast_fp16 = add(x = variance_15_cast_fp16, y = var_1953_to_fp16)[name = string("op_1954_cast_fp16")];
            fp32 var_1955_epsilon_0 = const()[name = string("op_1955_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [16, 1]> var_1955_cast_fp16 = rsqrt(epsilon = var_1955_epsilon_0, x = var_1954_cast_fp16)[name = string("op_1955_cast_fp16")];
            tensor<fp16, [16, 1024]> var_1956_cast_fp16 = mul(x = value_31_cast_fp16, y = var_1955_cast_fp16)[name = string("op_1956_cast_fp16")];
            tensor<fp16, [1024]> var_1958_to_fp16 = const()[name = string("op_1958_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(235844928)))];
            tensor<fp16, [16, 1024]> input_37_cast_fp16 = mul(x = var_1956_cast_fp16, y = var_1958_to_fp16)[name = string("input_37_cast_fp16")];
            tensor<fp16, [6144, 1024]> layers20_2_in_proj_qkv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(235847040))), lut = tensor<fp16, [384, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(240565696))))[name = string("layers20_2_in_proj_qkv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 6144]> linear_20_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = layers20_2_in_proj_qkv_weight_promoted_to_fp16_palettized, x = input_37_cast_fp16)[name = string("linear_20_cast_fp16")];
            tensor<int32, [2]> var_1962_perm_0 = const()[name = string("op_1962_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> mixed_axes_0 = const()[name = string("mixed_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [6144, 16]> var_1962_cast_fp16 = transpose(perm = var_1962_perm_0, x = linear_20_cast_fp16)[name = string("transpose_14")];
            tensor<fp16, [1, 6144, 16]> mixed_cast_fp16 = expand_dims(axes = mixed_axes_0, x = var_1962_cast_fp16)[name = string("mixed_cast_fp16")];
            bool convolution_input_interleave_0 = const()[name = string("convolution_input_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 6144, 19]> convolution_input_cast_fp16 = concat(axis = var_1935, interleave = convolution_input_interleave_0, values = (conv22, mixed_cast_fp16))[name = string("convolution_input_cast_fp16")];
            string input_39_pad_type_0 = const()[name = string("input_39_pad_type_0"), val = string("valid")];
            int32 input_39_groups_0 = const()[name = string("input_39_groups_0"), val = int32(6144)];
            tensor<int32, [1]> input_39_strides_0 = const()[name = string("input_39_strides_0"), val = tensor<int32, [1]>([1])];
            tensor<int32, [2]> input_39_pad_0 = const()[name = string("input_39_pad_0"), val = tensor<int32, [2]>([0, 0])];
            tensor<int32, [1]> input_39_dilations_0 = const()[name = string("input_39_dilations_0"), val = tensor<int32, [1]>([1])];
            tensor<fp16, [6144, 1, 4]> layers20_2_conv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1, 4]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(240614912))), lut = tensor<fp16, [384, 1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(240633408))))[name = string("layers20_2_conv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 6144, 16]> input_39_cast_fp16 = conv(dilations = input_39_dilations_0, groups = input_39_groups_0, pad = input_39_pad_0, pad_type = input_39_pad_type_0, strides = input_39_strides_0, weight = layers20_2_conv_weight_promoted_to_fp16_palettized, x = convolution_input_cast_fp16)[name = string("input_39_cast_fp16")];
            tensor<fp16, [1, 6144, 16]> var_1971_cast_fp16 = silu(x = input_39_cast_fp16)[name = string("op_1971_cast_fp16")];
            tensor<int32, [3]> var_1972_perm_0 = const()[name = string("op_1972_perm_0"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [3]> var_1975_begin_0 = const()[name = string("op_1975_begin_0"), val = tensor<int32, [3]>([0, 0, 16])];
            tensor<int32, [3]> var_1975_end_0 = const()[name = string("op_1975_end_0"), val = tensor<int32, [3]>([1, 6144, 19])];
            tensor<bool, [3]> var_1975_end_mask_0 = const()[name = string("op_1975_end_mask_0"), val = tensor<bool, [3]>([true, true, true])];
            tensor<fp16, [1, 6144, 3]> conv22_out = slice_by_index(begin = var_1975_begin_0, end = var_1975_end_0, end_mask = var_1975_end_mask_0, x = convolution_input_cast_fp16)[name = string("op_1975_cast_fp16")];
            tensor<int32, [3]> var_1976 = const()[name = string("op_1976"), val = tensor<int32, [3]>([2048, 2048, 2048])];
            int32 var_1977_axis_0 = const()[name = string("op_1977_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 16, 6144]> var_1972_cast_fp16 = transpose(perm = var_1972_perm_0, x = var_1971_cast_fp16)[name = string("transpose_13")];
            tensor<fp16, [1, 16, 2048]> var_1977_cast_fp16_0, tensor<fp16, [1, 16, 2048]> var_1977_cast_fp16_1, tensor<fp16, [1, 16, 2048]> var_1977_cast_fp16_2 = split(axis = var_1977_axis_0, split_sizes = var_1976, x = var_1972_cast_fp16)[name = string("op_1977_cast_fp16")];
            tensor<int32, [4]> var_1981 = const()[name = string("op_1981"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<fp16, [1, 16, 16, 128]> value_35_cast_fp16 = reshape(shape = var_1981, x = var_1977_cast_fp16_0)[name = string("value_35_cast_fp16")];
            tensor<int32, [4]> var_1983 = const()[name = string("op_1983"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<fp16, [1, 16, 16, 128]> value_37_cast_fp16 = reshape(shape = var_1983, x = var_1977_cast_fp16_1)[name = string("value_37_cast_fp16")];
            tensor<int32, [4]> var_1985 = const()[name = string("op_1985"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<fp16, [1, 16, 16, 128]> value_39_cast_fp16 = reshape(shape = var_1985, x = var_1977_cast_fp16_2)[name = string("value_39_cast_fp16")];
            tensor<fp16, [1, 16, 16, 128]> var_1987_cast_fp16 = mul(x = value_35_cast_fp16, y = value_35_cast_fp16)[name = string("op_1987_cast_fp16")];
            tensor<int32, [1]> var_1989_axes_0 = const()[name = string("op_1989_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_1989_keep_dims_0 = const()[name = string("op_1989_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 16, 16, 1]> var_1989_cast_fp16 = reduce_sum(axes = var_1989_axes_0, keep_dims = var_1989_keep_dims_0, x = var_1987_cast_fp16)[name = string("op_1989_cast_fp16")];
            fp16 var_1990_to_fp16 = const()[name = string("op_1990_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 16, 16, 1]> var_1991_cast_fp16 = add(x = var_1989_cast_fp16, y = var_1990_to_fp16)[name = string("op_1991_cast_fp16")];
            fp32 var_1992_epsilon_0 = const()[name = string("op_1992_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 16, 16, 1]> var_1992_cast_fp16 = rsqrt(epsilon = var_1992_epsilon_0, x = var_1991_cast_fp16)[name = string("op_1992_cast_fp16")];
            tensor<fp16, [1, 16, 16, 128]> var_1993_cast_fp16 = mul(x = value_35_cast_fp16, y = var_1992_cast_fp16)[name = string("op_1993_cast_fp16")];
            tensor<int32, [4]> var_1994_perm_0 = const()[name = string("op_1994_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            fp16 _inversed_query_11_y_0_to_fp16 = const()[name = string("_inversed_query_11_y_0_to_fp16"), val = fp16(0x1.6ap-4)];
            tensor<fp16, [1, 16, 16, 128]> var_1994_cast_fp16 = transpose(perm = var_1994_perm_0, x = var_1993_cast_fp16)[name = string("transpose_12")];
            tensor<fp16, [1, 16, 16, 128]> _inversed_query_11_cast_fp16 = mul(x = var_1994_cast_fp16, y = _inversed_query_11_y_0_to_fp16)[name = string("_inversed_query_11_cast_fp16")];
            tensor<fp16, [1, 16, 16, 128]> var_1997_cast_fp16 = mul(x = value_37_cast_fp16, y = value_37_cast_fp16)[name = string("op_1997_cast_fp16")];
            tensor<int32, [1]> var_1999_axes_0 = const()[name = string("op_1999_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_1999_keep_dims_0 = const()[name = string("op_1999_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 16, 16, 1]> var_1999_cast_fp16 = reduce_sum(axes = var_1999_axes_0, keep_dims = var_1999_keep_dims_0, x = var_1997_cast_fp16)[name = string("op_1999_cast_fp16")];
            fp16 var_2000_to_fp16 = const()[name = string("op_2000_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 16, 16, 1]> var_2001_cast_fp16 = add(x = var_1999_cast_fp16, y = var_2000_to_fp16)[name = string("op_2001_cast_fp16")];
            fp32 var_2002_epsilon_0 = const()[name = string("op_2002_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 16, 16, 1]> var_2002_cast_fp16 = rsqrt(epsilon = var_2002_epsilon_0, x = var_2001_cast_fp16)[name = string("op_2002_cast_fp16")];
            tensor<fp16, [1, 16, 16, 128]> var_2003_cast_fp16 = mul(x = value_37_cast_fp16, y = var_2002_cast_fp16)[name = string("op_2003_cast_fp16")];
            tensor<int32, [4]> key_11_perm_0 = const()[name = string("key_11_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<int32, [4]> value_41_perm_0 = const()[name = string("value_41_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<fp16, [16, 1024]> layers20_2_in_proj_b_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(240682624))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(240694976))))[name = string("layers20_2_in_proj_b_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16]> linear_21_bias_0_to_fp16 = const()[name = string("linear_21_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(207971904)))];
            tensor<fp16, [16, 16]> linear_21_cast_fp16 = linear(bias = linear_21_bias_0_to_fp16, weight = layers20_2_in_proj_b_weight_promoted_to_fp16_palettized, x = input_37_cast_fp16)[name = string("linear_21_cast_fp16")];
            tensor<fp16, [16, 16]> var_2008_cast_fp16 = sigmoid(x = linear_21_cast_fp16)[name = string("op_2008_cast_fp16")];
            tensor<int32, [2]> var_2009_perm_0 = const()[name = string("op_2009_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> beta_axes_0 = const()[name = string("beta_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 16]> var_2009_cast_fp16 = transpose(perm = var_2009_perm_0, x = var_2008_cast_fp16)[name = string("transpose_11")];
            tensor<fp16, [1, 16, 16]> beta_cast_fp16 = expand_dims(axes = beta_axes_0, x = var_2009_cast_fp16)[name = string("beta_cast_fp16")];
            tensor<fp16, [16, 1024]> op_2015_weight_0_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(240695168))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(240707520))))[name = string("op_2015_weight_0_to_fp16_palettized")];
            tensor<fp16, [16]> var_2015_bias_0_to_fp16 = const()[name = string("op_2015_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(240707712)))];
            tensor<fp16, [16, 16]> var_2015_cast_fp16 = linear(bias = var_2015_bias_0_to_fp16, weight = op_2015_weight_0_to_fp16_palettized, x = input_37_cast_fp16)[name = string("op_2015_cast_fp16")];
            tensor<fp16, [16, 16]> var_2016_cast_fp16 = softplus(x = var_2015_cast_fp16)[name = string("op_2016_cast_fp16")];
            tensor<fp16, [16]> var_2012_promoted_to_fp16 = const()[name = string("op_2012_promoted_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(240707840)))];
            tensor<fp16, [16, 16]> var_2017_cast_fp16 = mul(x = var_2012_promoted_to_fp16, y = var_2016_cast_fp16)[name = string("op_2017_cast_fp16")];
            tensor<int32, [2]> var_2018_perm_0 = const()[name = string("op_2018_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> decay_axes_0 = const()[name = string("decay_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 16]> var_2018_cast_fp16 = transpose(perm = var_2018_perm_0, x = var_2017_cast_fp16)[name = string("transpose_10")];
            tensor<fp16, [1, 16, 16]> decay_cast_fp16 = expand_dims(axes = decay_axes_0, x = var_2018_cast_fp16)[name = string("decay_cast_fp16")];
            tensor<fp16, [2048, 1024]> layers20_2_in_proj_z_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [2048, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(240707968))), lut = tensor<fp16, [128, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(242280896))))[name = string("layers20_2_in_proj_z_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 2048]> linear_23_cast_fp16 = linear(bias = linear_7_bias_0_to_fp16, weight = layers20_2_in_proj_z_weight_promoted_to_fp16_palettized, x = input_37_cast_fp16)[name = string("linear_23_cast_fp16")];
            tensor<int32, [3]> var_2026_begin_0 = const()[name = string("op_2026_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_2026_end_0 = const()[name = string("op_2026_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_2026_end_mask_0 = const()[name = string("op_2026_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_2026_squeeze_mask_0 = const()[name = string("op_2026_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_2026_cast_fp16 = slice_by_index(begin = var_2026_begin_0, end = var_2026_end_0, end_mask = var_2026_end_mask_0, squeeze_mask = var_2026_squeeze_mask_0, x = decay_cast_fp16)[name = string("op_2026_cast_fp16")];
            tensor<fp16, [1, 16]> var_2027_cast_fp16 = exp(x = var_2026_cast_fp16)[name = string("op_2027_cast_fp16")];
            tensor<int32, [1]> var_2028_axes_0 = const()[name = string("op_2028_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_2028_cast_fp16 = expand_dims(axes = var_2028_axes_0, x = var_2027_cast_fp16)[name = string("op_2028_cast_fp16")];
            tensor<int32, [1]> var_2029_axes_0 = const()[name = string("op_2029_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_2029_cast_fp16 = expand_dims(axes = var_2029_axes_0, x = var_2028_cast_fp16)[name = string("op_2029_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_129_cast_fp16 = mul(x = rec22, y = var_2029_cast_fp16)[name = string("state_129_cast_fp16")];
            tensor<int32, [4]> key_token_65_begin_0 = const()[name = string("key_token_65_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> key_token_65_end_0 = const()[name = string("key_token_65_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> key_token_65_end_mask_0 = const()[name = string("key_token_65_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_65_squeeze_mask_0 = const()[name = string("key_token_65_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 16, 128]> key_11_cast_fp16 = transpose(perm = key_11_perm_0, x = var_2003_cast_fp16)[name = string("transpose_9")];
            tensor<fp16, [1, 16, 128]> key_token_65_cast_fp16 = slice_by_index(begin = key_token_65_begin_0, end = key_token_65_end_0, end_mask = key_token_65_end_mask_0, squeeze_mask = key_token_65_squeeze_mask_0, x = key_11_cast_fp16)[name = string("key_token_65_cast_fp16")];
            tensor<int32, [4]> value_token_65_begin_0 = const()[name = string("value_token_65_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> value_token_65_end_0 = const()[name = string("value_token_65_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> value_token_65_end_mask_0 = const()[name = string("value_token_65_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_65_squeeze_mask_0 = const()[name = string("value_token_65_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 16, 128]> value_41_cast_fp16 = transpose(perm = value_41_perm_0, x = value_39_cast_fp16)[name = string("transpose_8")];
            tensor<fp16, [1, 16, 128]> value_token_65_cast_fp16 = slice_by_index(begin = value_token_65_begin_0, end = value_token_65_end_0, end_mask = value_token_65_end_mask_0, squeeze_mask = value_token_65_squeeze_mask_0, x = value_41_cast_fp16)[name = string("value_token_65_cast_fp16")];
            tensor<int32, [1]> var_2037_axes_0 = const()[name = string("op_2037_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_2037_cast_fp16 = expand_dims(axes = var_2037_axes_0, x = key_token_65_cast_fp16)[name = string("op_2037_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2038_cast_fp16 = mul(x = state_129_cast_fp16, y = var_2037_cast_fp16)[name = string("op_2038_cast_fp16")];
            tensor<int32, [1]> memory_65_axes_0 = const()[name = string("memory_65_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_65_keep_dims_0 = const()[name = string("memory_65_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_65_cast_fp16 = reduce_sum(axes = memory_65_axes_0, keep_dims = memory_65_keep_dims_0, x = var_2038_cast_fp16)[name = string("memory_65_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_2041_cast_fp16 = sub(x = value_token_65_cast_fp16, y = memory_65_cast_fp16)[name = string("op_2041_cast_fp16")];
            tensor<int32, [3]> var_2044_begin_0 = const()[name = string("op_2044_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_2044_end_0 = const()[name = string("op_2044_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_2044_end_mask_0 = const()[name = string("op_2044_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_2044_squeeze_mask_0 = const()[name = string("op_2044_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_2044_cast_fp16 = slice_by_index(begin = var_2044_begin_0, end = var_2044_end_0, end_mask = var_2044_end_mask_0, squeeze_mask = var_2044_squeeze_mask_0, x = beta_cast_fp16)[name = string("op_2044_cast_fp16")];
            tensor<int32, [1]> var_2045_axes_0 = const()[name = string("op_2045_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_2045_cast_fp16 = expand_dims(axes = var_2045_axes_0, x = var_2044_cast_fp16)[name = string("op_2045_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_65_cast_fp16 = mul(x = var_2041_cast_fp16, y = var_2045_cast_fp16)[name = string("delta_65_cast_fp16")];
            tensor<int32, [1]> var_2048_axes_0 = const()[name = string("op_2048_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_2048_cast_fp16 = expand_dims(axes = var_2048_axes_0, x = delta_65_cast_fp16)[name = string("op_2048_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2049_cast_fp16 = mul(x = var_2037_cast_fp16, y = var_2048_cast_fp16)[name = string("op_2049_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_131_cast_fp16 = add(x = state_129_cast_fp16, y = var_2049_cast_fp16)[name = string("state_131_cast_fp16")];
            tensor<int32, [4]> var_2053_begin_0 = const()[name = string("op_2053_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> var_2053_end_0 = const()[name = string("op_2053_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> var_2053_end_mask_0 = const()[name = string("op_2053_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_2053_squeeze_mask_0 = const()[name = string("op_2053_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_2053_cast_fp16 = slice_by_index(begin = var_2053_begin_0, end = var_2053_end_0, end_mask = var_2053_end_mask_0, squeeze_mask = var_2053_squeeze_mask_0, x = _inversed_query_11_cast_fp16)[name = string("op_2053_cast_fp16")];
            tensor<int32, [1]> var_2054_axes_0 = const()[name = string("op_2054_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_2054_cast_fp16 = expand_dims(axes = var_2054_axes_0, x = var_2053_cast_fp16)[name = string("op_2054_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2055_cast_fp16 = mul(x = state_131_cast_fp16, y = var_2054_cast_fp16)[name = string("op_2055_cast_fp16")];
            tensor<int32, [1]> var_2057_axes_0 = const()[name = string("op_2057_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_2057_keep_dims_0 = const()[name = string("op_2057_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_2057_cast_fp16 = reduce_sum(axes = var_2057_axes_0, keep_dims = var_2057_keep_dims_0, x = var_2055_cast_fp16)[name = string("op_2057_cast_fp16")];
            tensor<int32, [3]> var_2060_begin_0 = const()[name = string("op_2060_begin_0"), val = tensor<int32, [3]>([0, 0, 1])];
            tensor<int32, [3]> var_2060_end_0 = const()[name = string("op_2060_end_0"), val = tensor<int32, [3]>([1, 16, 2])];
            tensor<bool, [3]> var_2060_end_mask_0 = const()[name = string("op_2060_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_2060_squeeze_mask_0 = const()[name = string("op_2060_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_2060_cast_fp16 = slice_by_index(begin = var_2060_begin_0, end = var_2060_end_0, end_mask = var_2060_end_mask_0, squeeze_mask = var_2060_squeeze_mask_0, x = decay_cast_fp16)[name = string("op_2060_cast_fp16")];
            tensor<fp16, [1, 16]> var_2061_cast_fp16 = exp(x = var_2060_cast_fp16)[name = string("op_2061_cast_fp16")];
            tensor<int32, [1]> var_2062_axes_0 = const()[name = string("op_2062_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_2062_cast_fp16 = expand_dims(axes = var_2062_axes_0, x = var_2061_cast_fp16)[name = string("op_2062_cast_fp16")];
            tensor<int32, [1]> var_2063_axes_0 = const()[name = string("op_2063_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_2063_cast_fp16 = expand_dims(axes = var_2063_axes_0, x = var_2062_cast_fp16)[name = string("op_2063_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_133_cast_fp16 = mul(x = state_131_cast_fp16, y = var_2063_cast_fp16)[name = string("state_133_cast_fp16")];
            tensor<int32, [4]> key_token_67_begin_0 = const()[name = string("key_token_67_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> key_token_67_end_0 = const()[name = string("key_token_67_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> key_token_67_end_mask_0 = const()[name = string("key_token_67_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_67_squeeze_mask_0 = const()[name = string("key_token_67_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_67_cast_fp16 = slice_by_index(begin = key_token_67_begin_0, end = key_token_67_end_0, end_mask = key_token_67_end_mask_0, squeeze_mask = key_token_67_squeeze_mask_0, x = key_11_cast_fp16)[name = string("key_token_67_cast_fp16")];
            tensor<int32, [4]> value_token_67_begin_0 = const()[name = string("value_token_67_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> value_token_67_end_0 = const()[name = string("value_token_67_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> value_token_67_end_mask_0 = const()[name = string("value_token_67_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_67_squeeze_mask_0 = const()[name = string("value_token_67_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_67_cast_fp16 = slice_by_index(begin = value_token_67_begin_0, end = value_token_67_end_0, end_mask = value_token_67_end_mask_0, squeeze_mask = value_token_67_squeeze_mask_0, x = value_41_cast_fp16)[name = string("value_token_67_cast_fp16")];
            tensor<int32, [1]> var_2071_axes_0 = const()[name = string("op_2071_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_2071_cast_fp16 = expand_dims(axes = var_2071_axes_0, x = key_token_67_cast_fp16)[name = string("op_2071_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2072_cast_fp16 = mul(x = state_133_cast_fp16, y = var_2071_cast_fp16)[name = string("op_2072_cast_fp16")];
            tensor<int32, [1]> memory_67_axes_0 = const()[name = string("memory_67_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_67_keep_dims_0 = const()[name = string("memory_67_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_67_cast_fp16 = reduce_sum(axes = memory_67_axes_0, keep_dims = memory_67_keep_dims_0, x = var_2072_cast_fp16)[name = string("memory_67_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_2075_cast_fp16 = sub(x = value_token_67_cast_fp16, y = memory_67_cast_fp16)[name = string("op_2075_cast_fp16")];
            tensor<int32, [3]> var_2078_begin_0 = const()[name = string("op_2078_begin_0"), val = tensor<int32, [3]>([0, 0, 1])];
            tensor<int32, [3]> var_2078_end_0 = const()[name = string("op_2078_end_0"), val = tensor<int32, [3]>([1, 16, 2])];
            tensor<bool, [3]> var_2078_end_mask_0 = const()[name = string("op_2078_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_2078_squeeze_mask_0 = const()[name = string("op_2078_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_2078_cast_fp16 = slice_by_index(begin = var_2078_begin_0, end = var_2078_end_0, end_mask = var_2078_end_mask_0, squeeze_mask = var_2078_squeeze_mask_0, x = beta_cast_fp16)[name = string("op_2078_cast_fp16")];
            tensor<int32, [1]> var_2079_axes_0 = const()[name = string("op_2079_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_2079_cast_fp16 = expand_dims(axes = var_2079_axes_0, x = var_2078_cast_fp16)[name = string("op_2079_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_67_cast_fp16 = mul(x = var_2075_cast_fp16, y = var_2079_cast_fp16)[name = string("delta_67_cast_fp16")];
            tensor<int32, [1]> var_2082_axes_0 = const()[name = string("op_2082_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_2082_cast_fp16 = expand_dims(axes = var_2082_axes_0, x = delta_67_cast_fp16)[name = string("op_2082_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2083_cast_fp16 = mul(x = var_2071_cast_fp16, y = var_2082_cast_fp16)[name = string("op_2083_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_135_cast_fp16 = add(x = state_133_cast_fp16, y = var_2083_cast_fp16)[name = string("state_135_cast_fp16")];
            tensor<int32, [4]> var_2087_begin_0 = const()[name = string("op_2087_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> var_2087_end_0 = const()[name = string("op_2087_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> var_2087_end_mask_0 = const()[name = string("op_2087_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_2087_squeeze_mask_0 = const()[name = string("op_2087_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_2087_cast_fp16 = slice_by_index(begin = var_2087_begin_0, end = var_2087_end_0, end_mask = var_2087_end_mask_0, squeeze_mask = var_2087_squeeze_mask_0, x = _inversed_query_11_cast_fp16)[name = string("op_2087_cast_fp16")];
            tensor<int32, [1]> var_2088_axes_0 = const()[name = string("op_2088_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_2088_cast_fp16 = expand_dims(axes = var_2088_axes_0, x = var_2087_cast_fp16)[name = string("op_2088_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2089_cast_fp16 = mul(x = state_135_cast_fp16, y = var_2088_cast_fp16)[name = string("op_2089_cast_fp16")];
            tensor<int32, [1]> var_2091_axes_0 = const()[name = string("op_2091_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_2091_keep_dims_0 = const()[name = string("op_2091_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_2091_cast_fp16 = reduce_sum(axes = var_2091_axes_0, keep_dims = var_2091_keep_dims_0, x = var_2089_cast_fp16)[name = string("op_2091_cast_fp16")];
            tensor<int32, [3]> var_2094_begin_0 = const()[name = string("op_2094_begin_0"), val = tensor<int32, [3]>([0, 0, 2])];
            tensor<int32, [3]> var_2094_end_0 = const()[name = string("op_2094_end_0"), val = tensor<int32, [3]>([1, 16, 3])];
            tensor<bool, [3]> var_2094_end_mask_0 = const()[name = string("op_2094_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_2094_squeeze_mask_0 = const()[name = string("op_2094_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_2094_cast_fp16 = slice_by_index(begin = var_2094_begin_0, end = var_2094_end_0, end_mask = var_2094_end_mask_0, squeeze_mask = var_2094_squeeze_mask_0, x = decay_cast_fp16)[name = string("op_2094_cast_fp16")];
            tensor<fp16, [1, 16]> var_2095_cast_fp16 = exp(x = var_2094_cast_fp16)[name = string("op_2095_cast_fp16")];
            tensor<int32, [1]> var_2096_axes_0 = const()[name = string("op_2096_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_2096_cast_fp16 = expand_dims(axes = var_2096_axes_0, x = var_2095_cast_fp16)[name = string("op_2096_cast_fp16")];
            tensor<int32, [1]> var_2097_axes_0 = const()[name = string("op_2097_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_2097_cast_fp16 = expand_dims(axes = var_2097_axes_0, x = var_2096_cast_fp16)[name = string("op_2097_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_137_cast_fp16 = mul(x = state_135_cast_fp16, y = var_2097_cast_fp16)[name = string("state_137_cast_fp16")];
            tensor<int32, [4]> key_token_69_begin_0 = const()[name = string("key_token_69_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> key_token_69_end_0 = const()[name = string("key_token_69_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> key_token_69_end_mask_0 = const()[name = string("key_token_69_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_69_squeeze_mask_0 = const()[name = string("key_token_69_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_69_cast_fp16 = slice_by_index(begin = key_token_69_begin_0, end = key_token_69_end_0, end_mask = key_token_69_end_mask_0, squeeze_mask = key_token_69_squeeze_mask_0, x = key_11_cast_fp16)[name = string("key_token_69_cast_fp16")];
            tensor<int32, [4]> value_token_69_begin_0 = const()[name = string("value_token_69_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> value_token_69_end_0 = const()[name = string("value_token_69_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> value_token_69_end_mask_0 = const()[name = string("value_token_69_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_69_squeeze_mask_0 = const()[name = string("value_token_69_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_69_cast_fp16 = slice_by_index(begin = value_token_69_begin_0, end = value_token_69_end_0, end_mask = value_token_69_end_mask_0, squeeze_mask = value_token_69_squeeze_mask_0, x = value_41_cast_fp16)[name = string("value_token_69_cast_fp16")];
            tensor<int32, [1]> var_2105_axes_0 = const()[name = string("op_2105_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_2105_cast_fp16 = expand_dims(axes = var_2105_axes_0, x = key_token_69_cast_fp16)[name = string("op_2105_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2106_cast_fp16 = mul(x = state_137_cast_fp16, y = var_2105_cast_fp16)[name = string("op_2106_cast_fp16")];
            tensor<int32, [1]> memory_69_axes_0 = const()[name = string("memory_69_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_69_keep_dims_0 = const()[name = string("memory_69_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_69_cast_fp16 = reduce_sum(axes = memory_69_axes_0, keep_dims = memory_69_keep_dims_0, x = var_2106_cast_fp16)[name = string("memory_69_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_2109_cast_fp16 = sub(x = value_token_69_cast_fp16, y = memory_69_cast_fp16)[name = string("op_2109_cast_fp16")];
            tensor<int32, [3]> var_2112_begin_0 = const()[name = string("op_2112_begin_0"), val = tensor<int32, [3]>([0, 0, 2])];
            tensor<int32, [3]> var_2112_end_0 = const()[name = string("op_2112_end_0"), val = tensor<int32, [3]>([1, 16, 3])];
            tensor<bool, [3]> var_2112_end_mask_0 = const()[name = string("op_2112_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_2112_squeeze_mask_0 = const()[name = string("op_2112_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_2112_cast_fp16 = slice_by_index(begin = var_2112_begin_0, end = var_2112_end_0, end_mask = var_2112_end_mask_0, squeeze_mask = var_2112_squeeze_mask_0, x = beta_cast_fp16)[name = string("op_2112_cast_fp16")];
            tensor<int32, [1]> var_2113_axes_0 = const()[name = string("op_2113_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_2113_cast_fp16 = expand_dims(axes = var_2113_axes_0, x = var_2112_cast_fp16)[name = string("op_2113_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_69_cast_fp16 = mul(x = var_2109_cast_fp16, y = var_2113_cast_fp16)[name = string("delta_69_cast_fp16")];
            tensor<int32, [1]> var_2116_axes_0 = const()[name = string("op_2116_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_2116_cast_fp16 = expand_dims(axes = var_2116_axes_0, x = delta_69_cast_fp16)[name = string("op_2116_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2117_cast_fp16 = mul(x = var_2105_cast_fp16, y = var_2116_cast_fp16)[name = string("op_2117_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_139_cast_fp16 = add(x = state_137_cast_fp16, y = var_2117_cast_fp16)[name = string("state_139_cast_fp16")];
            tensor<int32, [4]> var_2121_begin_0 = const()[name = string("op_2121_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> var_2121_end_0 = const()[name = string("op_2121_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> var_2121_end_mask_0 = const()[name = string("op_2121_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_2121_squeeze_mask_0 = const()[name = string("op_2121_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_2121_cast_fp16 = slice_by_index(begin = var_2121_begin_0, end = var_2121_end_0, end_mask = var_2121_end_mask_0, squeeze_mask = var_2121_squeeze_mask_0, x = _inversed_query_11_cast_fp16)[name = string("op_2121_cast_fp16")];
            tensor<int32, [1]> var_2122_axes_0 = const()[name = string("op_2122_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_2122_cast_fp16 = expand_dims(axes = var_2122_axes_0, x = var_2121_cast_fp16)[name = string("op_2122_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2123_cast_fp16 = mul(x = state_139_cast_fp16, y = var_2122_cast_fp16)[name = string("op_2123_cast_fp16")];
            tensor<int32, [1]> var_2125_axes_0 = const()[name = string("op_2125_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_2125_keep_dims_0 = const()[name = string("op_2125_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_2125_cast_fp16 = reduce_sum(axes = var_2125_axes_0, keep_dims = var_2125_keep_dims_0, x = var_2123_cast_fp16)[name = string("op_2125_cast_fp16")];
            tensor<int32, [3]> var_2128_begin_0 = const()[name = string("op_2128_begin_0"), val = tensor<int32, [3]>([0, 0, 3])];
            tensor<int32, [3]> var_2128_end_0 = const()[name = string("op_2128_end_0"), val = tensor<int32, [3]>([1, 16, 4])];
            tensor<bool, [3]> var_2128_end_mask_0 = const()[name = string("op_2128_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_2128_squeeze_mask_0 = const()[name = string("op_2128_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_2128_cast_fp16 = slice_by_index(begin = var_2128_begin_0, end = var_2128_end_0, end_mask = var_2128_end_mask_0, squeeze_mask = var_2128_squeeze_mask_0, x = decay_cast_fp16)[name = string("op_2128_cast_fp16")];
            tensor<fp16, [1, 16]> var_2129_cast_fp16 = exp(x = var_2128_cast_fp16)[name = string("op_2129_cast_fp16")];
            tensor<int32, [1]> var_2130_axes_0 = const()[name = string("op_2130_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_2130_cast_fp16 = expand_dims(axes = var_2130_axes_0, x = var_2129_cast_fp16)[name = string("op_2130_cast_fp16")];
            tensor<int32, [1]> var_2131_axes_0 = const()[name = string("op_2131_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_2131_cast_fp16 = expand_dims(axes = var_2131_axes_0, x = var_2130_cast_fp16)[name = string("op_2131_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_141_cast_fp16 = mul(x = state_139_cast_fp16, y = var_2131_cast_fp16)[name = string("state_141_cast_fp16")];
            tensor<int32, [4]> key_token_71_begin_0 = const()[name = string("key_token_71_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> key_token_71_end_0 = const()[name = string("key_token_71_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> key_token_71_end_mask_0 = const()[name = string("key_token_71_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_71_squeeze_mask_0 = const()[name = string("key_token_71_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_71_cast_fp16 = slice_by_index(begin = key_token_71_begin_0, end = key_token_71_end_0, end_mask = key_token_71_end_mask_0, squeeze_mask = key_token_71_squeeze_mask_0, x = key_11_cast_fp16)[name = string("key_token_71_cast_fp16")];
            tensor<int32, [4]> value_token_71_begin_0 = const()[name = string("value_token_71_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> value_token_71_end_0 = const()[name = string("value_token_71_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> value_token_71_end_mask_0 = const()[name = string("value_token_71_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_71_squeeze_mask_0 = const()[name = string("value_token_71_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_71_cast_fp16 = slice_by_index(begin = value_token_71_begin_0, end = value_token_71_end_0, end_mask = value_token_71_end_mask_0, squeeze_mask = value_token_71_squeeze_mask_0, x = value_41_cast_fp16)[name = string("value_token_71_cast_fp16")];
            tensor<int32, [1]> var_2139_axes_0 = const()[name = string("op_2139_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_2139_cast_fp16 = expand_dims(axes = var_2139_axes_0, x = key_token_71_cast_fp16)[name = string("op_2139_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2140_cast_fp16 = mul(x = state_141_cast_fp16, y = var_2139_cast_fp16)[name = string("op_2140_cast_fp16")];
            tensor<int32, [1]> memory_71_axes_0 = const()[name = string("memory_71_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_71_keep_dims_0 = const()[name = string("memory_71_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_71_cast_fp16 = reduce_sum(axes = memory_71_axes_0, keep_dims = memory_71_keep_dims_0, x = var_2140_cast_fp16)[name = string("memory_71_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_2143_cast_fp16 = sub(x = value_token_71_cast_fp16, y = memory_71_cast_fp16)[name = string("op_2143_cast_fp16")];
            tensor<int32, [3]> var_2146_begin_0 = const()[name = string("op_2146_begin_0"), val = tensor<int32, [3]>([0, 0, 3])];
            tensor<int32, [3]> var_2146_end_0 = const()[name = string("op_2146_end_0"), val = tensor<int32, [3]>([1, 16, 4])];
            tensor<bool, [3]> var_2146_end_mask_0 = const()[name = string("op_2146_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_2146_squeeze_mask_0 = const()[name = string("op_2146_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_2146_cast_fp16 = slice_by_index(begin = var_2146_begin_0, end = var_2146_end_0, end_mask = var_2146_end_mask_0, squeeze_mask = var_2146_squeeze_mask_0, x = beta_cast_fp16)[name = string("op_2146_cast_fp16")];
            tensor<int32, [1]> var_2147_axes_0 = const()[name = string("op_2147_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_2147_cast_fp16 = expand_dims(axes = var_2147_axes_0, x = var_2146_cast_fp16)[name = string("op_2147_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_71_cast_fp16 = mul(x = var_2143_cast_fp16, y = var_2147_cast_fp16)[name = string("delta_71_cast_fp16")];
            tensor<int32, [1]> var_2150_axes_0 = const()[name = string("op_2150_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_2150_cast_fp16 = expand_dims(axes = var_2150_axes_0, x = delta_71_cast_fp16)[name = string("op_2150_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2151_cast_fp16 = mul(x = var_2139_cast_fp16, y = var_2150_cast_fp16)[name = string("op_2151_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_143_cast_fp16 = add(x = state_141_cast_fp16, y = var_2151_cast_fp16)[name = string("state_143_cast_fp16")];
            tensor<int32, [4]> var_2155_begin_0 = const()[name = string("op_2155_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> var_2155_end_0 = const()[name = string("op_2155_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> var_2155_end_mask_0 = const()[name = string("op_2155_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_2155_squeeze_mask_0 = const()[name = string("op_2155_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_2155_cast_fp16 = slice_by_index(begin = var_2155_begin_0, end = var_2155_end_0, end_mask = var_2155_end_mask_0, squeeze_mask = var_2155_squeeze_mask_0, x = _inversed_query_11_cast_fp16)[name = string("op_2155_cast_fp16")];
            tensor<int32, [1]> var_2156_axes_0 = const()[name = string("op_2156_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_2156_cast_fp16 = expand_dims(axes = var_2156_axes_0, x = var_2155_cast_fp16)[name = string("op_2156_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2157_cast_fp16 = mul(x = state_143_cast_fp16, y = var_2156_cast_fp16)[name = string("op_2157_cast_fp16")];
            tensor<int32, [1]> var_2159_axes_0 = const()[name = string("op_2159_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_2159_keep_dims_0 = const()[name = string("op_2159_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_2159_cast_fp16 = reduce_sum(axes = var_2159_axes_0, keep_dims = var_2159_keep_dims_0, x = var_2157_cast_fp16)[name = string("op_2159_cast_fp16")];
            tensor<int32, [3]> var_2162_begin_0 = const()[name = string("op_2162_begin_0"), val = tensor<int32, [3]>([0, 0, 4])];
            tensor<int32, [3]> var_2162_end_0 = const()[name = string("op_2162_end_0"), val = tensor<int32, [3]>([1, 16, 5])];
            tensor<bool, [3]> var_2162_end_mask_0 = const()[name = string("op_2162_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_2162_squeeze_mask_0 = const()[name = string("op_2162_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_2162_cast_fp16 = slice_by_index(begin = var_2162_begin_0, end = var_2162_end_0, end_mask = var_2162_end_mask_0, squeeze_mask = var_2162_squeeze_mask_0, x = decay_cast_fp16)[name = string("op_2162_cast_fp16")];
            tensor<fp16, [1, 16]> var_2163_cast_fp16 = exp(x = var_2162_cast_fp16)[name = string("op_2163_cast_fp16")];
            tensor<int32, [1]> var_2164_axes_0 = const()[name = string("op_2164_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_2164_cast_fp16 = expand_dims(axes = var_2164_axes_0, x = var_2163_cast_fp16)[name = string("op_2164_cast_fp16")];
            tensor<int32, [1]> var_2165_axes_0 = const()[name = string("op_2165_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_2165_cast_fp16 = expand_dims(axes = var_2165_axes_0, x = var_2164_cast_fp16)[name = string("op_2165_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_145_cast_fp16 = mul(x = state_143_cast_fp16, y = var_2165_cast_fp16)[name = string("state_145_cast_fp16")];
            tensor<int32, [4]> key_token_73_begin_0 = const()[name = string("key_token_73_begin_0"), val = tensor<int32, [4]>([0, 0, 4, 0])];
            tensor<int32, [4]> key_token_73_end_0 = const()[name = string("key_token_73_end_0"), val = tensor<int32, [4]>([1, 16, 5, 128])];
            tensor<bool, [4]> key_token_73_end_mask_0 = const()[name = string("key_token_73_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_73_squeeze_mask_0 = const()[name = string("key_token_73_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_73_cast_fp16 = slice_by_index(begin = key_token_73_begin_0, end = key_token_73_end_0, end_mask = key_token_73_end_mask_0, squeeze_mask = key_token_73_squeeze_mask_0, x = key_11_cast_fp16)[name = string("key_token_73_cast_fp16")];
            tensor<int32, [4]> value_token_73_begin_0 = const()[name = string("value_token_73_begin_0"), val = tensor<int32, [4]>([0, 0, 4, 0])];
            tensor<int32, [4]> value_token_73_end_0 = const()[name = string("value_token_73_end_0"), val = tensor<int32, [4]>([1, 16, 5, 128])];
            tensor<bool, [4]> value_token_73_end_mask_0 = const()[name = string("value_token_73_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_73_squeeze_mask_0 = const()[name = string("value_token_73_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_73_cast_fp16 = slice_by_index(begin = value_token_73_begin_0, end = value_token_73_end_0, end_mask = value_token_73_end_mask_0, squeeze_mask = value_token_73_squeeze_mask_0, x = value_41_cast_fp16)[name = string("value_token_73_cast_fp16")];
            tensor<int32, [1]> var_2173_axes_0 = const()[name = string("op_2173_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_2173_cast_fp16 = expand_dims(axes = var_2173_axes_0, x = key_token_73_cast_fp16)[name = string("op_2173_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2174_cast_fp16 = mul(x = state_145_cast_fp16, y = var_2173_cast_fp16)[name = string("op_2174_cast_fp16")];
            tensor<int32, [1]> memory_73_axes_0 = const()[name = string("memory_73_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_73_keep_dims_0 = const()[name = string("memory_73_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_73_cast_fp16 = reduce_sum(axes = memory_73_axes_0, keep_dims = memory_73_keep_dims_0, x = var_2174_cast_fp16)[name = string("memory_73_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_2177_cast_fp16 = sub(x = value_token_73_cast_fp16, y = memory_73_cast_fp16)[name = string("op_2177_cast_fp16")];
            tensor<int32, [3]> var_2180_begin_0 = const()[name = string("op_2180_begin_0"), val = tensor<int32, [3]>([0, 0, 4])];
            tensor<int32, [3]> var_2180_end_0 = const()[name = string("op_2180_end_0"), val = tensor<int32, [3]>([1, 16, 5])];
            tensor<bool, [3]> var_2180_end_mask_0 = const()[name = string("op_2180_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_2180_squeeze_mask_0 = const()[name = string("op_2180_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_2180_cast_fp16 = slice_by_index(begin = var_2180_begin_0, end = var_2180_end_0, end_mask = var_2180_end_mask_0, squeeze_mask = var_2180_squeeze_mask_0, x = beta_cast_fp16)[name = string("op_2180_cast_fp16")];
            tensor<int32, [1]> var_2181_axes_0 = const()[name = string("op_2181_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_2181_cast_fp16 = expand_dims(axes = var_2181_axes_0, x = var_2180_cast_fp16)[name = string("op_2181_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_73_cast_fp16 = mul(x = var_2177_cast_fp16, y = var_2181_cast_fp16)[name = string("delta_73_cast_fp16")];
            tensor<int32, [1]> var_2184_axes_0 = const()[name = string("op_2184_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_2184_cast_fp16 = expand_dims(axes = var_2184_axes_0, x = delta_73_cast_fp16)[name = string("op_2184_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2185_cast_fp16 = mul(x = var_2173_cast_fp16, y = var_2184_cast_fp16)[name = string("op_2185_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_147_cast_fp16 = add(x = state_145_cast_fp16, y = var_2185_cast_fp16)[name = string("state_147_cast_fp16")];
            tensor<int32, [4]> var_2189_begin_0 = const()[name = string("op_2189_begin_0"), val = tensor<int32, [4]>([0, 0, 4, 0])];
            tensor<int32, [4]> var_2189_end_0 = const()[name = string("op_2189_end_0"), val = tensor<int32, [4]>([1, 16, 5, 128])];
            tensor<bool, [4]> var_2189_end_mask_0 = const()[name = string("op_2189_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_2189_squeeze_mask_0 = const()[name = string("op_2189_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_2189_cast_fp16 = slice_by_index(begin = var_2189_begin_0, end = var_2189_end_0, end_mask = var_2189_end_mask_0, squeeze_mask = var_2189_squeeze_mask_0, x = _inversed_query_11_cast_fp16)[name = string("op_2189_cast_fp16")];
            tensor<int32, [1]> var_2190_axes_0 = const()[name = string("op_2190_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_2190_cast_fp16 = expand_dims(axes = var_2190_axes_0, x = var_2189_cast_fp16)[name = string("op_2190_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2191_cast_fp16 = mul(x = state_147_cast_fp16, y = var_2190_cast_fp16)[name = string("op_2191_cast_fp16")];
            tensor<int32, [1]> var_2193_axes_0 = const()[name = string("op_2193_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_2193_keep_dims_0 = const()[name = string("op_2193_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_2193_cast_fp16 = reduce_sum(axes = var_2193_axes_0, keep_dims = var_2193_keep_dims_0, x = var_2191_cast_fp16)[name = string("op_2193_cast_fp16")];
            tensor<int32, [3]> var_2196_begin_0 = const()[name = string("op_2196_begin_0"), val = tensor<int32, [3]>([0, 0, 5])];
            tensor<int32, [3]> var_2196_end_0 = const()[name = string("op_2196_end_0"), val = tensor<int32, [3]>([1, 16, 6])];
            tensor<bool, [3]> var_2196_end_mask_0 = const()[name = string("op_2196_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_2196_squeeze_mask_0 = const()[name = string("op_2196_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_2196_cast_fp16 = slice_by_index(begin = var_2196_begin_0, end = var_2196_end_0, end_mask = var_2196_end_mask_0, squeeze_mask = var_2196_squeeze_mask_0, x = decay_cast_fp16)[name = string("op_2196_cast_fp16")];
            tensor<fp16, [1, 16]> var_2197_cast_fp16 = exp(x = var_2196_cast_fp16)[name = string("op_2197_cast_fp16")];
            tensor<int32, [1]> var_2198_axes_0 = const()[name = string("op_2198_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_2198_cast_fp16 = expand_dims(axes = var_2198_axes_0, x = var_2197_cast_fp16)[name = string("op_2198_cast_fp16")];
            tensor<int32, [1]> var_2199_axes_0 = const()[name = string("op_2199_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_2199_cast_fp16 = expand_dims(axes = var_2199_axes_0, x = var_2198_cast_fp16)[name = string("op_2199_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_149_cast_fp16 = mul(x = state_147_cast_fp16, y = var_2199_cast_fp16)[name = string("state_149_cast_fp16")];
            tensor<int32, [4]> key_token_75_begin_0 = const()[name = string("key_token_75_begin_0"), val = tensor<int32, [4]>([0, 0, 5, 0])];
            tensor<int32, [4]> key_token_75_end_0 = const()[name = string("key_token_75_end_0"), val = tensor<int32, [4]>([1, 16, 6, 128])];
            tensor<bool, [4]> key_token_75_end_mask_0 = const()[name = string("key_token_75_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_75_squeeze_mask_0 = const()[name = string("key_token_75_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_75_cast_fp16 = slice_by_index(begin = key_token_75_begin_0, end = key_token_75_end_0, end_mask = key_token_75_end_mask_0, squeeze_mask = key_token_75_squeeze_mask_0, x = key_11_cast_fp16)[name = string("key_token_75_cast_fp16")];
            tensor<int32, [4]> value_token_75_begin_0 = const()[name = string("value_token_75_begin_0"), val = tensor<int32, [4]>([0, 0, 5, 0])];
            tensor<int32, [4]> value_token_75_end_0 = const()[name = string("value_token_75_end_0"), val = tensor<int32, [4]>([1, 16, 6, 128])];
            tensor<bool, [4]> value_token_75_end_mask_0 = const()[name = string("value_token_75_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_75_squeeze_mask_0 = const()[name = string("value_token_75_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_75_cast_fp16 = slice_by_index(begin = value_token_75_begin_0, end = value_token_75_end_0, end_mask = value_token_75_end_mask_0, squeeze_mask = value_token_75_squeeze_mask_0, x = value_41_cast_fp16)[name = string("value_token_75_cast_fp16")];
            tensor<int32, [1]> var_2207_axes_0 = const()[name = string("op_2207_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_2207_cast_fp16 = expand_dims(axes = var_2207_axes_0, x = key_token_75_cast_fp16)[name = string("op_2207_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2208_cast_fp16 = mul(x = state_149_cast_fp16, y = var_2207_cast_fp16)[name = string("op_2208_cast_fp16")];
            tensor<int32, [1]> memory_75_axes_0 = const()[name = string("memory_75_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_75_keep_dims_0 = const()[name = string("memory_75_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_75_cast_fp16 = reduce_sum(axes = memory_75_axes_0, keep_dims = memory_75_keep_dims_0, x = var_2208_cast_fp16)[name = string("memory_75_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_2211_cast_fp16 = sub(x = value_token_75_cast_fp16, y = memory_75_cast_fp16)[name = string("op_2211_cast_fp16")];
            tensor<int32, [3]> var_2214_begin_0 = const()[name = string("op_2214_begin_0"), val = tensor<int32, [3]>([0, 0, 5])];
            tensor<int32, [3]> var_2214_end_0 = const()[name = string("op_2214_end_0"), val = tensor<int32, [3]>([1, 16, 6])];
            tensor<bool, [3]> var_2214_end_mask_0 = const()[name = string("op_2214_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_2214_squeeze_mask_0 = const()[name = string("op_2214_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_2214_cast_fp16 = slice_by_index(begin = var_2214_begin_0, end = var_2214_end_0, end_mask = var_2214_end_mask_0, squeeze_mask = var_2214_squeeze_mask_0, x = beta_cast_fp16)[name = string("op_2214_cast_fp16")];
            tensor<int32, [1]> var_2215_axes_0 = const()[name = string("op_2215_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_2215_cast_fp16 = expand_dims(axes = var_2215_axes_0, x = var_2214_cast_fp16)[name = string("op_2215_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_75_cast_fp16 = mul(x = var_2211_cast_fp16, y = var_2215_cast_fp16)[name = string("delta_75_cast_fp16")];
            tensor<int32, [1]> var_2218_axes_0 = const()[name = string("op_2218_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_2218_cast_fp16 = expand_dims(axes = var_2218_axes_0, x = delta_75_cast_fp16)[name = string("op_2218_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2219_cast_fp16 = mul(x = var_2207_cast_fp16, y = var_2218_cast_fp16)[name = string("op_2219_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_151_cast_fp16 = add(x = state_149_cast_fp16, y = var_2219_cast_fp16)[name = string("state_151_cast_fp16")];
            tensor<int32, [4]> var_2223_begin_0 = const()[name = string("op_2223_begin_0"), val = tensor<int32, [4]>([0, 0, 5, 0])];
            tensor<int32, [4]> var_2223_end_0 = const()[name = string("op_2223_end_0"), val = tensor<int32, [4]>([1, 16, 6, 128])];
            tensor<bool, [4]> var_2223_end_mask_0 = const()[name = string("op_2223_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_2223_squeeze_mask_0 = const()[name = string("op_2223_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_2223_cast_fp16 = slice_by_index(begin = var_2223_begin_0, end = var_2223_end_0, end_mask = var_2223_end_mask_0, squeeze_mask = var_2223_squeeze_mask_0, x = _inversed_query_11_cast_fp16)[name = string("op_2223_cast_fp16")];
            tensor<int32, [1]> var_2224_axes_0 = const()[name = string("op_2224_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_2224_cast_fp16 = expand_dims(axes = var_2224_axes_0, x = var_2223_cast_fp16)[name = string("op_2224_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2225_cast_fp16 = mul(x = state_151_cast_fp16, y = var_2224_cast_fp16)[name = string("op_2225_cast_fp16")];
            tensor<int32, [1]> var_2227_axes_0 = const()[name = string("op_2227_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_2227_keep_dims_0 = const()[name = string("op_2227_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_2227_cast_fp16 = reduce_sum(axes = var_2227_axes_0, keep_dims = var_2227_keep_dims_0, x = var_2225_cast_fp16)[name = string("op_2227_cast_fp16")];
            tensor<int32, [3]> var_2230_begin_0 = const()[name = string("op_2230_begin_0"), val = tensor<int32, [3]>([0, 0, 6])];
            tensor<int32, [3]> var_2230_end_0 = const()[name = string("op_2230_end_0"), val = tensor<int32, [3]>([1, 16, 7])];
            tensor<bool, [3]> var_2230_end_mask_0 = const()[name = string("op_2230_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_2230_squeeze_mask_0 = const()[name = string("op_2230_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_2230_cast_fp16 = slice_by_index(begin = var_2230_begin_0, end = var_2230_end_0, end_mask = var_2230_end_mask_0, squeeze_mask = var_2230_squeeze_mask_0, x = decay_cast_fp16)[name = string("op_2230_cast_fp16")];
            tensor<fp16, [1, 16]> var_2231_cast_fp16 = exp(x = var_2230_cast_fp16)[name = string("op_2231_cast_fp16")];
            tensor<int32, [1]> var_2232_axes_0 = const()[name = string("op_2232_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_2232_cast_fp16 = expand_dims(axes = var_2232_axes_0, x = var_2231_cast_fp16)[name = string("op_2232_cast_fp16")];
            tensor<int32, [1]> var_2233_axes_0 = const()[name = string("op_2233_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_2233_cast_fp16 = expand_dims(axes = var_2233_axes_0, x = var_2232_cast_fp16)[name = string("op_2233_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_153_cast_fp16 = mul(x = state_151_cast_fp16, y = var_2233_cast_fp16)[name = string("state_153_cast_fp16")];
            tensor<int32, [4]> key_token_77_begin_0 = const()[name = string("key_token_77_begin_0"), val = tensor<int32, [4]>([0, 0, 6, 0])];
            tensor<int32, [4]> key_token_77_end_0 = const()[name = string("key_token_77_end_0"), val = tensor<int32, [4]>([1, 16, 7, 128])];
            tensor<bool, [4]> key_token_77_end_mask_0 = const()[name = string("key_token_77_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_77_squeeze_mask_0 = const()[name = string("key_token_77_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_77_cast_fp16 = slice_by_index(begin = key_token_77_begin_0, end = key_token_77_end_0, end_mask = key_token_77_end_mask_0, squeeze_mask = key_token_77_squeeze_mask_0, x = key_11_cast_fp16)[name = string("key_token_77_cast_fp16")];
            tensor<int32, [4]> value_token_77_begin_0 = const()[name = string("value_token_77_begin_0"), val = tensor<int32, [4]>([0, 0, 6, 0])];
            tensor<int32, [4]> value_token_77_end_0 = const()[name = string("value_token_77_end_0"), val = tensor<int32, [4]>([1, 16, 7, 128])];
            tensor<bool, [4]> value_token_77_end_mask_0 = const()[name = string("value_token_77_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_77_squeeze_mask_0 = const()[name = string("value_token_77_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_77_cast_fp16 = slice_by_index(begin = value_token_77_begin_0, end = value_token_77_end_0, end_mask = value_token_77_end_mask_0, squeeze_mask = value_token_77_squeeze_mask_0, x = value_41_cast_fp16)[name = string("value_token_77_cast_fp16")];
            tensor<int32, [1]> var_2241_axes_0 = const()[name = string("op_2241_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_2241_cast_fp16 = expand_dims(axes = var_2241_axes_0, x = key_token_77_cast_fp16)[name = string("op_2241_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2242_cast_fp16 = mul(x = state_153_cast_fp16, y = var_2241_cast_fp16)[name = string("op_2242_cast_fp16")];
            tensor<int32, [1]> memory_77_axes_0 = const()[name = string("memory_77_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_77_keep_dims_0 = const()[name = string("memory_77_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_77_cast_fp16 = reduce_sum(axes = memory_77_axes_0, keep_dims = memory_77_keep_dims_0, x = var_2242_cast_fp16)[name = string("memory_77_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_2245_cast_fp16 = sub(x = value_token_77_cast_fp16, y = memory_77_cast_fp16)[name = string("op_2245_cast_fp16")];
            tensor<int32, [3]> var_2248_begin_0 = const()[name = string("op_2248_begin_0"), val = tensor<int32, [3]>([0, 0, 6])];
            tensor<int32, [3]> var_2248_end_0 = const()[name = string("op_2248_end_0"), val = tensor<int32, [3]>([1, 16, 7])];
            tensor<bool, [3]> var_2248_end_mask_0 = const()[name = string("op_2248_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_2248_squeeze_mask_0 = const()[name = string("op_2248_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_2248_cast_fp16 = slice_by_index(begin = var_2248_begin_0, end = var_2248_end_0, end_mask = var_2248_end_mask_0, squeeze_mask = var_2248_squeeze_mask_0, x = beta_cast_fp16)[name = string("op_2248_cast_fp16")];
            tensor<int32, [1]> var_2249_axes_0 = const()[name = string("op_2249_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_2249_cast_fp16 = expand_dims(axes = var_2249_axes_0, x = var_2248_cast_fp16)[name = string("op_2249_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_77_cast_fp16 = mul(x = var_2245_cast_fp16, y = var_2249_cast_fp16)[name = string("delta_77_cast_fp16")];
            tensor<int32, [1]> var_2252_axes_0 = const()[name = string("op_2252_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_2252_cast_fp16 = expand_dims(axes = var_2252_axes_0, x = delta_77_cast_fp16)[name = string("op_2252_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2253_cast_fp16 = mul(x = var_2241_cast_fp16, y = var_2252_cast_fp16)[name = string("op_2253_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_155_cast_fp16 = add(x = state_153_cast_fp16, y = var_2253_cast_fp16)[name = string("state_155_cast_fp16")];
            tensor<int32, [4]> var_2257_begin_0 = const()[name = string("op_2257_begin_0"), val = tensor<int32, [4]>([0, 0, 6, 0])];
            tensor<int32, [4]> var_2257_end_0 = const()[name = string("op_2257_end_0"), val = tensor<int32, [4]>([1, 16, 7, 128])];
            tensor<bool, [4]> var_2257_end_mask_0 = const()[name = string("op_2257_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_2257_squeeze_mask_0 = const()[name = string("op_2257_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_2257_cast_fp16 = slice_by_index(begin = var_2257_begin_0, end = var_2257_end_0, end_mask = var_2257_end_mask_0, squeeze_mask = var_2257_squeeze_mask_0, x = _inversed_query_11_cast_fp16)[name = string("op_2257_cast_fp16")];
            tensor<int32, [1]> var_2258_axes_0 = const()[name = string("op_2258_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_2258_cast_fp16 = expand_dims(axes = var_2258_axes_0, x = var_2257_cast_fp16)[name = string("op_2258_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2259_cast_fp16 = mul(x = state_155_cast_fp16, y = var_2258_cast_fp16)[name = string("op_2259_cast_fp16")];
            tensor<int32, [1]> var_2261_axes_0 = const()[name = string("op_2261_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_2261_keep_dims_0 = const()[name = string("op_2261_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_2261_cast_fp16 = reduce_sum(axes = var_2261_axes_0, keep_dims = var_2261_keep_dims_0, x = var_2259_cast_fp16)[name = string("op_2261_cast_fp16")];
            tensor<int32, [3]> var_2264_begin_0 = const()[name = string("op_2264_begin_0"), val = tensor<int32, [3]>([0, 0, 7])];
            tensor<int32, [3]> var_2264_end_0 = const()[name = string("op_2264_end_0"), val = tensor<int32, [3]>([1, 16, 8])];
            tensor<bool, [3]> var_2264_end_mask_0 = const()[name = string("op_2264_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_2264_squeeze_mask_0 = const()[name = string("op_2264_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_2264_cast_fp16 = slice_by_index(begin = var_2264_begin_0, end = var_2264_end_0, end_mask = var_2264_end_mask_0, squeeze_mask = var_2264_squeeze_mask_0, x = decay_cast_fp16)[name = string("op_2264_cast_fp16")];
            tensor<fp16, [1, 16]> var_2265_cast_fp16 = exp(x = var_2264_cast_fp16)[name = string("op_2265_cast_fp16")];
            tensor<int32, [1]> var_2266_axes_0 = const()[name = string("op_2266_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_2266_cast_fp16 = expand_dims(axes = var_2266_axes_0, x = var_2265_cast_fp16)[name = string("op_2266_cast_fp16")];
            tensor<int32, [1]> var_2267_axes_0 = const()[name = string("op_2267_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_2267_cast_fp16 = expand_dims(axes = var_2267_axes_0, x = var_2266_cast_fp16)[name = string("op_2267_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_157_cast_fp16 = mul(x = state_155_cast_fp16, y = var_2267_cast_fp16)[name = string("state_157_cast_fp16")];
            tensor<int32, [4]> key_token_79_begin_0 = const()[name = string("key_token_79_begin_0"), val = tensor<int32, [4]>([0, 0, 7, 0])];
            tensor<int32, [4]> key_token_79_end_0 = const()[name = string("key_token_79_end_0"), val = tensor<int32, [4]>([1, 16, 8, 128])];
            tensor<bool, [4]> key_token_79_end_mask_0 = const()[name = string("key_token_79_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_79_squeeze_mask_0 = const()[name = string("key_token_79_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_79_cast_fp16 = slice_by_index(begin = key_token_79_begin_0, end = key_token_79_end_0, end_mask = key_token_79_end_mask_0, squeeze_mask = key_token_79_squeeze_mask_0, x = key_11_cast_fp16)[name = string("key_token_79_cast_fp16")];
            tensor<int32, [4]> value_token_79_begin_0 = const()[name = string("value_token_79_begin_0"), val = tensor<int32, [4]>([0, 0, 7, 0])];
            tensor<int32, [4]> value_token_79_end_0 = const()[name = string("value_token_79_end_0"), val = tensor<int32, [4]>([1, 16, 8, 128])];
            tensor<bool, [4]> value_token_79_end_mask_0 = const()[name = string("value_token_79_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_79_squeeze_mask_0 = const()[name = string("value_token_79_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_79_cast_fp16 = slice_by_index(begin = value_token_79_begin_0, end = value_token_79_end_0, end_mask = value_token_79_end_mask_0, squeeze_mask = value_token_79_squeeze_mask_0, x = value_41_cast_fp16)[name = string("value_token_79_cast_fp16")];
            tensor<int32, [1]> var_2275_axes_0 = const()[name = string("op_2275_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_2275_cast_fp16 = expand_dims(axes = var_2275_axes_0, x = key_token_79_cast_fp16)[name = string("op_2275_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2276_cast_fp16 = mul(x = state_157_cast_fp16, y = var_2275_cast_fp16)[name = string("op_2276_cast_fp16")];
            tensor<int32, [1]> memory_79_axes_0 = const()[name = string("memory_79_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_79_keep_dims_0 = const()[name = string("memory_79_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_79_cast_fp16 = reduce_sum(axes = memory_79_axes_0, keep_dims = memory_79_keep_dims_0, x = var_2276_cast_fp16)[name = string("memory_79_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_2279_cast_fp16 = sub(x = value_token_79_cast_fp16, y = memory_79_cast_fp16)[name = string("op_2279_cast_fp16")];
            tensor<int32, [3]> var_2282_begin_0 = const()[name = string("op_2282_begin_0"), val = tensor<int32, [3]>([0, 0, 7])];
            tensor<int32, [3]> var_2282_end_0 = const()[name = string("op_2282_end_0"), val = tensor<int32, [3]>([1, 16, 8])];
            tensor<bool, [3]> var_2282_end_mask_0 = const()[name = string("op_2282_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_2282_squeeze_mask_0 = const()[name = string("op_2282_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_2282_cast_fp16 = slice_by_index(begin = var_2282_begin_0, end = var_2282_end_0, end_mask = var_2282_end_mask_0, squeeze_mask = var_2282_squeeze_mask_0, x = beta_cast_fp16)[name = string("op_2282_cast_fp16")];
            tensor<int32, [1]> var_2283_axes_0 = const()[name = string("op_2283_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_2283_cast_fp16 = expand_dims(axes = var_2283_axes_0, x = var_2282_cast_fp16)[name = string("op_2283_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_79_cast_fp16 = mul(x = var_2279_cast_fp16, y = var_2283_cast_fp16)[name = string("delta_79_cast_fp16")];
            tensor<int32, [1]> var_2286_axes_0 = const()[name = string("op_2286_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_2286_cast_fp16 = expand_dims(axes = var_2286_axes_0, x = delta_79_cast_fp16)[name = string("op_2286_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2287_cast_fp16 = mul(x = var_2275_cast_fp16, y = var_2286_cast_fp16)[name = string("op_2287_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_159_cast_fp16 = add(x = state_157_cast_fp16, y = var_2287_cast_fp16)[name = string("state_159_cast_fp16")];
            tensor<int32, [4]> var_2291_begin_0 = const()[name = string("op_2291_begin_0"), val = tensor<int32, [4]>([0, 0, 7, 0])];
            tensor<int32, [4]> var_2291_end_0 = const()[name = string("op_2291_end_0"), val = tensor<int32, [4]>([1, 16, 8, 128])];
            tensor<bool, [4]> var_2291_end_mask_0 = const()[name = string("op_2291_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_2291_squeeze_mask_0 = const()[name = string("op_2291_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_2291_cast_fp16 = slice_by_index(begin = var_2291_begin_0, end = var_2291_end_0, end_mask = var_2291_end_mask_0, squeeze_mask = var_2291_squeeze_mask_0, x = _inversed_query_11_cast_fp16)[name = string("op_2291_cast_fp16")];
            tensor<int32, [1]> var_2292_axes_0 = const()[name = string("op_2292_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_2292_cast_fp16 = expand_dims(axes = var_2292_axes_0, x = var_2291_cast_fp16)[name = string("op_2292_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2293_cast_fp16 = mul(x = state_159_cast_fp16, y = var_2292_cast_fp16)[name = string("op_2293_cast_fp16")];
            tensor<int32, [1]> var_2295_axes_0 = const()[name = string("op_2295_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_2295_keep_dims_0 = const()[name = string("op_2295_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_2295_cast_fp16 = reduce_sum(axes = var_2295_axes_0, keep_dims = var_2295_keep_dims_0, x = var_2293_cast_fp16)[name = string("op_2295_cast_fp16")];
            tensor<int32, [3]> var_2298_begin_0 = const()[name = string("op_2298_begin_0"), val = tensor<int32, [3]>([0, 0, 8])];
            tensor<int32, [3]> var_2298_end_0 = const()[name = string("op_2298_end_0"), val = tensor<int32, [3]>([1, 16, 9])];
            tensor<bool, [3]> var_2298_end_mask_0 = const()[name = string("op_2298_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_2298_squeeze_mask_0 = const()[name = string("op_2298_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_2298_cast_fp16 = slice_by_index(begin = var_2298_begin_0, end = var_2298_end_0, end_mask = var_2298_end_mask_0, squeeze_mask = var_2298_squeeze_mask_0, x = decay_cast_fp16)[name = string("op_2298_cast_fp16")];
            tensor<fp16, [1, 16]> var_2299_cast_fp16 = exp(x = var_2298_cast_fp16)[name = string("op_2299_cast_fp16")];
            tensor<int32, [1]> var_2300_axes_0 = const()[name = string("op_2300_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_2300_cast_fp16 = expand_dims(axes = var_2300_axes_0, x = var_2299_cast_fp16)[name = string("op_2300_cast_fp16")];
            tensor<int32, [1]> var_2301_axes_0 = const()[name = string("op_2301_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_2301_cast_fp16 = expand_dims(axes = var_2301_axes_0, x = var_2300_cast_fp16)[name = string("op_2301_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_161_cast_fp16 = mul(x = state_159_cast_fp16, y = var_2301_cast_fp16)[name = string("state_161_cast_fp16")];
            tensor<int32, [4]> key_token_81_begin_0 = const()[name = string("key_token_81_begin_0"), val = tensor<int32, [4]>([0, 0, 8, 0])];
            tensor<int32, [4]> key_token_81_end_0 = const()[name = string("key_token_81_end_0"), val = tensor<int32, [4]>([1, 16, 9, 128])];
            tensor<bool, [4]> key_token_81_end_mask_0 = const()[name = string("key_token_81_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_81_squeeze_mask_0 = const()[name = string("key_token_81_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_81_cast_fp16 = slice_by_index(begin = key_token_81_begin_0, end = key_token_81_end_0, end_mask = key_token_81_end_mask_0, squeeze_mask = key_token_81_squeeze_mask_0, x = key_11_cast_fp16)[name = string("key_token_81_cast_fp16")];
            tensor<int32, [4]> value_token_81_begin_0 = const()[name = string("value_token_81_begin_0"), val = tensor<int32, [4]>([0, 0, 8, 0])];
            tensor<int32, [4]> value_token_81_end_0 = const()[name = string("value_token_81_end_0"), val = tensor<int32, [4]>([1, 16, 9, 128])];
            tensor<bool, [4]> value_token_81_end_mask_0 = const()[name = string("value_token_81_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_81_squeeze_mask_0 = const()[name = string("value_token_81_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_81_cast_fp16 = slice_by_index(begin = value_token_81_begin_0, end = value_token_81_end_0, end_mask = value_token_81_end_mask_0, squeeze_mask = value_token_81_squeeze_mask_0, x = value_41_cast_fp16)[name = string("value_token_81_cast_fp16")];
            tensor<int32, [1]> var_2309_axes_0 = const()[name = string("op_2309_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_2309_cast_fp16 = expand_dims(axes = var_2309_axes_0, x = key_token_81_cast_fp16)[name = string("op_2309_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2310_cast_fp16 = mul(x = state_161_cast_fp16, y = var_2309_cast_fp16)[name = string("op_2310_cast_fp16")];
            tensor<int32, [1]> memory_81_axes_0 = const()[name = string("memory_81_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_81_keep_dims_0 = const()[name = string("memory_81_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_81_cast_fp16 = reduce_sum(axes = memory_81_axes_0, keep_dims = memory_81_keep_dims_0, x = var_2310_cast_fp16)[name = string("memory_81_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_2313_cast_fp16 = sub(x = value_token_81_cast_fp16, y = memory_81_cast_fp16)[name = string("op_2313_cast_fp16")];
            tensor<int32, [3]> var_2316_begin_0 = const()[name = string("op_2316_begin_0"), val = tensor<int32, [3]>([0, 0, 8])];
            tensor<int32, [3]> var_2316_end_0 = const()[name = string("op_2316_end_0"), val = tensor<int32, [3]>([1, 16, 9])];
            tensor<bool, [3]> var_2316_end_mask_0 = const()[name = string("op_2316_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_2316_squeeze_mask_0 = const()[name = string("op_2316_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_2316_cast_fp16 = slice_by_index(begin = var_2316_begin_0, end = var_2316_end_0, end_mask = var_2316_end_mask_0, squeeze_mask = var_2316_squeeze_mask_0, x = beta_cast_fp16)[name = string("op_2316_cast_fp16")];
            tensor<int32, [1]> var_2317_axes_0 = const()[name = string("op_2317_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_2317_cast_fp16 = expand_dims(axes = var_2317_axes_0, x = var_2316_cast_fp16)[name = string("op_2317_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_81_cast_fp16 = mul(x = var_2313_cast_fp16, y = var_2317_cast_fp16)[name = string("delta_81_cast_fp16")];
            tensor<int32, [1]> var_2320_axes_0 = const()[name = string("op_2320_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_2320_cast_fp16 = expand_dims(axes = var_2320_axes_0, x = delta_81_cast_fp16)[name = string("op_2320_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2321_cast_fp16 = mul(x = var_2309_cast_fp16, y = var_2320_cast_fp16)[name = string("op_2321_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_163_cast_fp16 = add(x = state_161_cast_fp16, y = var_2321_cast_fp16)[name = string("state_163_cast_fp16")];
            tensor<int32, [4]> var_2325_begin_0 = const()[name = string("op_2325_begin_0"), val = tensor<int32, [4]>([0, 0, 8, 0])];
            tensor<int32, [4]> var_2325_end_0 = const()[name = string("op_2325_end_0"), val = tensor<int32, [4]>([1, 16, 9, 128])];
            tensor<bool, [4]> var_2325_end_mask_0 = const()[name = string("op_2325_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_2325_squeeze_mask_0 = const()[name = string("op_2325_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_2325_cast_fp16 = slice_by_index(begin = var_2325_begin_0, end = var_2325_end_0, end_mask = var_2325_end_mask_0, squeeze_mask = var_2325_squeeze_mask_0, x = _inversed_query_11_cast_fp16)[name = string("op_2325_cast_fp16")];
            tensor<int32, [1]> var_2326_axes_0 = const()[name = string("op_2326_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_2326_cast_fp16 = expand_dims(axes = var_2326_axes_0, x = var_2325_cast_fp16)[name = string("op_2326_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2327_cast_fp16 = mul(x = state_163_cast_fp16, y = var_2326_cast_fp16)[name = string("op_2327_cast_fp16")];
            tensor<int32, [1]> var_2329_axes_0 = const()[name = string("op_2329_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_2329_keep_dims_0 = const()[name = string("op_2329_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_2329_cast_fp16 = reduce_sum(axes = var_2329_axes_0, keep_dims = var_2329_keep_dims_0, x = var_2327_cast_fp16)[name = string("op_2329_cast_fp16")];
            tensor<int32, [3]> var_2332_begin_0 = const()[name = string("op_2332_begin_0"), val = tensor<int32, [3]>([0, 0, 9])];
            tensor<int32, [3]> var_2332_end_0 = const()[name = string("op_2332_end_0"), val = tensor<int32, [3]>([1, 16, 10])];
            tensor<bool, [3]> var_2332_end_mask_0 = const()[name = string("op_2332_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_2332_squeeze_mask_0 = const()[name = string("op_2332_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_2332_cast_fp16 = slice_by_index(begin = var_2332_begin_0, end = var_2332_end_0, end_mask = var_2332_end_mask_0, squeeze_mask = var_2332_squeeze_mask_0, x = decay_cast_fp16)[name = string("op_2332_cast_fp16")];
            tensor<fp16, [1, 16]> var_2333_cast_fp16 = exp(x = var_2332_cast_fp16)[name = string("op_2333_cast_fp16")];
            tensor<int32, [1]> var_2334_axes_0 = const()[name = string("op_2334_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_2334_cast_fp16 = expand_dims(axes = var_2334_axes_0, x = var_2333_cast_fp16)[name = string("op_2334_cast_fp16")];
            tensor<int32, [1]> var_2335_axes_0 = const()[name = string("op_2335_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_2335_cast_fp16 = expand_dims(axes = var_2335_axes_0, x = var_2334_cast_fp16)[name = string("op_2335_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_165_cast_fp16 = mul(x = state_163_cast_fp16, y = var_2335_cast_fp16)[name = string("state_165_cast_fp16")];
            tensor<int32, [4]> key_token_83_begin_0 = const()[name = string("key_token_83_begin_0"), val = tensor<int32, [4]>([0, 0, 9, 0])];
            tensor<int32, [4]> key_token_83_end_0 = const()[name = string("key_token_83_end_0"), val = tensor<int32, [4]>([1, 16, 10, 128])];
            tensor<bool, [4]> key_token_83_end_mask_0 = const()[name = string("key_token_83_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_83_squeeze_mask_0 = const()[name = string("key_token_83_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_83_cast_fp16 = slice_by_index(begin = key_token_83_begin_0, end = key_token_83_end_0, end_mask = key_token_83_end_mask_0, squeeze_mask = key_token_83_squeeze_mask_0, x = key_11_cast_fp16)[name = string("key_token_83_cast_fp16")];
            tensor<int32, [4]> value_token_83_begin_0 = const()[name = string("value_token_83_begin_0"), val = tensor<int32, [4]>([0, 0, 9, 0])];
            tensor<int32, [4]> value_token_83_end_0 = const()[name = string("value_token_83_end_0"), val = tensor<int32, [4]>([1, 16, 10, 128])];
            tensor<bool, [4]> value_token_83_end_mask_0 = const()[name = string("value_token_83_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_83_squeeze_mask_0 = const()[name = string("value_token_83_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_83_cast_fp16 = slice_by_index(begin = value_token_83_begin_0, end = value_token_83_end_0, end_mask = value_token_83_end_mask_0, squeeze_mask = value_token_83_squeeze_mask_0, x = value_41_cast_fp16)[name = string("value_token_83_cast_fp16")];
            tensor<int32, [1]> var_2343_axes_0 = const()[name = string("op_2343_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_2343_cast_fp16 = expand_dims(axes = var_2343_axes_0, x = key_token_83_cast_fp16)[name = string("op_2343_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2344_cast_fp16 = mul(x = state_165_cast_fp16, y = var_2343_cast_fp16)[name = string("op_2344_cast_fp16")];
            tensor<int32, [1]> memory_83_axes_0 = const()[name = string("memory_83_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_83_keep_dims_0 = const()[name = string("memory_83_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_83_cast_fp16 = reduce_sum(axes = memory_83_axes_0, keep_dims = memory_83_keep_dims_0, x = var_2344_cast_fp16)[name = string("memory_83_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_2347_cast_fp16 = sub(x = value_token_83_cast_fp16, y = memory_83_cast_fp16)[name = string("op_2347_cast_fp16")];
            tensor<int32, [3]> var_2350_begin_0 = const()[name = string("op_2350_begin_0"), val = tensor<int32, [3]>([0, 0, 9])];
            tensor<int32, [3]> var_2350_end_0 = const()[name = string("op_2350_end_0"), val = tensor<int32, [3]>([1, 16, 10])];
            tensor<bool, [3]> var_2350_end_mask_0 = const()[name = string("op_2350_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_2350_squeeze_mask_0 = const()[name = string("op_2350_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_2350_cast_fp16 = slice_by_index(begin = var_2350_begin_0, end = var_2350_end_0, end_mask = var_2350_end_mask_0, squeeze_mask = var_2350_squeeze_mask_0, x = beta_cast_fp16)[name = string("op_2350_cast_fp16")];
            tensor<int32, [1]> var_2351_axes_0 = const()[name = string("op_2351_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_2351_cast_fp16 = expand_dims(axes = var_2351_axes_0, x = var_2350_cast_fp16)[name = string("op_2351_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_83_cast_fp16 = mul(x = var_2347_cast_fp16, y = var_2351_cast_fp16)[name = string("delta_83_cast_fp16")];
            tensor<int32, [1]> var_2354_axes_0 = const()[name = string("op_2354_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_2354_cast_fp16 = expand_dims(axes = var_2354_axes_0, x = delta_83_cast_fp16)[name = string("op_2354_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2355_cast_fp16 = mul(x = var_2343_cast_fp16, y = var_2354_cast_fp16)[name = string("op_2355_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_167_cast_fp16 = add(x = state_165_cast_fp16, y = var_2355_cast_fp16)[name = string("state_167_cast_fp16")];
            tensor<int32, [4]> var_2359_begin_0 = const()[name = string("op_2359_begin_0"), val = tensor<int32, [4]>([0, 0, 9, 0])];
            tensor<int32, [4]> var_2359_end_0 = const()[name = string("op_2359_end_0"), val = tensor<int32, [4]>([1, 16, 10, 128])];
            tensor<bool, [4]> var_2359_end_mask_0 = const()[name = string("op_2359_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_2359_squeeze_mask_0 = const()[name = string("op_2359_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_2359_cast_fp16 = slice_by_index(begin = var_2359_begin_0, end = var_2359_end_0, end_mask = var_2359_end_mask_0, squeeze_mask = var_2359_squeeze_mask_0, x = _inversed_query_11_cast_fp16)[name = string("op_2359_cast_fp16")];
            tensor<int32, [1]> var_2360_axes_0 = const()[name = string("op_2360_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_2360_cast_fp16 = expand_dims(axes = var_2360_axes_0, x = var_2359_cast_fp16)[name = string("op_2360_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2361_cast_fp16 = mul(x = state_167_cast_fp16, y = var_2360_cast_fp16)[name = string("op_2361_cast_fp16")];
            tensor<int32, [1]> var_2363_axes_0 = const()[name = string("op_2363_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_2363_keep_dims_0 = const()[name = string("op_2363_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_2363_cast_fp16 = reduce_sum(axes = var_2363_axes_0, keep_dims = var_2363_keep_dims_0, x = var_2361_cast_fp16)[name = string("op_2363_cast_fp16")];
            tensor<int32, [3]> var_2366_begin_0 = const()[name = string("op_2366_begin_0"), val = tensor<int32, [3]>([0, 0, 10])];
            tensor<int32, [3]> var_2366_end_0 = const()[name = string("op_2366_end_0"), val = tensor<int32, [3]>([1, 16, 11])];
            tensor<bool, [3]> var_2366_end_mask_0 = const()[name = string("op_2366_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_2366_squeeze_mask_0 = const()[name = string("op_2366_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_2366_cast_fp16 = slice_by_index(begin = var_2366_begin_0, end = var_2366_end_0, end_mask = var_2366_end_mask_0, squeeze_mask = var_2366_squeeze_mask_0, x = decay_cast_fp16)[name = string("op_2366_cast_fp16")];
            tensor<fp16, [1, 16]> var_2367_cast_fp16 = exp(x = var_2366_cast_fp16)[name = string("op_2367_cast_fp16")];
            tensor<int32, [1]> var_2368_axes_0 = const()[name = string("op_2368_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_2368_cast_fp16 = expand_dims(axes = var_2368_axes_0, x = var_2367_cast_fp16)[name = string("op_2368_cast_fp16")];
            tensor<int32, [1]> var_2369_axes_0 = const()[name = string("op_2369_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_2369_cast_fp16 = expand_dims(axes = var_2369_axes_0, x = var_2368_cast_fp16)[name = string("op_2369_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_169_cast_fp16 = mul(x = state_167_cast_fp16, y = var_2369_cast_fp16)[name = string("state_169_cast_fp16")];
            tensor<int32, [4]> key_token_85_begin_0 = const()[name = string("key_token_85_begin_0"), val = tensor<int32, [4]>([0, 0, 10, 0])];
            tensor<int32, [4]> key_token_85_end_0 = const()[name = string("key_token_85_end_0"), val = tensor<int32, [4]>([1, 16, 11, 128])];
            tensor<bool, [4]> key_token_85_end_mask_0 = const()[name = string("key_token_85_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_85_squeeze_mask_0 = const()[name = string("key_token_85_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_85_cast_fp16 = slice_by_index(begin = key_token_85_begin_0, end = key_token_85_end_0, end_mask = key_token_85_end_mask_0, squeeze_mask = key_token_85_squeeze_mask_0, x = key_11_cast_fp16)[name = string("key_token_85_cast_fp16")];
            tensor<int32, [4]> value_token_85_begin_0 = const()[name = string("value_token_85_begin_0"), val = tensor<int32, [4]>([0, 0, 10, 0])];
            tensor<int32, [4]> value_token_85_end_0 = const()[name = string("value_token_85_end_0"), val = tensor<int32, [4]>([1, 16, 11, 128])];
            tensor<bool, [4]> value_token_85_end_mask_0 = const()[name = string("value_token_85_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_85_squeeze_mask_0 = const()[name = string("value_token_85_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_85_cast_fp16 = slice_by_index(begin = value_token_85_begin_0, end = value_token_85_end_0, end_mask = value_token_85_end_mask_0, squeeze_mask = value_token_85_squeeze_mask_0, x = value_41_cast_fp16)[name = string("value_token_85_cast_fp16")];
            tensor<int32, [1]> var_2377_axes_0 = const()[name = string("op_2377_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_2377_cast_fp16 = expand_dims(axes = var_2377_axes_0, x = key_token_85_cast_fp16)[name = string("op_2377_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2378_cast_fp16 = mul(x = state_169_cast_fp16, y = var_2377_cast_fp16)[name = string("op_2378_cast_fp16")];
            tensor<int32, [1]> memory_85_axes_0 = const()[name = string("memory_85_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_85_keep_dims_0 = const()[name = string("memory_85_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_85_cast_fp16 = reduce_sum(axes = memory_85_axes_0, keep_dims = memory_85_keep_dims_0, x = var_2378_cast_fp16)[name = string("memory_85_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_2381_cast_fp16 = sub(x = value_token_85_cast_fp16, y = memory_85_cast_fp16)[name = string("op_2381_cast_fp16")];
            tensor<int32, [3]> var_2384_begin_0 = const()[name = string("op_2384_begin_0"), val = tensor<int32, [3]>([0, 0, 10])];
            tensor<int32, [3]> var_2384_end_0 = const()[name = string("op_2384_end_0"), val = tensor<int32, [3]>([1, 16, 11])];
            tensor<bool, [3]> var_2384_end_mask_0 = const()[name = string("op_2384_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_2384_squeeze_mask_0 = const()[name = string("op_2384_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_2384_cast_fp16 = slice_by_index(begin = var_2384_begin_0, end = var_2384_end_0, end_mask = var_2384_end_mask_0, squeeze_mask = var_2384_squeeze_mask_0, x = beta_cast_fp16)[name = string("op_2384_cast_fp16")];
            tensor<int32, [1]> var_2385_axes_0 = const()[name = string("op_2385_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_2385_cast_fp16 = expand_dims(axes = var_2385_axes_0, x = var_2384_cast_fp16)[name = string("op_2385_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_85_cast_fp16 = mul(x = var_2381_cast_fp16, y = var_2385_cast_fp16)[name = string("delta_85_cast_fp16")];
            tensor<int32, [1]> var_2388_axes_0 = const()[name = string("op_2388_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_2388_cast_fp16 = expand_dims(axes = var_2388_axes_0, x = delta_85_cast_fp16)[name = string("op_2388_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2389_cast_fp16 = mul(x = var_2377_cast_fp16, y = var_2388_cast_fp16)[name = string("op_2389_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_171_cast_fp16 = add(x = state_169_cast_fp16, y = var_2389_cast_fp16)[name = string("state_171_cast_fp16")];
            tensor<int32, [4]> var_2393_begin_0 = const()[name = string("op_2393_begin_0"), val = tensor<int32, [4]>([0, 0, 10, 0])];
            tensor<int32, [4]> var_2393_end_0 = const()[name = string("op_2393_end_0"), val = tensor<int32, [4]>([1, 16, 11, 128])];
            tensor<bool, [4]> var_2393_end_mask_0 = const()[name = string("op_2393_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_2393_squeeze_mask_0 = const()[name = string("op_2393_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_2393_cast_fp16 = slice_by_index(begin = var_2393_begin_0, end = var_2393_end_0, end_mask = var_2393_end_mask_0, squeeze_mask = var_2393_squeeze_mask_0, x = _inversed_query_11_cast_fp16)[name = string("op_2393_cast_fp16")];
            tensor<int32, [1]> var_2394_axes_0 = const()[name = string("op_2394_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_2394_cast_fp16 = expand_dims(axes = var_2394_axes_0, x = var_2393_cast_fp16)[name = string("op_2394_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2395_cast_fp16 = mul(x = state_171_cast_fp16, y = var_2394_cast_fp16)[name = string("op_2395_cast_fp16")];
            tensor<int32, [1]> var_2397_axes_0 = const()[name = string("op_2397_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_2397_keep_dims_0 = const()[name = string("op_2397_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_2397_cast_fp16 = reduce_sum(axes = var_2397_axes_0, keep_dims = var_2397_keep_dims_0, x = var_2395_cast_fp16)[name = string("op_2397_cast_fp16")];
            tensor<int32, [3]> var_2400_begin_0 = const()[name = string("op_2400_begin_0"), val = tensor<int32, [3]>([0, 0, 11])];
            tensor<int32, [3]> var_2400_end_0 = const()[name = string("op_2400_end_0"), val = tensor<int32, [3]>([1, 16, 12])];
            tensor<bool, [3]> var_2400_end_mask_0 = const()[name = string("op_2400_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_2400_squeeze_mask_0 = const()[name = string("op_2400_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_2400_cast_fp16 = slice_by_index(begin = var_2400_begin_0, end = var_2400_end_0, end_mask = var_2400_end_mask_0, squeeze_mask = var_2400_squeeze_mask_0, x = decay_cast_fp16)[name = string("op_2400_cast_fp16")];
            tensor<fp16, [1, 16]> var_2401_cast_fp16 = exp(x = var_2400_cast_fp16)[name = string("op_2401_cast_fp16")];
            tensor<int32, [1]> var_2402_axes_0 = const()[name = string("op_2402_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_2402_cast_fp16 = expand_dims(axes = var_2402_axes_0, x = var_2401_cast_fp16)[name = string("op_2402_cast_fp16")];
            tensor<int32, [1]> var_2403_axes_0 = const()[name = string("op_2403_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_2403_cast_fp16 = expand_dims(axes = var_2403_axes_0, x = var_2402_cast_fp16)[name = string("op_2403_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_173_cast_fp16 = mul(x = state_171_cast_fp16, y = var_2403_cast_fp16)[name = string("state_173_cast_fp16")];
            tensor<int32, [4]> key_token_87_begin_0 = const()[name = string("key_token_87_begin_0"), val = tensor<int32, [4]>([0, 0, 11, 0])];
            tensor<int32, [4]> key_token_87_end_0 = const()[name = string("key_token_87_end_0"), val = tensor<int32, [4]>([1, 16, 12, 128])];
            tensor<bool, [4]> key_token_87_end_mask_0 = const()[name = string("key_token_87_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_87_squeeze_mask_0 = const()[name = string("key_token_87_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_87_cast_fp16 = slice_by_index(begin = key_token_87_begin_0, end = key_token_87_end_0, end_mask = key_token_87_end_mask_0, squeeze_mask = key_token_87_squeeze_mask_0, x = key_11_cast_fp16)[name = string("key_token_87_cast_fp16")];
            tensor<int32, [4]> value_token_87_begin_0 = const()[name = string("value_token_87_begin_0"), val = tensor<int32, [4]>([0, 0, 11, 0])];
            tensor<int32, [4]> value_token_87_end_0 = const()[name = string("value_token_87_end_0"), val = tensor<int32, [4]>([1, 16, 12, 128])];
            tensor<bool, [4]> value_token_87_end_mask_0 = const()[name = string("value_token_87_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_87_squeeze_mask_0 = const()[name = string("value_token_87_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_87_cast_fp16 = slice_by_index(begin = value_token_87_begin_0, end = value_token_87_end_0, end_mask = value_token_87_end_mask_0, squeeze_mask = value_token_87_squeeze_mask_0, x = value_41_cast_fp16)[name = string("value_token_87_cast_fp16")];
            tensor<int32, [1]> var_2411_axes_0 = const()[name = string("op_2411_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_2411_cast_fp16 = expand_dims(axes = var_2411_axes_0, x = key_token_87_cast_fp16)[name = string("op_2411_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2412_cast_fp16 = mul(x = state_173_cast_fp16, y = var_2411_cast_fp16)[name = string("op_2412_cast_fp16")];
            tensor<int32, [1]> memory_87_axes_0 = const()[name = string("memory_87_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_87_keep_dims_0 = const()[name = string("memory_87_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_87_cast_fp16 = reduce_sum(axes = memory_87_axes_0, keep_dims = memory_87_keep_dims_0, x = var_2412_cast_fp16)[name = string("memory_87_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_2415_cast_fp16 = sub(x = value_token_87_cast_fp16, y = memory_87_cast_fp16)[name = string("op_2415_cast_fp16")];
            tensor<int32, [3]> var_2418_begin_0 = const()[name = string("op_2418_begin_0"), val = tensor<int32, [3]>([0, 0, 11])];
            tensor<int32, [3]> var_2418_end_0 = const()[name = string("op_2418_end_0"), val = tensor<int32, [3]>([1, 16, 12])];
            tensor<bool, [3]> var_2418_end_mask_0 = const()[name = string("op_2418_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_2418_squeeze_mask_0 = const()[name = string("op_2418_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_2418_cast_fp16 = slice_by_index(begin = var_2418_begin_0, end = var_2418_end_0, end_mask = var_2418_end_mask_0, squeeze_mask = var_2418_squeeze_mask_0, x = beta_cast_fp16)[name = string("op_2418_cast_fp16")];
            tensor<int32, [1]> var_2419_axes_0 = const()[name = string("op_2419_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_2419_cast_fp16 = expand_dims(axes = var_2419_axes_0, x = var_2418_cast_fp16)[name = string("op_2419_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_87_cast_fp16 = mul(x = var_2415_cast_fp16, y = var_2419_cast_fp16)[name = string("delta_87_cast_fp16")];
            tensor<int32, [1]> var_2422_axes_0 = const()[name = string("op_2422_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_2422_cast_fp16 = expand_dims(axes = var_2422_axes_0, x = delta_87_cast_fp16)[name = string("op_2422_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2423_cast_fp16 = mul(x = var_2411_cast_fp16, y = var_2422_cast_fp16)[name = string("op_2423_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_175_cast_fp16 = add(x = state_173_cast_fp16, y = var_2423_cast_fp16)[name = string("state_175_cast_fp16")];
            tensor<int32, [4]> var_2427_begin_0 = const()[name = string("op_2427_begin_0"), val = tensor<int32, [4]>([0, 0, 11, 0])];
            tensor<int32, [4]> var_2427_end_0 = const()[name = string("op_2427_end_0"), val = tensor<int32, [4]>([1, 16, 12, 128])];
            tensor<bool, [4]> var_2427_end_mask_0 = const()[name = string("op_2427_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_2427_squeeze_mask_0 = const()[name = string("op_2427_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_2427_cast_fp16 = slice_by_index(begin = var_2427_begin_0, end = var_2427_end_0, end_mask = var_2427_end_mask_0, squeeze_mask = var_2427_squeeze_mask_0, x = _inversed_query_11_cast_fp16)[name = string("op_2427_cast_fp16")];
            tensor<int32, [1]> var_2428_axes_0 = const()[name = string("op_2428_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_2428_cast_fp16 = expand_dims(axes = var_2428_axes_0, x = var_2427_cast_fp16)[name = string("op_2428_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2429_cast_fp16 = mul(x = state_175_cast_fp16, y = var_2428_cast_fp16)[name = string("op_2429_cast_fp16")];
            tensor<int32, [1]> var_2431_axes_0 = const()[name = string("op_2431_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_2431_keep_dims_0 = const()[name = string("op_2431_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_2431_cast_fp16 = reduce_sum(axes = var_2431_axes_0, keep_dims = var_2431_keep_dims_0, x = var_2429_cast_fp16)[name = string("op_2431_cast_fp16")];
            tensor<int32, [3]> var_2434_begin_0 = const()[name = string("op_2434_begin_0"), val = tensor<int32, [3]>([0, 0, 12])];
            tensor<int32, [3]> var_2434_end_0 = const()[name = string("op_2434_end_0"), val = tensor<int32, [3]>([1, 16, 13])];
            tensor<bool, [3]> var_2434_end_mask_0 = const()[name = string("op_2434_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_2434_squeeze_mask_0 = const()[name = string("op_2434_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_2434_cast_fp16 = slice_by_index(begin = var_2434_begin_0, end = var_2434_end_0, end_mask = var_2434_end_mask_0, squeeze_mask = var_2434_squeeze_mask_0, x = decay_cast_fp16)[name = string("op_2434_cast_fp16")];
            tensor<fp16, [1, 16]> var_2435_cast_fp16 = exp(x = var_2434_cast_fp16)[name = string("op_2435_cast_fp16")];
            tensor<int32, [1]> var_2436_axes_0 = const()[name = string("op_2436_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_2436_cast_fp16 = expand_dims(axes = var_2436_axes_0, x = var_2435_cast_fp16)[name = string("op_2436_cast_fp16")];
            tensor<int32, [1]> var_2437_axes_0 = const()[name = string("op_2437_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_2437_cast_fp16 = expand_dims(axes = var_2437_axes_0, x = var_2436_cast_fp16)[name = string("op_2437_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_177_cast_fp16 = mul(x = state_175_cast_fp16, y = var_2437_cast_fp16)[name = string("state_177_cast_fp16")];
            tensor<int32, [4]> key_token_89_begin_0 = const()[name = string("key_token_89_begin_0"), val = tensor<int32, [4]>([0, 0, 12, 0])];
            tensor<int32, [4]> key_token_89_end_0 = const()[name = string("key_token_89_end_0"), val = tensor<int32, [4]>([1, 16, 13, 128])];
            tensor<bool, [4]> key_token_89_end_mask_0 = const()[name = string("key_token_89_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_89_squeeze_mask_0 = const()[name = string("key_token_89_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_89_cast_fp16 = slice_by_index(begin = key_token_89_begin_0, end = key_token_89_end_0, end_mask = key_token_89_end_mask_0, squeeze_mask = key_token_89_squeeze_mask_0, x = key_11_cast_fp16)[name = string("key_token_89_cast_fp16")];
            tensor<int32, [4]> value_token_89_begin_0 = const()[name = string("value_token_89_begin_0"), val = tensor<int32, [4]>([0, 0, 12, 0])];
            tensor<int32, [4]> value_token_89_end_0 = const()[name = string("value_token_89_end_0"), val = tensor<int32, [4]>([1, 16, 13, 128])];
            tensor<bool, [4]> value_token_89_end_mask_0 = const()[name = string("value_token_89_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_89_squeeze_mask_0 = const()[name = string("value_token_89_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_89_cast_fp16 = slice_by_index(begin = value_token_89_begin_0, end = value_token_89_end_0, end_mask = value_token_89_end_mask_0, squeeze_mask = value_token_89_squeeze_mask_0, x = value_41_cast_fp16)[name = string("value_token_89_cast_fp16")];
            tensor<int32, [1]> var_2445_axes_0 = const()[name = string("op_2445_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_2445_cast_fp16 = expand_dims(axes = var_2445_axes_0, x = key_token_89_cast_fp16)[name = string("op_2445_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2446_cast_fp16 = mul(x = state_177_cast_fp16, y = var_2445_cast_fp16)[name = string("op_2446_cast_fp16")];
            tensor<int32, [1]> memory_89_axes_0 = const()[name = string("memory_89_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_89_keep_dims_0 = const()[name = string("memory_89_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_89_cast_fp16 = reduce_sum(axes = memory_89_axes_0, keep_dims = memory_89_keep_dims_0, x = var_2446_cast_fp16)[name = string("memory_89_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_2449_cast_fp16 = sub(x = value_token_89_cast_fp16, y = memory_89_cast_fp16)[name = string("op_2449_cast_fp16")];
            tensor<int32, [3]> var_2452_begin_0 = const()[name = string("op_2452_begin_0"), val = tensor<int32, [3]>([0, 0, 12])];
            tensor<int32, [3]> var_2452_end_0 = const()[name = string("op_2452_end_0"), val = tensor<int32, [3]>([1, 16, 13])];
            tensor<bool, [3]> var_2452_end_mask_0 = const()[name = string("op_2452_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_2452_squeeze_mask_0 = const()[name = string("op_2452_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_2452_cast_fp16 = slice_by_index(begin = var_2452_begin_0, end = var_2452_end_0, end_mask = var_2452_end_mask_0, squeeze_mask = var_2452_squeeze_mask_0, x = beta_cast_fp16)[name = string("op_2452_cast_fp16")];
            tensor<int32, [1]> var_2453_axes_0 = const()[name = string("op_2453_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_2453_cast_fp16 = expand_dims(axes = var_2453_axes_0, x = var_2452_cast_fp16)[name = string("op_2453_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_89_cast_fp16 = mul(x = var_2449_cast_fp16, y = var_2453_cast_fp16)[name = string("delta_89_cast_fp16")];
            tensor<int32, [1]> var_2456_axes_0 = const()[name = string("op_2456_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_2456_cast_fp16 = expand_dims(axes = var_2456_axes_0, x = delta_89_cast_fp16)[name = string("op_2456_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2457_cast_fp16 = mul(x = var_2445_cast_fp16, y = var_2456_cast_fp16)[name = string("op_2457_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_179_cast_fp16 = add(x = state_177_cast_fp16, y = var_2457_cast_fp16)[name = string("state_179_cast_fp16")];
            tensor<int32, [4]> var_2461_begin_0 = const()[name = string("op_2461_begin_0"), val = tensor<int32, [4]>([0, 0, 12, 0])];
            tensor<int32, [4]> var_2461_end_0 = const()[name = string("op_2461_end_0"), val = tensor<int32, [4]>([1, 16, 13, 128])];
            tensor<bool, [4]> var_2461_end_mask_0 = const()[name = string("op_2461_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_2461_squeeze_mask_0 = const()[name = string("op_2461_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_2461_cast_fp16 = slice_by_index(begin = var_2461_begin_0, end = var_2461_end_0, end_mask = var_2461_end_mask_0, squeeze_mask = var_2461_squeeze_mask_0, x = _inversed_query_11_cast_fp16)[name = string("op_2461_cast_fp16")];
            tensor<int32, [1]> var_2462_axes_0 = const()[name = string("op_2462_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_2462_cast_fp16 = expand_dims(axes = var_2462_axes_0, x = var_2461_cast_fp16)[name = string("op_2462_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2463_cast_fp16 = mul(x = state_179_cast_fp16, y = var_2462_cast_fp16)[name = string("op_2463_cast_fp16")];
            tensor<int32, [1]> var_2465_axes_0 = const()[name = string("op_2465_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_2465_keep_dims_0 = const()[name = string("op_2465_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_2465_cast_fp16 = reduce_sum(axes = var_2465_axes_0, keep_dims = var_2465_keep_dims_0, x = var_2463_cast_fp16)[name = string("op_2465_cast_fp16")];
            tensor<int32, [3]> var_2468_begin_0 = const()[name = string("op_2468_begin_0"), val = tensor<int32, [3]>([0, 0, 13])];
            tensor<int32, [3]> var_2468_end_0 = const()[name = string("op_2468_end_0"), val = tensor<int32, [3]>([1, 16, 14])];
            tensor<bool, [3]> var_2468_end_mask_0 = const()[name = string("op_2468_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_2468_squeeze_mask_0 = const()[name = string("op_2468_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_2468_cast_fp16 = slice_by_index(begin = var_2468_begin_0, end = var_2468_end_0, end_mask = var_2468_end_mask_0, squeeze_mask = var_2468_squeeze_mask_0, x = decay_cast_fp16)[name = string("op_2468_cast_fp16")];
            tensor<fp16, [1, 16]> var_2469_cast_fp16 = exp(x = var_2468_cast_fp16)[name = string("op_2469_cast_fp16")];
            tensor<int32, [1]> var_2470_axes_0 = const()[name = string("op_2470_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_2470_cast_fp16 = expand_dims(axes = var_2470_axes_0, x = var_2469_cast_fp16)[name = string("op_2470_cast_fp16")];
            tensor<int32, [1]> var_2471_axes_0 = const()[name = string("op_2471_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_2471_cast_fp16 = expand_dims(axes = var_2471_axes_0, x = var_2470_cast_fp16)[name = string("op_2471_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_181_cast_fp16 = mul(x = state_179_cast_fp16, y = var_2471_cast_fp16)[name = string("state_181_cast_fp16")];
            tensor<int32, [4]> key_token_91_begin_0 = const()[name = string("key_token_91_begin_0"), val = tensor<int32, [4]>([0, 0, 13, 0])];
            tensor<int32, [4]> key_token_91_end_0 = const()[name = string("key_token_91_end_0"), val = tensor<int32, [4]>([1, 16, 14, 128])];
            tensor<bool, [4]> key_token_91_end_mask_0 = const()[name = string("key_token_91_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_91_squeeze_mask_0 = const()[name = string("key_token_91_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_91_cast_fp16 = slice_by_index(begin = key_token_91_begin_0, end = key_token_91_end_0, end_mask = key_token_91_end_mask_0, squeeze_mask = key_token_91_squeeze_mask_0, x = key_11_cast_fp16)[name = string("key_token_91_cast_fp16")];
            tensor<int32, [4]> value_token_91_begin_0 = const()[name = string("value_token_91_begin_0"), val = tensor<int32, [4]>([0, 0, 13, 0])];
            tensor<int32, [4]> value_token_91_end_0 = const()[name = string("value_token_91_end_0"), val = tensor<int32, [4]>([1, 16, 14, 128])];
            tensor<bool, [4]> value_token_91_end_mask_0 = const()[name = string("value_token_91_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_91_squeeze_mask_0 = const()[name = string("value_token_91_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_91_cast_fp16 = slice_by_index(begin = value_token_91_begin_0, end = value_token_91_end_0, end_mask = value_token_91_end_mask_0, squeeze_mask = value_token_91_squeeze_mask_0, x = value_41_cast_fp16)[name = string("value_token_91_cast_fp16")];
            tensor<int32, [1]> var_2479_axes_0 = const()[name = string("op_2479_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_2479_cast_fp16 = expand_dims(axes = var_2479_axes_0, x = key_token_91_cast_fp16)[name = string("op_2479_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2480_cast_fp16 = mul(x = state_181_cast_fp16, y = var_2479_cast_fp16)[name = string("op_2480_cast_fp16")];
            tensor<int32, [1]> memory_91_axes_0 = const()[name = string("memory_91_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_91_keep_dims_0 = const()[name = string("memory_91_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_91_cast_fp16 = reduce_sum(axes = memory_91_axes_0, keep_dims = memory_91_keep_dims_0, x = var_2480_cast_fp16)[name = string("memory_91_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_2483_cast_fp16 = sub(x = value_token_91_cast_fp16, y = memory_91_cast_fp16)[name = string("op_2483_cast_fp16")];
            tensor<int32, [3]> var_2486_begin_0 = const()[name = string("op_2486_begin_0"), val = tensor<int32, [3]>([0, 0, 13])];
            tensor<int32, [3]> var_2486_end_0 = const()[name = string("op_2486_end_0"), val = tensor<int32, [3]>([1, 16, 14])];
            tensor<bool, [3]> var_2486_end_mask_0 = const()[name = string("op_2486_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_2486_squeeze_mask_0 = const()[name = string("op_2486_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_2486_cast_fp16 = slice_by_index(begin = var_2486_begin_0, end = var_2486_end_0, end_mask = var_2486_end_mask_0, squeeze_mask = var_2486_squeeze_mask_0, x = beta_cast_fp16)[name = string("op_2486_cast_fp16")];
            tensor<int32, [1]> var_2487_axes_0 = const()[name = string("op_2487_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_2487_cast_fp16 = expand_dims(axes = var_2487_axes_0, x = var_2486_cast_fp16)[name = string("op_2487_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_91_cast_fp16 = mul(x = var_2483_cast_fp16, y = var_2487_cast_fp16)[name = string("delta_91_cast_fp16")];
            tensor<int32, [1]> var_2490_axes_0 = const()[name = string("op_2490_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_2490_cast_fp16 = expand_dims(axes = var_2490_axes_0, x = delta_91_cast_fp16)[name = string("op_2490_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2491_cast_fp16 = mul(x = var_2479_cast_fp16, y = var_2490_cast_fp16)[name = string("op_2491_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_183_cast_fp16 = add(x = state_181_cast_fp16, y = var_2491_cast_fp16)[name = string("state_183_cast_fp16")];
            tensor<int32, [4]> var_2495_begin_0 = const()[name = string("op_2495_begin_0"), val = tensor<int32, [4]>([0, 0, 13, 0])];
            tensor<int32, [4]> var_2495_end_0 = const()[name = string("op_2495_end_0"), val = tensor<int32, [4]>([1, 16, 14, 128])];
            tensor<bool, [4]> var_2495_end_mask_0 = const()[name = string("op_2495_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_2495_squeeze_mask_0 = const()[name = string("op_2495_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_2495_cast_fp16 = slice_by_index(begin = var_2495_begin_0, end = var_2495_end_0, end_mask = var_2495_end_mask_0, squeeze_mask = var_2495_squeeze_mask_0, x = _inversed_query_11_cast_fp16)[name = string("op_2495_cast_fp16")];
            tensor<int32, [1]> var_2496_axes_0 = const()[name = string("op_2496_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_2496_cast_fp16 = expand_dims(axes = var_2496_axes_0, x = var_2495_cast_fp16)[name = string("op_2496_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2497_cast_fp16 = mul(x = state_183_cast_fp16, y = var_2496_cast_fp16)[name = string("op_2497_cast_fp16")];
            tensor<int32, [1]> var_2499_axes_0 = const()[name = string("op_2499_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_2499_keep_dims_0 = const()[name = string("op_2499_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_2499_cast_fp16 = reduce_sum(axes = var_2499_axes_0, keep_dims = var_2499_keep_dims_0, x = var_2497_cast_fp16)[name = string("op_2499_cast_fp16")];
            tensor<int32, [3]> var_2502_begin_0 = const()[name = string("op_2502_begin_0"), val = tensor<int32, [3]>([0, 0, 14])];
            tensor<int32, [3]> var_2502_end_0 = const()[name = string("op_2502_end_0"), val = tensor<int32, [3]>([1, 16, 15])];
            tensor<bool, [3]> var_2502_end_mask_0 = const()[name = string("op_2502_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_2502_squeeze_mask_0 = const()[name = string("op_2502_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_2502_cast_fp16 = slice_by_index(begin = var_2502_begin_0, end = var_2502_end_0, end_mask = var_2502_end_mask_0, squeeze_mask = var_2502_squeeze_mask_0, x = decay_cast_fp16)[name = string("op_2502_cast_fp16")];
            tensor<fp16, [1, 16]> var_2503_cast_fp16 = exp(x = var_2502_cast_fp16)[name = string("op_2503_cast_fp16")];
            tensor<int32, [1]> var_2504_axes_0 = const()[name = string("op_2504_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_2504_cast_fp16 = expand_dims(axes = var_2504_axes_0, x = var_2503_cast_fp16)[name = string("op_2504_cast_fp16")];
            tensor<int32, [1]> var_2505_axes_0 = const()[name = string("op_2505_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_2505_cast_fp16 = expand_dims(axes = var_2505_axes_0, x = var_2504_cast_fp16)[name = string("op_2505_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_185_cast_fp16 = mul(x = state_183_cast_fp16, y = var_2505_cast_fp16)[name = string("state_185_cast_fp16")];
            tensor<int32, [4]> key_token_93_begin_0 = const()[name = string("key_token_93_begin_0"), val = tensor<int32, [4]>([0, 0, 14, 0])];
            tensor<int32, [4]> key_token_93_end_0 = const()[name = string("key_token_93_end_0"), val = tensor<int32, [4]>([1, 16, 15, 128])];
            tensor<bool, [4]> key_token_93_end_mask_0 = const()[name = string("key_token_93_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_93_squeeze_mask_0 = const()[name = string("key_token_93_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_93_cast_fp16 = slice_by_index(begin = key_token_93_begin_0, end = key_token_93_end_0, end_mask = key_token_93_end_mask_0, squeeze_mask = key_token_93_squeeze_mask_0, x = key_11_cast_fp16)[name = string("key_token_93_cast_fp16")];
            tensor<int32, [4]> value_token_93_begin_0 = const()[name = string("value_token_93_begin_0"), val = tensor<int32, [4]>([0, 0, 14, 0])];
            tensor<int32, [4]> value_token_93_end_0 = const()[name = string("value_token_93_end_0"), val = tensor<int32, [4]>([1, 16, 15, 128])];
            tensor<bool, [4]> value_token_93_end_mask_0 = const()[name = string("value_token_93_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_93_squeeze_mask_0 = const()[name = string("value_token_93_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_93_cast_fp16 = slice_by_index(begin = value_token_93_begin_0, end = value_token_93_end_0, end_mask = value_token_93_end_mask_0, squeeze_mask = value_token_93_squeeze_mask_0, x = value_41_cast_fp16)[name = string("value_token_93_cast_fp16")];
            tensor<int32, [1]> var_2513_axes_0 = const()[name = string("op_2513_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_2513_cast_fp16 = expand_dims(axes = var_2513_axes_0, x = key_token_93_cast_fp16)[name = string("op_2513_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2514_cast_fp16 = mul(x = state_185_cast_fp16, y = var_2513_cast_fp16)[name = string("op_2514_cast_fp16")];
            tensor<int32, [1]> memory_93_axes_0 = const()[name = string("memory_93_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_93_keep_dims_0 = const()[name = string("memory_93_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_93_cast_fp16 = reduce_sum(axes = memory_93_axes_0, keep_dims = memory_93_keep_dims_0, x = var_2514_cast_fp16)[name = string("memory_93_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_2517_cast_fp16 = sub(x = value_token_93_cast_fp16, y = memory_93_cast_fp16)[name = string("op_2517_cast_fp16")];
            tensor<int32, [3]> var_2520_begin_0 = const()[name = string("op_2520_begin_0"), val = tensor<int32, [3]>([0, 0, 14])];
            tensor<int32, [3]> var_2520_end_0 = const()[name = string("op_2520_end_0"), val = tensor<int32, [3]>([1, 16, 15])];
            tensor<bool, [3]> var_2520_end_mask_0 = const()[name = string("op_2520_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_2520_squeeze_mask_0 = const()[name = string("op_2520_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_2520_cast_fp16 = slice_by_index(begin = var_2520_begin_0, end = var_2520_end_0, end_mask = var_2520_end_mask_0, squeeze_mask = var_2520_squeeze_mask_0, x = beta_cast_fp16)[name = string("op_2520_cast_fp16")];
            tensor<int32, [1]> var_2521_axes_0 = const()[name = string("op_2521_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_2521_cast_fp16 = expand_dims(axes = var_2521_axes_0, x = var_2520_cast_fp16)[name = string("op_2521_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_93_cast_fp16 = mul(x = var_2517_cast_fp16, y = var_2521_cast_fp16)[name = string("delta_93_cast_fp16")];
            tensor<int32, [1]> var_2524_axes_0 = const()[name = string("op_2524_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_2524_cast_fp16 = expand_dims(axes = var_2524_axes_0, x = delta_93_cast_fp16)[name = string("op_2524_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2525_cast_fp16 = mul(x = var_2513_cast_fp16, y = var_2524_cast_fp16)[name = string("op_2525_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_187_cast_fp16 = add(x = state_185_cast_fp16, y = var_2525_cast_fp16)[name = string("state_187_cast_fp16")];
            tensor<int32, [4]> var_2529_begin_0 = const()[name = string("op_2529_begin_0"), val = tensor<int32, [4]>([0, 0, 14, 0])];
            tensor<int32, [4]> var_2529_end_0 = const()[name = string("op_2529_end_0"), val = tensor<int32, [4]>([1, 16, 15, 128])];
            tensor<bool, [4]> var_2529_end_mask_0 = const()[name = string("op_2529_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_2529_squeeze_mask_0 = const()[name = string("op_2529_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_2529_cast_fp16 = slice_by_index(begin = var_2529_begin_0, end = var_2529_end_0, end_mask = var_2529_end_mask_0, squeeze_mask = var_2529_squeeze_mask_0, x = _inversed_query_11_cast_fp16)[name = string("op_2529_cast_fp16")];
            tensor<int32, [1]> var_2530_axes_0 = const()[name = string("op_2530_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_2530_cast_fp16 = expand_dims(axes = var_2530_axes_0, x = var_2529_cast_fp16)[name = string("op_2530_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2531_cast_fp16 = mul(x = state_187_cast_fp16, y = var_2530_cast_fp16)[name = string("op_2531_cast_fp16")];
            tensor<int32, [1]> var_2533_axes_0 = const()[name = string("op_2533_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_2533_keep_dims_0 = const()[name = string("op_2533_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_2533_cast_fp16 = reduce_sum(axes = var_2533_axes_0, keep_dims = var_2533_keep_dims_0, x = var_2531_cast_fp16)[name = string("op_2533_cast_fp16")];
            tensor<int32, [3]> var_2536_begin_0 = const()[name = string("op_2536_begin_0"), val = tensor<int32, [3]>([0, 0, 15])];
            tensor<int32, [3]> var_2536_end_0 = const()[name = string("op_2536_end_0"), val = tensor<int32, [3]>([1, 16, 16])];
            tensor<bool, [3]> var_2536_end_mask_0 = const()[name = string("op_2536_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_2536_squeeze_mask_0 = const()[name = string("op_2536_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_2536_cast_fp16 = slice_by_index(begin = var_2536_begin_0, end = var_2536_end_0, end_mask = var_2536_end_mask_0, squeeze_mask = var_2536_squeeze_mask_0, x = decay_cast_fp16)[name = string("op_2536_cast_fp16")];
            tensor<fp16, [1, 16]> var_2537_cast_fp16 = exp(x = var_2536_cast_fp16)[name = string("op_2537_cast_fp16")];
            tensor<int32, [1]> var_2538_axes_0 = const()[name = string("op_2538_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_2538_cast_fp16 = expand_dims(axes = var_2538_axes_0, x = var_2537_cast_fp16)[name = string("op_2538_cast_fp16")];
            tensor<int32, [1]> var_2539_axes_0 = const()[name = string("op_2539_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_2539_cast_fp16 = expand_dims(axes = var_2539_axes_0, x = var_2538_cast_fp16)[name = string("op_2539_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_189_cast_fp16 = mul(x = state_187_cast_fp16, y = var_2539_cast_fp16)[name = string("state_189_cast_fp16")];
            tensor<int32, [4]> key_token_begin_0 = const()[name = string("key_token_begin_0"), val = tensor<int32, [4]>([0, 0, 15, 0])];
            tensor<int32, [4]> key_token_end_0 = const()[name = string("key_token_end_0"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<bool, [4]> key_token_end_mask_0 = const()[name = string("key_token_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_squeeze_mask_0 = const()[name = string("key_token_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_cast_fp16 = slice_by_index(begin = key_token_begin_0, end = key_token_end_0, end_mask = key_token_end_mask_0, squeeze_mask = key_token_squeeze_mask_0, x = key_11_cast_fp16)[name = string("key_token_cast_fp16")];
            tensor<int32, [4]> value_token_begin_0 = const()[name = string("value_token_begin_0"), val = tensor<int32, [4]>([0, 0, 15, 0])];
            tensor<int32, [4]> value_token_end_0 = const()[name = string("value_token_end_0"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<bool, [4]> value_token_end_mask_0 = const()[name = string("value_token_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_squeeze_mask_0 = const()[name = string("value_token_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_cast_fp16 = slice_by_index(begin = value_token_begin_0, end = value_token_end_0, end_mask = value_token_end_mask_0, squeeze_mask = value_token_squeeze_mask_0, x = value_41_cast_fp16)[name = string("value_token_cast_fp16")];
            tensor<int32, [1]> var_2547_axes_0 = const()[name = string("op_2547_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_2547_cast_fp16 = expand_dims(axes = var_2547_axes_0, x = key_token_cast_fp16)[name = string("op_2547_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2548_cast_fp16 = mul(x = state_189_cast_fp16, y = var_2547_cast_fp16)[name = string("op_2548_cast_fp16")];
            tensor<int32, [1]> memory_axes_0 = const()[name = string("memory_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_keep_dims_0 = const()[name = string("memory_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_cast_fp16 = reduce_sum(axes = memory_axes_0, keep_dims = memory_keep_dims_0, x = var_2548_cast_fp16)[name = string("memory_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_2551_cast_fp16 = sub(x = value_token_cast_fp16, y = memory_cast_fp16)[name = string("op_2551_cast_fp16")];
            tensor<int32, [3]> var_2554_begin_0 = const()[name = string("op_2554_begin_0"), val = tensor<int32, [3]>([0, 0, 15])];
            tensor<int32, [3]> var_2554_end_0 = const()[name = string("op_2554_end_0"), val = tensor<int32, [3]>([1, 16, 16])];
            tensor<bool, [3]> var_2554_end_mask_0 = const()[name = string("op_2554_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_2554_squeeze_mask_0 = const()[name = string("op_2554_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_2554_cast_fp16 = slice_by_index(begin = var_2554_begin_0, end = var_2554_end_0, end_mask = var_2554_end_mask_0, squeeze_mask = var_2554_squeeze_mask_0, x = beta_cast_fp16)[name = string("op_2554_cast_fp16")];
            tensor<int32, [1]> var_2555_axes_0 = const()[name = string("op_2555_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_2555_cast_fp16 = expand_dims(axes = var_2555_axes_0, x = var_2554_cast_fp16)[name = string("op_2555_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_cast_fp16 = mul(x = var_2551_cast_fp16, y = var_2555_cast_fp16)[name = string("delta_cast_fp16")];
            tensor<int32, [1]> var_2558_axes_0 = const()[name = string("op_2558_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_2558_cast_fp16 = expand_dims(axes = var_2558_axes_0, x = delta_cast_fp16)[name = string("op_2558_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2559_cast_fp16 = mul(x = var_2547_cast_fp16, y = var_2558_cast_fp16)[name = string("op_2559_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> rec22_out = add(x = state_189_cast_fp16, y = var_2559_cast_fp16)[name = string("state_cast_fp16")];
            tensor<int32, [4]> var_2563_begin_0 = const()[name = string("op_2563_begin_0"), val = tensor<int32, [4]>([0, 0, 15, 0])];
            tensor<int32, [4]> var_2563_end_0 = const()[name = string("op_2563_end_0"), val = tensor<int32, [4]>([1, 16, 16, 128])];
            tensor<bool, [4]> var_2563_end_mask_0 = const()[name = string("op_2563_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_2563_squeeze_mask_0 = const()[name = string("op_2563_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_2563_cast_fp16 = slice_by_index(begin = var_2563_begin_0, end = var_2563_end_0, end_mask = var_2563_end_mask_0, squeeze_mask = var_2563_squeeze_mask_0, x = _inversed_query_11_cast_fp16)[name = string("op_2563_cast_fp16")];
            tensor<int32, [1]> var_2564_axes_0 = const()[name = string("op_2564_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_2564_cast_fp16 = expand_dims(axes = var_2564_axes_0, x = var_2563_cast_fp16)[name = string("op_2564_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_2565_cast_fp16 = mul(x = rec22_out, y = var_2564_cast_fp16)[name = string("op_2565_cast_fp16")];
            tensor<int32, [1]> var_2567_axes_0 = const()[name = string("op_2567_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_2567_keep_dims_0 = const()[name = string("op_2567_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_2567_cast_fp16 = reduce_sum(axes = var_2567_axes_0, keep_dims = var_2567_keep_dims_0, x = var_2565_cast_fp16)[name = string("op_2567_cast_fp16")];
            int32 attention_21_axis_0 = const()[name = string("attention_21_axis_0"), val = int32(1)];
            tensor<fp16, [1, 16, 16, 128]> attention_21_cast_fp16 = stack(axis = attention_21_axis_0, values = (var_2057_cast_fp16, var_2091_cast_fp16, var_2125_cast_fp16, var_2159_cast_fp16, var_2193_cast_fp16, var_2227_cast_fp16, var_2261_cast_fp16, var_2295_cast_fp16, var_2329_cast_fp16, var_2363_cast_fp16, var_2397_cast_fp16, var_2431_cast_fp16, var_2465_cast_fp16, var_2499_cast_fp16, var_2533_cast_fp16, var_2567_cast_fp16))[name = string("attention_21_cast_fp16")];
            tensor<int32, [2]> var_2570 = const()[name = string("op_2570"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [256, 128]> attention_23_cast_fp16 = reshape(shape = var_2570, x = attention_21_cast_fp16)[name = string("attention_23_cast_fp16")];
            tensor<int32, [2]> var_2572 = const()[name = string("op_2572"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [256, 128]> input_41_cast_fp16 = reshape(shape = var_2572, x = linear_23_cast_fp16)[name = string("input_41_cast_fp16")];
            tensor<fp16, [256, 128]> var_2574_cast_fp16 = mul(x = attention_23_cast_fp16, y = attention_23_cast_fp16)[name = string("op_2574_cast_fp16")];
            tensor<int32, [1]> variance_17_axes_0 = const()[name = string("variance_17_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_17_keep_dims_0 = const()[name = string("variance_17_keep_dims_0"), val = bool(true)];
            tensor<fp16, [256, 1]> variance_17_cast_fp16 = reduce_mean(axes = variance_17_axes_0, keep_dims = variance_17_keep_dims_0, x = var_2574_cast_fp16)[name = string("variance_17_cast_fp16")];
            fp16 var_2577_to_fp16 = const()[name = string("op_2577_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [256, 1]> var_2578_cast_fp16 = add(x = variance_17_cast_fp16, y = var_2577_to_fp16)[name = string("op_2578_cast_fp16")];
            fp32 var_2579_epsilon_0 = const()[name = string("op_2579_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [256, 1]> var_2579_cast_fp16 = rsqrt(epsilon = var_2579_epsilon_0, x = var_2578_cast_fp16)[name = string("op_2579_cast_fp16")];
            tensor<fp16, [256, 128]> attention_25_cast_fp16 = mul(x = attention_23_cast_fp16, y = var_2579_cast_fp16)[name = string("attention_25_cast_fp16")];
            tensor<fp16, [128]> layers20_2_gated_norm_promoted_to_fp16 = const()[name = string("layers20_2_gated_norm_promoted_to_fp16"), val = tensor<fp16, [128]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(242297344)))];
            tensor<fp16, [256, 128]> attention_27_cast_fp16 = mul(x = attention_25_cast_fp16, y = layers20_2_gated_norm_promoted_to_fp16)[name = string("attention_27_cast_fp16")];
            tensor<fp16, [256, 128]> var_2582_cast_fp16 = silu(x = input_41_cast_fp16)[name = string("op_2582_cast_fp16")];
            tensor<fp16, [256, 128]> attention_29_cast_fp16 = mul(x = attention_27_cast_fp16, y = var_2582_cast_fp16)[name = string("attention_29_cast_fp16")];
            tensor<int32, [2]> var_2584 = const()[name = string("op_2584"), val = tensor<int32, [2]>([16, 2048])];
            tensor<fp16, [16, 2048]> input_43_cast_fp16 = reshape(shape = var_2584, x = attention_29_cast_fp16)[name = string("input_43_cast_fp16")];
            tensor<fp16, [1024, 2048]> layers20_2_out_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 2048]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(242297664))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(243870592))))[name = string("layers20_2_out_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 1024]> linear_24_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = layers20_2_out_proj_weight_promoted_to_fp16_palettized, x = input_43_cast_fp16)[name = string("linear_24_cast_fp16")];
            tensor<fp16, [16, 1024]> value_43_cast_fp16 = add(x = value_31_cast_fp16, y = linear_24_cast_fp16)[name = string("value_43_cast_fp16")];
            tensor<fp16, [16, 1024]> var_2589_cast_fp16 = mul(x = value_43_cast_fp16, y = value_43_cast_fp16)[name = string("op_2589_cast_fp16")];
            tensor<int32, [1]> variance_19_axes_0 = const()[name = string("variance_19_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_19_keep_dims_0 = const()[name = string("variance_19_keep_dims_0"), val = bool(true)];
            tensor<fp16, [16, 1]> variance_19_cast_fp16 = reduce_mean(axes = variance_19_axes_0, keep_dims = variance_19_keep_dims_0, x = var_2589_cast_fp16)[name = string("variance_19_cast_fp16")];
            fp16 var_2592_to_fp16 = const()[name = string("op_2592_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [16, 1]> var_2593_cast_fp16 = add(x = variance_19_cast_fp16, y = var_2592_to_fp16)[name = string("op_2593_cast_fp16")];
            fp32 var_2594_epsilon_0 = const()[name = string("op_2594_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [16, 1]> var_2594_cast_fp16 = rsqrt(epsilon = var_2594_epsilon_0, x = var_2593_cast_fp16)[name = string("op_2594_cast_fp16")];
            tensor<fp16, [16, 1024]> var_2595_cast_fp16 = mul(x = value_43_cast_fp16, y = var_2594_cast_fp16)[name = string("op_2595_cast_fp16")];
            tensor<fp16, [1024]> var_2597_to_fp16 = const()[name = string("op_2597_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(243878848)))];
            tensor<fp16, [16, 1024]> input_45_cast_fp16 = mul(x = var_2595_cast_fp16, y = var_2597_to_fp16)[name = string("input_45_cast_fp16")];
            tensor<fp16, [3584, 1024]> layers20_2_gate_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(243880960))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(246633536))))[name = string("layers20_2_gate_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 3584]> linear_25_cast_fp16 = linear(bias = linear_1_bias_0_to_fp16, weight = layers20_2_gate_proj_weight_promoted_to_fp16_palettized, x = input_45_cast_fp16)[name = string("linear_25_cast_fp16")];
            tensor<fp16, [16, 3584]> var_2601_cast_fp16 = silu(x = linear_25_cast_fp16)[name = string("op_2601_cast_fp16")];
            tensor<fp16, [3584, 1024]> layers20_2_up_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(246662272))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(249414848))))[name = string("layers20_2_up_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 3584]> linear_26_cast_fp16 = linear(bias = linear_1_bias_0_to_fp16, weight = layers20_2_up_proj_weight_promoted_to_fp16_palettized, x = input_45_cast_fp16)[name = string("linear_26_cast_fp16")];
            tensor<fp16, [16, 3584]> input_49_cast_fp16 = mul(x = var_2601_cast_fp16, y = linear_26_cast_fp16)[name = string("input_49_cast_fp16")];
            tensor<fp16, [1024, 3584]> layers20_2_down_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 3584]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(249443584))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(252196160))))[name = string("layers20_2_down_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 1024]> linear_27_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = layers20_2_down_proj_weight_promoted_to_fp16_palettized, x = input_49_cast_fp16)[name = string("linear_27_cast_fp16")];
            tensor<fp16, [16, 1024]> value_45_cast_fp16 = add(x = value_43_cast_fp16, y = linear_27_cast_fp16)[name = string("value_45_cast_fp16")];
            int32 var_2626 = const()[name = string("op_2626"), val = int32(-1)];
            tensor<fp16, [16, 1024]> var_2633_cast_fp16 = mul(x = value_45_cast_fp16, y = value_45_cast_fp16)[name = string("op_2633_cast_fp16")];
            tensor<int32, [1]> variance_21_axes_0 = const()[name = string("variance_21_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_21_keep_dims_0 = const()[name = string("variance_21_keep_dims_0"), val = bool(true)];
            tensor<fp16, [16, 1]> variance_21_cast_fp16 = reduce_mean(axes = variance_21_axes_0, keep_dims = variance_21_keep_dims_0, x = var_2633_cast_fp16)[name = string("variance_21_cast_fp16")];
            fp16 var_2636_to_fp16 = const()[name = string("op_2636_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [16, 1]> var_2637_cast_fp16 = add(x = variance_21_cast_fp16, y = var_2636_to_fp16)[name = string("op_2637_cast_fp16")];
            fp32 var_2638_epsilon_0 = const()[name = string("op_2638_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [16, 1]> var_2638_cast_fp16 = rsqrt(epsilon = var_2638_epsilon_0, x = var_2637_cast_fp16)[name = string("op_2638_cast_fp16")];
            tensor<fp16, [16, 1024]> var_2639_cast_fp16 = mul(x = value_45_cast_fp16, y = var_2638_cast_fp16)[name = string("op_2639_cast_fp16")];
            tensor<fp16, [1024]> var_2641_to_fp16 = const()[name = string("op_2641_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(252204416)))];
            tensor<fp16, [16, 1024]> input_51_cast_fp16 = mul(x = var_2639_cast_fp16, y = var_2641_to_fp16)[name = string("input_51_cast_fp16")];
            tensor<fp16, [4096, 1024]> projection23_q_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [4096, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(252206528))), lut = tensor<fp16, [256, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(255352320))))[name = string("projection23_q_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4096]> linear_28_bias_0_to_fp16 = const()[name = string("linear_28_bias_0_to_fp16"), val = tensor<fp16, [4096]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(255385152)))];
            tensor<fp16, [16, 4096]> linear_28_cast_fp16 = linear(bias = linear_28_bias_0_to_fp16, weight = projection23_q_proj_weight_promoted_to_fp16_palettized, x = input_51_cast_fp16)[name = string("linear_28_cast_fp16")];
            tensor<int32, [3]> var_2645 = const()[name = string("op_2645"), val = tensor<int32, [3]>([16, 8, 512])];
            tensor<fp16, [16, 8, 512]> query_and_gate_cast_fp16 = reshape(shape = var_2645, x = linear_28_cast_fp16)[name = string("query_and_gate_cast_fp16")];
            tensor<int32, [2]> var_2647_split_sizes_0 = const()[name = string("op_2647_split_sizes_0"), val = tensor<int32, [2]>([256, 256])];
            int32 var_2647_axis_0 = const()[name = string("op_2647_axis_0"), val = int32(-1)];
            tensor<fp16, [16, 8, 256]> var_2647_cast_fp16_0, tensor<fp16, [16, 8, 256]> var_2647_cast_fp16_1 = split(axis = var_2647_axis_0, split_sizes = var_2647_split_sizes_0, x = query_and_gate_cast_fp16)[name = string("op_2647_cast_fp16")];
            tensor<fp16, [512, 1024]> projection23_k_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [512, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(255393408))), lut = tensor<fp16, [32, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(255786688))))[name = string("projection23_k_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [512]> linear_29_bias_0_to_fp16 = const()[name = string("linear_29_bias_0_to_fp16"), val = tensor<fp16, [512]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(255790848)))];
            tensor<fp16, [16, 512]> linear_29_cast_fp16 = linear(bias = linear_29_bias_0_to_fp16, weight = projection23_k_proj_weight_promoted_to_fp16_palettized, x = input_51_cast_fp16)[name = string("linear_29_cast_fp16")];
            tensor<int32, [3]> var_2651 = const()[name = string("op_2651"), val = tensor<int32, [3]>([16, 2, 256])];
            tensor<fp16, [16, 2, 256]> value_49_cast_fp16 = reshape(shape = var_2651, x = linear_29_cast_fp16)[name = string("value_49_cast_fp16")];
            tensor<fp16, [512, 1024]> projection23_v_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [512, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(255791936))), lut = tensor<fp16, [32, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(256185216))))[name = string("projection23_v_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 512]> linear_30_cast_fp16 = linear(bias = linear_29_bias_0_to_fp16, weight = projection23_v_proj_weight_promoted_to_fp16_palettized, x = input_51_cast_fp16)[name = string("linear_30_cast_fp16")];
            tensor<int32, [3]> var_2655 = const()[name = string("op_2655"), val = tensor<int32, [3]>([16, 2, 256])];
            tensor<fp16, [16, 2, 256]> value_55_cast_fp16 = reshape(shape = var_2655, x = linear_30_cast_fp16)[name = string("value_55_cast_fp16")];
            tensor<fp16, [16, 8, 256]> var_2657_cast_fp16 = mul(x = var_2647_cast_fp16_0, y = var_2647_cast_fp16_0)[name = string("op_2657_cast_fp16")];
            tensor<int32, [1]> variance_23_axes_0 = const()[name = string("variance_23_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_23_keep_dims_0 = const()[name = string("variance_23_keep_dims_0"), val = bool(true)];
            tensor<fp16, [16, 8, 1]> variance_23_cast_fp16 = reduce_mean(axes = variance_23_axes_0, keep_dims = variance_23_keep_dims_0, x = var_2657_cast_fp16)[name = string("variance_23_cast_fp16")];
            fp16 var_2660_to_fp16 = const()[name = string("op_2660_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [16, 8, 1]> var_2661_cast_fp16 = add(x = variance_23_cast_fp16, y = var_2660_to_fp16)[name = string("op_2661_cast_fp16")];
            fp32 var_2662_epsilon_0 = const()[name = string("op_2662_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [16, 8, 1]> var_2662_cast_fp16 = rsqrt(epsilon = var_2662_epsilon_0, x = var_2661_cast_fp16)[name = string("op_2662_cast_fp16")];
            tensor<fp16, [16, 8, 256]> var_2663_cast_fp16 = mul(x = var_2647_cast_fp16_0, y = var_2662_cast_fp16)[name = string("op_2663_cast_fp16")];
            tensor<fp16, [256]> var_2665_to_fp16 = const()[name = string("op_2665_to_fp16"), val = tensor<fp16, [256]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(256189376)))];
            tensor<fp16, [16, 8, 256]> var_2666_cast_fp16 = mul(x = var_2663_cast_fp16, y = var_2665_to_fp16)[name = string("op_2666_cast_fp16")];
            tensor<int32, [3]> var_2667 = const()[name = string("op_2667"), val = tensor<int32, [3]>([1, 0, 2])];
            tensor<int32, [1]> value_51_axes_0 = const()[name = string("value_51_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [8, 16, 256]> var_2668_cast_fp16 = transpose(perm = var_2667, x = var_2666_cast_fp16)[name = string("transpose_7")];
            tensor<fp16, [1, 8, 16, 256]> value_51_cast_fp16 = expand_dims(axes = value_51_axes_0, x = var_2668_cast_fp16)[name = string("value_51_cast_fp16")];
            tensor<fp16, [16, 2, 256]> var_2670_cast_fp16 = mul(x = value_49_cast_fp16, y = value_49_cast_fp16)[name = string("op_2670_cast_fp16")];
            tensor<int32, [1]> variance_25_axes_0 = const()[name = string("variance_25_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_25_keep_dims_0 = const()[name = string("variance_25_keep_dims_0"), val = bool(true)];
            tensor<fp16, [16, 2, 1]> variance_25_cast_fp16 = reduce_mean(axes = variance_25_axes_0, keep_dims = variance_25_keep_dims_0, x = var_2670_cast_fp16)[name = string("variance_25_cast_fp16")];
            fp16 var_2673_to_fp16 = const()[name = string("op_2673_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [16, 2, 1]> var_2674_cast_fp16 = add(x = variance_25_cast_fp16, y = var_2673_to_fp16)[name = string("op_2674_cast_fp16")];
            fp32 var_2675_epsilon_0 = const()[name = string("op_2675_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [16, 2, 1]> var_2675_cast_fp16 = rsqrt(epsilon = var_2675_epsilon_0, x = var_2674_cast_fp16)[name = string("op_2675_cast_fp16")];
            tensor<fp16, [16, 2, 256]> var_2676_cast_fp16 = mul(x = value_49_cast_fp16, y = var_2675_cast_fp16)[name = string("op_2676_cast_fp16")];
            tensor<fp16, [256]> var_2678_to_fp16 = const()[name = string("op_2678_to_fp16"), val = tensor<fp16, [256]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(256189952)))];
            tensor<fp16, [16, 2, 256]> var_2679_cast_fp16 = mul(x = var_2676_cast_fp16, y = var_2678_to_fp16)[name = string("op_2679_cast_fp16")];
            tensor<int32, [3]> var_2680 = const()[name = string("op_2680"), val = tensor<int32, [3]>([1, 0, 2])];
            tensor<int32, [1]> value_53_axes_0 = const()[name = string("value_53_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [2, 16, 256]> var_2681_cast_fp16 = transpose(perm = var_2680, x = var_2679_cast_fp16)[name = string("transpose_6")];
            tensor<fp16, [1, 2, 16, 256]> value_53_cast_fp16 = expand_dims(axes = value_53_axes_0, x = var_2681_cast_fp16)[name = string("value_53_cast_fp16")];
            tensor<int32, [4]> rotated_1_begin_0 = const()[name = string("rotated_1_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> rotated_1_end_0 = const()[name = string("rotated_1_end_0"), val = tensor<int32, [4]>([1, 8, 16, 64])];
            tensor<bool, [4]> rotated_1_end_mask_0 = const()[name = string("rotated_1_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
            tensor<fp16, [1, 8, 16, 64]> rotated_1_cast_fp16 = slice_by_index(begin = rotated_1_begin_0, end = rotated_1_end_0, end_mask = rotated_1_end_mask_0, x = value_51_cast_fp16)[name = string("rotated_1_cast_fp16")];
            tensor<int32, [4]> passthrough_1_begin_0 = const()[name = string("passthrough_1_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 64])];
            tensor<int32, [4]> passthrough_1_end_0 = const()[name = string("passthrough_1_end_0"), val = tensor<int32, [4]>([1, 8, 16, 256])];
            tensor<bool, [4]> passthrough_1_end_mask_0 = const()[name = string("passthrough_1_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
            tensor<fp16, [1, 8, 16, 192]> passthrough_1_cast_fp16 = slice_by_index(begin = passthrough_1_begin_0, end = passthrough_1_end_0, end_mask = passthrough_1_end_mask_0, x = value_51_cast_fp16)[name = string("passthrough_1_cast_fp16")];
            tensor<int32, [2]> var_2685_split_sizes_0 = const()[name = string("op_2685_split_sizes_0"), val = tensor<int32, [2]>([32, 32])];
            int32 var_2685_axis_0 = const()[name = string("op_2685_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 8, 16, 32]> var_2685_cast_fp16_0, tensor<fp16, [1, 8, 16, 32]> var_2685_cast_fp16_1 = split(axis = var_2685_axis_0, split_sizes = var_2685_split_sizes_0, x = rotated_1_cast_fp16)[name = string("op_2685_cast_fp16")];
            fp16 const_41_promoted_to_fp16 = const()[name = string("const_41_promoted_to_fp16"), val = fp16(-0x1p+0)];
            tensor<fp16, [1, 8, 16, 32]> var_2687_cast_fp16 = mul(x = var_2685_cast_fp16_1, y = const_41_promoted_to_fp16)[name = string("op_2687_cast_fp16")];
            bool rotated_half_1_interleave_0 = const()[name = string("rotated_half_1_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 8, 16, 64]> rotated_half_1_cast_fp16 = concat(axis = var_2626, interleave = rotated_half_1_interleave_0, values = (var_2687_cast_fp16, var_2685_cast_fp16_0))[name = string("rotated_half_1_cast_fp16")];
            tensor<fp16, [1, 8, 16, 64]> var_2690_cast_fp16 = mul(x = rotated_1_cast_fp16, y = cos)[name = string("op_2690_cast_fp16")];
            tensor<fp16, [1, 8, 16, 64]> var_2691_cast_fp16 = mul(x = rotated_half_1_cast_fp16, y = sin)[name = string("op_2691_cast_fp16")];
            tensor<fp16, [1, 8, 16, 64]> var_2692_cast_fp16 = add(x = var_2690_cast_fp16, y = var_2691_cast_fp16)[name = string("op_2692_cast_fp16")];
            bool query_13_interleave_0 = const()[name = string("query_13_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 8, 16, 256]> query_13_cast_fp16 = concat(axis = var_2626, interleave = query_13_interleave_0, values = (var_2692_cast_fp16, passthrough_1_cast_fp16))[name = string("query_13_cast_fp16")];
            tensor<int32, [4]> rotated_begin_0 = const()[name = string("rotated_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> rotated_end_0 = const()[name = string("rotated_end_0"), val = tensor<int32, [4]>([1, 2, 16, 64])];
            tensor<bool, [4]> rotated_end_mask_0 = const()[name = string("rotated_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
            tensor<fp16, [1, 2, 16, 64]> rotated_cast_fp16 = slice_by_index(begin = rotated_begin_0, end = rotated_end_0, end_mask = rotated_end_mask_0, x = value_53_cast_fp16)[name = string("rotated_cast_fp16")];
            tensor<int32, [4]> passthrough_begin_0 = const()[name = string("passthrough_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 64])];
            tensor<int32, [4]> passthrough_end_0 = const()[name = string("passthrough_end_0"), val = tensor<int32, [4]>([1, 2, 16, 256])];
            tensor<bool, [4]> passthrough_end_mask_0 = const()[name = string("passthrough_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
            tensor<fp16, [1, 2, 16, 192]> passthrough_cast_fp16 = slice_by_index(begin = passthrough_begin_0, end = passthrough_end_0, end_mask = passthrough_end_mask_0, x = value_53_cast_fp16)[name = string("passthrough_cast_fp16")];
            tensor<int32, [2]> var_2697_split_sizes_0 = const()[name = string("op_2697_split_sizes_0"), val = tensor<int32, [2]>([32, 32])];
            int32 var_2697_axis_0 = const()[name = string("op_2697_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 2, 16, 32]> var_2697_cast_fp16_0, tensor<fp16, [1, 2, 16, 32]> var_2697_cast_fp16_1 = split(axis = var_2697_axis_0, split_sizes = var_2697_split_sizes_0, x = rotated_cast_fp16)[name = string("op_2697_cast_fp16")];
            fp16 const_42_promoted_to_fp16 = const()[name = string("const_42_promoted_to_fp16"), val = fp16(-0x1p+0)];
            tensor<fp16, [1, 2, 16, 32]> var_2699_cast_fp16 = mul(x = var_2697_cast_fp16_1, y = const_42_promoted_to_fp16)[name = string("op_2699_cast_fp16")];
            bool rotated_half_interleave_0 = const()[name = string("rotated_half_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 2, 16, 64]> rotated_half_cast_fp16 = concat(axis = var_2626, interleave = rotated_half_interleave_0, values = (var_2699_cast_fp16, var_2697_cast_fp16_0))[name = string("rotated_half_cast_fp16")];
            tensor<fp16, [1, 2, 16, 64]> var_2702_cast_fp16 = mul(x = rotated_cast_fp16, y = cos)[name = string("op_2702_cast_fp16")];
            tensor<fp16, [1, 2, 16, 64]> var_2703_cast_fp16 = mul(x = rotated_half_cast_fp16, y = sin)[name = string("op_2703_cast_fp16")];
            tensor<fp16, [1, 2, 16, 64]> var_2704_cast_fp16 = add(x = var_2702_cast_fp16, y = var_2703_cast_fp16)[name = string("op_2704_cast_fp16")];
            bool key_13_interleave_0 = const()[name = string("key_13_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 2, 16, 256]> key_13_cast_fp16 = concat(axis = var_2626, interleave = key_13_interleave_0, values = (var_2704_cast_fp16, passthrough_cast_fp16))[name = string("key_13_cast_fp16")];
            tensor<int32, [4]> var_2707 = const()[name = string("op_2707"), val = tensor<int32, [4]>([2, 4, 16, 256])];
            tensor<fp16, [2, 4, 16, 256]> var_2708_cast_fp16 = reshape(shape = var_2707, x = query_13_cast_fp16)[name = string("op_2708_cast_fp16")];
            tensor<int32, [4]> transpose_1_perm_0 = const()[name = string("transpose_1_perm_0"), val = tensor<int32, [4]>([0, 1, 3, 2])];
            tensor<int32, [3]> var_2711 = const()[name = string("op_2711"), val = tensor<int32, [3]>([2, 16, 256])];
            tensor<fp16, [2, 16, 256]> key_cast_fp16 = reshape(shape = var_2711, x = key_13_cast_fp16)[name = string("key_cast_fp16")];
            tensor<int32, [3]> var_2713 = const()[name = string("op_2713"), val = tensor<int32, [3]>([1, 0, 2])];
            tensor<int32, [2]> var_2715 = const()[name = string("op_2715"), val = tensor<int32, [2]>([16, 2048])];
            tensor<fp16, [16, 2048]> gate_cast_fp16 = reshape(shape = var_2715, x = var_2647_cast_fp16_1)[name = string("gate_cast_fp16")];
            tensor<int32, [1]> var_2723_axes_0 = const()[name = string("op_2723_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [1, 2, 16, 256]> var_2723_cast_fp16 = expand_dims(axes = var_2723_axes_0, x = key_cast_fp16)[name = string("op_2723_cast_fp16")];
            tensor<int32, [1]> var_2725_axes_0 = const()[name = string("op_2725_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [1, 1, 2, 16, 256]> var_2725_cast_fp16 = expand_dims(axes = var_2725_axes_0, x = var_2723_cast_fp16)[name = string("op_2725_cast_fp16")];
            tensor<int32, [1]> expand_dims_16 = const()[name = string("expand_dims_16"), val = tensor<int32, [1]>([1])];
            tensor<int32, [1]> expand_dims_17 = const()[name = string("expand_dims_17"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_18 = const()[name = string("expand_dims_18"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_20 = const()[name = string("expand_dims_20"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_21 = const()[name = string("expand_dims_21"), val = tensor<int32, [1]>([2])];
            tensor<int32, [1]> expand_dims_22 = const()[name = string("expand_dims_22"), val = tensor<int32, [1]>([1])];
            int32 concat_10_axis_0 = const()[name = string("concat_10_axis_0"), val = int32(0)];
            bool concat_10_interleave_0 = const()[name = string("concat_10_interleave_0"), val = bool(false)];
            tensor<int32, [5]> concat_10 = concat(axis = concat_10_axis_0, interleave = concat_10_interleave_0, values = (expand_dims_16, expand_dims_17, expand_dims_18, current_pos, expand_dims_20))[name = string("concat_10")];
            tensor<int32, [1]> concat_11_values2_0 = const()[name = string("concat_11_values2_0"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> concat_11_values4_0 = const()[name = string("concat_11_values4_0"), val = tensor<int32, [1]>([0])];
            int32 concat_11_axis_0 = const()[name = string("concat_11_axis_0"), val = int32(0)];
            bool concat_11_interleave_0 = const()[name = string("concat_11_interleave_0"), val = bool(false)];
            tensor<int32, [5]> concat_11 = concat(axis = concat_11_axis_0, interleave = concat_11_interleave_0, values = (expand_dims_21, expand_dims_22, concat_11_values2_0, var_38, concat_11_values4_0))[name = string("concat_11")];
            tensor<int32, [5]> kv_cache_internal_tensor_assign_3_stride_0 = const()[name = string("kv_cache_internal_tensor_assign_3_stride_0"), val = tensor<int32, [5]>([1, 1, 1, 1, 1])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_3_begin_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_3_begin_mask_0"), val = tensor<bool, [5]>([false, false, false, false, false])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_3_end_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_3_end_mask_0"), val = tensor<bool, [5]>([false, false, true, false, true])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_3_squeeze_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_3_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, false, false, false])];
            tensor<fp16, [2, 2, 2, 2048, 256]> kv_cache_internal_tensor_assign_3_cast_fp16 = slice_update(begin = concat_10, begin_mask = kv_cache_internal_tensor_assign_3_begin_mask_0, end = concat_11, end_mask = kv_cache_internal_tensor_assign_3_end_mask_0, squeeze_mask = kv_cache_internal_tensor_assign_3_squeeze_mask_0, stride = kv_cache_internal_tensor_assign_3_stride_0, update = var_2725_cast_fp16, x = coreml_update_state_5)[name = string("kv_cache_internal_tensor_assign_3_cast_fp16")];
            write_state(data = kv_cache_internal_tensor_assign_3_cast_fp16, input = kv_cache)[name = string("coreml_update_state_6_write_state")];
            tensor<fp16, [2, 2, 2, 2048, 256]> coreml_update_state_6 = read_state(input = kv_cache)[name = string("coreml_update_state_6")];
            tensor<int32, [1]> var_2756_axes_0 = const()[name = string("op_2756_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [2, 16, 256]> value_57_cast_fp16 = transpose(perm = var_2713, x = value_55_cast_fp16)[name = string("transpose_5")];
            tensor<fp16, [1, 2, 16, 256]> var_2756_cast_fp16 = expand_dims(axes = var_2756_axes_0, x = value_57_cast_fp16)[name = string("op_2756_cast_fp16")];
            tensor<int32, [1]> var_2758_axes_0 = const()[name = string("op_2758_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [1, 1, 2, 16, 256]> var_2758_cast_fp16 = expand_dims(axes = var_2758_axes_0, x = var_2756_cast_fp16)[name = string("op_2758_cast_fp16")];
            tensor<int32, [1]> expand_dims_24 = const()[name = string("expand_dims_24"), val = tensor<int32, [1]>([1])];
            tensor<int32, [1]> expand_dims_25 = const()[name = string("expand_dims_25"), val = tensor<int32, [1]>([1])];
            tensor<int32, [1]> expand_dims_26 = const()[name = string("expand_dims_26"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_28 = const()[name = string("expand_dims_28"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_29 = const()[name = string("expand_dims_29"), val = tensor<int32, [1]>([2])];
            tensor<int32, [1]> expand_dims_30 = const()[name = string("expand_dims_30"), val = tensor<int32, [1]>([2])];
            int32 concat_14_axis_0 = const()[name = string("concat_14_axis_0"), val = int32(0)];
            bool concat_14_interleave_0 = const()[name = string("concat_14_interleave_0"), val = bool(false)];
            tensor<int32, [5]> concat_14 = concat(axis = concat_14_axis_0, interleave = concat_14_interleave_0, values = (expand_dims_24, expand_dims_25, expand_dims_26, current_pos, expand_dims_28))[name = string("concat_14")];
            tensor<int32, [1]> concat_15_values2_0 = const()[name = string("concat_15_values2_0"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> concat_15_values4_0 = const()[name = string("concat_15_values4_0"), val = tensor<int32, [1]>([0])];
            int32 concat_15_axis_0 = const()[name = string("concat_15_axis_0"), val = int32(0)];
            bool concat_15_interleave_0 = const()[name = string("concat_15_interleave_0"), val = bool(false)];
            tensor<int32, [5]> concat_15 = concat(axis = concat_15_axis_0, interleave = concat_15_interleave_0, values = (expand_dims_29, expand_dims_30, concat_15_values2_0, var_38, concat_15_values4_0))[name = string("concat_15")];
            tensor<int32, [5]> kv_cache_internal_tensor_assign_4_stride_0 = const()[name = string("kv_cache_internal_tensor_assign_4_stride_0"), val = tensor<int32, [5]>([1, 1, 1, 1, 1])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_4_begin_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_4_begin_mask_0"), val = tensor<bool, [5]>([false, false, false, false, false])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_4_end_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_4_end_mask_0"), val = tensor<bool, [5]>([false, false, true, false, true])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_4_squeeze_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_4_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, false, false, false])];
            tensor<fp16, [2, 2, 2, 2048, 256]> kv_cache_internal_tensor_assign_4_cast_fp16 = slice_update(begin = concat_14, begin_mask = kv_cache_internal_tensor_assign_4_begin_mask_0, end = concat_15, end_mask = kv_cache_internal_tensor_assign_4_end_mask_0, squeeze_mask = kv_cache_internal_tensor_assign_4_squeeze_mask_0, stride = kv_cache_internal_tensor_assign_4_stride_0, update = var_2758_cast_fp16, x = coreml_update_state_6)[name = string("kv_cache_internal_tensor_assign_4_cast_fp16")];
            write_state(data = kv_cache_internal_tensor_assign_4_cast_fp16, input = kv_cache)[name = string("coreml_update_state_7_write_state")];
            tensor<fp16, [2, 2, 2, 2048, 256]> coreml_update_state_7 = read_state(input = kv_cache)[name = string("coreml_update_state_7")];
            tensor<int32, [5]> var_2790_begin_0 = const()[name = string("op_2790_begin_0"), val = tensor<int32, [5]>([1, 0, 0, 0, 0])];
            tensor<int32, [5]> var_2790_end_0 = const()[name = string("op_2790_end_0"), val = tensor<int32, [5]>([2, 2, 2, 2048, 256])];
            tensor<bool, [5]> var_2790_end_mask_0 = const()[name = string("op_2790_end_mask_0"), val = tensor<bool, [5]>([false, true, true, true, true])];
            tensor<bool, [5]> var_2790_squeeze_mask_0 = const()[name = string("op_2790_squeeze_mask_0"), val = tensor<bool, [5]>([true, false, false, false, false])];
            tensor<fp16, [2, 2, 2048, 256]> var_2790_cast_fp16 = slice_by_index(begin = var_2790_begin_0, end = var_2790_end_0, end_mask = var_2790_end_mask_0, squeeze_mask = var_2790_squeeze_mask_0, x = coreml_update_state_7)[name = string("op_2790_cast_fp16")];
            tensor<int32, [4]> keys_begin_0 = const()[name = string("keys_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> keys_end_0 = const()[name = string("keys_end_0"), val = tensor<int32, [4]>([1, 2, 2048, 256])];
            tensor<bool, [4]> keys_end_mask_0 = const()[name = string("keys_end_mask_0"), val = tensor<bool, [4]>([false, true, true, true])];
            tensor<bool, [4]> keys_squeeze_mask_0 = const()[name = string("keys_squeeze_mask_0"), val = tensor<bool, [4]>([true, false, false, false])];
            tensor<fp16, [2, 2048, 256]> keys_cast_fp16 = slice_by_index(begin = keys_begin_0, end = keys_end_0, end_mask = keys_end_mask_0, squeeze_mask = keys_squeeze_mask_0, x = var_2790_cast_fp16)[name = string("keys_cast_fp16")];
            tensor<int32, [4]> values_begin_0 = const()[name = string("values_begin_0"), val = tensor<int32, [4]>([1, 0, 0, 0])];
            tensor<int32, [4]> values_end_0 = const()[name = string("values_end_0"), val = tensor<int32, [4]>([2, 2, 2048, 256])];
            tensor<bool, [4]> values_end_mask_0 = const()[name = string("values_end_mask_0"), val = tensor<bool, [4]>([false, true, true, true])];
            tensor<bool, [4]> values_squeeze_mask_0 = const()[name = string("values_squeeze_mask_0"), val = tensor<bool, [4]>([true, false, false, false])];
            tensor<fp16, [2, 2048, 256]> values_cast_fp16 = slice_by_index(begin = values_begin_0, end = values_end_0, end_mask = values_end_mask_0, squeeze_mask = values_squeeze_mask_0, x = var_2790_cast_fp16)[name = string("values_cast_fp16")];
            int32 var_2808 = const()[name = string("op_2808"), val = int32(1)];
            tensor<int32, [4]> var_2815 = const()[name = string("op_2815"), val = tensor<int32, [4]>([1, 2048, 1, 16])];
            tensor<fp16, [2, 4, 256, 16]> transpose_1_cast_fp16 = transpose(perm = transpose_1_perm_0, x = var_2708_cast_fp16)[name = string("transpose_4")];
            tensor<fp16, [1, 2048, 1, 16]> var_2816_cast_fp16 = reshape(shape = var_2815, x = transpose_1_cast_fp16)[name = string("op_2816_cast_fp16")];
            tensor<int32, [3]> var_2817 = const()[name = string("op_2817"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [4]> var_2820 = const()[name = string("op_2820"), val = tensor<int32, [4]>([1, 512, 1, 2048])];
            tensor<fp16, [2, 256, 2048]> var_2818_cast_fp16 = transpose(perm = var_2817, x = keys_cast_fp16)[name = string("transpose_3")];
            tensor<fp16, [1, 512, 1, 2048]> key_native_cast_fp16 = reshape(shape = var_2820, x = var_2818_cast_fp16)[name = string("key_native_cast_fp16")];
            tensor<int32, [3]> var_2822 = const()[name = string("op_2822"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [4]> var_2825 = const()[name = string("op_2825"), val = tensor<int32, [4]>([1, 512, 1, 2048])];
            tensor<fp16, [2, 256, 2048]> var_2823_cast_fp16 = transpose(perm = var_2822, x = values_cast_fp16)[name = string("transpose_2")];
            tensor<fp16, [1, 512, 1, 2048]> var_2826_cast_fp16 = reshape(shape = var_2825, x = var_2823_cast_fp16)[name = string("op_2826_cast_fp16")];
            tensor<int32, [8]> tile_19 = const()[name = string("tile_19"), val = tensor<int32, [8]>([256, 256, 256, 256, 256, 256, 256, 256])];
            int32 var_2830_axis_0 = const()[name = string("op_2830_axis_0"), val = int32(1)];
            tensor<fp16, [1, 256, 1, 16]> var_2830_cast_fp16_0, tensor<fp16, [1, 256, 1, 16]> var_2830_cast_fp16_1, tensor<fp16, [1, 256, 1, 16]> var_2830_cast_fp16_2, tensor<fp16, [1, 256, 1, 16]> var_2830_cast_fp16_3, tensor<fp16, [1, 256, 1, 16]> var_2830_cast_fp16_4, tensor<fp16, [1, 256, 1, 16]> var_2830_cast_fp16_5, tensor<fp16, [1, 256, 1, 16]> var_2830_cast_fp16_6, tensor<fp16, [1, 256, 1, 16]> var_2830_cast_fp16_7 = split(axis = var_2830_axis_0, split_sizes = tile_19, x = var_2816_cast_fp16)[name = string("op_2830_cast_fp16")];
            tensor<int32, [4]> var_2839_perm_0 = const()[name = string("op_2839_perm_0"), val = tensor<int32, [4]>([0, 3, 2, 1])];
            tensor<int32, [2]> tile_20 = const()[name = string("tile_20"), val = tensor<int32, [2]>([256, 256])];
            int32 var_2840_axis_0 = const()[name = string("op_2840_axis_0"), val = int32(3)];
            tensor<fp16, [1, 2048, 1, 512]> var_2839_cast_fp16 = transpose(perm = var_2839_perm_0, x = key_native_cast_fp16)[name = string("transpose_1")];
            tensor<fp16, [1, 2048, 1, 256]> var_2840_cast_fp16_0, tensor<fp16, [1, 2048, 1, 256]> var_2840_cast_fp16_1 = split(axis = var_2840_axis_0, split_sizes = tile_20, x = var_2839_cast_fp16)[name = string("op_2840_cast_fp16")];
            tensor<int32, [2]> tile_21 = const()[name = string("tile_21"), val = tensor<int32, [2]>([256, 256])];
            int32 var_2843_axis_0 = const()[name = string("op_2843_axis_0"), val = int32(1)];
            tensor<fp16, [1, 256, 1, 2048]> var_2843_cast_fp16_0, tensor<fp16, [1, 256, 1, 2048]> var_2843_cast_fp16_1 = split(axis = var_2843_axis_0, split_sizes = tile_21, x = var_2826_cast_fp16)[name = string("op_2843_cast_fp16")];
            string scores_17_equation_0 = const()[name = string("scores_17_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 16]> scores_17_cast_fp16 = einsum(equation = scores_17_equation_0, values = (var_2840_cast_fp16_0, var_2830_cast_fp16_0))[name = string("scores_17_cast_fp16")];
            fp16 var_2848_to_fp16 = const()[name = string("op_2848_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 16]> var_2849_cast_fp16 = mul(x = scores_17_cast_fp16, y = var_2848_to_fp16)[name = string("op_2849_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_2850_cast_fp16 = add(x = var_2849_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_2850_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_2851_cast_fp16 = softmax(axis = var_2808, x = var_2850_cast_fp16)[name = string("op_2851_cast_fp16")];
            tensor<int32, [4]> tile_22 = const()[name = string("tile_22"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_2852_axis_0 = const()[name = string("op_2852_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 16]> var_2852_cast_fp16_0, tensor<fp16, [1, 512, 1, 16]> var_2852_cast_fp16_1, tensor<fp16, [1, 512, 1, 16]> var_2852_cast_fp16_2, tensor<fp16, [1, 512, 1, 16]> var_2852_cast_fp16_3 = split(axis = var_2852_axis_0, split_sizes = tile_22, x = var_2851_cast_fp16)[name = string("op_2852_cast_fp16")];
            tensor<int32, [4]> tile_23 = const()[name = string("tile_23"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_2857_axis_0 = const()[name = string("op_2857_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_2857_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_2857_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_2857_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_2857_cast_fp16_3 = split(axis = var_2857_axis_0, split_sizes = tile_23, x = var_2843_cast_fp16_0)[name = string("op_2857_cast_fp16")];
            fp16 var_2862_to_fp16 = const()[name = string("op_2862_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_2863_cast_fp16 = mul(x = var_2852_cast_fp16_0, y = var_2862_to_fp16)[name = string("op_2863_cast_fp16")];
            string var_2865_equation_0 = const()[name = string("op_2865_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_2865_cast_fp16 = einsum(equation = var_2865_equation_0, values = (var_2857_cast_fp16_0, var_2863_cast_fp16))[name = string("op_2865_cast_fp16")];
            fp16 var_2866_to_fp16 = const()[name = string("op_2866_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_2867_cast_fp16 = mul(x = var_2852_cast_fp16_1, y = var_2866_to_fp16)[name = string("op_2867_cast_fp16")];
            string var_2869_equation_0 = const()[name = string("op_2869_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_2869_cast_fp16 = einsum(equation = var_2869_equation_0, values = (var_2857_cast_fp16_1, var_2867_cast_fp16))[name = string("op_2869_cast_fp16")];
            fp16 var_2870_to_fp16 = const()[name = string("op_2870_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_2871_cast_fp16 = mul(x = var_2852_cast_fp16_2, y = var_2870_to_fp16)[name = string("op_2871_cast_fp16")];
            string var_2873_equation_0 = const()[name = string("op_2873_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_2873_cast_fp16 = einsum(equation = var_2873_equation_0, values = (var_2857_cast_fp16_2, var_2871_cast_fp16))[name = string("op_2873_cast_fp16")];
            fp16 var_2874_to_fp16 = const()[name = string("op_2874_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_2875_cast_fp16 = mul(x = var_2852_cast_fp16_3, y = var_2874_to_fp16)[name = string("op_2875_cast_fp16")];
            string var_2877_equation_0 = const()[name = string("op_2877_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_2877_cast_fp16 = einsum(equation = var_2877_equation_0, values = (var_2857_cast_fp16_3, var_2875_cast_fp16))[name = string("op_2877_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_2878_cast_fp16 = add(x = var_2865_cast_fp16, y = var_2869_cast_fp16)[name = string("op_2878_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_2879_cast_fp16 = add(x = var_2873_cast_fp16, y = var_2877_cast_fp16)[name = string("op_2879_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_2880_cast_fp16 = add(x = var_2878_cast_fp16, y = var_2879_cast_fp16)[name = string("op_2880_cast_fp16")];
            fp16 _inversed_2882_y_0_to_fp16 = const()[name = string("_inversed_2882_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 16]> _inversed_2882_cast_fp16 = mul(x = var_2880_cast_fp16, y = _inversed_2882_y_0_to_fp16)[name = string("_inversed_2882_cast_fp16")];
            string scores_19_equation_0 = const()[name = string("scores_19_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 16]> scores_19_cast_fp16 = einsum(equation = scores_19_equation_0, values = (var_2840_cast_fp16_0, var_2830_cast_fp16_1))[name = string("scores_19_cast_fp16")];
            fp16 var_2885_to_fp16 = const()[name = string("op_2885_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 16]> var_2886_cast_fp16 = mul(x = scores_19_cast_fp16, y = var_2885_to_fp16)[name = string("op_2886_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_2887_cast_fp16 = add(x = var_2886_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_2887_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_2888_cast_fp16 = softmax(axis = var_2808, x = var_2887_cast_fp16)[name = string("op_2888_cast_fp16")];
            tensor<int32, [4]> tile_24 = const()[name = string("tile_24"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_2889_axis_0 = const()[name = string("op_2889_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 16]> var_2889_cast_fp16_0, tensor<fp16, [1, 512, 1, 16]> var_2889_cast_fp16_1, tensor<fp16, [1, 512, 1, 16]> var_2889_cast_fp16_2, tensor<fp16, [1, 512, 1, 16]> var_2889_cast_fp16_3 = split(axis = var_2889_axis_0, split_sizes = tile_24, x = var_2888_cast_fp16)[name = string("op_2889_cast_fp16")];
            tensor<int32, [4]> tile_25 = const()[name = string("tile_25"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_2894_axis_0 = const()[name = string("op_2894_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_2894_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_2894_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_2894_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_2894_cast_fp16_3 = split(axis = var_2894_axis_0, split_sizes = tile_25, x = var_2843_cast_fp16_0)[name = string("op_2894_cast_fp16")];
            fp16 var_2899_to_fp16 = const()[name = string("op_2899_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_2900_cast_fp16 = mul(x = var_2889_cast_fp16_0, y = var_2899_to_fp16)[name = string("op_2900_cast_fp16")];
            string var_2902_equation_0 = const()[name = string("op_2902_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_2902_cast_fp16 = einsum(equation = var_2902_equation_0, values = (var_2894_cast_fp16_0, var_2900_cast_fp16))[name = string("op_2902_cast_fp16")];
            fp16 var_2903_to_fp16 = const()[name = string("op_2903_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_2904_cast_fp16 = mul(x = var_2889_cast_fp16_1, y = var_2903_to_fp16)[name = string("op_2904_cast_fp16")];
            string var_2906_equation_0 = const()[name = string("op_2906_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_2906_cast_fp16 = einsum(equation = var_2906_equation_0, values = (var_2894_cast_fp16_1, var_2904_cast_fp16))[name = string("op_2906_cast_fp16")];
            fp16 var_2907_to_fp16 = const()[name = string("op_2907_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_2908_cast_fp16 = mul(x = var_2889_cast_fp16_2, y = var_2907_to_fp16)[name = string("op_2908_cast_fp16")];
            string var_2910_equation_0 = const()[name = string("op_2910_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_2910_cast_fp16 = einsum(equation = var_2910_equation_0, values = (var_2894_cast_fp16_2, var_2908_cast_fp16))[name = string("op_2910_cast_fp16")];
            fp16 var_2911_to_fp16 = const()[name = string("op_2911_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_2912_cast_fp16 = mul(x = var_2889_cast_fp16_3, y = var_2911_to_fp16)[name = string("op_2912_cast_fp16")];
            string var_2914_equation_0 = const()[name = string("op_2914_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_2914_cast_fp16 = einsum(equation = var_2914_equation_0, values = (var_2894_cast_fp16_3, var_2912_cast_fp16))[name = string("op_2914_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_2915_cast_fp16 = add(x = var_2902_cast_fp16, y = var_2906_cast_fp16)[name = string("op_2915_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_2916_cast_fp16 = add(x = var_2910_cast_fp16, y = var_2914_cast_fp16)[name = string("op_2916_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_2917_cast_fp16 = add(x = var_2915_cast_fp16, y = var_2916_cast_fp16)[name = string("op_2917_cast_fp16")];
            fp16 _inversed_2919_y_0_to_fp16 = const()[name = string("_inversed_2919_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 16]> _inversed_2919_cast_fp16 = mul(x = var_2917_cast_fp16, y = _inversed_2919_y_0_to_fp16)[name = string("_inversed_2919_cast_fp16")];
            string scores_21_equation_0 = const()[name = string("scores_21_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 16]> scores_21_cast_fp16 = einsum(equation = scores_21_equation_0, values = (var_2840_cast_fp16_0, var_2830_cast_fp16_2))[name = string("scores_21_cast_fp16")];
            fp16 var_2922_to_fp16 = const()[name = string("op_2922_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 16]> var_2923_cast_fp16 = mul(x = scores_21_cast_fp16, y = var_2922_to_fp16)[name = string("op_2923_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_2924_cast_fp16 = add(x = var_2923_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_2924_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_2925_cast_fp16 = softmax(axis = var_2808, x = var_2924_cast_fp16)[name = string("op_2925_cast_fp16")];
            tensor<int32, [4]> tile_26 = const()[name = string("tile_26"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_2926_axis_0 = const()[name = string("op_2926_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 16]> var_2926_cast_fp16_0, tensor<fp16, [1, 512, 1, 16]> var_2926_cast_fp16_1, tensor<fp16, [1, 512, 1, 16]> var_2926_cast_fp16_2, tensor<fp16, [1, 512, 1, 16]> var_2926_cast_fp16_3 = split(axis = var_2926_axis_0, split_sizes = tile_26, x = var_2925_cast_fp16)[name = string("op_2926_cast_fp16")];
            tensor<int32, [4]> tile_27 = const()[name = string("tile_27"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_2931_axis_0 = const()[name = string("op_2931_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_2931_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_2931_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_2931_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_2931_cast_fp16_3 = split(axis = var_2931_axis_0, split_sizes = tile_27, x = var_2843_cast_fp16_0)[name = string("op_2931_cast_fp16")];
            fp16 var_2936_to_fp16 = const()[name = string("op_2936_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_2937_cast_fp16 = mul(x = var_2926_cast_fp16_0, y = var_2936_to_fp16)[name = string("op_2937_cast_fp16")];
            string var_2939_equation_0 = const()[name = string("op_2939_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_2939_cast_fp16 = einsum(equation = var_2939_equation_0, values = (var_2931_cast_fp16_0, var_2937_cast_fp16))[name = string("op_2939_cast_fp16")];
            fp16 var_2940_to_fp16 = const()[name = string("op_2940_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_2941_cast_fp16 = mul(x = var_2926_cast_fp16_1, y = var_2940_to_fp16)[name = string("op_2941_cast_fp16")];
            string var_2943_equation_0 = const()[name = string("op_2943_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_2943_cast_fp16 = einsum(equation = var_2943_equation_0, values = (var_2931_cast_fp16_1, var_2941_cast_fp16))[name = string("op_2943_cast_fp16")];
            fp16 var_2944_to_fp16 = const()[name = string("op_2944_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_2945_cast_fp16 = mul(x = var_2926_cast_fp16_2, y = var_2944_to_fp16)[name = string("op_2945_cast_fp16")];
            string var_2947_equation_0 = const()[name = string("op_2947_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_2947_cast_fp16 = einsum(equation = var_2947_equation_0, values = (var_2931_cast_fp16_2, var_2945_cast_fp16))[name = string("op_2947_cast_fp16")];
            fp16 var_2948_to_fp16 = const()[name = string("op_2948_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_2949_cast_fp16 = mul(x = var_2926_cast_fp16_3, y = var_2948_to_fp16)[name = string("op_2949_cast_fp16")];
            string var_2951_equation_0 = const()[name = string("op_2951_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_2951_cast_fp16 = einsum(equation = var_2951_equation_0, values = (var_2931_cast_fp16_3, var_2949_cast_fp16))[name = string("op_2951_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_2952_cast_fp16 = add(x = var_2939_cast_fp16, y = var_2943_cast_fp16)[name = string("op_2952_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_2953_cast_fp16 = add(x = var_2947_cast_fp16, y = var_2951_cast_fp16)[name = string("op_2953_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_2954_cast_fp16 = add(x = var_2952_cast_fp16, y = var_2953_cast_fp16)[name = string("op_2954_cast_fp16")];
            fp16 _inversed_2956_y_0_to_fp16 = const()[name = string("_inversed_2956_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 16]> _inversed_2956_cast_fp16 = mul(x = var_2954_cast_fp16, y = _inversed_2956_y_0_to_fp16)[name = string("_inversed_2956_cast_fp16")];
            string scores_23_equation_0 = const()[name = string("scores_23_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 16]> scores_23_cast_fp16 = einsum(equation = scores_23_equation_0, values = (var_2840_cast_fp16_0, var_2830_cast_fp16_3))[name = string("scores_23_cast_fp16")];
            fp16 var_2959_to_fp16 = const()[name = string("op_2959_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 16]> var_2960_cast_fp16 = mul(x = scores_23_cast_fp16, y = var_2959_to_fp16)[name = string("op_2960_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_2961_cast_fp16 = add(x = var_2960_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_2961_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_2962_cast_fp16 = softmax(axis = var_2808, x = var_2961_cast_fp16)[name = string("op_2962_cast_fp16")];
            tensor<int32, [4]> tile_28 = const()[name = string("tile_28"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_2963_axis_0 = const()[name = string("op_2963_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 16]> var_2963_cast_fp16_0, tensor<fp16, [1, 512, 1, 16]> var_2963_cast_fp16_1, tensor<fp16, [1, 512, 1, 16]> var_2963_cast_fp16_2, tensor<fp16, [1, 512, 1, 16]> var_2963_cast_fp16_3 = split(axis = var_2963_axis_0, split_sizes = tile_28, x = var_2962_cast_fp16)[name = string("op_2963_cast_fp16")];
            tensor<int32, [4]> tile_29 = const()[name = string("tile_29"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_2968_axis_0 = const()[name = string("op_2968_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_2968_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_2968_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_2968_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_2968_cast_fp16_3 = split(axis = var_2968_axis_0, split_sizes = tile_29, x = var_2843_cast_fp16_0)[name = string("op_2968_cast_fp16")];
            fp16 var_2973_to_fp16 = const()[name = string("op_2973_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_2974_cast_fp16 = mul(x = var_2963_cast_fp16_0, y = var_2973_to_fp16)[name = string("op_2974_cast_fp16")];
            string var_2976_equation_0 = const()[name = string("op_2976_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_2976_cast_fp16 = einsum(equation = var_2976_equation_0, values = (var_2968_cast_fp16_0, var_2974_cast_fp16))[name = string("op_2976_cast_fp16")];
            fp16 var_2977_to_fp16 = const()[name = string("op_2977_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_2978_cast_fp16 = mul(x = var_2963_cast_fp16_1, y = var_2977_to_fp16)[name = string("op_2978_cast_fp16")];
            string var_2980_equation_0 = const()[name = string("op_2980_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_2980_cast_fp16 = einsum(equation = var_2980_equation_0, values = (var_2968_cast_fp16_1, var_2978_cast_fp16))[name = string("op_2980_cast_fp16")];
            fp16 var_2981_to_fp16 = const()[name = string("op_2981_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_2982_cast_fp16 = mul(x = var_2963_cast_fp16_2, y = var_2981_to_fp16)[name = string("op_2982_cast_fp16")];
            string var_2984_equation_0 = const()[name = string("op_2984_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_2984_cast_fp16 = einsum(equation = var_2984_equation_0, values = (var_2968_cast_fp16_2, var_2982_cast_fp16))[name = string("op_2984_cast_fp16")];
            fp16 var_2985_to_fp16 = const()[name = string("op_2985_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_2986_cast_fp16 = mul(x = var_2963_cast_fp16_3, y = var_2985_to_fp16)[name = string("op_2986_cast_fp16")];
            string var_2988_equation_0 = const()[name = string("op_2988_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_2988_cast_fp16 = einsum(equation = var_2988_equation_0, values = (var_2968_cast_fp16_3, var_2986_cast_fp16))[name = string("op_2988_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_2989_cast_fp16 = add(x = var_2976_cast_fp16, y = var_2980_cast_fp16)[name = string("op_2989_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_2990_cast_fp16 = add(x = var_2984_cast_fp16, y = var_2988_cast_fp16)[name = string("op_2990_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_2991_cast_fp16 = add(x = var_2989_cast_fp16, y = var_2990_cast_fp16)[name = string("op_2991_cast_fp16")];
            fp16 _inversed_2993_y_0_to_fp16 = const()[name = string("_inversed_2993_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 16]> _inversed_2993_cast_fp16 = mul(x = var_2991_cast_fp16, y = _inversed_2993_y_0_to_fp16)[name = string("_inversed_2993_cast_fp16")];
            string scores_25_equation_0 = const()[name = string("scores_25_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 16]> scores_25_cast_fp16 = einsum(equation = scores_25_equation_0, values = (var_2840_cast_fp16_1, var_2830_cast_fp16_4))[name = string("scores_25_cast_fp16")];
            fp16 var_2996_to_fp16 = const()[name = string("op_2996_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 16]> var_2997_cast_fp16 = mul(x = scores_25_cast_fp16, y = var_2996_to_fp16)[name = string("op_2997_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_2998_cast_fp16 = add(x = var_2997_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_2998_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_2999_cast_fp16 = softmax(axis = var_2808, x = var_2998_cast_fp16)[name = string("op_2999_cast_fp16")];
            tensor<int32, [4]> tile_30 = const()[name = string("tile_30"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_3000_axis_0 = const()[name = string("op_3000_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 16]> var_3000_cast_fp16_0, tensor<fp16, [1, 512, 1, 16]> var_3000_cast_fp16_1, tensor<fp16, [1, 512, 1, 16]> var_3000_cast_fp16_2, tensor<fp16, [1, 512, 1, 16]> var_3000_cast_fp16_3 = split(axis = var_3000_axis_0, split_sizes = tile_30, x = var_2999_cast_fp16)[name = string("op_3000_cast_fp16")];
            tensor<int32, [4]> tile_31 = const()[name = string("tile_31"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_3005_axis_0 = const()[name = string("op_3005_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_3005_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_3005_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_3005_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_3005_cast_fp16_3 = split(axis = var_3005_axis_0, split_sizes = tile_31, x = var_2843_cast_fp16_1)[name = string("op_3005_cast_fp16")];
            fp16 var_3010_to_fp16 = const()[name = string("op_3010_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_3011_cast_fp16 = mul(x = var_3000_cast_fp16_0, y = var_3010_to_fp16)[name = string("op_3011_cast_fp16")];
            string var_3013_equation_0 = const()[name = string("op_3013_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_3013_cast_fp16 = einsum(equation = var_3013_equation_0, values = (var_3005_cast_fp16_0, var_3011_cast_fp16))[name = string("op_3013_cast_fp16")];
            fp16 var_3014_to_fp16 = const()[name = string("op_3014_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_3015_cast_fp16 = mul(x = var_3000_cast_fp16_1, y = var_3014_to_fp16)[name = string("op_3015_cast_fp16")];
            string var_3017_equation_0 = const()[name = string("op_3017_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_3017_cast_fp16 = einsum(equation = var_3017_equation_0, values = (var_3005_cast_fp16_1, var_3015_cast_fp16))[name = string("op_3017_cast_fp16")];
            fp16 var_3018_to_fp16 = const()[name = string("op_3018_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_3019_cast_fp16 = mul(x = var_3000_cast_fp16_2, y = var_3018_to_fp16)[name = string("op_3019_cast_fp16")];
            string var_3021_equation_0 = const()[name = string("op_3021_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_3021_cast_fp16 = einsum(equation = var_3021_equation_0, values = (var_3005_cast_fp16_2, var_3019_cast_fp16))[name = string("op_3021_cast_fp16")];
            fp16 var_3022_to_fp16 = const()[name = string("op_3022_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_3023_cast_fp16 = mul(x = var_3000_cast_fp16_3, y = var_3022_to_fp16)[name = string("op_3023_cast_fp16")];
            string var_3025_equation_0 = const()[name = string("op_3025_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_3025_cast_fp16 = einsum(equation = var_3025_equation_0, values = (var_3005_cast_fp16_3, var_3023_cast_fp16))[name = string("op_3025_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_3026_cast_fp16 = add(x = var_3013_cast_fp16, y = var_3017_cast_fp16)[name = string("op_3026_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_3027_cast_fp16 = add(x = var_3021_cast_fp16, y = var_3025_cast_fp16)[name = string("op_3027_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_3028_cast_fp16 = add(x = var_3026_cast_fp16, y = var_3027_cast_fp16)[name = string("op_3028_cast_fp16")];
            fp16 _inversed_3030_y_0_to_fp16 = const()[name = string("_inversed_3030_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 16]> _inversed_3030_cast_fp16 = mul(x = var_3028_cast_fp16, y = _inversed_3030_y_0_to_fp16)[name = string("_inversed_3030_cast_fp16")];
            string scores_27_equation_0 = const()[name = string("scores_27_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 16]> scores_27_cast_fp16 = einsum(equation = scores_27_equation_0, values = (var_2840_cast_fp16_1, var_2830_cast_fp16_5))[name = string("scores_27_cast_fp16")];
            fp16 var_3033_to_fp16 = const()[name = string("op_3033_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 16]> var_3034_cast_fp16 = mul(x = scores_27_cast_fp16, y = var_3033_to_fp16)[name = string("op_3034_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_3035_cast_fp16 = add(x = var_3034_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_3035_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_3036_cast_fp16 = softmax(axis = var_2808, x = var_3035_cast_fp16)[name = string("op_3036_cast_fp16")];
            tensor<int32, [4]> tile_32 = const()[name = string("tile_32"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_3037_axis_0 = const()[name = string("op_3037_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 16]> var_3037_cast_fp16_0, tensor<fp16, [1, 512, 1, 16]> var_3037_cast_fp16_1, tensor<fp16, [1, 512, 1, 16]> var_3037_cast_fp16_2, tensor<fp16, [1, 512, 1, 16]> var_3037_cast_fp16_3 = split(axis = var_3037_axis_0, split_sizes = tile_32, x = var_3036_cast_fp16)[name = string("op_3037_cast_fp16")];
            tensor<int32, [4]> tile_33 = const()[name = string("tile_33"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_3042_axis_0 = const()[name = string("op_3042_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_3042_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_3042_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_3042_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_3042_cast_fp16_3 = split(axis = var_3042_axis_0, split_sizes = tile_33, x = var_2843_cast_fp16_1)[name = string("op_3042_cast_fp16")];
            fp16 var_3047_to_fp16 = const()[name = string("op_3047_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_3048_cast_fp16 = mul(x = var_3037_cast_fp16_0, y = var_3047_to_fp16)[name = string("op_3048_cast_fp16")];
            string var_3050_equation_0 = const()[name = string("op_3050_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_3050_cast_fp16 = einsum(equation = var_3050_equation_0, values = (var_3042_cast_fp16_0, var_3048_cast_fp16))[name = string("op_3050_cast_fp16")];
            fp16 var_3051_to_fp16 = const()[name = string("op_3051_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_3052_cast_fp16 = mul(x = var_3037_cast_fp16_1, y = var_3051_to_fp16)[name = string("op_3052_cast_fp16")];
            string var_3054_equation_0 = const()[name = string("op_3054_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_3054_cast_fp16 = einsum(equation = var_3054_equation_0, values = (var_3042_cast_fp16_1, var_3052_cast_fp16))[name = string("op_3054_cast_fp16")];
            fp16 var_3055_to_fp16 = const()[name = string("op_3055_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_3056_cast_fp16 = mul(x = var_3037_cast_fp16_2, y = var_3055_to_fp16)[name = string("op_3056_cast_fp16")];
            string var_3058_equation_0 = const()[name = string("op_3058_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_3058_cast_fp16 = einsum(equation = var_3058_equation_0, values = (var_3042_cast_fp16_2, var_3056_cast_fp16))[name = string("op_3058_cast_fp16")];
            fp16 var_3059_to_fp16 = const()[name = string("op_3059_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_3060_cast_fp16 = mul(x = var_3037_cast_fp16_3, y = var_3059_to_fp16)[name = string("op_3060_cast_fp16")];
            string var_3062_equation_0 = const()[name = string("op_3062_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_3062_cast_fp16 = einsum(equation = var_3062_equation_0, values = (var_3042_cast_fp16_3, var_3060_cast_fp16))[name = string("op_3062_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_3063_cast_fp16 = add(x = var_3050_cast_fp16, y = var_3054_cast_fp16)[name = string("op_3063_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_3064_cast_fp16 = add(x = var_3058_cast_fp16, y = var_3062_cast_fp16)[name = string("op_3064_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_3065_cast_fp16 = add(x = var_3063_cast_fp16, y = var_3064_cast_fp16)[name = string("op_3065_cast_fp16")];
            fp16 _inversed_3067_y_0_to_fp16 = const()[name = string("_inversed_3067_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 16]> _inversed_3067_cast_fp16 = mul(x = var_3065_cast_fp16, y = _inversed_3067_y_0_to_fp16)[name = string("_inversed_3067_cast_fp16")];
            string scores_29_equation_0 = const()[name = string("scores_29_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 16]> scores_29_cast_fp16 = einsum(equation = scores_29_equation_0, values = (var_2840_cast_fp16_1, var_2830_cast_fp16_6))[name = string("scores_29_cast_fp16")];
            fp16 var_3070_to_fp16 = const()[name = string("op_3070_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 16]> var_3071_cast_fp16 = mul(x = scores_29_cast_fp16, y = var_3070_to_fp16)[name = string("op_3071_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_3072_cast_fp16 = add(x = var_3071_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_3072_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_3073_cast_fp16 = softmax(axis = var_2808, x = var_3072_cast_fp16)[name = string("op_3073_cast_fp16")];
            tensor<int32, [4]> tile_34 = const()[name = string("tile_34"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_3074_axis_0 = const()[name = string("op_3074_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 16]> var_3074_cast_fp16_0, tensor<fp16, [1, 512, 1, 16]> var_3074_cast_fp16_1, tensor<fp16, [1, 512, 1, 16]> var_3074_cast_fp16_2, tensor<fp16, [1, 512, 1, 16]> var_3074_cast_fp16_3 = split(axis = var_3074_axis_0, split_sizes = tile_34, x = var_3073_cast_fp16)[name = string("op_3074_cast_fp16")];
            tensor<int32, [4]> tile_35 = const()[name = string("tile_35"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_3079_axis_0 = const()[name = string("op_3079_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_3079_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_3079_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_3079_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_3079_cast_fp16_3 = split(axis = var_3079_axis_0, split_sizes = tile_35, x = var_2843_cast_fp16_1)[name = string("op_3079_cast_fp16")];
            fp16 var_3084_to_fp16 = const()[name = string("op_3084_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_3085_cast_fp16 = mul(x = var_3074_cast_fp16_0, y = var_3084_to_fp16)[name = string("op_3085_cast_fp16")];
            string var_3087_equation_0 = const()[name = string("op_3087_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_3087_cast_fp16 = einsum(equation = var_3087_equation_0, values = (var_3079_cast_fp16_0, var_3085_cast_fp16))[name = string("op_3087_cast_fp16")];
            fp16 var_3088_to_fp16 = const()[name = string("op_3088_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_3089_cast_fp16 = mul(x = var_3074_cast_fp16_1, y = var_3088_to_fp16)[name = string("op_3089_cast_fp16")];
            string var_3091_equation_0 = const()[name = string("op_3091_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_3091_cast_fp16 = einsum(equation = var_3091_equation_0, values = (var_3079_cast_fp16_1, var_3089_cast_fp16))[name = string("op_3091_cast_fp16")];
            fp16 var_3092_to_fp16 = const()[name = string("op_3092_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_3093_cast_fp16 = mul(x = var_3074_cast_fp16_2, y = var_3092_to_fp16)[name = string("op_3093_cast_fp16")];
            string var_3095_equation_0 = const()[name = string("op_3095_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_3095_cast_fp16 = einsum(equation = var_3095_equation_0, values = (var_3079_cast_fp16_2, var_3093_cast_fp16))[name = string("op_3095_cast_fp16")];
            fp16 var_3096_to_fp16 = const()[name = string("op_3096_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_3097_cast_fp16 = mul(x = var_3074_cast_fp16_3, y = var_3096_to_fp16)[name = string("op_3097_cast_fp16")];
            string var_3099_equation_0 = const()[name = string("op_3099_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_3099_cast_fp16 = einsum(equation = var_3099_equation_0, values = (var_3079_cast_fp16_3, var_3097_cast_fp16))[name = string("op_3099_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_3100_cast_fp16 = add(x = var_3087_cast_fp16, y = var_3091_cast_fp16)[name = string("op_3100_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_3101_cast_fp16 = add(x = var_3095_cast_fp16, y = var_3099_cast_fp16)[name = string("op_3101_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_3102_cast_fp16 = add(x = var_3100_cast_fp16, y = var_3101_cast_fp16)[name = string("op_3102_cast_fp16")];
            fp16 _inversed_3104_y_0_to_fp16 = const()[name = string("_inversed_3104_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 16]> _inversed_3104_cast_fp16 = mul(x = var_3102_cast_fp16, y = _inversed_3104_y_0_to_fp16)[name = string("_inversed_3104_cast_fp16")];
            string scores_equation_0 = const()[name = string("scores_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 16]> scores_cast_fp16 = einsum(equation = scores_equation_0, values = (var_2840_cast_fp16_1, var_2830_cast_fp16_7))[name = string("scores_cast_fp16")];
            fp16 var_3107_to_fp16 = const()[name = string("op_3107_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 16]> var_3108_cast_fp16 = mul(x = scores_cast_fp16, y = var_3107_to_fp16)[name = string("op_3108_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_3109_cast_fp16 = add(x = var_3108_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_3109_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 16]> var_3110_cast_fp16 = softmax(axis = var_2808, x = var_3109_cast_fp16)[name = string("op_3110_cast_fp16")];
            tensor<int32, [4]> tile_36 = const()[name = string("tile_36"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_3111_axis_0 = const()[name = string("op_3111_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 16]> var_3111_cast_fp16_0, tensor<fp16, [1, 512, 1, 16]> var_3111_cast_fp16_1, tensor<fp16, [1, 512, 1, 16]> var_3111_cast_fp16_2, tensor<fp16, [1, 512, 1, 16]> var_3111_cast_fp16_3 = split(axis = var_3111_axis_0, split_sizes = tile_36, x = var_3110_cast_fp16)[name = string("op_3111_cast_fp16")];
            tensor<int32, [4]> tile_37 = const()[name = string("tile_37"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_3116_axis_0 = const()[name = string("op_3116_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_3116_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_3116_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_3116_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_3116_cast_fp16_3 = split(axis = var_3116_axis_0, split_sizes = tile_37, x = var_2843_cast_fp16_1)[name = string("op_3116_cast_fp16")];
            fp16 var_3121_to_fp16 = const()[name = string("op_3121_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_3122_cast_fp16 = mul(x = var_3111_cast_fp16_0, y = var_3121_to_fp16)[name = string("op_3122_cast_fp16")];
            string var_3124_equation_0 = const()[name = string("op_3124_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_3124_cast_fp16 = einsum(equation = var_3124_equation_0, values = (var_3116_cast_fp16_0, var_3122_cast_fp16))[name = string("op_3124_cast_fp16")];
            fp16 var_3125_to_fp16 = const()[name = string("op_3125_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_3126_cast_fp16 = mul(x = var_3111_cast_fp16_1, y = var_3125_to_fp16)[name = string("op_3126_cast_fp16")];
            string var_3128_equation_0 = const()[name = string("op_3128_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_3128_cast_fp16 = einsum(equation = var_3128_equation_0, values = (var_3116_cast_fp16_1, var_3126_cast_fp16))[name = string("op_3128_cast_fp16")];
            fp16 var_3129_to_fp16 = const()[name = string("op_3129_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_3130_cast_fp16 = mul(x = var_3111_cast_fp16_2, y = var_3129_to_fp16)[name = string("op_3130_cast_fp16")];
            string var_3132_equation_0 = const()[name = string("op_3132_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_3132_cast_fp16 = einsum(equation = var_3132_equation_0, values = (var_3116_cast_fp16_2, var_3130_cast_fp16))[name = string("op_3132_cast_fp16")];
            fp16 var_3133_to_fp16 = const()[name = string("op_3133_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 16]> var_3134_cast_fp16 = mul(x = var_3111_cast_fp16_3, y = var_3133_to_fp16)[name = string("op_3134_cast_fp16")];
            string var_3136_equation_0 = const()[name = string("op_3136_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 16]> var_3136_cast_fp16 = einsum(equation = var_3136_equation_0, values = (var_3116_cast_fp16_3, var_3134_cast_fp16))[name = string("op_3136_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_3137_cast_fp16 = add(x = var_3124_cast_fp16, y = var_3128_cast_fp16)[name = string("op_3137_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_3138_cast_fp16 = add(x = var_3132_cast_fp16, y = var_3136_cast_fp16)[name = string("op_3138_cast_fp16")];
            tensor<fp16, [1, 256, 1, 16]> var_3139_cast_fp16 = add(x = var_3137_cast_fp16, y = var_3138_cast_fp16)[name = string("op_3139_cast_fp16")];
            fp16 _inversed_3141_y_0_to_fp16 = const()[name = string("_inversed_3141_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 16]> _inversed_3141_cast_fp16 = mul(x = var_3139_cast_fp16, y = _inversed_3141_y_0_to_fp16)[name = string("_inversed_3141_cast_fp16")];
            bool var_3143_interleave_0 = const()[name = string("op_3143_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 2048, 1, 16]> var_3143_cast_fp16 = concat(axis = var_2808, interleave = var_3143_interleave_0, values = (_inversed_2882_cast_fp16, _inversed_2919_cast_fp16, _inversed_2956_cast_fp16, _inversed_2993_cast_fp16, _inversed_3030_cast_fp16, _inversed_3067_cast_fp16, _inversed_3104_cast_fp16, _inversed_3141_cast_fp16))[name = string("op_3143_cast_fp16")];
            tensor<int32, [4]> var_3144 = const()[name = string("op_3144"), val = tensor<int32, [4]>([2, 4, 256, 16])];
            tensor<fp16, [2, 4, 256, 16]> var_3145_cast_fp16 = reshape(shape = var_3144, x = var_3143_cast_fp16)[name = string("op_3145_cast_fp16")];
            tensor<int32, [4]> transpose_2_perm_0 = const()[name = string("transpose_2_perm_0"), val = tensor<int32, [4]>([3, 0, 1, 2])];
            tensor<int32, [2]> var_3164 = const()[name = string("op_3164"), val = tensor<int32, [2]>([16, 2048])];
            tensor<fp16, [16, 2, 4, 256]> transpose_2_cast_fp16 = transpose(perm = transpose_2_perm_0, x = var_3145_cast_fp16)[name = string("transpose_0")];
            tensor<fp16, [16, 2048]> attention_output_cast_fp16 = reshape(shape = var_3164, x = transpose_2_cast_fp16)[name = string("attention_output_cast_fp16")];
            tensor<fp16, [16, 2048]> var_3166_cast_fp16 = sigmoid(x = gate_cast_fp16)[name = string("op_3166_cast_fp16")];
            tensor<fp16, [16, 2048]> input_53_cast_fp16 = mul(x = attention_output_cast_fp16, y = var_3166_cast_fp16)[name = string("input_53_cast_fp16")];
            tensor<fp16, [1024, 2048]> completion23_o_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 2048]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(256190528))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(257763456))))[name = string("completion23_o_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 1024]> linear_31_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = completion23_o_proj_weight_promoted_to_fp16_palettized, x = input_53_cast_fp16)[name = string("linear_31_cast_fp16")];
            tensor<fp16, [16, 1024]> value_59_cast_fp16 = add(x = value_45_cast_fp16, y = linear_31_cast_fp16)[name = string("value_59_cast_fp16")];
            tensor<fp16, [16, 1024]> var_3171_cast_fp16 = mul(x = value_59_cast_fp16, y = value_59_cast_fp16)[name = string("op_3171_cast_fp16")];
            tensor<int32, [1]> variance_27_axes_0 = const()[name = string("variance_27_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_27_keep_dims_0 = const()[name = string("variance_27_keep_dims_0"), val = bool(true)];
            tensor<fp16, [16, 1]> variance_27_cast_fp16 = reduce_mean(axes = variance_27_axes_0, keep_dims = variance_27_keep_dims_0, x = var_3171_cast_fp16)[name = string("variance_27_cast_fp16")];
            fp16 var_3174_to_fp16 = const()[name = string("op_3174_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [16, 1]> var_3175_cast_fp16 = add(x = variance_27_cast_fp16, y = var_3174_to_fp16)[name = string("op_3175_cast_fp16")];
            fp32 var_3176_epsilon_0 = const()[name = string("op_3176_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [16, 1]> var_3176_cast_fp16 = rsqrt(epsilon = var_3176_epsilon_0, x = var_3175_cast_fp16)[name = string("op_3176_cast_fp16")];
            tensor<fp16, [16, 1024]> var_3177_cast_fp16 = mul(x = value_59_cast_fp16, y = var_3176_cast_fp16)[name = string("op_3177_cast_fp16")];
            tensor<fp16, [1024]> var_3179_to_fp16 = const()[name = string("op_3179_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(257771712)))];
            tensor<fp16, [16, 1024]> input_55_cast_fp16 = mul(x = var_3177_cast_fp16, y = var_3179_to_fp16)[name = string("input_55_cast_fp16")];
            tensor<fp16, [3584, 1024]> completion23_gate_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(257773824))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(260526400))))[name = string("completion23_gate_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 3584]> linear_32_cast_fp16 = linear(bias = linear_1_bias_0_to_fp16, weight = completion23_gate_proj_weight_promoted_to_fp16_palettized, x = input_55_cast_fp16)[name = string("linear_32_cast_fp16")];
            tensor<fp16, [16, 3584]> var_3183_cast_fp16 = silu(x = linear_32_cast_fp16)[name = string("op_3183_cast_fp16")];
            tensor<fp16, [3584, 1024]> completion23_up_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(260555136))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(263307712))))[name = string("completion23_up_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 3584]> linear_33_cast_fp16 = linear(bias = linear_1_bias_0_to_fp16, weight = completion23_up_proj_weight_promoted_to_fp16_palettized, x = input_55_cast_fp16)[name = string("linear_33_cast_fp16")];
            tensor<fp16, [16, 3584]> input_cast_fp16 = mul(x = var_3183_cast_fp16, y = linear_33_cast_fp16)[name = string("input_cast_fp16")];
            tensor<fp16, [1024, 3584]> completion23_down_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 3584]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(263336448))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(266089024))))[name = string("completion23_down_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16, 1024]> linear_34_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = completion23_down_proj_weight_promoted_to_fp16_palettized, x = input_cast_fp16)[name = string("linear_34_cast_fp16")];
            tensor<fp16, [16, 1024]> value_cast_fp16 = add(x = value_59_cast_fp16, y = linear_34_cast_fp16)[name = string("value_cast_fp16")];
            tensor<fp16, [16, 1024]> var_3190_cast_fp16 = mul(x = value_cast_fp16, y = value_cast_fp16)[name = string("op_3190_cast_fp16")];
            tensor<int32, [1]> variance_axes_0 = const()[name = string("variance_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_keep_dims_0 = const()[name = string("variance_keep_dims_0"), val = bool(true)];
            tensor<fp16, [16, 1]> variance_cast_fp16 = reduce_mean(axes = variance_axes_0, keep_dims = variance_keep_dims_0, x = var_3190_cast_fp16)[name = string("variance_cast_fp16")];
            fp16 var_3197_to_fp16 = const()[name = string("op_3197_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [16, 1]> var_3198_cast_fp16 = add(x = variance_cast_fp16, y = var_3197_to_fp16)[name = string("op_3198_cast_fp16")];
            fp32 var_3199_epsilon_0 = const()[name = string("op_3199_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [16, 1]> var_3199_cast_fp16 = rsqrt(epsilon = var_3199_epsilon_0, x = var_3198_cast_fp16)[name = string("op_3199_cast_fp16")];
            tensor<fp16, [16, 1024]> var_3200_cast_fp16 = mul(x = value_cast_fp16, y = var_3199_cast_fp16)[name = string("op_3200_cast_fp16")];
            tensor<fp16, [1024]> var_3203_to_fp16 = const()[name = string("op_3203_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(266097280)))];
            tensor<fp16, [16, 1024]> hidden_out = mul(x = var_3200_cast_fp16, y = var_3203_to_fp16)[name = string("hidden_cast_fp16")];
            tensor<int32, [2]> var_3209_begin_0 = const()[name = string("op_3209_begin_0"), val = tensor<int32, [2]>([-1, 0])];
            tensor<int32, [2]> var_3209_end_0 = const()[name = string("op_3209_end_0"), val = tensor<int32, [2]>([16, 1024])];
            tensor<bool, [2]> var_3209_end_mask_0 = const()[name = string("op_3209_end_mask_0"), val = tensor<bool, [2]>([true, true])];
            tensor<fp16, [1, 1024]> var_3209_cast_fp16 = slice_by_index(begin = var_3209_begin_0, end = var_3209_end_0, end_mask = var_3209_end_mask_0, x = hidden_out)[name = string("op_3209_cast_fp16")];
            tensor<fp16, [248320, 1024]> head_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [248320, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(64))), lut = tensor<fp16, [15520, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(190709888))))[name = string("head_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [248320]> linear_35_bias_0_to_fp16 = const()[name = string("linear_35_bias_0_to_fp16"), val = tensor<fp16, [248320]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(192696512)))];
            tensor<fp16, [1, 248320]> linear_35_cast_fp16 = linear(bias = linear_35_bias_0_to_fp16, weight = head_weight_promoted_to_fp16_palettized, x = var_3209_cast_fp16)[name = string("linear_35_cast_fp16")];
            int32 var_3214_axis_0 = const()[name = string("op_3214_axis_0"), val = int32(-1)];
            bool var_3214_keep_dims_0 = const()[name = string("op_3214_keep_dims_0"), val = bool(false)];
            string var_3214_output_dtype_0 = const()[name = string("op_3214_output_dtype_0"), val = string("int32")];
            tensor<int32, [1]> next_token = reduce_argmax(axis = var_3214_axis_0, keep_dims = var_3214_keep_dims_0, output_dtype = var_3214_output_dtype_0, x = linear_35_cast_fp16)[name = string("op_3214_cast_fp16")];
        } -> (hidden_out, conv20_out, rec20_out, conv21_out, rec21_out, conv22_out, rec22_out, next_token);
    func prefill4<ios18>(tensor<fp16, [1, 6144, 3]> conv20, tensor<fp16, [1, 6144, 3]> conv21, tensor<fp16, [1, 6144, 3]> conv22, tensor<fp16, [1, 1, 4, 64]> cos, tensor<int32, [1]> current_pos, tensor<fp16, [4, 2048]> gate19, tensor<fp16, [4, 1024]> hidden, tensor<fp16, [2, 4, 256]> k19, state<tensor<fp16, [2, 2, 2, 2048, 256]>> kv_cache, tensor<fp16, [1, 4, 2048]> mask19, tensor<fp16, [2, 4, 4, 256]> q19, tensor<fp16, [1, 16, 128, 128]> rec20, tensor<fp16, [1, 16, 128, 128]> rec21, tensor<fp16, [1, 16, 128, 128]> rec22, tensor<fp16, [1, 1, 4, 64]> sin, tensor<fp16, [2, 4, 256]> v19) {
            tensor<int32, [1]> var_33_axes_0 = const()[name = string("op_33_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [1, 2, 4, 256]> var_33_cast_fp16 = expand_dims(axes = var_33_axes_0, x = k19)[name = string("op_33_cast_fp16")];
            tensor<int32, [1]> var_35_axes_0 = const()[name = string("op_35_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [1, 1, 2, 4, 256]> var_35_cast_fp16 = expand_dims(axes = var_35_axes_0, x = var_33_cast_fp16)[name = string("op_35_cast_fp16")];
            int32 var_37 = const()[name = string("op_37"), val = int32(4)];
            tensor<int32, [1]> var_38 = add(x = current_pos, y = var_37)[name = string("op_38")];
            tensor<fp16, [2, 2, 2, 2048, 256]> read_state_0 = read_state(input = kv_cache)[name = string("read_state_0")];
            tensor<int32, [1]> expand_dims_0 = const()[name = string("expand_dims_0"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_1 = const()[name = string("expand_dims_1"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_2 = const()[name = string("expand_dims_2"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_4 = const()[name = string("expand_dims_4"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_5 = const()[name = string("expand_dims_5"), val = tensor<int32, [1]>([1])];
            tensor<int32, [1]> expand_dims_6 = const()[name = string("expand_dims_6"), val = tensor<int32, [1]>([1])];
            int32 concat_2_axis_0 = const()[name = string("concat_2_axis_0"), val = int32(0)];
            bool concat_2_interleave_0 = const()[name = string("concat_2_interleave_0"), val = bool(false)];
            tensor<int32, [5]> concat_2 = concat(axis = concat_2_axis_0, interleave = concat_2_interleave_0, values = (expand_dims_0, expand_dims_1, expand_dims_2, current_pos, expand_dims_4))[name = string("concat_2")];
            tensor<int32, [1]> concat_3_values2_0 = const()[name = string("concat_3_values2_0"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> concat_3_values4_0 = const()[name = string("concat_3_values4_0"), val = tensor<int32, [1]>([0])];
            int32 concat_3_axis_0 = const()[name = string("concat_3_axis_0"), val = int32(0)];
            bool concat_3_interleave_0 = const()[name = string("concat_3_interleave_0"), val = bool(false)];
            tensor<int32, [5]> concat_3 = concat(axis = concat_3_axis_0, interleave = concat_3_interleave_0, values = (expand_dims_5, expand_dims_6, concat_3_values2_0, var_38, concat_3_values4_0))[name = string("concat_3")];
            tensor<int32, [5]> kv_cache_internal_tensor_assign_1_stride_0 = const()[name = string("kv_cache_internal_tensor_assign_1_stride_0"), val = tensor<int32, [5]>([1, 1, 1, 1, 1])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_1_begin_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_1_begin_mask_0"), val = tensor<bool, [5]>([false, false, false, false, false])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_1_end_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_1_end_mask_0"), val = tensor<bool, [5]>([false, false, true, false, true])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_1_squeeze_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_1_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, false, false, false])];
            tensor<fp16, [2, 2, 2, 2048, 256]> kv_cache_internal_tensor_assign_1_cast_fp16 = slice_update(begin = concat_2, begin_mask = kv_cache_internal_tensor_assign_1_begin_mask_0, end = concat_3, end_mask = kv_cache_internal_tensor_assign_1_end_mask_0, squeeze_mask = kv_cache_internal_tensor_assign_1_squeeze_mask_0, stride = kv_cache_internal_tensor_assign_1_stride_0, update = var_35_cast_fp16, x = read_state_0)[name = string("kv_cache_internal_tensor_assign_1_cast_fp16")];
            write_state(data = kv_cache_internal_tensor_assign_1_cast_fp16, input = kv_cache)[name = string("coreml_update_state_8_write_state")];
            tensor<fp16, [2, 2, 2, 2048, 256]> coreml_update_state_4 = read_state(input = kv_cache)[name = string("coreml_update_state_8")];
            tensor<int32, [1]> var_66_axes_0 = const()[name = string("op_66_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [1, 2, 4, 256]> var_66_cast_fp16 = expand_dims(axes = var_66_axes_0, x = v19)[name = string("op_66_cast_fp16")];
            tensor<int32, [1]> var_68_axes_0 = const()[name = string("op_68_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [1, 1, 2, 4, 256]> var_68_cast_fp16 = expand_dims(axes = var_68_axes_0, x = var_66_cast_fp16)[name = string("op_68_cast_fp16")];
            tensor<int32, [1]> expand_dims_8 = const()[name = string("expand_dims_8"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_9 = const()[name = string("expand_dims_9"), val = tensor<int32, [1]>([1])];
            tensor<int32, [1]> expand_dims_10 = const()[name = string("expand_dims_10"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_12 = const()[name = string("expand_dims_12"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_13 = const()[name = string("expand_dims_13"), val = tensor<int32, [1]>([1])];
            tensor<int32, [1]> expand_dims_14 = const()[name = string("expand_dims_14"), val = tensor<int32, [1]>([2])];
            int32 concat_6_axis_0 = const()[name = string("concat_6_axis_0"), val = int32(0)];
            bool concat_6_interleave_0 = const()[name = string("concat_6_interleave_0"), val = bool(false)];
            tensor<int32, [5]> concat_6 = concat(axis = concat_6_axis_0, interleave = concat_6_interleave_0, values = (expand_dims_8, expand_dims_9, expand_dims_10, current_pos, expand_dims_12))[name = string("concat_6")];
            tensor<int32, [1]> concat_7_values2_0 = const()[name = string("concat_7_values2_0"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> concat_7_values4_0 = const()[name = string("concat_7_values4_0"), val = tensor<int32, [1]>([0])];
            int32 concat_7_axis_0 = const()[name = string("concat_7_axis_0"), val = int32(0)];
            bool concat_7_interleave_0 = const()[name = string("concat_7_interleave_0"), val = bool(false)];
            tensor<int32, [5]> concat_7 = concat(axis = concat_7_axis_0, interleave = concat_7_interleave_0, values = (expand_dims_13, expand_dims_14, concat_7_values2_0, var_38, concat_7_values4_0))[name = string("concat_7")];
            tensor<int32, [5]> kv_cache_internal_tensor_assign_2_stride_0 = const()[name = string("kv_cache_internal_tensor_assign_2_stride_0"), val = tensor<int32, [5]>([1, 1, 1, 1, 1])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_2_begin_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_2_begin_mask_0"), val = tensor<bool, [5]>([false, false, false, false, false])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_2_end_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_2_end_mask_0"), val = tensor<bool, [5]>([false, false, true, false, true])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_2_squeeze_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_2_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, false, false, false])];
            tensor<fp16, [2, 2, 2, 2048, 256]> kv_cache_internal_tensor_assign_2_cast_fp16 = slice_update(begin = concat_6, begin_mask = kv_cache_internal_tensor_assign_2_begin_mask_0, end = concat_7, end_mask = kv_cache_internal_tensor_assign_2_end_mask_0, squeeze_mask = kv_cache_internal_tensor_assign_2_squeeze_mask_0, stride = kv_cache_internal_tensor_assign_2_stride_0, update = var_68_cast_fp16, x = coreml_update_state_4)[name = string("kv_cache_internal_tensor_assign_2_cast_fp16")];
            write_state(data = kv_cache_internal_tensor_assign_2_cast_fp16, input = kv_cache)[name = string("coreml_update_state_9_write_state")];
            tensor<fp16, [2, 2, 2, 2048, 256]> coreml_update_state_5 = read_state(input = kv_cache)[name = string("coreml_update_state_9")];
            tensor<int32, [5]> var_100_begin_0 = const()[name = string("op_100_begin_0"), val = tensor<int32, [5]>([0, 0, 0, 0, 0])];
            tensor<int32, [5]> var_100_end_0 = const()[name = string("op_100_end_0"), val = tensor<int32, [5]>([1, 2, 2, 2048, 256])];
            tensor<bool, [5]> var_100_end_mask_0 = const()[name = string("op_100_end_mask_0"), val = tensor<bool, [5]>([false, true, true, true, true])];
            tensor<bool, [5]> var_100_squeeze_mask_0 = const()[name = string("op_100_squeeze_mask_0"), val = tensor<bool, [5]>([true, false, false, false, false])];
            tensor<fp16, [2, 2, 2048, 256]> var_100_cast_fp16 = slice_by_index(begin = var_100_begin_0, end = var_100_end_0, end_mask = var_100_end_mask_0, squeeze_mask = var_100_squeeze_mask_0, x = coreml_update_state_5)[name = string("op_100_cast_fp16")];
            tensor<int32, [4]> keys_1_begin_0 = const()[name = string("keys_1_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> keys_1_end_0 = const()[name = string("keys_1_end_0"), val = tensor<int32, [4]>([1, 2, 2048, 256])];
            tensor<bool, [4]> keys_1_end_mask_0 = const()[name = string("keys_1_end_mask_0"), val = tensor<bool, [4]>([false, true, true, true])];
            tensor<bool, [4]> keys_1_squeeze_mask_0 = const()[name = string("keys_1_squeeze_mask_0"), val = tensor<bool, [4]>([true, false, false, false])];
            tensor<fp16, [2, 2048, 256]> keys_1_cast_fp16 = slice_by_index(begin = keys_1_begin_0, end = keys_1_end_0, end_mask = keys_1_end_mask_0, squeeze_mask = keys_1_squeeze_mask_0, x = var_100_cast_fp16)[name = string("keys_1_cast_fp16")];
            tensor<int32, [4]> values_1_begin_0 = const()[name = string("values_1_begin_0"), val = tensor<int32, [4]>([1, 0, 0, 0])];
            tensor<int32, [4]> values_1_end_0 = const()[name = string("values_1_end_0"), val = tensor<int32, [4]>([2, 2, 2048, 256])];
            tensor<bool, [4]> values_1_end_mask_0 = const()[name = string("values_1_end_mask_0"), val = tensor<bool, [4]>([false, true, true, true])];
            tensor<bool, [4]> values_1_squeeze_mask_0 = const()[name = string("values_1_squeeze_mask_0"), val = tensor<bool, [4]>([true, false, false, false])];
            tensor<fp16, [2, 2048, 256]> values_1_cast_fp16 = slice_by_index(begin = values_1_begin_0, end = values_1_end_0, end_mask = values_1_end_mask_0, squeeze_mask = values_1_squeeze_mask_0, x = var_100_cast_fp16)[name = string("values_1_cast_fp16")];
            int32 var_117 = const()[name = string("op_117"), val = int32(1)];
            tensor<int32, [4]> var_121 = const()[name = string("op_121"), val = tensor<int32, [4]>([0, 2, 3, 1])];
            tensor<int32, [4]> var_124 = const()[name = string("op_124"), val = tensor<int32, [4]>([1, 2048, 1, 4])];
            tensor<fp16, [2, 4, 256, 4]> var_122_cast_fp16 = transpose(perm = var_121, x = q19)[name = string("transpose_34")];
            tensor<fp16, [1, 2048, 1, 4]> var_125_cast_fp16 = reshape(shape = var_124, x = var_122_cast_fp16)[name = string("op_125_cast_fp16")];
            tensor<int32, [3]> var_126 = const()[name = string("op_126"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [4]> var_129 = const()[name = string("op_129"), val = tensor<int32, [4]>([1, 512, 1, 2048])];
            tensor<fp16, [2, 256, 2048]> var_127_cast_fp16 = transpose(perm = var_126, x = keys_1_cast_fp16)[name = string("transpose_33")];
            tensor<fp16, [1, 512, 1, 2048]> key_native_1_cast_fp16 = reshape(shape = var_129, x = var_127_cast_fp16)[name = string("key_native_1_cast_fp16")];
            tensor<int32, [3]> var_131 = const()[name = string("op_131"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [4]> var_134 = const()[name = string("op_134"), val = tensor<int32, [4]>([1, 512, 1, 2048])];
            tensor<fp16, [2, 256, 2048]> var_132_cast_fp16 = transpose(perm = var_131, x = values_1_cast_fp16)[name = string("transpose_32")];
            tensor<fp16, [1, 512, 1, 2048]> var_135_cast_fp16 = reshape(shape = var_134, x = var_132_cast_fp16)[name = string("op_135_cast_fp16")];
            tensor<int32, [3]> var_136 = const()[name = string("op_136"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [1]> mask_native_1_axes_0 = const()[name = string("mask_native_1_axes_0"), val = tensor<int32, [1]>([2])];
            tensor<fp16, [1, 2048, 4]> var_137_cast_fp16 = transpose(perm = var_136, x = mask19)[name = string("transpose_31")];
            tensor<fp16, [1, 2048, 1, 4]> mask_native_1_cast_fp16 = expand_dims(axes = mask_native_1_axes_0, x = var_137_cast_fp16)[name = string("mask_native_1_cast_fp16")];
            tensor<int32, [8]> tile_0 = const()[name = string("tile_0"), val = tensor<int32, [8]>([256, 256, 256, 256, 256, 256, 256, 256])];
            int32 var_139_axis_0 = const()[name = string("op_139_axis_0"), val = int32(1)];
            tensor<fp16, [1, 256, 1, 4]> var_139_cast_fp16_0, tensor<fp16, [1, 256, 1, 4]> var_139_cast_fp16_1, tensor<fp16, [1, 256, 1, 4]> var_139_cast_fp16_2, tensor<fp16, [1, 256, 1, 4]> var_139_cast_fp16_3, tensor<fp16, [1, 256, 1, 4]> var_139_cast_fp16_4, tensor<fp16, [1, 256, 1, 4]> var_139_cast_fp16_5, tensor<fp16, [1, 256, 1, 4]> var_139_cast_fp16_6, tensor<fp16, [1, 256, 1, 4]> var_139_cast_fp16_7 = split(axis = var_139_axis_0, split_sizes = tile_0, x = var_125_cast_fp16)[name = string("op_139_cast_fp16")];
            tensor<int32, [4]> var_148_perm_0 = const()[name = string("op_148_perm_0"), val = tensor<int32, [4]>([0, 3, 2, 1])];
            tensor<int32, [2]> tile_1 = const()[name = string("tile_1"), val = tensor<int32, [2]>([256, 256])];
            int32 var_149_axis_0 = const()[name = string("op_149_axis_0"), val = int32(3)];
            tensor<fp16, [1, 2048, 1, 512]> var_148_cast_fp16 = transpose(perm = var_148_perm_0, x = key_native_1_cast_fp16)[name = string("transpose_30")];
            tensor<fp16, [1, 2048, 1, 256]> var_149_cast_fp16_0, tensor<fp16, [1, 2048, 1, 256]> var_149_cast_fp16_1 = split(axis = var_149_axis_0, split_sizes = tile_1, x = var_148_cast_fp16)[name = string("op_149_cast_fp16")];
            tensor<int32, [2]> tile_2 = const()[name = string("tile_2"), val = tensor<int32, [2]>([256, 256])];
            int32 var_152_axis_0 = const()[name = string("op_152_axis_0"), val = int32(1)];
            tensor<fp16, [1, 256, 1, 2048]> var_152_cast_fp16_0, tensor<fp16, [1, 256, 1, 2048]> var_152_cast_fp16_1 = split(axis = var_152_axis_0, split_sizes = tile_2, x = var_135_cast_fp16)[name = string("op_152_cast_fp16")];
            string scores_1_equation_0 = const()[name = string("scores_1_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 4]> scores_1_cast_fp16 = einsum(equation = scores_1_equation_0, values = (var_149_cast_fp16_0, var_139_cast_fp16_0))[name = string("scores_1_cast_fp16")];
            fp16 var_157_to_fp16 = const()[name = string("op_157_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 4]> var_158_cast_fp16 = mul(x = scores_1_cast_fp16, y = var_157_to_fp16)[name = string("op_158_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_159_cast_fp16 = add(x = var_158_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_159_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_160_cast_fp16 = softmax(axis = var_117, x = var_159_cast_fp16)[name = string("op_160_cast_fp16")];
            tensor<int32, [4]> tile_3 = const()[name = string("tile_3"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_161_axis_0 = const()[name = string("op_161_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 4]> var_161_cast_fp16_0, tensor<fp16, [1, 512, 1, 4]> var_161_cast_fp16_1, tensor<fp16, [1, 512, 1, 4]> var_161_cast_fp16_2, tensor<fp16, [1, 512, 1, 4]> var_161_cast_fp16_3 = split(axis = var_161_axis_0, split_sizes = tile_3, x = var_160_cast_fp16)[name = string("op_161_cast_fp16")];
            tensor<int32, [4]> tile_4 = const()[name = string("tile_4"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_166_axis_0 = const()[name = string("op_166_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_166_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_166_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_166_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_166_cast_fp16_3 = split(axis = var_166_axis_0, split_sizes = tile_4, x = var_152_cast_fp16_0)[name = string("op_166_cast_fp16")];
            fp16 var_171_to_fp16 = const()[name = string("op_171_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_172_cast_fp16 = mul(x = var_161_cast_fp16_0, y = var_171_to_fp16)[name = string("op_172_cast_fp16")];
            string var_174_equation_0 = const()[name = string("op_174_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_174_cast_fp16 = einsum(equation = var_174_equation_0, values = (var_166_cast_fp16_0, var_172_cast_fp16))[name = string("op_174_cast_fp16")];
            fp16 var_175_to_fp16 = const()[name = string("op_175_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_176_cast_fp16 = mul(x = var_161_cast_fp16_1, y = var_175_to_fp16)[name = string("op_176_cast_fp16")];
            string var_178_equation_0 = const()[name = string("op_178_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_178_cast_fp16 = einsum(equation = var_178_equation_0, values = (var_166_cast_fp16_1, var_176_cast_fp16))[name = string("op_178_cast_fp16")];
            fp16 var_179_to_fp16 = const()[name = string("op_179_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_180_cast_fp16 = mul(x = var_161_cast_fp16_2, y = var_179_to_fp16)[name = string("op_180_cast_fp16")];
            string var_182_equation_0 = const()[name = string("op_182_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_182_cast_fp16 = einsum(equation = var_182_equation_0, values = (var_166_cast_fp16_2, var_180_cast_fp16))[name = string("op_182_cast_fp16")];
            fp16 var_183_to_fp16 = const()[name = string("op_183_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_184_cast_fp16 = mul(x = var_161_cast_fp16_3, y = var_183_to_fp16)[name = string("op_184_cast_fp16")];
            string var_186_equation_0 = const()[name = string("op_186_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_186_cast_fp16 = einsum(equation = var_186_equation_0, values = (var_166_cast_fp16_3, var_184_cast_fp16))[name = string("op_186_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_187_cast_fp16 = add(x = var_174_cast_fp16, y = var_178_cast_fp16)[name = string("op_187_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_188_cast_fp16 = add(x = var_182_cast_fp16, y = var_186_cast_fp16)[name = string("op_188_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_189_cast_fp16 = add(x = var_187_cast_fp16, y = var_188_cast_fp16)[name = string("op_189_cast_fp16")];
            fp16 _inversed_191_y_0_to_fp16 = const()[name = string("_inversed_191_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 4]> _inversed_191_cast_fp16 = mul(x = var_189_cast_fp16, y = _inversed_191_y_0_to_fp16)[name = string("_inversed_191_cast_fp16")];
            string scores_3_equation_0 = const()[name = string("scores_3_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 4]> scores_3_cast_fp16 = einsum(equation = scores_3_equation_0, values = (var_149_cast_fp16_0, var_139_cast_fp16_1))[name = string("scores_3_cast_fp16")];
            fp16 var_194_to_fp16 = const()[name = string("op_194_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 4]> var_195_cast_fp16 = mul(x = scores_3_cast_fp16, y = var_194_to_fp16)[name = string("op_195_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_196_cast_fp16 = add(x = var_195_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_196_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_197_cast_fp16 = softmax(axis = var_117, x = var_196_cast_fp16)[name = string("op_197_cast_fp16")];
            tensor<int32, [4]> tile_5 = const()[name = string("tile_5"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_198_axis_0 = const()[name = string("op_198_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 4]> var_198_cast_fp16_0, tensor<fp16, [1, 512, 1, 4]> var_198_cast_fp16_1, tensor<fp16, [1, 512, 1, 4]> var_198_cast_fp16_2, tensor<fp16, [1, 512, 1, 4]> var_198_cast_fp16_3 = split(axis = var_198_axis_0, split_sizes = tile_5, x = var_197_cast_fp16)[name = string("op_198_cast_fp16")];
            tensor<int32, [4]> tile_6 = const()[name = string("tile_6"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_203_axis_0 = const()[name = string("op_203_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_203_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_203_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_203_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_203_cast_fp16_3 = split(axis = var_203_axis_0, split_sizes = tile_6, x = var_152_cast_fp16_0)[name = string("op_203_cast_fp16")];
            fp16 var_208_to_fp16 = const()[name = string("op_208_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_209_cast_fp16 = mul(x = var_198_cast_fp16_0, y = var_208_to_fp16)[name = string("op_209_cast_fp16")];
            string var_211_equation_0 = const()[name = string("op_211_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_211_cast_fp16 = einsum(equation = var_211_equation_0, values = (var_203_cast_fp16_0, var_209_cast_fp16))[name = string("op_211_cast_fp16")];
            fp16 var_212_to_fp16 = const()[name = string("op_212_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_213_cast_fp16 = mul(x = var_198_cast_fp16_1, y = var_212_to_fp16)[name = string("op_213_cast_fp16")];
            string var_215_equation_0 = const()[name = string("op_215_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_215_cast_fp16 = einsum(equation = var_215_equation_0, values = (var_203_cast_fp16_1, var_213_cast_fp16))[name = string("op_215_cast_fp16")];
            fp16 var_216_to_fp16 = const()[name = string("op_216_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_217_cast_fp16 = mul(x = var_198_cast_fp16_2, y = var_216_to_fp16)[name = string("op_217_cast_fp16")];
            string var_219_equation_0 = const()[name = string("op_219_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_219_cast_fp16 = einsum(equation = var_219_equation_0, values = (var_203_cast_fp16_2, var_217_cast_fp16))[name = string("op_219_cast_fp16")];
            fp16 var_220_to_fp16 = const()[name = string("op_220_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_221_cast_fp16 = mul(x = var_198_cast_fp16_3, y = var_220_to_fp16)[name = string("op_221_cast_fp16")];
            string var_223_equation_0 = const()[name = string("op_223_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_223_cast_fp16 = einsum(equation = var_223_equation_0, values = (var_203_cast_fp16_3, var_221_cast_fp16))[name = string("op_223_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_224_cast_fp16 = add(x = var_211_cast_fp16, y = var_215_cast_fp16)[name = string("op_224_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_225_cast_fp16 = add(x = var_219_cast_fp16, y = var_223_cast_fp16)[name = string("op_225_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_226_cast_fp16 = add(x = var_224_cast_fp16, y = var_225_cast_fp16)[name = string("op_226_cast_fp16")];
            fp16 _inversed_228_y_0_to_fp16 = const()[name = string("_inversed_228_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 4]> _inversed_228_cast_fp16 = mul(x = var_226_cast_fp16, y = _inversed_228_y_0_to_fp16)[name = string("_inversed_228_cast_fp16")];
            string scores_5_equation_0 = const()[name = string("scores_5_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 4]> scores_5_cast_fp16 = einsum(equation = scores_5_equation_0, values = (var_149_cast_fp16_0, var_139_cast_fp16_2))[name = string("scores_5_cast_fp16")];
            fp16 var_231_to_fp16 = const()[name = string("op_231_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 4]> var_232_cast_fp16 = mul(x = scores_5_cast_fp16, y = var_231_to_fp16)[name = string("op_232_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_233_cast_fp16 = add(x = var_232_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_233_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_234_cast_fp16 = softmax(axis = var_117, x = var_233_cast_fp16)[name = string("op_234_cast_fp16")];
            tensor<int32, [4]> tile_7 = const()[name = string("tile_7"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_235_axis_0 = const()[name = string("op_235_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 4]> var_235_cast_fp16_0, tensor<fp16, [1, 512, 1, 4]> var_235_cast_fp16_1, tensor<fp16, [1, 512, 1, 4]> var_235_cast_fp16_2, tensor<fp16, [1, 512, 1, 4]> var_235_cast_fp16_3 = split(axis = var_235_axis_0, split_sizes = tile_7, x = var_234_cast_fp16)[name = string("op_235_cast_fp16")];
            tensor<int32, [4]> tile_8 = const()[name = string("tile_8"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_240_axis_0 = const()[name = string("op_240_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_240_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_240_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_240_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_240_cast_fp16_3 = split(axis = var_240_axis_0, split_sizes = tile_8, x = var_152_cast_fp16_0)[name = string("op_240_cast_fp16")];
            fp16 var_245_to_fp16 = const()[name = string("op_245_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_246_cast_fp16 = mul(x = var_235_cast_fp16_0, y = var_245_to_fp16)[name = string("op_246_cast_fp16")];
            string var_248_equation_0 = const()[name = string("op_248_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_248_cast_fp16 = einsum(equation = var_248_equation_0, values = (var_240_cast_fp16_0, var_246_cast_fp16))[name = string("op_248_cast_fp16")];
            fp16 var_249_to_fp16 = const()[name = string("op_249_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_250_cast_fp16 = mul(x = var_235_cast_fp16_1, y = var_249_to_fp16)[name = string("op_250_cast_fp16")];
            string var_252_equation_0 = const()[name = string("op_252_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_252_cast_fp16 = einsum(equation = var_252_equation_0, values = (var_240_cast_fp16_1, var_250_cast_fp16))[name = string("op_252_cast_fp16")];
            fp16 var_253_to_fp16 = const()[name = string("op_253_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_254_cast_fp16 = mul(x = var_235_cast_fp16_2, y = var_253_to_fp16)[name = string("op_254_cast_fp16")];
            string var_256_equation_0 = const()[name = string("op_256_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_256_cast_fp16 = einsum(equation = var_256_equation_0, values = (var_240_cast_fp16_2, var_254_cast_fp16))[name = string("op_256_cast_fp16")];
            fp16 var_257_to_fp16 = const()[name = string("op_257_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_258_cast_fp16 = mul(x = var_235_cast_fp16_3, y = var_257_to_fp16)[name = string("op_258_cast_fp16")];
            string var_260_equation_0 = const()[name = string("op_260_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_260_cast_fp16 = einsum(equation = var_260_equation_0, values = (var_240_cast_fp16_3, var_258_cast_fp16))[name = string("op_260_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_261_cast_fp16 = add(x = var_248_cast_fp16, y = var_252_cast_fp16)[name = string("op_261_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_262_cast_fp16 = add(x = var_256_cast_fp16, y = var_260_cast_fp16)[name = string("op_262_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_263_cast_fp16 = add(x = var_261_cast_fp16, y = var_262_cast_fp16)[name = string("op_263_cast_fp16")];
            fp16 _inversed_265_y_0_to_fp16 = const()[name = string("_inversed_265_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 4]> _inversed_265_cast_fp16 = mul(x = var_263_cast_fp16, y = _inversed_265_y_0_to_fp16)[name = string("_inversed_265_cast_fp16")];
            string scores_7_equation_0 = const()[name = string("scores_7_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 4]> scores_7_cast_fp16 = einsum(equation = scores_7_equation_0, values = (var_149_cast_fp16_0, var_139_cast_fp16_3))[name = string("scores_7_cast_fp16")];
            fp16 var_268_to_fp16 = const()[name = string("op_268_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 4]> var_269_cast_fp16 = mul(x = scores_7_cast_fp16, y = var_268_to_fp16)[name = string("op_269_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_270_cast_fp16 = add(x = var_269_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_270_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_271_cast_fp16 = softmax(axis = var_117, x = var_270_cast_fp16)[name = string("op_271_cast_fp16")];
            tensor<int32, [4]> tile_9 = const()[name = string("tile_9"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_272_axis_0 = const()[name = string("op_272_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 4]> var_272_cast_fp16_0, tensor<fp16, [1, 512, 1, 4]> var_272_cast_fp16_1, tensor<fp16, [1, 512, 1, 4]> var_272_cast_fp16_2, tensor<fp16, [1, 512, 1, 4]> var_272_cast_fp16_3 = split(axis = var_272_axis_0, split_sizes = tile_9, x = var_271_cast_fp16)[name = string("op_272_cast_fp16")];
            tensor<int32, [4]> tile_10 = const()[name = string("tile_10"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_277_axis_0 = const()[name = string("op_277_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_277_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_277_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_277_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_277_cast_fp16_3 = split(axis = var_277_axis_0, split_sizes = tile_10, x = var_152_cast_fp16_0)[name = string("op_277_cast_fp16")];
            fp16 var_282_to_fp16 = const()[name = string("op_282_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_283_cast_fp16 = mul(x = var_272_cast_fp16_0, y = var_282_to_fp16)[name = string("op_283_cast_fp16")];
            string var_285_equation_0 = const()[name = string("op_285_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_285_cast_fp16 = einsum(equation = var_285_equation_0, values = (var_277_cast_fp16_0, var_283_cast_fp16))[name = string("op_285_cast_fp16")];
            fp16 var_286_to_fp16 = const()[name = string("op_286_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_287_cast_fp16 = mul(x = var_272_cast_fp16_1, y = var_286_to_fp16)[name = string("op_287_cast_fp16")];
            string var_289_equation_0 = const()[name = string("op_289_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_289_cast_fp16 = einsum(equation = var_289_equation_0, values = (var_277_cast_fp16_1, var_287_cast_fp16))[name = string("op_289_cast_fp16")];
            fp16 var_290_to_fp16 = const()[name = string("op_290_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_291_cast_fp16 = mul(x = var_272_cast_fp16_2, y = var_290_to_fp16)[name = string("op_291_cast_fp16")];
            string var_293_equation_0 = const()[name = string("op_293_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_293_cast_fp16 = einsum(equation = var_293_equation_0, values = (var_277_cast_fp16_2, var_291_cast_fp16))[name = string("op_293_cast_fp16")];
            fp16 var_294_to_fp16 = const()[name = string("op_294_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_295_cast_fp16 = mul(x = var_272_cast_fp16_3, y = var_294_to_fp16)[name = string("op_295_cast_fp16")];
            string var_297_equation_0 = const()[name = string("op_297_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_297_cast_fp16 = einsum(equation = var_297_equation_0, values = (var_277_cast_fp16_3, var_295_cast_fp16))[name = string("op_297_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_298_cast_fp16 = add(x = var_285_cast_fp16, y = var_289_cast_fp16)[name = string("op_298_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_299_cast_fp16 = add(x = var_293_cast_fp16, y = var_297_cast_fp16)[name = string("op_299_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_300_cast_fp16 = add(x = var_298_cast_fp16, y = var_299_cast_fp16)[name = string("op_300_cast_fp16")];
            fp16 _inversed_302_y_0_to_fp16 = const()[name = string("_inversed_302_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 4]> _inversed_302_cast_fp16 = mul(x = var_300_cast_fp16, y = _inversed_302_y_0_to_fp16)[name = string("_inversed_302_cast_fp16")];
            string scores_9_equation_0 = const()[name = string("scores_9_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 4]> scores_9_cast_fp16 = einsum(equation = scores_9_equation_0, values = (var_149_cast_fp16_1, var_139_cast_fp16_4))[name = string("scores_9_cast_fp16")];
            fp16 var_305_to_fp16 = const()[name = string("op_305_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 4]> var_306_cast_fp16 = mul(x = scores_9_cast_fp16, y = var_305_to_fp16)[name = string("op_306_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_307_cast_fp16 = add(x = var_306_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_307_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_308_cast_fp16 = softmax(axis = var_117, x = var_307_cast_fp16)[name = string("op_308_cast_fp16")];
            tensor<int32, [4]> tile_11 = const()[name = string("tile_11"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_309_axis_0 = const()[name = string("op_309_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 4]> var_309_cast_fp16_0, tensor<fp16, [1, 512, 1, 4]> var_309_cast_fp16_1, tensor<fp16, [1, 512, 1, 4]> var_309_cast_fp16_2, tensor<fp16, [1, 512, 1, 4]> var_309_cast_fp16_3 = split(axis = var_309_axis_0, split_sizes = tile_11, x = var_308_cast_fp16)[name = string("op_309_cast_fp16")];
            tensor<int32, [4]> tile_12 = const()[name = string("tile_12"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_314_axis_0 = const()[name = string("op_314_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_314_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_314_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_314_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_314_cast_fp16_3 = split(axis = var_314_axis_0, split_sizes = tile_12, x = var_152_cast_fp16_1)[name = string("op_314_cast_fp16")];
            fp16 var_319_to_fp16 = const()[name = string("op_319_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_320_cast_fp16 = mul(x = var_309_cast_fp16_0, y = var_319_to_fp16)[name = string("op_320_cast_fp16")];
            string var_322_equation_0 = const()[name = string("op_322_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_322_cast_fp16 = einsum(equation = var_322_equation_0, values = (var_314_cast_fp16_0, var_320_cast_fp16))[name = string("op_322_cast_fp16")];
            fp16 var_323_to_fp16 = const()[name = string("op_323_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_324_cast_fp16 = mul(x = var_309_cast_fp16_1, y = var_323_to_fp16)[name = string("op_324_cast_fp16")];
            string var_326_equation_0 = const()[name = string("op_326_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_326_cast_fp16 = einsum(equation = var_326_equation_0, values = (var_314_cast_fp16_1, var_324_cast_fp16))[name = string("op_326_cast_fp16")];
            fp16 var_327_to_fp16 = const()[name = string("op_327_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_328_cast_fp16 = mul(x = var_309_cast_fp16_2, y = var_327_to_fp16)[name = string("op_328_cast_fp16")];
            string var_330_equation_0 = const()[name = string("op_330_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_330_cast_fp16 = einsum(equation = var_330_equation_0, values = (var_314_cast_fp16_2, var_328_cast_fp16))[name = string("op_330_cast_fp16")];
            fp16 var_331_to_fp16 = const()[name = string("op_331_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_332_cast_fp16 = mul(x = var_309_cast_fp16_3, y = var_331_to_fp16)[name = string("op_332_cast_fp16")];
            string var_334_equation_0 = const()[name = string("op_334_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_334_cast_fp16 = einsum(equation = var_334_equation_0, values = (var_314_cast_fp16_3, var_332_cast_fp16))[name = string("op_334_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_335_cast_fp16 = add(x = var_322_cast_fp16, y = var_326_cast_fp16)[name = string("op_335_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_336_cast_fp16 = add(x = var_330_cast_fp16, y = var_334_cast_fp16)[name = string("op_336_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_337_cast_fp16 = add(x = var_335_cast_fp16, y = var_336_cast_fp16)[name = string("op_337_cast_fp16")];
            fp16 _inversed_339_y_0_to_fp16 = const()[name = string("_inversed_339_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 4]> _inversed_339_cast_fp16 = mul(x = var_337_cast_fp16, y = _inversed_339_y_0_to_fp16)[name = string("_inversed_339_cast_fp16")];
            string scores_11_equation_0 = const()[name = string("scores_11_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 4]> scores_11_cast_fp16 = einsum(equation = scores_11_equation_0, values = (var_149_cast_fp16_1, var_139_cast_fp16_5))[name = string("scores_11_cast_fp16")];
            fp16 var_342_to_fp16 = const()[name = string("op_342_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 4]> var_343_cast_fp16 = mul(x = scores_11_cast_fp16, y = var_342_to_fp16)[name = string("op_343_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_344_cast_fp16 = add(x = var_343_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_344_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_345_cast_fp16 = softmax(axis = var_117, x = var_344_cast_fp16)[name = string("op_345_cast_fp16")];
            tensor<int32, [4]> tile_13 = const()[name = string("tile_13"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_346_axis_0 = const()[name = string("op_346_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 4]> var_346_cast_fp16_0, tensor<fp16, [1, 512, 1, 4]> var_346_cast_fp16_1, tensor<fp16, [1, 512, 1, 4]> var_346_cast_fp16_2, tensor<fp16, [1, 512, 1, 4]> var_346_cast_fp16_3 = split(axis = var_346_axis_0, split_sizes = tile_13, x = var_345_cast_fp16)[name = string("op_346_cast_fp16")];
            tensor<int32, [4]> tile_14 = const()[name = string("tile_14"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_351_axis_0 = const()[name = string("op_351_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_351_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_351_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_351_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_351_cast_fp16_3 = split(axis = var_351_axis_0, split_sizes = tile_14, x = var_152_cast_fp16_1)[name = string("op_351_cast_fp16")];
            fp16 var_356_to_fp16 = const()[name = string("op_356_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_357_cast_fp16 = mul(x = var_346_cast_fp16_0, y = var_356_to_fp16)[name = string("op_357_cast_fp16")];
            string var_359_equation_0 = const()[name = string("op_359_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_359_cast_fp16 = einsum(equation = var_359_equation_0, values = (var_351_cast_fp16_0, var_357_cast_fp16))[name = string("op_359_cast_fp16")];
            fp16 var_360_to_fp16 = const()[name = string("op_360_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_361_cast_fp16 = mul(x = var_346_cast_fp16_1, y = var_360_to_fp16)[name = string("op_361_cast_fp16")];
            string var_363_equation_0 = const()[name = string("op_363_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_363_cast_fp16 = einsum(equation = var_363_equation_0, values = (var_351_cast_fp16_1, var_361_cast_fp16))[name = string("op_363_cast_fp16")];
            fp16 var_364_to_fp16 = const()[name = string("op_364_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_365_cast_fp16 = mul(x = var_346_cast_fp16_2, y = var_364_to_fp16)[name = string("op_365_cast_fp16")];
            string var_367_equation_0 = const()[name = string("op_367_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_367_cast_fp16 = einsum(equation = var_367_equation_0, values = (var_351_cast_fp16_2, var_365_cast_fp16))[name = string("op_367_cast_fp16")];
            fp16 var_368_to_fp16 = const()[name = string("op_368_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_369_cast_fp16 = mul(x = var_346_cast_fp16_3, y = var_368_to_fp16)[name = string("op_369_cast_fp16")];
            string var_371_equation_0 = const()[name = string("op_371_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_371_cast_fp16 = einsum(equation = var_371_equation_0, values = (var_351_cast_fp16_3, var_369_cast_fp16))[name = string("op_371_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_372_cast_fp16 = add(x = var_359_cast_fp16, y = var_363_cast_fp16)[name = string("op_372_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_373_cast_fp16 = add(x = var_367_cast_fp16, y = var_371_cast_fp16)[name = string("op_373_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_374_cast_fp16 = add(x = var_372_cast_fp16, y = var_373_cast_fp16)[name = string("op_374_cast_fp16")];
            fp16 _inversed_376_y_0_to_fp16 = const()[name = string("_inversed_376_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 4]> _inversed_376_cast_fp16 = mul(x = var_374_cast_fp16, y = _inversed_376_y_0_to_fp16)[name = string("_inversed_376_cast_fp16")];
            string scores_13_equation_0 = const()[name = string("scores_13_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 4]> scores_13_cast_fp16 = einsum(equation = scores_13_equation_0, values = (var_149_cast_fp16_1, var_139_cast_fp16_6))[name = string("scores_13_cast_fp16")];
            fp16 var_379_to_fp16 = const()[name = string("op_379_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 4]> var_380_cast_fp16 = mul(x = scores_13_cast_fp16, y = var_379_to_fp16)[name = string("op_380_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_381_cast_fp16 = add(x = var_380_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_381_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_382_cast_fp16 = softmax(axis = var_117, x = var_381_cast_fp16)[name = string("op_382_cast_fp16")];
            tensor<int32, [4]> tile_15 = const()[name = string("tile_15"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_383_axis_0 = const()[name = string("op_383_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 4]> var_383_cast_fp16_0, tensor<fp16, [1, 512, 1, 4]> var_383_cast_fp16_1, tensor<fp16, [1, 512, 1, 4]> var_383_cast_fp16_2, tensor<fp16, [1, 512, 1, 4]> var_383_cast_fp16_3 = split(axis = var_383_axis_0, split_sizes = tile_15, x = var_382_cast_fp16)[name = string("op_383_cast_fp16")];
            tensor<int32, [4]> tile_16 = const()[name = string("tile_16"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_388_axis_0 = const()[name = string("op_388_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_388_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_388_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_388_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_388_cast_fp16_3 = split(axis = var_388_axis_0, split_sizes = tile_16, x = var_152_cast_fp16_1)[name = string("op_388_cast_fp16")];
            fp16 var_393_to_fp16 = const()[name = string("op_393_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_394_cast_fp16 = mul(x = var_383_cast_fp16_0, y = var_393_to_fp16)[name = string("op_394_cast_fp16")];
            string var_396_equation_0 = const()[name = string("op_396_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_396_cast_fp16 = einsum(equation = var_396_equation_0, values = (var_388_cast_fp16_0, var_394_cast_fp16))[name = string("op_396_cast_fp16")];
            fp16 var_397_to_fp16 = const()[name = string("op_397_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_398_cast_fp16 = mul(x = var_383_cast_fp16_1, y = var_397_to_fp16)[name = string("op_398_cast_fp16")];
            string var_400_equation_0 = const()[name = string("op_400_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_400_cast_fp16 = einsum(equation = var_400_equation_0, values = (var_388_cast_fp16_1, var_398_cast_fp16))[name = string("op_400_cast_fp16")];
            fp16 var_401_to_fp16 = const()[name = string("op_401_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_402_cast_fp16 = mul(x = var_383_cast_fp16_2, y = var_401_to_fp16)[name = string("op_402_cast_fp16")];
            string var_404_equation_0 = const()[name = string("op_404_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_404_cast_fp16 = einsum(equation = var_404_equation_0, values = (var_388_cast_fp16_2, var_402_cast_fp16))[name = string("op_404_cast_fp16")];
            fp16 var_405_to_fp16 = const()[name = string("op_405_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_406_cast_fp16 = mul(x = var_383_cast_fp16_3, y = var_405_to_fp16)[name = string("op_406_cast_fp16")];
            string var_408_equation_0 = const()[name = string("op_408_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_408_cast_fp16 = einsum(equation = var_408_equation_0, values = (var_388_cast_fp16_3, var_406_cast_fp16))[name = string("op_408_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_409_cast_fp16 = add(x = var_396_cast_fp16, y = var_400_cast_fp16)[name = string("op_409_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_410_cast_fp16 = add(x = var_404_cast_fp16, y = var_408_cast_fp16)[name = string("op_410_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_411_cast_fp16 = add(x = var_409_cast_fp16, y = var_410_cast_fp16)[name = string("op_411_cast_fp16")];
            fp16 _inversed_413_y_0_to_fp16 = const()[name = string("_inversed_413_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 4]> _inversed_413_cast_fp16 = mul(x = var_411_cast_fp16, y = _inversed_413_y_0_to_fp16)[name = string("_inversed_413_cast_fp16")];
            string scores_15_equation_0 = const()[name = string("scores_15_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 4]> scores_15_cast_fp16 = einsum(equation = scores_15_equation_0, values = (var_149_cast_fp16_1, var_139_cast_fp16_7))[name = string("scores_15_cast_fp16")];
            fp16 var_416_to_fp16 = const()[name = string("op_416_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 4]> var_417_cast_fp16 = mul(x = scores_15_cast_fp16, y = var_416_to_fp16)[name = string("op_417_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_418_cast_fp16 = add(x = var_417_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_418_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_419_cast_fp16 = softmax(axis = var_117, x = var_418_cast_fp16)[name = string("op_419_cast_fp16")];
            tensor<int32, [4]> tile_17 = const()[name = string("tile_17"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_420_axis_0 = const()[name = string("op_420_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 4]> var_420_cast_fp16_0, tensor<fp16, [1, 512, 1, 4]> var_420_cast_fp16_1, tensor<fp16, [1, 512, 1, 4]> var_420_cast_fp16_2, tensor<fp16, [1, 512, 1, 4]> var_420_cast_fp16_3 = split(axis = var_420_axis_0, split_sizes = tile_17, x = var_419_cast_fp16)[name = string("op_420_cast_fp16")];
            tensor<int32, [4]> tile_18 = const()[name = string("tile_18"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_425_axis_0 = const()[name = string("op_425_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_425_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_425_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_425_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_425_cast_fp16_3 = split(axis = var_425_axis_0, split_sizes = tile_18, x = var_152_cast_fp16_1)[name = string("op_425_cast_fp16")];
            fp16 var_430_to_fp16 = const()[name = string("op_430_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_431_cast_fp16 = mul(x = var_420_cast_fp16_0, y = var_430_to_fp16)[name = string("op_431_cast_fp16")];
            string var_433_equation_0 = const()[name = string("op_433_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_433_cast_fp16 = einsum(equation = var_433_equation_0, values = (var_425_cast_fp16_0, var_431_cast_fp16))[name = string("op_433_cast_fp16")];
            fp16 var_434_to_fp16 = const()[name = string("op_434_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_435_cast_fp16 = mul(x = var_420_cast_fp16_1, y = var_434_to_fp16)[name = string("op_435_cast_fp16")];
            string var_437_equation_0 = const()[name = string("op_437_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_437_cast_fp16 = einsum(equation = var_437_equation_0, values = (var_425_cast_fp16_1, var_435_cast_fp16))[name = string("op_437_cast_fp16")];
            fp16 var_438_to_fp16 = const()[name = string("op_438_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_439_cast_fp16 = mul(x = var_420_cast_fp16_2, y = var_438_to_fp16)[name = string("op_439_cast_fp16")];
            string var_441_equation_0 = const()[name = string("op_441_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_441_cast_fp16 = einsum(equation = var_441_equation_0, values = (var_425_cast_fp16_2, var_439_cast_fp16))[name = string("op_441_cast_fp16")];
            fp16 var_442_to_fp16 = const()[name = string("op_442_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_443_cast_fp16 = mul(x = var_420_cast_fp16_3, y = var_442_to_fp16)[name = string("op_443_cast_fp16")];
            string var_445_equation_0 = const()[name = string("op_445_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_445_cast_fp16 = einsum(equation = var_445_equation_0, values = (var_425_cast_fp16_3, var_443_cast_fp16))[name = string("op_445_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_446_cast_fp16 = add(x = var_433_cast_fp16, y = var_437_cast_fp16)[name = string("op_446_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_447_cast_fp16 = add(x = var_441_cast_fp16, y = var_445_cast_fp16)[name = string("op_447_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_448_cast_fp16 = add(x = var_446_cast_fp16, y = var_447_cast_fp16)[name = string("op_448_cast_fp16")];
            fp16 _inversed_450_y_0_to_fp16 = const()[name = string("_inversed_450_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 4]> _inversed_450_cast_fp16 = mul(x = var_448_cast_fp16, y = _inversed_450_y_0_to_fp16)[name = string("_inversed_450_cast_fp16")];
            bool var_452_interleave_0 = const()[name = string("op_452_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 2048, 1, 4]> var_452_cast_fp16 = concat(axis = var_117, interleave = var_452_interleave_0, values = (_inversed_191_cast_fp16, _inversed_228_cast_fp16, _inversed_265_cast_fp16, _inversed_302_cast_fp16, _inversed_339_cast_fp16, _inversed_376_cast_fp16, _inversed_413_cast_fp16, _inversed_450_cast_fp16))[name = string("op_452_cast_fp16")];
            tensor<int32, [4]> var_453 = const()[name = string("op_453"), val = tensor<int32, [4]>([2, 4, 256, 4])];
            tensor<fp16, [2, 4, 256, 4]> var_454_cast_fp16 = reshape(shape = var_453, x = var_452_cast_fp16)[name = string("op_454_cast_fp16")];
            tensor<int32, [4]> transpose_0_perm_0 = const()[name = string("transpose_0_perm_0"), val = tensor<int32, [4]>([3, 0, 1, 2])];
            tensor<int32, [2]> var_473 = const()[name = string("op_473"), val = tensor<int32, [2]>([4, 2048])];
            tensor<fp16, [4, 2, 4, 256]> transpose_0_cast_fp16 = transpose(perm = transpose_0_perm_0, x = var_454_cast_fp16)[name = string("transpose_29")];
            tensor<fp16, [4, 2048]> attention_output_3_cast_fp16 = reshape(shape = var_473, x = transpose_0_cast_fp16)[name = string("attention_output_3_cast_fp16")];
            tensor<fp16, [4, 2048]> var_475_cast_fp16 = sigmoid(x = gate19)[name = string("op_475_cast_fp16")];
            tensor<fp16, [4, 2048]> input_1_cast_fp16 = mul(x = attention_output_3_cast_fp16, y = var_475_cast_fp16)[name = string("input_1_cast_fp16")];
            tensor<fp16, [1024, 2048]> completion19_o_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 2048]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(193193216))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(194766144))))[name = string("completion19_o_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1024]> linear_0_bias_0_to_fp16 = const()[name = string("linear_0_bias_0_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(194774400)))];
            tensor<fp16, [4, 1024]> linear_0_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = completion19_o_proj_weight_promoted_to_fp16_palettized, x = input_1_cast_fp16)[name = string("linear_0_cast_fp16")];
            tensor<fp16, [4, 1024]> value_1_cast_fp16 = add(x = hidden, y = linear_0_cast_fp16)[name = string("value_1_cast_fp16")];
            tensor<fp16, [4, 1024]> var_480_cast_fp16 = mul(x = value_1_cast_fp16, y = value_1_cast_fp16)[name = string("op_480_cast_fp16")];
            tensor<int32, [1]> variance_1_axes_0 = const()[name = string("variance_1_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_1_keep_dims_0 = const()[name = string("variance_1_keep_dims_0"), val = bool(true)];
            tensor<fp16, [4, 1]> variance_1_cast_fp16 = reduce_mean(axes = variance_1_axes_0, keep_dims = variance_1_keep_dims_0, x = var_480_cast_fp16)[name = string("variance_1_cast_fp16")];
            fp16 var_483_to_fp16 = const()[name = string("op_483_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [4, 1]> var_484_cast_fp16 = add(x = variance_1_cast_fp16, y = var_483_to_fp16)[name = string("op_484_cast_fp16")];
            fp32 var_485_epsilon_0 = const()[name = string("op_485_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [4, 1]> var_485_cast_fp16 = rsqrt(epsilon = var_485_epsilon_0, x = var_484_cast_fp16)[name = string("op_485_cast_fp16")];
            tensor<fp16, [4, 1024]> var_486_cast_fp16 = mul(x = value_1_cast_fp16, y = var_485_cast_fp16)[name = string("op_486_cast_fp16")];
            tensor<fp16, [1024]> var_488_to_fp16 = const()[name = string("op_488_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(194776512)))];
            tensor<fp16, [4, 1024]> input_3_cast_fp16 = mul(x = var_486_cast_fp16, y = var_488_to_fp16)[name = string("input_3_cast_fp16")];
            tensor<fp16, [3584, 1024]> completion19_gate_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(194778624))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(197531200))))[name = string("completion19_gate_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [3584]> linear_1_bias_0_to_fp16 = const()[name = string("linear_1_bias_0_to_fp16"), val = tensor<fp16, [3584]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(197559936)))];
            tensor<fp16, [4, 3584]> linear_1_cast_fp16 = linear(bias = linear_1_bias_0_to_fp16, weight = completion19_gate_proj_weight_promoted_to_fp16_palettized, x = input_3_cast_fp16)[name = string("linear_1_cast_fp16")];
            tensor<fp16, [4, 3584]> var_492_cast_fp16 = silu(x = linear_1_cast_fp16)[name = string("op_492_cast_fp16")];
            tensor<fp16, [3584, 1024]> completion19_up_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(197567168))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(200319744))))[name = string("completion19_up_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 3584]> linear_2_cast_fp16 = linear(bias = linear_1_bias_0_to_fp16, weight = completion19_up_proj_weight_promoted_to_fp16_palettized, x = input_3_cast_fp16)[name = string("linear_2_cast_fp16")];
            tensor<fp16, [4, 3584]> input_7_cast_fp16 = mul(x = var_492_cast_fp16, y = linear_2_cast_fp16)[name = string("input_7_cast_fp16")];
            tensor<fp16, [1024, 3584]> completion19_down_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 3584]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(200348480))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(203101056))))[name = string("completion19_down_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 1024]> linear_3_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = completion19_down_proj_weight_promoted_to_fp16_palettized, x = input_7_cast_fp16)[name = string("linear_3_cast_fp16")];
            tensor<fp16, [4, 1024]> value_3_cast_fp16 = add(x = value_1_cast_fp16, y = linear_3_cast_fp16)[name = string("value_3_cast_fp16")];
            int32 var_515 = const()[name = string("op_515"), val = int32(-1)];
            tensor<fp16, [4, 1024]> var_530_cast_fp16 = mul(x = value_3_cast_fp16, y = value_3_cast_fp16)[name = string("op_530_cast_fp16")];
            tensor<int32, [1]> variance_3_axes_0 = const()[name = string("variance_3_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_3_keep_dims_0 = const()[name = string("variance_3_keep_dims_0"), val = bool(true)];
            tensor<fp16, [4, 1]> variance_3_cast_fp16 = reduce_mean(axes = variance_3_axes_0, keep_dims = variance_3_keep_dims_0, x = var_530_cast_fp16)[name = string("variance_3_cast_fp16")];
            fp16 var_533_to_fp16 = const()[name = string("op_533_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [4, 1]> var_534_cast_fp16 = add(x = variance_3_cast_fp16, y = var_533_to_fp16)[name = string("op_534_cast_fp16")];
            fp32 var_535_epsilon_0 = const()[name = string("op_535_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [4, 1]> var_535_cast_fp16 = rsqrt(epsilon = var_535_epsilon_0, x = var_534_cast_fp16)[name = string("op_535_cast_fp16")];
            tensor<fp16, [4, 1024]> var_536_cast_fp16 = mul(x = value_3_cast_fp16, y = var_535_cast_fp16)[name = string("op_536_cast_fp16")];
            tensor<fp16, [1024]> var_538_to_fp16 = const()[name = string("op_538_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(203109312)))];
            tensor<fp16, [4, 1024]> input_9_cast_fp16 = mul(x = var_536_cast_fp16, y = var_538_to_fp16)[name = string("input_9_cast_fp16")];
            tensor<fp16, [6144, 1024]> layers20_0_in_proj_qkv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(203111424))), lut = tensor<fp16, [384, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(207830080))))[name = string("layers20_0_in_proj_qkv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [6144]> linear_4_bias_0_to_fp16 = const()[name = string("linear_4_bias_0_to_fp16"), val = tensor<fp16, [6144]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(207879296)))];
            tensor<fp16, [4, 6144]> linear_4_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = layers20_0_in_proj_qkv_weight_promoted_to_fp16_palettized, x = input_9_cast_fp16)[name = string("linear_4_cast_fp16")];
            tensor<int32, [2]> var_542_perm_0 = const()[name = string("op_542_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> mixed_1_axes_0 = const()[name = string("mixed_1_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [6144, 4]> var_542_cast_fp16 = transpose(perm = var_542_perm_0, x = linear_4_cast_fp16)[name = string("transpose_28")];
            tensor<fp16, [1, 6144, 4]> mixed_1_cast_fp16 = expand_dims(axes = mixed_1_axes_0, x = var_542_cast_fp16)[name = string("mixed_1_cast_fp16")];
            bool convolution_input_1_interleave_0 = const()[name = string("convolution_input_1_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 6144, 7]> convolution_input_1_cast_fp16 = concat(axis = var_515, interleave = convolution_input_1_interleave_0, values = (conv20, mixed_1_cast_fp16))[name = string("convolution_input_1_cast_fp16")];
            string input_11_pad_type_0 = const()[name = string("input_11_pad_type_0"), val = string("valid")];
            int32 input_11_groups_0 = const()[name = string("input_11_groups_0"), val = int32(6144)];
            tensor<int32, [1]> input_11_strides_0 = const()[name = string("input_11_strides_0"), val = tensor<int32, [1]>([1])];
            tensor<int32, [2]> input_11_pad_0 = const()[name = string("input_11_pad_0"), val = tensor<int32, [2]>([0, 0])];
            tensor<int32, [1]> input_11_dilations_0 = const()[name = string("input_11_dilations_0"), val = tensor<int32, [1]>([1])];
            tensor<fp16, [6144, 1, 4]> layers20_0_conv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1, 4]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(207891648))), lut = tensor<fp16, [384, 1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(207910144))))[name = string("layers20_0_conv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 6144, 4]> input_11_cast_fp16 = conv(dilations = input_11_dilations_0, groups = input_11_groups_0, pad = input_11_pad_0, pad_type = input_11_pad_type_0, strides = input_11_strides_0, weight = layers20_0_conv_weight_promoted_to_fp16_palettized, x = convolution_input_1_cast_fp16)[name = string("input_11_cast_fp16")];
            tensor<fp16, [1, 6144, 4]> var_551_cast_fp16 = silu(x = input_11_cast_fp16)[name = string("op_551_cast_fp16")];
            tensor<int32, [3]> var_552_perm_0 = const()[name = string("op_552_perm_0"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [3]> var_555_begin_0 = const()[name = string("op_555_begin_0"), val = tensor<int32, [3]>([0, 0, 4])];
            tensor<int32, [3]> var_555_end_0 = const()[name = string("op_555_end_0"), val = tensor<int32, [3]>([1, 6144, 7])];
            tensor<bool, [3]> var_555_end_mask_0 = const()[name = string("op_555_end_mask_0"), val = tensor<bool, [3]>([true, true, true])];
            tensor<fp16, [1, 6144, 3]> conv20_out = slice_by_index(begin = var_555_begin_0, end = var_555_end_0, end_mask = var_555_end_mask_0, x = convolution_input_1_cast_fp16)[name = string("op_555_cast_fp16")];
            tensor<int32, [3]> var_556 = const()[name = string("op_556"), val = tensor<int32, [3]>([2048, 2048, 2048])];
            int32 var_557_axis_0 = const()[name = string("op_557_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 4, 6144]> var_552_cast_fp16 = transpose(perm = var_552_perm_0, x = var_551_cast_fp16)[name = string("transpose_27")];
            tensor<fp16, [1, 4, 2048]> var_557_cast_fp16_0, tensor<fp16, [1, 4, 2048]> var_557_cast_fp16_1, tensor<fp16, [1, 4, 2048]> var_557_cast_fp16_2 = split(axis = var_557_axis_0, split_sizes = var_556, x = var_552_cast_fp16)[name = string("op_557_cast_fp16")];
            tensor<int32, [4]> var_561 = const()[name = string("op_561"), val = tensor<int32, [4]>([1, 4, 16, 128])];
            tensor<fp16, [1, 4, 16, 128]> value_7_cast_fp16 = reshape(shape = var_561, x = var_557_cast_fp16_0)[name = string("value_7_cast_fp16")];
            tensor<int32, [4]> var_563 = const()[name = string("op_563"), val = tensor<int32, [4]>([1, 4, 16, 128])];
            tensor<fp16, [1, 4, 16, 128]> value_9_cast_fp16 = reshape(shape = var_563, x = var_557_cast_fp16_1)[name = string("value_9_cast_fp16")];
            tensor<int32, [4]> var_565 = const()[name = string("op_565"), val = tensor<int32, [4]>([1, 4, 16, 128])];
            tensor<fp16, [1, 4, 16, 128]> value_11_cast_fp16 = reshape(shape = var_565, x = var_557_cast_fp16_2)[name = string("value_11_cast_fp16")];
            tensor<fp16, [1, 4, 16, 128]> var_567_cast_fp16 = mul(x = value_7_cast_fp16, y = value_7_cast_fp16)[name = string("op_567_cast_fp16")];
            tensor<int32, [1]> var_569_axes_0 = const()[name = string("op_569_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_569_keep_dims_0 = const()[name = string("op_569_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 4, 16, 1]> var_569_cast_fp16 = reduce_sum(axes = var_569_axes_0, keep_dims = var_569_keep_dims_0, x = var_567_cast_fp16)[name = string("op_569_cast_fp16")];
            fp16 var_570_to_fp16 = const()[name = string("op_570_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 4, 16, 1]> var_571_cast_fp16 = add(x = var_569_cast_fp16, y = var_570_to_fp16)[name = string("op_571_cast_fp16")];
            fp32 var_572_epsilon_0 = const()[name = string("op_572_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 4, 16, 1]> var_572_cast_fp16 = rsqrt(epsilon = var_572_epsilon_0, x = var_571_cast_fp16)[name = string("op_572_cast_fp16")];
            tensor<fp16, [1, 4, 16, 128]> var_573_cast_fp16 = mul(x = value_7_cast_fp16, y = var_572_cast_fp16)[name = string("op_573_cast_fp16")];
            tensor<int32, [4]> var_574_perm_0 = const()[name = string("op_574_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            fp16 _inversed_query_3_y_0_to_fp16 = const()[name = string("_inversed_query_3_y_0_to_fp16"), val = fp16(0x1.6ap-4)];
            tensor<fp16, [1, 16, 4, 128]> var_574_cast_fp16 = transpose(perm = var_574_perm_0, x = var_573_cast_fp16)[name = string("transpose_26")];
            tensor<fp16, [1, 16, 4, 128]> _inversed_query_3_cast_fp16 = mul(x = var_574_cast_fp16, y = _inversed_query_3_y_0_to_fp16)[name = string("_inversed_query_3_cast_fp16")];
            tensor<fp16, [1, 4, 16, 128]> var_577_cast_fp16 = mul(x = value_9_cast_fp16, y = value_9_cast_fp16)[name = string("op_577_cast_fp16")];
            tensor<int32, [1]> var_579_axes_0 = const()[name = string("op_579_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_579_keep_dims_0 = const()[name = string("op_579_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 4, 16, 1]> var_579_cast_fp16 = reduce_sum(axes = var_579_axes_0, keep_dims = var_579_keep_dims_0, x = var_577_cast_fp16)[name = string("op_579_cast_fp16")];
            fp16 var_580_to_fp16 = const()[name = string("op_580_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 4, 16, 1]> var_581_cast_fp16 = add(x = var_579_cast_fp16, y = var_580_to_fp16)[name = string("op_581_cast_fp16")];
            fp32 var_582_epsilon_0 = const()[name = string("op_582_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 4, 16, 1]> var_582_cast_fp16 = rsqrt(epsilon = var_582_epsilon_0, x = var_581_cast_fp16)[name = string("op_582_cast_fp16")];
            tensor<fp16, [1, 4, 16, 128]> var_583_cast_fp16 = mul(x = value_9_cast_fp16, y = var_582_cast_fp16)[name = string("op_583_cast_fp16")];
            tensor<int32, [4]> key_3_perm_0 = const()[name = string("key_3_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<int32, [4]> value_13_perm_0 = const()[name = string("value_13_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<fp16, [16, 1024]> layers20_0_in_proj_b_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(207959360))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(207971712))))[name = string("layers20_0_in_proj_b_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16]> linear_5_bias_0_to_fp16 = const()[name = string("linear_5_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(207971904)))];
            tensor<fp16, [4, 16]> linear_5_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = layers20_0_in_proj_b_weight_promoted_to_fp16_palettized, x = input_9_cast_fp16)[name = string("linear_5_cast_fp16")];
            tensor<fp16, [4, 16]> var_588_cast_fp16 = sigmoid(x = linear_5_cast_fp16)[name = string("op_588_cast_fp16")];
            tensor<int32, [2]> var_589_perm_0 = const()[name = string("op_589_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> beta_1_axes_0 = const()[name = string("beta_1_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 4]> var_589_cast_fp16 = transpose(perm = var_589_perm_0, x = var_588_cast_fp16)[name = string("transpose_25")];
            tensor<fp16, [1, 16, 4]> beta_1_cast_fp16 = expand_dims(axes = beta_1_axes_0, x = var_589_cast_fp16)[name = string("beta_1_cast_fp16")];
            tensor<fp16, [16, 1024]> op_595_weight_0_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(207972032))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(207984384))))[name = string("op_595_weight_0_to_fp16_palettized")];
            tensor<fp16, [16]> var_595_bias_0_to_fp16 = const()[name = string("op_595_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(207984576)))];
            tensor<fp16, [4, 16]> var_595_cast_fp16 = linear(bias = var_595_bias_0_to_fp16, weight = op_595_weight_0_to_fp16_palettized, x = input_9_cast_fp16)[name = string("op_595_cast_fp16")];
            tensor<fp16, [4, 16]> var_596_cast_fp16 = softplus(x = var_595_cast_fp16)[name = string("op_596_cast_fp16")];
            tensor<fp16, [16]> var_592_promoted_to_fp16 = const()[name = string("op_592_promoted_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(207984704)))];
            tensor<fp16, [4, 16]> var_597_cast_fp16 = mul(x = var_592_promoted_to_fp16, y = var_596_cast_fp16)[name = string("op_597_cast_fp16")];
            tensor<int32, [2]> var_598_perm_0 = const()[name = string("op_598_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> decay_1_axes_0 = const()[name = string("decay_1_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 4]> var_598_cast_fp16 = transpose(perm = var_598_perm_0, x = var_597_cast_fp16)[name = string("transpose_24")];
            tensor<fp16, [1, 16, 4]> decay_1_cast_fp16 = expand_dims(axes = decay_1_axes_0, x = var_598_cast_fp16)[name = string("decay_1_cast_fp16")];
            tensor<fp16, [2048, 1024]> layers20_0_in_proj_z_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [2048, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(207984832))), lut = tensor<fp16, [128, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(209557760))))[name = string("layers20_0_in_proj_z_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [2048]> linear_7_bias_0_to_fp16 = const()[name = string("linear_7_bias_0_to_fp16"), val = tensor<fp16, [2048]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(209574208)))];
            tensor<fp16, [4, 2048]> linear_7_cast_fp16 = linear(bias = linear_7_bias_0_to_fp16, weight = layers20_0_in_proj_z_weight_promoted_to_fp16_palettized, x = input_9_cast_fp16)[name = string("linear_7_cast_fp16")];
            tensor<int32, [3]> var_606_begin_0 = const()[name = string("op_606_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_606_end_0 = const()[name = string("op_606_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_606_end_mask_0 = const()[name = string("op_606_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_606_squeeze_mask_0 = const()[name = string("op_606_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_606_cast_fp16 = slice_by_index(begin = var_606_begin_0, end = var_606_end_0, end_mask = var_606_end_mask_0, squeeze_mask = var_606_squeeze_mask_0, x = decay_1_cast_fp16)[name = string("op_606_cast_fp16")];
            tensor<fp16, [1, 16]> var_607_cast_fp16 = exp(x = var_606_cast_fp16)[name = string("op_607_cast_fp16")];
            tensor<int32, [1]> var_608_axes_0 = const()[name = string("op_608_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_608_cast_fp16 = expand_dims(axes = var_608_axes_0, x = var_607_cast_fp16)[name = string("op_608_cast_fp16")];
            tensor<int32, [1]> var_609_axes_0 = const()[name = string("op_609_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_609_cast_fp16 = expand_dims(axes = var_609_axes_0, x = var_608_cast_fp16)[name = string("op_609_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_1_cast_fp16 = mul(x = rec20, y = var_609_cast_fp16)[name = string("state_1_cast_fp16")];
            tensor<int32, [4]> key_token_1_begin_0 = const()[name = string("key_token_1_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> key_token_1_end_0 = const()[name = string("key_token_1_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> key_token_1_end_mask_0 = const()[name = string("key_token_1_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_1_squeeze_mask_0 = const()[name = string("key_token_1_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 4, 128]> key_3_cast_fp16 = transpose(perm = key_3_perm_0, x = var_583_cast_fp16)[name = string("transpose_23")];
            tensor<fp16, [1, 16, 128]> key_token_1_cast_fp16 = slice_by_index(begin = key_token_1_begin_0, end = key_token_1_end_0, end_mask = key_token_1_end_mask_0, squeeze_mask = key_token_1_squeeze_mask_0, x = key_3_cast_fp16)[name = string("key_token_1_cast_fp16")];
            tensor<int32, [4]> value_token_1_begin_0 = const()[name = string("value_token_1_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> value_token_1_end_0 = const()[name = string("value_token_1_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> value_token_1_end_mask_0 = const()[name = string("value_token_1_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_1_squeeze_mask_0 = const()[name = string("value_token_1_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 4, 128]> value_13_cast_fp16 = transpose(perm = value_13_perm_0, x = value_11_cast_fp16)[name = string("transpose_22")];
            tensor<fp16, [1, 16, 128]> value_token_1_cast_fp16 = slice_by_index(begin = value_token_1_begin_0, end = value_token_1_end_0, end_mask = value_token_1_end_mask_0, squeeze_mask = value_token_1_squeeze_mask_0, x = value_13_cast_fp16)[name = string("value_token_1_cast_fp16")];
            tensor<int32, [1]> var_617_axes_0 = const()[name = string("op_617_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_617_cast_fp16 = expand_dims(axes = var_617_axes_0, x = key_token_1_cast_fp16)[name = string("op_617_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_618_cast_fp16 = mul(x = state_1_cast_fp16, y = var_617_cast_fp16)[name = string("op_618_cast_fp16")];
            tensor<int32, [1]> memory_1_axes_0 = const()[name = string("memory_1_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_1_keep_dims_0 = const()[name = string("memory_1_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_1_cast_fp16 = reduce_sum(axes = memory_1_axes_0, keep_dims = memory_1_keep_dims_0, x = var_618_cast_fp16)[name = string("memory_1_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_621_cast_fp16 = sub(x = value_token_1_cast_fp16, y = memory_1_cast_fp16)[name = string("op_621_cast_fp16")];
            tensor<int32, [3]> var_624_begin_0 = const()[name = string("op_624_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_624_end_0 = const()[name = string("op_624_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_624_end_mask_0 = const()[name = string("op_624_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_624_squeeze_mask_0 = const()[name = string("op_624_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_624_cast_fp16 = slice_by_index(begin = var_624_begin_0, end = var_624_end_0, end_mask = var_624_end_mask_0, squeeze_mask = var_624_squeeze_mask_0, x = beta_1_cast_fp16)[name = string("op_624_cast_fp16")];
            tensor<int32, [1]> var_625_axes_0 = const()[name = string("op_625_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_625_cast_fp16 = expand_dims(axes = var_625_axes_0, x = var_624_cast_fp16)[name = string("op_625_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_1_cast_fp16 = mul(x = var_621_cast_fp16, y = var_625_cast_fp16)[name = string("delta_1_cast_fp16")];
            tensor<int32, [1]> var_628_axes_0 = const()[name = string("op_628_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_628_cast_fp16 = expand_dims(axes = var_628_axes_0, x = delta_1_cast_fp16)[name = string("op_628_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_629_cast_fp16 = mul(x = var_617_cast_fp16, y = var_628_cast_fp16)[name = string("op_629_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_3_cast_fp16 = add(x = state_1_cast_fp16, y = var_629_cast_fp16)[name = string("state_3_cast_fp16")];
            tensor<int32, [4]> var_633_begin_0 = const()[name = string("op_633_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> var_633_end_0 = const()[name = string("op_633_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> var_633_end_mask_0 = const()[name = string("op_633_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_633_squeeze_mask_0 = const()[name = string("op_633_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_633_cast_fp16 = slice_by_index(begin = var_633_begin_0, end = var_633_end_0, end_mask = var_633_end_mask_0, squeeze_mask = var_633_squeeze_mask_0, x = _inversed_query_3_cast_fp16)[name = string("op_633_cast_fp16")];
            tensor<int32, [1]> var_634_axes_0 = const()[name = string("op_634_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_634_cast_fp16 = expand_dims(axes = var_634_axes_0, x = var_633_cast_fp16)[name = string("op_634_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_635_cast_fp16 = mul(x = state_3_cast_fp16, y = var_634_cast_fp16)[name = string("op_635_cast_fp16")];
            tensor<int32, [1]> var_637_axes_0 = const()[name = string("op_637_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_637_keep_dims_0 = const()[name = string("op_637_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_637_cast_fp16 = reduce_sum(axes = var_637_axes_0, keep_dims = var_637_keep_dims_0, x = var_635_cast_fp16)[name = string("op_637_cast_fp16")];
            tensor<int32, [3]> var_640_begin_0 = const()[name = string("op_640_begin_0"), val = tensor<int32, [3]>([0, 0, 1])];
            tensor<int32, [3]> var_640_end_0 = const()[name = string("op_640_end_0"), val = tensor<int32, [3]>([1, 16, 2])];
            tensor<bool, [3]> var_640_end_mask_0 = const()[name = string("op_640_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_640_squeeze_mask_0 = const()[name = string("op_640_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_640_cast_fp16 = slice_by_index(begin = var_640_begin_0, end = var_640_end_0, end_mask = var_640_end_mask_0, squeeze_mask = var_640_squeeze_mask_0, x = decay_1_cast_fp16)[name = string("op_640_cast_fp16")];
            tensor<fp16, [1, 16]> var_641_cast_fp16 = exp(x = var_640_cast_fp16)[name = string("op_641_cast_fp16")];
            tensor<int32, [1]> var_642_axes_0 = const()[name = string("op_642_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_642_cast_fp16 = expand_dims(axes = var_642_axes_0, x = var_641_cast_fp16)[name = string("op_642_cast_fp16")];
            tensor<int32, [1]> var_643_axes_0 = const()[name = string("op_643_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_643_cast_fp16 = expand_dims(axes = var_643_axes_0, x = var_642_cast_fp16)[name = string("op_643_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_5_cast_fp16 = mul(x = state_3_cast_fp16, y = var_643_cast_fp16)[name = string("state_5_cast_fp16")];
            tensor<int32, [4]> key_token_3_begin_0 = const()[name = string("key_token_3_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> key_token_3_end_0 = const()[name = string("key_token_3_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> key_token_3_end_mask_0 = const()[name = string("key_token_3_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_3_squeeze_mask_0 = const()[name = string("key_token_3_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_3_cast_fp16 = slice_by_index(begin = key_token_3_begin_0, end = key_token_3_end_0, end_mask = key_token_3_end_mask_0, squeeze_mask = key_token_3_squeeze_mask_0, x = key_3_cast_fp16)[name = string("key_token_3_cast_fp16")];
            tensor<int32, [4]> value_token_3_begin_0 = const()[name = string("value_token_3_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> value_token_3_end_0 = const()[name = string("value_token_3_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> value_token_3_end_mask_0 = const()[name = string("value_token_3_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_3_squeeze_mask_0 = const()[name = string("value_token_3_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_3_cast_fp16 = slice_by_index(begin = value_token_3_begin_0, end = value_token_3_end_0, end_mask = value_token_3_end_mask_0, squeeze_mask = value_token_3_squeeze_mask_0, x = value_13_cast_fp16)[name = string("value_token_3_cast_fp16")];
            tensor<int32, [1]> var_651_axes_0 = const()[name = string("op_651_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_651_cast_fp16 = expand_dims(axes = var_651_axes_0, x = key_token_3_cast_fp16)[name = string("op_651_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_652_cast_fp16 = mul(x = state_5_cast_fp16, y = var_651_cast_fp16)[name = string("op_652_cast_fp16")];
            tensor<int32, [1]> memory_3_axes_0 = const()[name = string("memory_3_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_3_keep_dims_0 = const()[name = string("memory_3_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_3_cast_fp16 = reduce_sum(axes = memory_3_axes_0, keep_dims = memory_3_keep_dims_0, x = var_652_cast_fp16)[name = string("memory_3_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_655_cast_fp16 = sub(x = value_token_3_cast_fp16, y = memory_3_cast_fp16)[name = string("op_655_cast_fp16")];
            tensor<int32, [3]> var_658_begin_0 = const()[name = string("op_658_begin_0"), val = tensor<int32, [3]>([0, 0, 1])];
            tensor<int32, [3]> var_658_end_0 = const()[name = string("op_658_end_0"), val = tensor<int32, [3]>([1, 16, 2])];
            tensor<bool, [3]> var_658_end_mask_0 = const()[name = string("op_658_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_658_squeeze_mask_0 = const()[name = string("op_658_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_658_cast_fp16 = slice_by_index(begin = var_658_begin_0, end = var_658_end_0, end_mask = var_658_end_mask_0, squeeze_mask = var_658_squeeze_mask_0, x = beta_1_cast_fp16)[name = string("op_658_cast_fp16")];
            tensor<int32, [1]> var_659_axes_0 = const()[name = string("op_659_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_659_cast_fp16 = expand_dims(axes = var_659_axes_0, x = var_658_cast_fp16)[name = string("op_659_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_3_cast_fp16 = mul(x = var_655_cast_fp16, y = var_659_cast_fp16)[name = string("delta_3_cast_fp16")];
            tensor<int32, [1]> var_662_axes_0 = const()[name = string("op_662_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_662_cast_fp16 = expand_dims(axes = var_662_axes_0, x = delta_3_cast_fp16)[name = string("op_662_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_663_cast_fp16 = mul(x = var_651_cast_fp16, y = var_662_cast_fp16)[name = string("op_663_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_7_cast_fp16 = add(x = state_5_cast_fp16, y = var_663_cast_fp16)[name = string("state_7_cast_fp16")];
            tensor<int32, [4]> var_667_begin_0 = const()[name = string("op_667_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> var_667_end_0 = const()[name = string("op_667_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> var_667_end_mask_0 = const()[name = string("op_667_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_667_squeeze_mask_0 = const()[name = string("op_667_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_667_cast_fp16 = slice_by_index(begin = var_667_begin_0, end = var_667_end_0, end_mask = var_667_end_mask_0, squeeze_mask = var_667_squeeze_mask_0, x = _inversed_query_3_cast_fp16)[name = string("op_667_cast_fp16")];
            tensor<int32, [1]> var_668_axes_0 = const()[name = string("op_668_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_668_cast_fp16 = expand_dims(axes = var_668_axes_0, x = var_667_cast_fp16)[name = string("op_668_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_669_cast_fp16 = mul(x = state_7_cast_fp16, y = var_668_cast_fp16)[name = string("op_669_cast_fp16")];
            tensor<int32, [1]> var_671_axes_0 = const()[name = string("op_671_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_671_keep_dims_0 = const()[name = string("op_671_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_671_cast_fp16 = reduce_sum(axes = var_671_axes_0, keep_dims = var_671_keep_dims_0, x = var_669_cast_fp16)[name = string("op_671_cast_fp16")];
            tensor<int32, [3]> var_674_begin_0 = const()[name = string("op_674_begin_0"), val = tensor<int32, [3]>([0, 0, 2])];
            tensor<int32, [3]> var_674_end_0 = const()[name = string("op_674_end_0"), val = tensor<int32, [3]>([1, 16, 3])];
            tensor<bool, [3]> var_674_end_mask_0 = const()[name = string("op_674_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_674_squeeze_mask_0 = const()[name = string("op_674_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_674_cast_fp16 = slice_by_index(begin = var_674_begin_0, end = var_674_end_0, end_mask = var_674_end_mask_0, squeeze_mask = var_674_squeeze_mask_0, x = decay_1_cast_fp16)[name = string("op_674_cast_fp16")];
            tensor<fp16, [1, 16]> var_675_cast_fp16 = exp(x = var_674_cast_fp16)[name = string("op_675_cast_fp16")];
            tensor<int32, [1]> var_676_axes_0 = const()[name = string("op_676_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_676_cast_fp16 = expand_dims(axes = var_676_axes_0, x = var_675_cast_fp16)[name = string("op_676_cast_fp16")];
            tensor<int32, [1]> var_677_axes_0 = const()[name = string("op_677_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_677_cast_fp16 = expand_dims(axes = var_677_axes_0, x = var_676_cast_fp16)[name = string("op_677_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_9_cast_fp16 = mul(x = state_7_cast_fp16, y = var_677_cast_fp16)[name = string("state_9_cast_fp16")];
            tensor<int32, [4]> key_token_5_begin_0 = const()[name = string("key_token_5_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> key_token_5_end_0 = const()[name = string("key_token_5_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> key_token_5_end_mask_0 = const()[name = string("key_token_5_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_5_squeeze_mask_0 = const()[name = string("key_token_5_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_5_cast_fp16 = slice_by_index(begin = key_token_5_begin_0, end = key_token_5_end_0, end_mask = key_token_5_end_mask_0, squeeze_mask = key_token_5_squeeze_mask_0, x = key_3_cast_fp16)[name = string("key_token_5_cast_fp16")];
            tensor<int32, [4]> value_token_5_begin_0 = const()[name = string("value_token_5_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> value_token_5_end_0 = const()[name = string("value_token_5_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> value_token_5_end_mask_0 = const()[name = string("value_token_5_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_5_squeeze_mask_0 = const()[name = string("value_token_5_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_5_cast_fp16 = slice_by_index(begin = value_token_5_begin_0, end = value_token_5_end_0, end_mask = value_token_5_end_mask_0, squeeze_mask = value_token_5_squeeze_mask_0, x = value_13_cast_fp16)[name = string("value_token_5_cast_fp16")];
            tensor<int32, [1]> var_685_axes_0 = const()[name = string("op_685_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_685_cast_fp16 = expand_dims(axes = var_685_axes_0, x = key_token_5_cast_fp16)[name = string("op_685_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_686_cast_fp16 = mul(x = state_9_cast_fp16, y = var_685_cast_fp16)[name = string("op_686_cast_fp16")];
            tensor<int32, [1]> memory_5_axes_0 = const()[name = string("memory_5_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_5_keep_dims_0 = const()[name = string("memory_5_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_5_cast_fp16 = reduce_sum(axes = memory_5_axes_0, keep_dims = memory_5_keep_dims_0, x = var_686_cast_fp16)[name = string("memory_5_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_689_cast_fp16 = sub(x = value_token_5_cast_fp16, y = memory_5_cast_fp16)[name = string("op_689_cast_fp16")];
            tensor<int32, [3]> var_692_begin_0 = const()[name = string("op_692_begin_0"), val = tensor<int32, [3]>([0, 0, 2])];
            tensor<int32, [3]> var_692_end_0 = const()[name = string("op_692_end_0"), val = tensor<int32, [3]>([1, 16, 3])];
            tensor<bool, [3]> var_692_end_mask_0 = const()[name = string("op_692_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_692_squeeze_mask_0 = const()[name = string("op_692_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_692_cast_fp16 = slice_by_index(begin = var_692_begin_0, end = var_692_end_0, end_mask = var_692_end_mask_0, squeeze_mask = var_692_squeeze_mask_0, x = beta_1_cast_fp16)[name = string("op_692_cast_fp16")];
            tensor<int32, [1]> var_693_axes_0 = const()[name = string("op_693_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_693_cast_fp16 = expand_dims(axes = var_693_axes_0, x = var_692_cast_fp16)[name = string("op_693_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_5_cast_fp16 = mul(x = var_689_cast_fp16, y = var_693_cast_fp16)[name = string("delta_5_cast_fp16")];
            tensor<int32, [1]> var_696_axes_0 = const()[name = string("op_696_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_696_cast_fp16 = expand_dims(axes = var_696_axes_0, x = delta_5_cast_fp16)[name = string("op_696_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_697_cast_fp16 = mul(x = var_685_cast_fp16, y = var_696_cast_fp16)[name = string("op_697_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_11_cast_fp16 = add(x = state_9_cast_fp16, y = var_697_cast_fp16)[name = string("state_11_cast_fp16")];
            tensor<int32, [4]> var_701_begin_0 = const()[name = string("op_701_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> var_701_end_0 = const()[name = string("op_701_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> var_701_end_mask_0 = const()[name = string("op_701_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_701_squeeze_mask_0 = const()[name = string("op_701_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_701_cast_fp16 = slice_by_index(begin = var_701_begin_0, end = var_701_end_0, end_mask = var_701_end_mask_0, squeeze_mask = var_701_squeeze_mask_0, x = _inversed_query_3_cast_fp16)[name = string("op_701_cast_fp16")];
            tensor<int32, [1]> var_702_axes_0 = const()[name = string("op_702_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_702_cast_fp16 = expand_dims(axes = var_702_axes_0, x = var_701_cast_fp16)[name = string("op_702_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_703_cast_fp16 = mul(x = state_11_cast_fp16, y = var_702_cast_fp16)[name = string("op_703_cast_fp16")];
            tensor<int32, [1]> var_705_axes_0 = const()[name = string("op_705_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_705_keep_dims_0 = const()[name = string("op_705_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_705_cast_fp16 = reduce_sum(axes = var_705_axes_0, keep_dims = var_705_keep_dims_0, x = var_703_cast_fp16)[name = string("op_705_cast_fp16")];
            tensor<int32, [3]> var_708_begin_0 = const()[name = string("op_708_begin_0"), val = tensor<int32, [3]>([0, 0, 3])];
            tensor<int32, [3]> var_708_end_0 = const()[name = string("op_708_end_0"), val = tensor<int32, [3]>([1, 16, 4])];
            tensor<bool, [3]> var_708_end_mask_0 = const()[name = string("op_708_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_708_squeeze_mask_0 = const()[name = string("op_708_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_708_cast_fp16 = slice_by_index(begin = var_708_begin_0, end = var_708_end_0, end_mask = var_708_end_mask_0, squeeze_mask = var_708_squeeze_mask_0, x = decay_1_cast_fp16)[name = string("op_708_cast_fp16")];
            tensor<fp16, [1, 16]> var_709_cast_fp16 = exp(x = var_708_cast_fp16)[name = string("op_709_cast_fp16")];
            tensor<int32, [1]> var_710_axes_0 = const()[name = string("op_710_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_710_cast_fp16 = expand_dims(axes = var_710_axes_0, x = var_709_cast_fp16)[name = string("op_710_cast_fp16")];
            tensor<int32, [1]> var_711_axes_0 = const()[name = string("op_711_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_711_cast_fp16 = expand_dims(axes = var_711_axes_0, x = var_710_cast_fp16)[name = string("op_711_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_13_cast_fp16 = mul(x = state_11_cast_fp16, y = var_711_cast_fp16)[name = string("state_13_cast_fp16")];
            tensor<int32, [4]> key_token_7_begin_0 = const()[name = string("key_token_7_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> key_token_7_end_0 = const()[name = string("key_token_7_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> key_token_7_end_mask_0 = const()[name = string("key_token_7_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_7_squeeze_mask_0 = const()[name = string("key_token_7_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_7_cast_fp16 = slice_by_index(begin = key_token_7_begin_0, end = key_token_7_end_0, end_mask = key_token_7_end_mask_0, squeeze_mask = key_token_7_squeeze_mask_0, x = key_3_cast_fp16)[name = string("key_token_7_cast_fp16")];
            tensor<int32, [4]> value_token_7_begin_0 = const()[name = string("value_token_7_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> value_token_7_end_0 = const()[name = string("value_token_7_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> value_token_7_end_mask_0 = const()[name = string("value_token_7_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_7_squeeze_mask_0 = const()[name = string("value_token_7_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_7_cast_fp16 = slice_by_index(begin = value_token_7_begin_0, end = value_token_7_end_0, end_mask = value_token_7_end_mask_0, squeeze_mask = value_token_7_squeeze_mask_0, x = value_13_cast_fp16)[name = string("value_token_7_cast_fp16")];
            tensor<int32, [1]> var_719_axes_0 = const()[name = string("op_719_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_719_cast_fp16 = expand_dims(axes = var_719_axes_0, x = key_token_7_cast_fp16)[name = string("op_719_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_720_cast_fp16 = mul(x = state_13_cast_fp16, y = var_719_cast_fp16)[name = string("op_720_cast_fp16")];
            tensor<int32, [1]> memory_7_axes_0 = const()[name = string("memory_7_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_7_keep_dims_0 = const()[name = string("memory_7_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_7_cast_fp16 = reduce_sum(axes = memory_7_axes_0, keep_dims = memory_7_keep_dims_0, x = var_720_cast_fp16)[name = string("memory_7_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_723_cast_fp16 = sub(x = value_token_7_cast_fp16, y = memory_7_cast_fp16)[name = string("op_723_cast_fp16")];
            tensor<int32, [3]> var_726_begin_0 = const()[name = string("op_726_begin_0"), val = tensor<int32, [3]>([0, 0, 3])];
            tensor<int32, [3]> var_726_end_0 = const()[name = string("op_726_end_0"), val = tensor<int32, [3]>([1, 16, 4])];
            tensor<bool, [3]> var_726_end_mask_0 = const()[name = string("op_726_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_726_squeeze_mask_0 = const()[name = string("op_726_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_726_cast_fp16 = slice_by_index(begin = var_726_begin_0, end = var_726_end_0, end_mask = var_726_end_mask_0, squeeze_mask = var_726_squeeze_mask_0, x = beta_1_cast_fp16)[name = string("op_726_cast_fp16")];
            tensor<int32, [1]> var_727_axes_0 = const()[name = string("op_727_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_727_cast_fp16 = expand_dims(axes = var_727_axes_0, x = var_726_cast_fp16)[name = string("op_727_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_7_cast_fp16 = mul(x = var_723_cast_fp16, y = var_727_cast_fp16)[name = string("delta_7_cast_fp16")];
            tensor<int32, [1]> var_730_axes_0 = const()[name = string("op_730_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_730_cast_fp16 = expand_dims(axes = var_730_axes_0, x = delta_7_cast_fp16)[name = string("op_730_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_731_cast_fp16 = mul(x = var_719_cast_fp16, y = var_730_cast_fp16)[name = string("op_731_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> rec20_out = add(x = state_13_cast_fp16, y = var_731_cast_fp16)[name = string("state_15_cast_fp16")];
            tensor<int32, [4]> var_735_begin_0 = const()[name = string("op_735_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> var_735_end_0 = const()[name = string("op_735_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> var_735_end_mask_0 = const()[name = string("op_735_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_735_squeeze_mask_0 = const()[name = string("op_735_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_735_cast_fp16 = slice_by_index(begin = var_735_begin_0, end = var_735_end_0, end_mask = var_735_end_mask_0, squeeze_mask = var_735_squeeze_mask_0, x = _inversed_query_3_cast_fp16)[name = string("op_735_cast_fp16")];
            tensor<int32, [1]> var_736_axes_0 = const()[name = string("op_736_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_736_cast_fp16 = expand_dims(axes = var_736_axes_0, x = var_735_cast_fp16)[name = string("op_736_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_737_cast_fp16 = mul(x = rec20_out, y = var_736_cast_fp16)[name = string("op_737_cast_fp16")];
            tensor<int32, [1]> var_739_axes_0 = const()[name = string("op_739_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_739_keep_dims_0 = const()[name = string("op_739_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_739_cast_fp16 = reduce_sum(axes = var_739_axes_0, keep_dims = var_739_keep_dims_0, x = var_737_cast_fp16)[name = string("op_739_cast_fp16")];
            int32 attention_1_axis_0 = const()[name = string("attention_1_axis_0"), val = int32(1)];
            tensor<fp16, [1, 4, 16, 128]> attention_1_cast_fp16 = stack(axis = attention_1_axis_0, values = (var_637_cast_fp16, var_671_cast_fp16, var_705_cast_fp16, var_739_cast_fp16))[name = string("attention_1_cast_fp16")];
            tensor<int32, [2]> var_742 = const()[name = string("op_742"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [64, 128]> attention_3_cast_fp16 = reshape(shape = var_742, x = attention_1_cast_fp16)[name = string("attention_3_cast_fp16")];
            tensor<int32, [2]> var_744 = const()[name = string("op_744"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [64, 128]> input_13_cast_fp16 = reshape(shape = var_744, x = linear_7_cast_fp16)[name = string("input_13_cast_fp16")];
            tensor<fp16, [64, 128]> var_746_cast_fp16 = mul(x = attention_3_cast_fp16, y = attention_3_cast_fp16)[name = string("op_746_cast_fp16")];
            tensor<int32, [1]> variance_5_axes_0 = const()[name = string("variance_5_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_5_keep_dims_0 = const()[name = string("variance_5_keep_dims_0"), val = bool(true)];
            tensor<fp16, [64, 1]> variance_5_cast_fp16 = reduce_mean(axes = variance_5_axes_0, keep_dims = variance_5_keep_dims_0, x = var_746_cast_fp16)[name = string("variance_5_cast_fp16")];
            fp16 var_749_to_fp16 = const()[name = string("op_749_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [64, 1]> var_750_cast_fp16 = add(x = variance_5_cast_fp16, y = var_749_to_fp16)[name = string("op_750_cast_fp16")];
            fp32 var_751_epsilon_0 = const()[name = string("op_751_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [64, 1]> var_751_cast_fp16 = rsqrt(epsilon = var_751_epsilon_0, x = var_750_cast_fp16)[name = string("op_751_cast_fp16")];
            tensor<fp16, [64, 128]> attention_5_cast_fp16 = mul(x = attention_3_cast_fp16, y = var_751_cast_fp16)[name = string("attention_5_cast_fp16")];
            tensor<fp16, [128]> layers20_0_gated_norm_promoted_to_fp16 = const()[name = string("layers20_0_gated_norm_promoted_to_fp16"), val = tensor<fp16, [128]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(209578368)))];
            tensor<fp16, [64, 128]> attention_7_cast_fp16 = mul(x = attention_5_cast_fp16, y = layers20_0_gated_norm_promoted_to_fp16)[name = string("attention_7_cast_fp16")];
            tensor<fp16, [64, 128]> var_754_cast_fp16 = silu(x = input_13_cast_fp16)[name = string("op_754_cast_fp16")];
            tensor<fp16, [64, 128]> attention_9_cast_fp16 = mul(x = attention_7_cast_fp16, y = var_754_cast_fp16)[name = string("attention_9_cast_fp16")];
            tensor<int32, [2]> var_756 = const()[name = string("op_756"), val = tensor<int32, [2]>([4, 2048])];
            tensor<fp16, [4, 2048]> input_15_cast_fp16 = reshape(shape = var_756, x = attention_9_cast_fp16)[name = string("input_15_cast_fp16")];
            tensor<fp16, [1024, 2048]> layers20_0_out_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 2048]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(209578688))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(211151616))))[name = string("layers20_0_out_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 1024]> linear_8_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = layers20_0_out_proj_weight_promoted_to_fp16_palettized, x = input_15_cast_fp16)[name = string("linear_8_cast_fp16")];
            tensor<fp16, [4, 1024]> value_15_cast_fp16 = add(x = value_3_cast_fp16, y = linear_8_cast_fp16)[name = string("value_15_cast_fp16")];
            tensor<fp16, [4, 1024]> var_761_cast_fp16 = mul(x = value_15_cast_fp16, y = value_15_cast_fp16)[name = string("op_761_cast_fp16")];
            tensor<int32, [1]> variance_7_axes_0 = const()[name = string("variance_7_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_7_keep_dims_0 = const()[name = string("variance_7_keep_dims_0"), val = bool(true)];
            tensor<fp16, [4, 1]> variance_7_cast_fp16 = reduce_mean(axes = variance_7_axes_0, keep_dims = variance_7_keep_dims_0, x = var_761_cast_fp16)[name = string("variance_7_cast_fp16")];
            fp16 var_764_to_fp16 = const()[name = string("op_764_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [4, 1]> var_765_cast_fp16 = add(x = variance_7_cast_fp16, y = var_764_to_fp16)[name = string("op_765_cast_fp16")];
            fp32 var_766_epsilon_0 = const()[name = string("op_766_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [4, 1]> var_766_cast_fp16 = rsqrt(epsilon = var_766_epsilon_0, x = var_765_cast_fp16)[name = string("op_766_cast_fp16")];
            tensor<fp16, [4, 1024]> var_767_cast_fp16 = mul(x = value_15_cast_fp16, y = var_766_cast_fp16)[name = string("op_767_cast_fp16")];
            tensor<fp16, [1024]> var_769_to_fp16 = const()[name = string("op_769_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(211159872)))];
            tensor<fp16, [4, 1024]> input_17_cast_fp16 = mul(x = var_767_cast_fp16, y = var_769_to_fp16)[name = string("input_17_cast_fp16")];
            tensor<fp16, [3584, 1024]> layers20_0_gate_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(211161984))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(213914560))))[name = string("layers20_0_gate_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 3584]> linear_9_cast_fp16 = linear(bias = linear_1_bias_0_to_fp16, weight = layers20_0_gate_proj_weight_promoted_to_fp16_palettized, x = input_17_cast_fp16)[name = string("linear_9_cast_fp16")];
            tensor<fp16, [4, 3584]> var_773_cast_fp16 = silu(x = linear_9_cast_fp16)[name = string("op_773_cast_fp16")];
            tensor<fp16, [3584, 1024]> layers20_0_up_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(213943296))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(216695872))))[name = string("layers20_0_up_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 3584]> linear_10_cast_fp16 = linear(bias = linear_1_bias_0_to_fp16, weight = layers20_0_up_proj_weight_promoted_to_fp16_palettized, x = input_17_cast_fp16)[name = string("linear_10_cast_fp16")];
            tensor<fp16, [4, 3584]> input_21_cast_fp16 = mul(x = var_773_cast_fp16, y = linear_10_cast_fp16)[name = string("input_21_cast_fp16")];
            tensor<fp16, [1024, 3584]> layers20_0_down_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 3584]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(216724608))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(219477184))))[name = string("layers20_0_down_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 1024]> linear_11_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = layers20_0_down_proj_weight_promoted_to_fp16_palettized, x = input_21_cast_fp16)[name = string("linear_11_cast_fp16")];
            tensor<fp16, [4, 1024]> value_17_cast_fp16 = add(x = value_15_cast_fp16, y = linear_11_cast_fp16)[name = string("value_17_cast_fp16")];
            int32 var_800 = const()[name = string("op_800"), val = int32(-1)];
            tensor<fp16, [4, 1024]> var_815_cast_fp16 = mul(x = value_17_cast_fp16, y = value_17_cast_fp16)[name = string("op_815_cast_fp16")];
            tensor<int32, [1]> variance_9_axes_0 = const()[name = string("variance_9_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_9_keep_dims_0 = const()[name = string("variance_9_keep_dims_0"), val = bool(true)];
            tensor<fp16, [4, 1]> variance_9_cast_fp16 = reduce_mean(axes = variance_9_axes_0, keep_dims = variance_9_keep_dims_0, x = var_815_cast_fp16)[name = string("variance_9_cast_fp16")];
            fp16 var_818_to_fp16 = const()[name = string("op_818_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [4, 1]> var_819_cast_fp16 = add(x = variance_9_cast_fp16, y = var_818_to_fp16)[name = string("op_819_cast_fp16")];
            fp32 var_820_epsilon_0 = const()[name = string("op_820_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [4, 1]> var_820_cast_fp16 = rsqrt(epsilon = var_820_epsilon_0, x = var_819_cast_fp16)[name = string("op_820_cast_fp16")];
            tensor<fp16, [4, 1024]> var_821_cast_fp16 = mul(x = value_17_cast_fp16, y = var_820_cast_fp16)[name = string("op_821_cast_fp16")];
            tensor<fp16, [1024]> var_823_to_fp16 = const()[name = string("op_823_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(219485440)))];
            tensor<fp16, [4, 1024]> input_23_cast_fp16 = mul(x = var_821_cast_fp16, y = var_823_to_fp16)[name = string("input_23_cast_fp16")];
            tensor<fp16, [6144, 1024]> layers20_1_in_proj_qkv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(219487552))), lut = tensor<fp16, [384, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(224206208))))[name = string("layers20_1_in_proj_qkv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 6144]> linear_12_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = layers20_1_in_proj_qkv_weight_promoted_to_fp16_palettized, x = input_23_cast_fp16)[name = string("linear_12_cast_fp16")];
            tensor<int32, [2]> var_827_perm_0 = const()[name = string("op_827_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> mixed_3_axes_0 = const()[name = string("mixed_3_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [6144, 4]> var_827_cast_fp16 = transpose(perm = var_827_perm_0, x = linear_12_cast_fp16)[name = string("transpose_21")];
            tensor<fp16, [1, 6144, 4]> mixed_3_cast_fp16 = expand_dims(axes = mixed_3_axes_0, x = var_827_cast_fp16)[name = string("mixed_3_cast_fp16")];
            bool convolution_input_3_interleave_0 = const()[name = string("convolution_input_3_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 6144, 7]> convolution_input_3_cast_fp16 = concat(axis = var_800, interleave = convolution_input_3_interleave_0, values = (conv21, mixed_3_cast_fp16))[name = string("convolution_input_3_cast_fp16")];
            string input_25_pad_type_0 = const()[name = string("input_25_pad_type_0"), val = string("valid")];
            int32 input_25_groups_0 = const()[name = string("input_25_groups_0"), val = int32(6144)];
            tensor<int32, [1]> input_25_strides_0 = const()[name = string("input_25_strides_0"), val = tensor<int32, [1]>([1])];
            tensor<int32, [2]> input_25_pad_0 = const()[name = string("input_25_pad_0"), val = tensor<int32, [2]>([0, 0])];
            tensor<int32, [1]> input_25_dilations_0 = const()[name = string("input_25_dilations_0"), val = tensor<int32, [1]>([1])];
            tensor<fp16, [6144, 1, 4]> layers20_1_conv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1, 4]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(224255424))), lut = tensor<fp16, [384, 1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(224273920))))[name = string("layers20_1_conv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 6144, 4]> input_25_cast_fp16 = conv(dilations = input_25_dilations_0, groups = input_25_groups_0, pad = input_25_pad_0, pad_type = input_25_pad_type_0, strides = input_25_strides_0, weight = layers20_1_conv_weight_promoted_to_fp16_palettized, x = convolution_input_3_cast_fp16)[name = string("input_25_cast_fp16")];
            tensor<fp16, [1, 6144, 4]> var_836_cast_fp16 = silu(x = input_25_cast_fp16)[name = string("op_836_cast_fp16")];
            tensor<int32, [3]> var_837_perm_0 = const()[name = string("op_837_perm_0"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [3]> var_840_begin_0 = const()[name = string("op_840_begin_0"), val = tensor<int32, [3]>([0, 0, 4])];
            tensor<int32, [3]> var_840_end_0 = const()[name = string("op_840_end_0"), val = tensor<int32, [3]>([1, 6144, 7])];
            tensor<bool, [3]> var_840_end_mask_0 = const()[name = string("op_840_end_mask_0"), val = tensor<bool, [3]>([true, true, true])];
            tensor<fp16, [1, 6144, 3]> conv21_out = slice_by_index(begin = var_840_begin_0, end = var_840_end_0, end_mask = var_840_end_mask_0, x = convolution_input_3_cast_fp16)[name = string("op_840_cast_fp16")];
            tensor<int32, [3]> var_841 = const()[name = string("op_841"), val = tensor<int32, [3]>([2048, 2048, 2048])];
            int32 var_842_axis_0 = const()[name = string("op_842_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 4, 6144]> var_837_cast_fp16 = transpose(perm = var_837_perm_0, x = var_836_cast_fp16)[name = string("transpose_20")];
            tensor<fp16, [1, 4, 2048]> var_842_cast_fp16_0, tensor<fp16, [1, 4, 2048]> var_842_cast_fp16_1, tensor<fp16, [1, 4, 2048]> var_842_cast_fp16_2 = split(axis = var_842_axis_0, split_sizes = var_841, x = var_837_cast_fp16)[name = string("op_842_cast_fp16")];
            tensor<int32, [4]> var_846 = const()[name = string("op_846"), val = tensor<int32, [4]>([1, 4, 16, 128])];
            tensor<fp16, [1, 4, 16, 128]> value_21_cast_fp16 = reshape(shape = var_846, x = var_842_cast_fp16_0)[name = string("value_21_cast_fp16")];
            tensor<int32, [4]> var_848 = const()[name = string("op_848"), val = tensor<int32, [4]>([1, 4, 16, 128])];
            tensor<fp16, [1, 4, 16, 128]> value_23_cast_fp16 = reshape(shape = var_848, x = var_842_cast_fp16_1)[name = string("value_23_cast_fp16")];
            tensor<int32, [4]> var_850 = const()[name = string("op_850"), val = tensor<int32, [4]>([1, 4, 16, 128])];
            tensor<fp16, [1, 4, 16, 128]> value_25_cast_fp16 = reshape(shape = var_850, x = var_842_cast_fp16_2)[name = string("value_25_cast_fp16")];
            tensor<fp16, [1, 4, 16, 128]> var_852_cast_fp16 = mul(x = value_21_cast_fp16, y = value_21_cast_fp16)[name = string("op_852_cast_fp16")];
            tensor<int32, [1]> var_854_axes_0 = const()[name = string("op_854_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_854_keep_dims_0 = const()[name = string("op_854_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 4, 16, 1]> var_854_cast_fp16 = reduce_sum(axes = var_854_axes_0, keep_dims = var_854_keep_dims_0, x = var_852_cast_fp16)[name = string("op_854_cast_fp16")];
            fp16 var_855_to_fp16 = const()[name = string("op_855_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 4, 16, 1]> var_856_cast_fp16 = add(x = var_854_cast_fp16, y = var_855_to_fp16)[name = string("op_856_cast_fp16")];
            fp32 var_857_epsilon_0 = const()[name = string("op_857_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 4, 16, 1]> var_857_cast_fp16 = rsqrt(epsilon = var_857_epsilon_0, x = var_856_cast_fp16)[name = string("op_857_cast_fp16")];
            tensor<fp16, [1, 4, 16, 128]> var_858_cast_fp16 = mul(x = value_21_cast_fp16, y = var_857_cast_fp16)[name = string("op_858_cast_fp16")];
            tensor<int32, [4]> var_859_perm_0 = const()[name = string("op_859_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            fp16 _inversed_query_7_y_0_to_fp16 = const()[name = string("_inversed_query_7_y_0_to_fp16"), val = fp16(0x1.6ap-4)];
            tensor<fp16, [1, 16, 4, 128]> var_859_cast_fp16 = transpose(perm = var_859_perm_0, x = var_858_cast_fp16)[name = string("transpose_19")];
            tensor<fp16, [1, 16, 4, 128]> _inversed_query_7_cast_fp16 = mul(x = var_859_cast_fp16, y = _inversed_query_7_y_0_to_fp16)[name = string("_inversed_query_7_cast_fp16")];
            tensor<fp16, [1, 4, 16, 128]> var_862_cast_fp16 = mul(x = value_23_cast_fp16, y = value_23_cast_fp16)[name = string("op_862_cast_fp16")];
            tensor<int32, [1]> var_864_axes_0 = const()[name = string("op_864_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_864_keep_dims_0 = const()[name = string("op_864_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 4, 16, 1]> var_864_cast_fp16 = reduce_sum(axes = var_864_axes_0, keep_dims = var_864_keep_dims_0, x = var_862_cast_fp16)[name = string("op_864_cast_fp16")];
            fp16 var_865_to_fp16 = const()[name = string("op_865_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 4, 16, 1]> var_866_cast_fp16 = add(x = var_864_cast_fp16, y = var_865_to_fp16)[name = string("op_866_cast_fp16")];
            fp32 var_867_epsilon_0 = const()[name = string("op_867_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 4, 16, 1]> var_867_cast_fp16 = rsqrt(epsilon = var_867_epsilon_0, x = var_866_cast_fp16)[name = string("op_867_cast_fp16")];
            tensor<fp16, [1, 4, 16, 128]> var_868_cast_fp16 = mul(x = value_23_cast_fp16, y = var_867_cast_fp16)[name = string("op_868_cast_fp16")];
            tensor<int32, [4]> key_7_perm_0 = const()[name = string("key_7_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<int32, [4]> value_27_perm_0 = const()[name = string("value_27_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<fp16, [16, 1024]> layers20_1_in_proj_b_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(224323136))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(224335488))))[name = string("layers20_1_in_proj_b_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16]> linear_13_bias_0_to_fp16 = const()[name = string("linear_13_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(207971904)))];
            tensor<fp16, [4, 16]> linear_13_cast_fp16 = linear(bias = linear_13_bias_0_to_fp16, weight = layers20_1_in_proj_b_weight_promoted_to_fp16_palettized, x = input_23_cast_fp16)[name = string("linear_13_cast_fp16")];
            tensor<fp16, [4, 16]> var_873_cast_fp16 = sigmoid(x = linear_13_cast_fp16)[name = string("op_873_cast_fp16")];
            tensor<int32, [2]> var_874_perm_0 = const()[name = string("op_874_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> beta_3_axes_0 = const()[name = string("beta_3_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 4]> var_874_cast_fp16 = transpose(perm = var_874_perm_0, x = var_873_cast_fp16)[name = string("transpose_18")];
            tensor<fp16, [1, 16, 4]> beta_3_cast_fp16 = expand_dims(axes = beta_3_axes_0, x = var_874_cast_fp16)[name = string("beta_3_cast_fp16")];
            tensor<fp16, [16, 1024]> op_880_weight_0_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(224335680))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(224348032))))[name = string("op_880_weight_0_to_fp16_palettized")];
            tensor<fp16, [16]> var_880_bias_0_to_fp16 = const()[name = string("op_880_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(224348224)))];
            tensor<fp16, [4, 16]> var_880_cast_fp16 = linear(bias = var_880_bias_0_to_fp16, weight = op_880_weight_0_to_fp16_palettized, x = input_23_cast_fp16)[name = string("op_880_cast_fp16")];
            tensor<fp16, [4, 16]> var_881_cast_fp16 = softplus(x = var_880_cast_fp16)[name = string("op_881_cast_fp16")];
            tensor<fp16, [16]> var_877_promoted_to_fp16 = const()[name = string("op_877_promoted_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(224348352)))];
            tensor<fp16, [4, 16]> var_882_cast_fp16 = mul(x = var_877_promoted_to_fp16, y = var_881_cast_fp16)[name = string("op_882_cast_fp16")];
            tensor<int32, [2]> var_883_perm_0 = const()[name = string("op_883_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> decay_3_axes_0 = const()[name = string("decay_3_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 4]> var_883_cast_fp16 = transpose(perm = var_883_perm_0, x = var_882_cast_fp16)[name = string("transpose_17")];
            tensor<fp16, [1, 16, 4]> decay_3_cast_fp16 = expand_dims(axes = decay_3_axes_0, x = var_883_cast_fp16)[name = string("decay_3_cast_fp16")];
            tensor<fp16, [2048, 1024]> layers20_1_in_proj_z_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [2048, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(224348480))), lut = tensor<fp16, [128, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(225921408))))[name = string("layers20_1_in_proj_z_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 2048]> linear_15_cast_fp16 = linear(bias = linear_7_bias_0_to_fp16, weight = layers20_1_in_proj_z_weight_promoted_to_fp16_palettized, x = input_23_cast_fp16)[name = string("linear_15_cast_fp16")];
            tensor<int32, [3]> var_891_begin_0 = const()[name = string("op_891_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_891_end_0 = const()[name = string("op_891_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_891_end_mask_0 = const()[name = string("op_891_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_891_squeeze_mask_0 = const()[name = string("op_891_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_891_cast_fp16 = slice_by_index(begin = var_891_begin_0, end = var_891_end_0, end_mask = var_891_end_mask_0, squeeze_mask = var_891_squeeze_mask_0, x = decay_3_cast_fp16)[name = string("op_891_cast_fp16")];
            tensor<fp16, [1, 16]> var_892_cast_fp16 = exp(x = var_891_cast_fp16)[name = string("op_892_cast_fp16")];
            tensor<int32, [1]> var_893_axes_0 = const()[name = string("op_893_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_893_cast_fp16 = expand_dims(axes = var_893_axes_0, x = var_892_cast_fp16)[name = string("op_893_cast_fp16")];
            tensor<int32, [1]> var_894_axes_0 = const()[name = string("op_894_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_894_cast_fp16 = expand_dims(axes = var_894_axes_0, x = var_893_cast_fp16)[name = string("op_894_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_17_cast_fp16 = mul(x = rec21, y = var_894_cast_fp16)[name = string("state_17_cast_fp16")];
            tensor<int32, [4]> key_token_9_begin_0 = const()[name = string("key_token_9_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> key_token_9_end_0 = const()[name = string("key_token_9_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> key_token_9_end_mask_0 = const()[name = string("key_token_9_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_9_squeeze_mask_0 = const()[name = string("key_token_9_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 4, 128]> key_7_cast_fp16 = transpose(perm = key_7_perm_0, x = var_868_cast_fp16)[name = string("transpose_16")];
            tensor<fp16, [1, 16, 128]> key_token_9_cast_fp16 = slice_by_index(begin = key_token_9_begin_0, end = key_token_9_end_0, end_mask = key_token_9_end_mask_0, squeeze_mask = key_token_9_squeeze_mask_0, x = key_7_cast_fp16)[name = string("key_token_9_cast_fp16")];
            tensor<int32, [4]> value_token_9_begin_0 = const()[name = string("value_token_9_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> value_token_9_end_0 = const()[name = string("value_token_9_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> value_token_9_end_mask_0 = const()[name = string("value_token_9_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_9_squeeze_mask_0 = const()[name = string("value_token_9_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 4, 128]> value_27_cast_fp16 = transpose(perm = value_27_perm_0, x = value_25_cast_fp16)[name = string("transpose_15")];
            tensor<fp16, [1, 16, 128]> value_token_9_cast_fp16 = slice_by_index(begin = value_token_9_begin_0, end = value_token_9_end_0, end_mask = value_token_9_end_mask_0, squeeze_mask = value_token_9_squeeze_mask_0, x = value_27_cast_fp16)[name = string("value_token_9_cast_fp16")];
            tensor<int32, [1]> var_902_axes_0 = const()[name = string("op_902_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_902_cast_fp16 = expand_dims(axes = var_902_axes_0, x = key_token_9_cast_fp16)[name = string("op_902_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_903_cast_fp16 = mul(x = state_17_cast_fp16, y = var_902_cast_fp16)[name = string("op_903_cast_fp16")];
            tensor<int32, [1]> memory_9_axes_0 = const()[name = string("memory_9_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_9_keep_dims_0 = const()[name = string("memory_9_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_9_cast_fp16 = reduce_sum(axes = memory_9_axes_0, keep_dims = memory_9_keep_dims_0, x = var_903_cast_fp16)[name = string("memory_9_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_906_cast_fp16 = sub(x = value_token_9_cast_fp16, y = memory_9_cast_fp16)[name = string("op_906_cast_fp16")];
            tensor<int32, [3]> var_909_begin_0 = const()[name = string("op_909_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_909_end_0 = const()[name = string("op_909_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_909_end_mask_0 = const()[name = string("op_909_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_909_squeeze_mask_0 = const()[name = string("op_909_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_909_cast_fp16 = slice_by_index(begin = var_909_begin_0, end = var_909_end_0, end_mask = var_909_end_mask_0, squeeze_mask = var_909_squeeze_mask_0, x = beta_3_cast_fp16)[name = string("op_909_cast_fp16")];
            tensor<int32, [1]> var_910_axes_0 = const()[name = string("op_910_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_910_cast_fp16 = expand_dims(axes = var_910_axes_0, x = var_909_cast_fp16)[name = string("op_910_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_9_cast_fp16 = mul(x = var_906_cast_fp16, y = var_910_cast_fp16)[name = string("delta_9_cast_fp16")];
            tensor<int32, [1]> var_913_axes_0 = const()[name = string("op_913_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_913_cast_fp16 = expand_dims(axes = var_913_axes_0, x = delta_9_cast_fp16)[name = string("op_913_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_914_cast_fp16 = mul(x = var_902_cast_fp16, y = var_913_cast_fp16)[name = string("op_914_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_19_cast_fp16 = add(x = state_17_cast_fp16, y = var_914_cast_fp16)[name = string("state_19_cast_fp16")];
            tensor<int32, [4]> var_918_begin_0 = const()[name = string("op_918_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> var_918_end_0 = const()[name = string("op_918_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> var_918_end_mask_0 = const()[name = string("op_918_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_918_squeeze_mask_0 = const()[name = string("op_918_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_918_cast_fp16 = slice_by_index(begin = var_918_begin_0, end = var_918_end_0, end_mask = var_918_end_mask_0, squeeze_mask = var_918_squeeze_mask_0, x = _inversed_query_7_cast_fp16)[name = string("op_918_cast_fp16")];
            tensor<int32, [1]> var_919_axes_0 = const()[name = string("op_919_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_919_cast_fp16 = expand_dims(axes = var_919_axes_0, x = var_918_cast_fp16)[name = string("op_919_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_920_cast_fp16 = mul(x = state_19_cast_fp16, y = var_919_cast_fp16)[name = string("op_920_cast_fp16")];
            tensor<int32, [1]> var_922_axes_0 = const()[name = string("op_922_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_922_keep_dims_0 = const()[name = string("op_922_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_922_cast_fp16 = reduce_sum(axes = var_922_axes_0, keep_dims = var_922_keep_dims_0, x = var_920_cast_fp16)[name = string("op_922_cast_fp16")];
            tensor<int32, [3]> var_925_begin_0 = const()[name = string("op_925_begin_0"), val = tensor<int32, [3]>([0, 0, 1])];
            tensor<int32, [3]> var_925_end_0 = const()[name = string("op_925_end_0"), val = tensor<int32, [3]>([1, 16, 2])];
            tensor<bool, [3]> var_925_end_mask_0 = const()[name = string("op_925_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_925_squeeze_mask_0 = const()[name = string("op_925_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_925_cast_fp16 = slice_by_index(begin = var_925_begin_0, end = var_925_end_0, end_mask = var_925_end_mask_0, squeeze_mask = var_925_squeeze_mask_0, x = decay_3_cast_fp16)[name = string("op_925_cast_fp16")];
            tensor<fp16, [1, 16]> var_926_cast_fp16 = exp(x = var_925_cast_fp16)[name = string("op_926_cast_fp16")];
            tensor<int32, [1]> var_927_axes_0 = const()[name = string("op_927_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_927_cast_fp16 = expand_dims(axes = var_927_axes_0, x = var_926_cast_fp16)[name = string("op_927_cast_fp16")];
            tensor<int32, [1]> var_928_axes_0 = const()[name = string("op_928_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_928_cast_fp16 = expand_dims(axes = var_928_axes_0, x = var_927_cast_fp16)[name = string("op_928_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_21_cast_fp16 = mul(x = state_19_cast_fp16, y = var_928_cast_fp16)[name = string("state_21_cast_fp16")];
            tensor<int32, [4]> key_token_11_begin_0 = const()[name = string("key_token_11_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> key_token_11_end_0 = const()[name = string("key_token_11_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> key_token_11_end_mask_0 = const()[name = string("key_token_11_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_11_squeeze_mask_0 = const()[name = string("key_token_11_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_11_cast_fp16 = slice_by_index(begin = key_token_11_begin_0, end = key_token_11_end_0, end_mask = key_token_11_end_mask_0, squeeze_mask = key_token_11_squeeze_mask_0, x = key_7_cast_fp16)[name = string("key_token_11_cast_fp16")];
            tensor<int32, [4]> value_token_11_begin_0 = const()[name = string("value_token_11_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> value_token_11_end_0 = const()[name = string("value_token_11_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> value_token_11_end_mask_0 = const()[name = string("value_token_11_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_11_squeeze_mask_0 = const()[name = string("value_token_11_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_11_cast_fp16 = slice_by_index(begin = value_token_11_begin_0, end = value_token_11_end_0, end_mask = value_token_11_end_mask_0, squeeze_mask = value_token_11_squeeze_mask_0, x = value_27_cast_fp16)[name = string("value_token_11_cast_fp16")];
            tensor<int32, [1]> var_936_axes_0 = const()[name = string("op_936_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_936_cast_fp16 = expand_dims(axes = var_936_axes_0, x = key_token_11_cast_fp16)[name = string("op_936_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_937_cast_fp16 = mul(x = state_21_cast_fp16, y = var_936_cast_fp16)[name = string("op_937_cast_fp16")];
            tensor<int32, [1]> memory_11_axes_0 = const()[name = string("memory_11_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_11_keep_dims_0 = const()[name = string("memory_11_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_11_cast_fp16 = reduce_sum(axes = memory_11_axes_0, keep_dims = memory_11_keep_dims_0, x = var_937_cast_fp16)[name = string("memory_11_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_940_cast_fp16 = sub(x = value_token_11_cast_fp16, y = memory_11_cast_fp16)[name = string("op_940_cast_fp16")];
            tensor<int32, [3]> var_943_begin_0 = const()[name = string("op_943_begin_0"), val = tensor<int32, [3]>([0, 0, 1])];
            tensor<int32, [3]> var_943_end_0 = const()[name = string("op_943_end_0"), val = tensor<int32, [3]>([1, 16, 2])];
            tensor<bool, [3]> var_943_end_mask_0 = const()[name = string("op_943_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_943_squeeze_mask_0 = const()[name = string("op_943_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_943_cast_fp16 = slice_by_index(begin = var_943_begin_0, end = var_943_end_0, end_mask = var_943_end_mask_0, squeeze_mask = var_943_squeeze_mask_0, x = beta_3_cast_fp16)[name = string("op_943_cast_fp16")];
            tensor<int32, [1]> var_944_axes_0 = const()[name = string("op_944_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_944_cast_fp16 = expand_dims(axes = var_944_axes_0, x = var_943_cast_fp16)[name = string("op_944_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_11_cast_fp16 = mul(x = var_940_cast_fp16, y = var_944_cast_fp16)[name = string("delta_11_cast_fp16")];
            tensor<int32, [1]> var_947_axes_0 = const()[name = string("op_947_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_947_cast_fp16 = expand_dims(axes = var_947_axes_0, x = delta_11_cast_fp16)[name = string("op_947_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_948_cast_fp16 = mul(x = var_936_cast_fp16, y = var_947_cast_fp16)[name = string("op_948_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_23_cast_fp16 = add(x = state_21_cast_fp16, y = var_948_cast_fp16)[name = string("state_23_cast_fp16")];
            tensor<int32, [4]> var_952_begin_0 = const()[name = string("op_952_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> var_952_end_0 = const()[name = string("op_952_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> var_952_end_mask_0 = const()[name = string("op_952_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_952_squeeze_mask_0 = const()[name = string("op_952_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_952_cast_fp16 = slice_by_index(begin = var_952_begin_0, end = var_952_end_0, end_mask = var_952_end_mask_0, squeeze_mask = var_952_squeeze_mask_0, x = _inversed_query_7_cast_fp16)[name = string("op_952_cast_fp16")];
            tensor<int32, [1]> var_953_axes_0 = const()[name = string("op_953_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_953_cast_fp16 = expand_dims(axes = var_953_axes_0, x = var_952_cast_fp16)[name = string("op_953_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_954_cast_fp16 = mul(x = state_23_cast_fp16, y = var_953_cast_fp16)[name = string("op_954_cast_fp16")];
            tensor<int32, [1]> var_956_axes_0 = const()[name = string("op_956_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_956_keep_dims_0 = const()[name = string("op_956_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_956_cast_fp16 = reduce_sum(axes = var_956_axes_0, keep_dims = var_956_keep_dims_0, x = var_954_cast_fp16)[name = string("op_956_cast_fp16")];
            tensor<int32, [3]> var_959_begin_0 = const()[name = string("op_959_begin_0"), val = tensor<int32, [3]>([0, 0, 2])];
            tensor<int32, [3]> var_959_end_0 = const()[name = string("op_959_end_0"), val = tensor<int32, [3]>([1, 16, 3])];
            tensor<bool, [3]> var_959_end_mask_0 = const()[name = string("op_959_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_959_squeeze_mask_0 = const()[name = string("op_959_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_959_cast_fp16 = slice_by_index(begin = var_959_begin_0, end = var_959_end_0, end_mask = var_959_end_mask_0, squeeze_mask = var_959_squeeze_mask_0, x = decay_3_cast_fp16)[name = string("op_959_cast_fp16")];
            tensor<fp16, [1, 16]> var_960_cast_fp16 = exp(x = var_959_cast_fp16)[name = string("op_960_cast_fp16")];
            tensor<int32, [1]> var_961_axes_0 = const()[name = string("op_961_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_961_cast_fp16 = expand_dims(axes = var_961_axes_0, x = var_960_cast_fp16)[name = string("op_961_cast_fp16")];
            tensor<int32, [1]> var_962_axes_0 = const()[name = string("op_962_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_962_cast_fp16 = expand_dims(axes = var_962_axes_0, x = var_961_cast_fp16)[name = string("op_962_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_25_cast_fp16 = mul(x = state_23_cast_fp16, y = var_962_cast_fp16)[name = string("state_25_cast_fp16")];
            tensor<int32, [4]> key_token_13_begin_0 = const()[name = string("key_token_13_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> key_token_13_end_0 = const()[name = string("key_token_13_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> key_token_13_end_mask_0 = const()[name = string("key_token_13_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_13_squeeze_mask_0 = const()[name = string("key_token_13_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_13_cast_fp16 = slice_by_index(begin = key_token_13_begin_0, end = key_token_13_end_0, end_mask = key_token_13_end_mask_0, squeeze_mask = key_token_13_squeeze_mask_0, x = key_7_cast_fp16)[name = string("key_token_13_cast_fp16")];
            tensor<int32, [4]> value_token_13_begin_0 = const()[name = string("value_token_13_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> value_token_13_end_0 = const()[name = string("value_token_13_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> value_token_13_end_mask_0 = const()[name = string("value_token_13_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_13_squeeze_mask_0 = const()[name = string("value_token_13_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_13_cast_fp16 = slice_by_index(begin = value_token_13_begin_0, end = value_token_13_end_0, end_mask = value_token_13_end_mask_0, squeeze_mask = value_token_13_squeeze_mask_0, x = value_27_cast_fp16)[name = string("value_token_13_cast_fp16")];
            tensor<int32, [1]> var_970_axes_0 = const()[name = string("op_970_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_970_cast_fp16 = expand_dims(axes = var_970_axes_0, x = key_token_13_cast_fp16)[name = string("op_970_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_971_cast_fp16 = mul(x = state_25_cast_fp16, y = var_970_cast_fp16)[name = string("op_971_cast_fp16")];
            tensor<int32, [1]> memory_13_axes_0 = const()[name = string("memory_13_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_13_keep_dims_0 = const()[name = string("memory_13_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_13_cast_fp16 = reduce_sum(axes = memory_13_axes_0, keep_dims = memory_13_keep_dims_0, x = var_971_cast_fp16)[name = string("memory_13_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_974_cast_fp16 = sub(x = value_token_13_cast_fp16, y = memory_13_cast_fp16)[name = string("op_974_cast_fp16")];
            tensor<int32, [3]> var_977_begin_0 = const()[name = string("op_977_begin_0"), val = tensor<int32, [3]>([0, 0, 2])];
            tensor<int32, [3]> var_977_end_0 = const()[name = string("op_977_end_0"), val = tensor<int32, [3]>([1, 16, 3])];
            tensor<bool, [3]> var_977_end_mask_0 = const()[name = string("op_977_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_977_squeeze_mask_0 = const()[name = string("op_977_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_977_cast_fp16 = slice_by_index(begin = var_977_begin_0, end = var_977_end_0, end_mask = var_977_end_mask_0, squeeze_mask = var_977_squeeze_mask_0, x = beta_3_cast_fp16)[name = string("op_977_cast_fp16")];
            tensor<int32, [1]> var_978_axes_0 = const()[name = string("op_978_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_978_cast_fp16 = expand_dims(axes = var_978_axes_0, x = var_977_cast_fp16)[name = string("op_978_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_13_cast_fp16 = mul(x = var_974_cast_fp16, y = var_978_cast_fp16)[name = string("delta_13_cast_fp16")];
            tensor<int32, [1]> var_981_axes_0 = const()[name = string("op_981_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_981_cast_fp16 = expand_dims(axes = var_981_axes_0, x = delta_13_cast_fp16)[name = string("op_981_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_982_cast_fp16 = mul(x = var_970_cast_fp16, y = var_981_cast_fp16)[name = string("op_982_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_27_cast_fp16 = add(x = state_25_cast_fp16, y = var_982_cast_fp16)[name = string("state_27_cast_fp16")];
            tensor<int32, [4]> var_986_begin_0 = const()[name = string("op_986_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> var_986_end_0 = const()[name = string("op_986_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> var_986_end_mask_0 = const()[name = string("op_986_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_986_squeeze_mask_0 = const()[name = string("op_986_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_986_cast_fp16 = slice_by_index(begin = var_986_begin_0, end = var_986_end_0, end_mask = var_986_end_mask_0, squeeze_mask = var_986_squeeze_mask_0, x = _inversed_query_7_cast_fp16)[name = string("op_986_cast_fp16")];
            tensor<int32, [1]> var_987_axes_0 = const()[name = string("op_987_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_987_cast_fp16 = expand_dims(axes = var_987_axes_0, x = var_986_cast_fp16)[name = string("op_987_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_988_cast_fp16 = mul(x = state_27_cast_fp16, y = var_987_cast_fp16)[name = string("op_988_cast_fp16")];
            tensor<int32, [1]> var_990_axes_0 = const()[name = string("op_990_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_990_keep_dims_0 = const()[name = string("op_990_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_990_cast_fp16 = reduce_sum(axes = var_990_axes_0, keep_dims = var_990_keep_dims_0, x = var_988_cast_fp16)[name = string("op_990_cast_fp16")];
            tensor<int32, [3]> var_993_begin_0 = const()[name = string("op_993_begin_0"), val = tensor<int32, [3]>([0, 0, 3])];
            tensor<int32, [3]> var_993_end_0 = const()[name = string("op_993_end_0"), val = tensor<int32, [3]>([1, 16, 4])];
            tensor<bool, [3]> var_993_end_mask_0 = const()[name = string("op_993_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_993_squeeze_mask_0 = const()[name = string("op_993_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_993_cast_fp16 = slice_by_index(begin = var_993_begin_0, end = var_993_end_0, end_mask = var_993_end_mask_0, squeeze_mask = var_993_squeeze_mask_0, x = decay_3_cast_fp16)[name = string("op_993_cast_fp16")];
            tensor<fp16, [1, 16]> var_994_cast_fp16 = exp(x = var_993_cast_fp16)[name = string("op_994_cast_fp16")];
            tensor<int32, [1]> var_995_axes_0 = const()[name = string("op_995_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_995_cast_fp16 = expand_dims(axes = var_995_axes_0, x = var_994_cast_fp16)[name = string("op_995_cast_fp16")];
            tensor<int32, [1]> var_996_axes_0 = const()[name = string("op_996_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_996_cast_fp16 = expand_dims(axes = var_996_axes_0, x = var_995_cast_fp16)[name = string("op_996_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_29_cast_fp16 = mul(x = state_27_cast_fp16, y = var_996_cast_fp16)[name = string("state_29_cast_fp16")];
            tensor<int32, [4]> key_token_15_begin_0 = const()[name = string("key_token_15_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> key_token_15_end_0 = const()[name = string("key_token_15_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> key_token_15_end_mask_0 = const()[name = string("key_token_15_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_15_squeeze_mask_0 = const()[name = string("key_token_15_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_15_cast_fp16 = slice_by_index(begin = key_token_15_begin_0, end = key_token_15_end_0, end_mask = key_token_15_end_mask_0, squeeze_mask = key_token_15_squeeze_mask_0, x = key_7_cast_fp16)[name = string("key_token_15_cast_fp16")];
            tensor<int32, [4]> value_token_15_begin_0 = const()[name = string("value_token_15_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> value_token_15_end_0 = const()[name = string("value_token_15_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> value_token_15_end_mask_0 = const()[name = string("value_token_15_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_15_squeeze_mask_0 = const()[name = string("value_token_15_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_15_cast_fp16 = slice_by_index(begin = value_token_15_begin_0, end = value_token_15_end_0, end_mask = value_token_15_end_mask_0, squeeze_mask = value_token_15_squeeze_mask_0, x = value_27_cast_fp16)[name = string("value_token_15_cast_fp16")];
            tensor<int32, [1]> var_1004_axes_0 = const()[name = string("op_1004_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1004_cast_fp16 = expand_dims(axes = var_1004_axes_0, x = key_token_15_cast_fp16)[name = string("op_1004_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1005_cast_fp16 = mul(x = state_29_cast_fp16, y = var_1004_cast_fp16)[name = string("op_1005_cast_fp16")];
            tensor<int32, [1]> memory_15_axes_0 = const()[name = string("memory_15_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_15_keep_dims_0 = const()[name = string("memory_15_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_15_cast_fp16 = reduce_sum(axes = memory_15_axes_0, keep_dims = memory_15_keep_dims_0, x = var_1005_cast_fp16)[name = string("memory_15_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_1008_cast_fp16 = sub(x = value_token_15_cast_fp16, y = memory_15_cast_fp16)[name = string("op_1008_cast_fp16")];
            tensor<int32, [3]> var_1011_begin_0 = const()[name = string("op_1011_begin_0"), val = tensor<int32, [3]>([0, 0, 3])];
            tensor<int32, [3]> var_1011_end_0 = const()[name = string("op_1011_end_0"), val = tensor<int32, [3]>([1, 16, 4])];
            tensor<bool, [3]> var_1011_end_mask_0 = const()[name = string("op_1011_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1011_squeeze_mask_0 = const()[name = string("op_1011_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1011_cast_fp16 = slice_by_index(begin = var_1011_begin_0, end = var_1011_end_0, end_mask = var_1011_end_mask_0, squeeze_mask = var_1011_squeeze_mask_0, x = beta_3_cast_fp16)[name = string("op_1011_cast_fp16")];
            tensor<int32, [1]> var_1012_axes_0 = const()[name = string("op_1012_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1012_cast_fp16 = expand_dims(axes = var_1012_axes_0, x = var_1011_cast_fp16)[name = string("op_1012_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_15_cast_fp16 = mul(x = var_1008_cast_fp16, y = var_1012_cast_fp16)[name = string("delta_15_cast_fp16")];
            tensor<int32, [1]> var_1015_axes_0 = const()[name = string("op_1015_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_1015_cast_fp16 = expand_dims(axes = var_1015_axes_0, x = delta_15_cast_fp16)[name = string("op_1015_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1016_cast_fp16 = mul(x = var_1004_cast_fp16, y = var_1015_cast_fp16)[name = string("op_1016_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> rec21_out = add(x = state_29_cast_fp16, y = var_1016_cast_fp16)[name = string("state_31_cast_fp16")];
            tensor<int32, [4]> var_1020_begin_0 = const()[name = string("op_1020_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> var_1020_end_0 = const()[name = string("op_1020_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> var_1020_end_mask_0 = const()[name = string("op_1020_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_1020_squeeze_mask_0 = const()[name = string("op_1020_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_1020_cast_fp16 = slice_by_index(begin = var_1020_begin_0, end = var_1020_end_0, end_mask = var_1020_end_mask_0, squeeze_mask = var_1020_squeeze_mask_0, x = _inversed_query_7_cast_fp16)[name = string("op_1020_cast_fp16")];
            tensor<int32, [1]> var_1021_axes_0 = const()[name = string("op_1021_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1021_cast_fp16 = expand_dims(axes = var_1021_axes_0, x = var_1020_cast_fp16)[name = string("op_1021_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1022_cast_fp16 = mul(x = rec21_out, y = var_1021_cast_fp16)[name = string("op_1022_cast_fp16")];
            tensor<int32, [1]> var_1024_axes_0 = const()[name = string("op_1024_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_1024_keep_dims_0 = const()[name = string("op_1024_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_1024_cast_fp16 = reduce_sum(axes = var_1024_axes_0, keep_dims = var_1024_keep_dims_0, x = var_1022_cast_fp16)[name = string("op_1024_cast_fp16")];
            int32 attention_11_axis_0 = const()[name = string("attention_11_axis_0"), val = int32(1)];
            tensor<fp16, [1, 4, 16, 128]> attention_11_cast_fp16 = stack(axis = attention_11_axis_0, values = (var_922_cast_fp16, var_956_cast_fp16, var_990_cast_fp16, var_1024_cast_fp16))[name = string("attention_11_cast_fp16")];
            tensor<int32, [2]> var_1027 = const()[name = string("op_1027"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [64, 128]> attention_13_cast_fp16 = reshape(shape = var_1027, x = attention_11_cast_fp16)[name = string("attention_13_cast_fp16")];
            tensor<int32, [2]> var_1029 = const()[name = string("op_1029"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [64, 128]> input_27_cast_fp16 = reshape(shape = var_1029, x = linear_15_cast_fp16)[name = string("input_27_cast_fp16")];
            tensor<fp16, [64, 128]> var_1031_cast_fp16 = mul(x = attention_13_cast_fp16, y = attention_13_cast_fp16)[name = string("op_1031_cast_fp16")];
            tensor<int32, [1]> variance_11_axes_0 = const()[name = string("variance_11_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_11_keep_dims_0 = const()[name = string("variance_11_keep_dims_0"), val = bool(true)];
            tensor<fp16, [64, 1]> variance_11_cast_fp16 = reduce_mean(axes = variance_11_axes_0, keep_dims = variance_11_keep_dims_0, x = var_1031_cast_fp16)[name = string("variance_11_cast_fp16")];
            fp16 var_1034_to_fp16 = const()[name = string("op_1034_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [64, 1]> var_1035_cast_fp16 = add(x = variance_11_cast_fp16, y = var_1034_to_fp16)[name = string("op_1035_cast_fp16")];
            fp32 var_1036_epsilon_0 = const()[name = string("op_1036_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [64, 1]> var_1036_cast_fp16 = rsqrt(epsilon = var_1036_epsilon_0, x = var_1035_cast_fp16)[name = string("op_1036_cast_fp16")];
            tensor<fp16, [64, 128]> attention_15_cast_fp16 = mul(x = attention_13_cast_fp16, y = var_1036_cast_fp16)[name = string("attention_15_cast_fp16")];
            tensor<fp16, [128]> layers20_1_gated_norm_promoted_to_fp16 = const()[name = string("layers20_1_gated_norm_promoted_to_fp16"), val = tensor<fp16, [128]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(225937856)))];
            tensor<fp16, [64, 128]> attention_17_cast_fp16 = mul(x = attention_15_cast_fp16, y = layers20_1_gated_norm_promoted_to_fp16)[name = string("attention_17_cast_fp16")];
            tensor<fp16, [64, 128]> var_1039_cast_fp16 = silu(x = input_27_cast_fp16)[name = string("op_1039_cast_fp16")];
            tensor<fp16, [64, 128]> attention_19_cast_fp16 = mul(x = attention_17_cast_fp16, y = var_1039_cast_fp16)[name = string("attention_19_cast_fp16")];
            tensor<int32, [2]> var_1041 = const()[name = string("op_1041"), val = tensor<int32, [2]>([4, 2048])];
            tensor<fp16, [4, 2048]> input_29_cast_fp16 = reshape(shape = var_1041, x = attention_19_cast_fp16)[name = string("input_29_cast_fp16")];
            tensor<fp16, [1024, 2048]> layers20_1_out_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 2048]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(225938176))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(227511104))))[name = string("layers20_1_out_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 1024]> linear_16_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = layers20_1_out_proj_weight_promoted_to_fp16_palettized, x = input_29_cast_fp16)[name = string("linear_16_cast_fp16")];
            tensor<fp16, [4, 1024]> value_29_cast_fp16 = add(x = value_17_cast_fp16, y = linear_16_cast_fp16)[name = string("value_29_cast_fp16")];
            tensor<fp16, [4, 1024]> var_1046_cast_fp16 = mul(x = value_29_cast_fp16, y = value_29_cast_fp16)[name = string("op_1046_cast_fp16")];
            tensor<int32, [1]> variance_13_axes_0 = const()[name = string("variance_13_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_13_keep_dims_0 = const()[name = string("variance_13_keep_dims_0"), val = bool(true)];
            tensor<fp16, [4, 1]> variance_13_cast_fp16 = reduce_mean(axes = variance_13_axes_0, keep_dims = variance_13_keep_dims_0, x = var_1046_cast_fp16)[name = string("variance_13_cast_fp16")];
            fp16 var_1049_to_fp16 = const()[name = string("op_1049_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [4, 1]> var_1050_cast_fp16 = add(x = variance_13_cast_fp16, y = var_1049_to_fp16)[name = string("op_1050_cast_fp16")];
            fp32 var_1051_epsilon_0 = const()[name = string("op_1051_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [4, 1]> var_1051_cast_fp16 = rsqrt(epsilon = var_1051_epsilon_0, x = var_1050_cast_fp16)[name = string("op_1051_cast_fp16")];
            tensor<fp16, [4, 1024]> var_1052_cast_fp16 = mul(x = value_29_cast_fp16, y = var_1051_cast_fp16)[name = string("op_1052_cast_fp16")];
            tensor<fp16, [1024]> var_1054_to_fp16 = const()[name = string("op_1054_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(227519360)))];
            tensor<fp16, [4, 1024]> input_31_cast_fp16 = mul(x = var_1052_cast_fp16, y = var_1054_to_fp16)[name = string("input_31_cast_fp16")];
            tensor<fp16, [3584, 1024]> layers20_1_gate_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(227521472))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(230274048))))[name = string("layers20_1_gate_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 3584]> linear_17_cast_fp16 = linear(bias = linear_1_bias_0_to_fp16, weight = layers20_1_gate_proj_weight_promoted_to_fp16_palettized, x = input_31_cast_fp16)[name = string("linear_17_cast_fp16")];
            tensor<fp16, [4, 3584]> var_1058_cast_fp16 = silu(x = linear_17_cast_fp16)[name = string("op_1058_cast_fp16")];
            tensor<fp16, [3584, 1024]> layers20_1_up_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(230302784))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(233055360))))[name = string("layers20_1_up_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 3584]> linear_18_cast_fp16 = linear(bias = linear_1_bias_0_to_fp16, weight = layers20_1_up_proj_weight_promoted_to_fp16_palettized, x = input_31_cast_fp16)[name = string("linear_18_cast_fp16")];
            tensor<fp16, [4, 3584]> input_35_cast_fp16 = mul(x = var_1058_cast_fp16, y = linear_18_cast_fp16)[name = string("input_35_cast_fp16")];
            tensor<fp16, [1024, 3584]> layers20_1_down_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 3584]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(233084096))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(235836672))))[name = string("layers20_1_down_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 1024]> linear_19_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = layers20_1_down_proj_weight_promoted_to_fp16_palettized, x = input_35_cast_fp16)[name = string("linear_19_cast_fp16")];
            tensor<fp16, [4, 1024]> value_31_cast_fp16 = add(x = value_29_cast_fp16, y = linear_19_cast_fp16)[name = string("value_31_cast_fp16")];
            int32 var_1085 = const()[name = string("op_1085"), val = int32(-1)];
            tensor<fp16, [4, 1024]> var_1100_cast_fp16 = mul(x = value_31_cast_fp16, y = value_31_cast_fp16)[name = string("op_1100_cast_fp16")];
            tensor<int32, [1]> variance_15_axes_0 = const()[name = string("variance_15_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_15_keep_dims_0 = const()[name = string("variance_15_keep_dims_0"), val = bool(true)];
            tensor<fp16, [4, 1]> variance_15_cast_fp16 = reduce_mean(axes = variance_15_axes_0, keep_dims = variance_15_keep_dims_0, x = var_1100_cast_fp16)[name = string("variance_15_cast_fp16")];
            fp16 var_1103_to_fp16 = const()[name = string("op_1103_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [4, 1]> var_1104_cast_fp16 = add(x = variance_15_cast_fp16, y = var_1103_to_fp16)[name = string("op_1104_cast_fp16")];
            fp32 var_1105_epsilon_0 = const()[name = string("op_1105_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [4, 1]> var_1105_cast_fp16 = rsqrt(epsilon = var_1105_epsilon_0, x = var_1104_cast_fp16)[name = string("op_1105_cast_fp16")];
            tensor<fp16, [4, 1024]> var_1106_cast_fp16 = mul(x = value_31_cast_fp16, y = var_1105_cast_fp16)[name = string("op_1106_cast_fp16")];
            tensor<fp16, [1024]> var_1108_to_fp16 = const()[name = string("op_1108_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(235844928)))];
            tensor<fp16, [4, 1024]> input_37_cast_fp16 = mul(x = var_1106_cast_fp16, y = var_1108_to_fp16)[name = string("input_37_cast_fp16")];
            tensor<fp16, [6144, 1024]> layers20_2_in_proj_qkv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(235847040))), lut = tensor<fp16, [384, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(240565696))))[name = string("layers20_2_in_proj_qkv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 6144]> linear_20_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = layers20_2_in_proj_qkv_weight_promoted_to_fp16_palettized, x = input_37_cast_fp16)[name = string("linear_20_cast_fp16")];
            tensor<int32, [2]> var_1112_perm_0 = const()[name = string("op_1112_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> mixed_axes_0 = const()[name = string("mixed_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [6144, 4]> var_1112_cast_fp16 = transpose(perm = var_1112_perm_0, x = linear_20_cast_fp16)[name = string("transpose_14")];
            tensor<fp16, [1, 6144, 4]> mixed_cast_fp16 = expand_dims(axes = mixed_axes_0, x = var_1112_cast_fp16)[name = string("mixed_cast_fp16")];
            bool convolution_input_interleave_0 = const()[name = string("convolution_input_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 6144, 7]> convolution_input_cast_fp16 = concat(axis = var_1085, interleave = convolution_input_interleave_0, values = (conv22, mixed_cast_fp16))[name = string("convolution_input_cast_fp16")];
            string input_39_pad_type_0 = const()[name = string("input_39_pad_type_0"), val = string("valid")];
            int32 input_39_groups_0 = const()[name = string("input_39_groups_0"), val = int32(6144)];
            tensor<int32, [1]> input_39_strides_0 = const()[name = string("input_39_strides_0"), val = tensor<int32, [1]>([1])];
            tensor<int32, [2]> input_39_pad_0 = const()[name = string("input_39_pad_0"), val = tensor<int32, [2]>([0, 0])];
            tensor<int32, [1]> input_39_dilations_0 = const()[name = string("input_39_dilations_0"), val = tensor<int32, [1]>([1])];
            tensor<fp16, [6144, 1, 4]> layers20_2_conv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1, 4]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(240614912))), lut = tensor<fp16, [384, 1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(240633408))))[name = string("layers20_2_conv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 6144, 4]> input_39_cast_fp16 = conv(dilations = input_39_dilations_0, groups = input_39_groups_0, pad = input_39_pad_0, pad_type = input_39_pad_type_0, strides = input_39_strides_0, weight = layers20_2_conv_weight_promoted_to_fp16_palettized, x = convolution_input_cast_fp16)[name = string("input_39_cast_fp16")];
            tensor<fp16, [1, 6144, 4]> var_1121_cast_fp16 = silu(x = input_39_cast_fp16)[name = string("op_1121_cast_fp16")];
            tensor<int32, [3]> var_1122_perm_0 = const()[name = string("op_1122_perm_0"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [3]> var_1125_begin_0 = const()[name = string("op_1125_begin_0"), val = tensor<int32, [3]>([0, 0, 4])];
            tensor<int32, [3]> var_1125_end_0 = const()[name = string("op_1125_end_0"), val = tensor<int32, [3]>([1, 6144, 7])];
            tensor<bool, [3]> var_1125_end_mask_0 = const()[name = string("op_1125_end_mask_0"), val = tensor<bool, [3]>([true, true, true])];
            tensor<fp16, [1, 6144, 3]> conv22_out = slice_by_index(begin = var_1125_begin_0, end = var_1125_end_0, end_mask = var_1125_end_mask_0, x = convolution_input_cast_fp16)[name = string("op_1125_cast_fp16")];
            tensor<int32, [3]> var_1126 = const()[name = string("op_1126"), val = tensor<int32, [3]>([2048, 2048, 2048])];
            int32 var_1127_axis_0 = const()[name = string("op_1127_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 4, 6144]> var_1122_cast_fp16 = transpose(perm = var_1122_perm_0, x = var_1121_cast_fp16)[name = string("transpose_13")];
            tensor<fp16, [1, 4, 2048]> var_1127_cast_fp16_0, tensor<fp16, [1, 4, 2048]> var_1127_cast_fp16_1, tensor<fp16, [1, 4, 2048]> var_1127_cast_fp16_2 = split(axis = var_1127_axis_0, split_sizes = var_1126, x = var_1122_cast_fp16)[name = string("op_1127_cast_fp16")];
            tensor<int32, [4]> var_1131 = const()[name = string("op_1131"), val = tensor<int32, [4]>([1, 4, 16, 128])];
            tensor<fp16, [1, 4, 16, 128]> value_35_cast_fp16 = reshape(shape = var_1131, x = var_1127_cast_fp16_0)[name = string("value_35_cast_fp16")];
            tensor<int32, [4]> var_1133 = const()[name = string("op_1133"), val = tensor<int32, [4]>([1, 4, 16, 128])];
            tensor<fp16, [1, 4, 16, 128]> value_37_cast_fp16 = reshape(shape = var_1133, x = var_1127_cast_fp16_1)[name = string("value_37_cast_fp16")];
            tensor<int32, [4]> var_1135 = const()[name = string("op_1135"), val = tensor<int32, [4]>([1, 4, 16, 128])];
            tensor<fp16, [1, 4, 16, 128]> value_39_cast_fp16 = reshape(shape = var_1135, x = var_1127_cast_fp16_2)[name = string("value_39_cast_fp16")];
            tensor<fp16, [1, 4, 16, 128]> var_1137_cast_fp16 = mul(x = value_35_cast_fp16, y = value_35_cast_fp16)[name = string("op_1137_cast_fp16")];
            tensor<int32, [1]> var_1139_axes_0 = const()[name = string("op_1139_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_1139_keep_dims_0 = const()[name = string("op_1139_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 4, 16, 1]> var_1139_cast_fp16 = reduce_sum(axes = var_1139_axes_0, keep_dims = var_1139_keep_dims_0, x = var_1137_cast_fp16)[name = string("op_1139_cast_fp16")];
            fp16 var_1140_to_fp16 = const()[name = string("op_1140_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 4, 16, 1]> var_1141_cast_fp16 = add(x = var_1139_cast_fp16, y = var_1140_to_fp16)[name = string("op_1141_cast_fp16")];
            fp32 var_1142_epsilon_0 = const()[name = string("op_1142_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 4, 16, 1]> var_1142_cast_fp16 = rsqrt(epsilon = var_1142_epsilon_0, x = var_1141_cast_fp16)[name = string("op_1142_cast_fp16")];
            tensor<fp16, [1, 4, 16, 128]> var_1143_cast_fp16 = mul(x = value_35_cast_fp16, y = var_1142_cast_fp16)[name = string("op_1143_cast_fp16")];
            tensor<int32, [4]> var_1144_perm_0 = const()[name = string("op_1144_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            fp16 _inversed_query_11_y_0_to_fp16 = const()[name = string("_inversed_query_11_y_0_to_fp16"), val = fp16(0x1.6ap-4)];
            tensor<fp16, [1, 16, 4, 128]> var_1144_cast_fp16 = transpose(perm = var_1144_perm_0, x = var_1143_cast_fp16)[name = string("transpose_12")];
            tensor<fp16, [1, 16, 4, 128]> _inversed_query_11_cast_fp16 = mul(x = var_1144_cast_fp16, y = _inversed_query_11_y_0_to_fp16)[name = string("_inversed_query_11_cast_fp16")];
            tensor<fp16, [1, 4, 16, 128]> var_1147_cast_fp16 = mul(x = value_37_cast_fp16, y = value_37_cast_fp16)[name = string("op_1147_cast_fp16")];
            tensor<int32, [1]> var_1149_axes_0 = const()[name = string("op_1149_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_1149_keep_dims_0 = const()[name = string("op_1149_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 4, 16, 1]> var_1149_cast_fp16 = reduce_sum(axes = var_1149_axes_0, keep_dims = var_1149_keep_dims_0, x = var_1147_cast_fp16)[name = string("op_1149_cast_fp16")];
            fp16 var_1150_to_fp16 = const()[name = string("op_1150_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 4, 16, 1]> var_1151_cast_fp16 = add(x = var_1149_cast_fp16, y = var_1150_to_fp16)[name = string("op_1151_cast_fp16")];
            fp32 var_1152_epsilon_0 = const()[name = string("op_1152_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 4, 16, 1]> var_1152_cast_fp16 = rsqrt(epsilon = var_1152_epsilon_0, x = var_1151_cast_fp16)[name = string("op_1152_cast_fp16")];
            tensor<fp16, [1, 4, 16, 128]> var_1153_cast_fp16 = mul(x = value_37_cast_fp16, y = var_1152_cast_fp16)[name = string("op_1153_cast_fp16")];
            tensor<int32, [4]> key_11_perm_0 = const()[name = string("key_11_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<int32, [4]> value_41_perm_0 = const()[name = string("value_41_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<fp16, [16, 1024]> layers20_2_in_proj_b_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(240682624))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(240694976))))[name = string("layers20_2_in_proj_b_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16]> linear_21_bias_0_to_fp16 = const()[name = string("linear_21_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(207971904)))];
            tensor<fp16, [4, 16]> linear_21_cast_fp16 = linear(bias = linear_21_bias_0_to_fp16, weight = layers20_2_in_proj_b_weight_promoted_to_fp16_palettized, x = input_37_cast_fp16)[name = string("linear_21_cast_fp16")];
            tensor<fp16, [4, 16]> var_1158_cast_fp16 = sigmoid(x = linear_21_cast_fp16)[name = string("op_1158_cast_fp16")];
            tensor<int32, [2]> var_1159_perm_0 = const()[name = string("op_1159_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> beta_axes_0 = const()[name = string("beta_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 4]> var_1159_cast_fp16 = transpose(perm = var_1159_perm_0, x = var_1158_cast_fp16)[name = string("transpose_11")];
            tensor<fp16, [1, 16, 4]> beta_cast_fp16 = expand_dims(axes = beta_axes_0, x = var_1159_cast_fp16)[name = string("beta_cast_fp16")];
            tensor<fp16, [16, 1024]> op_1165_weight_0_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(240695168))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(240707520))))[name = string("op_1165_weight_0_to_fp16_palettized")];
            tensor<fp16, [16]> var_1165_bias_0_to_fp16 = const()[name = string("op_1165_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(240707712)))];
            tensor<fp16, [4, 16]> var_1165_cast_fp16 = linear(bias = var_1165_bias_0_to_fp16, weight = op_1165_weight_0_to_fp16_palettized, x = input_37_cast_fp16)[name = string("op_1165_cast_fp16")];
            tensor<fp16, [4, 16]> var_1166_cast_fp16 = softplus(x = var_1165_cast_fp16)[name = string("op_1166_cast_fp16")];
            tensor<fp16, [16]> var_1162_promoted_to_fp16 = const()[name = string("op_1162_promoted_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(240707840)))];
            tensor<fp16, [4, 16]> var_1167_cast_fp16 = mul(x = var_1162_promoted_to_fp16, y = var_1166_cast_fp16)[name = string("op_1167_cast_fp16")];
            tensor<int32, [2]> var_1168_perm_0 = const()[name = string("op_1168_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> decay_axes_0 = const()[name = string("decay_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 4]> var_1168_cast_fp16 = transpose(perm = var_1168_perm_0, x = var_1167_cast_fp16)[name = string("transpose_10")];
            tensor<fp16, [1, 16, 4]> decay_cast_fp16 = expand_dims(axes = decay_axes_0, x = var_1168_cast_fp16)[name = string("decay_cast_fp16")];
            tensor<fp16, [2048, 1024]> layers20_2_in_proj_z_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [2048, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(240707968))), lut = tensor<fp16, [128, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(242280896))))[name = string("layers20_2_in_proj_z_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 2048]> linear_23_cast_fp16 = linear(bias = linear_7_bias_0_to_fp16, weight = layers20_2_in_proj_z_weight_promoted_to_fp16_palettized, x = input_37_cast_fp16)[name = string("linear_23_cast_fp16")];
            tensor<int32, [3]> var_1176_begin_0 = const()[name = string("op_1176_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_1176_end_0 = const()[name = string("op_1176_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_1176_end_mask_0 = const()[name = string("op_1176_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1176_squeeze_mask_0 = const()[name = string("op_1176_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1176_cast_fp16 = slice_by_index(begin = var_1176_begin_0, end = var_1176_end_0, end_mask = var_1176_end_mask_0, squeeze_mask = var_1176_squeeze_mask_0, x = decay_cast_fp16)[name = string("op_1176_cast_fp16")];
            tensor<fp16, [1, 16]> var_1177_cast_fp16 = exp(x = var_1176_cast_fp16)[name = string("op_1177_cast_fp16")];
            tensor<int32, [1]> var_1178_axes_0 = const()[name = string("op_1178_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1178_cast_fp16 = expand_dims(axes = var_1178_axes_0, x = var_1177_cast_fp16)[name = string("op_1178_cast_fp16")];
            tensor<int32, [1]> var_1179_axes_0 = const()[name = string("op_1179_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_1179_cast_fp16 = expand_dims(axes = var_1179_axes_0, x = var_1178_cast_fp16)[name = string("op_1179_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_33_cast_fp16 = mul(x = rec22, y = var_1179_cast_fp16)[name = string("state_33_cast_fp16")];
            tensor<int32, [4]> key_token_17_begin_0 = const()[name = string("key_token_17_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> key_token_17_end_0 = const()[name = string("key_token_17_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> key_token_17_end_mask_0 = const()[name = string("key_token_17_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_17_squeeze_mask_0 = const()[name = string("key_token_17_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 4, 128]> key_11_cast_fp16 = transpose(perm = key_11_perm_0, x = var_1153_cast_fp16)[name = string("transpose_9")];
            tensor<fp16, [1, 16, 128]> key_token_17_cast_fp16 = slice_by_index(begin = key_token_17_begin_0, end = key_token_17_end_0, end_mask = key_token_17_end_mask_0, squeeze_mask = key_token_17_squeeze_mask_0, x = key_11_cast_fp16)[name = string("key_token_17_cast_fp16")];
            tensor<int32, [4]> value_token_17_begin_0 = const()[name = string("value_token_17_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> value_token_17_end_0 = const()[name = string("value_token_17_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> value_token_17_end_mask_0 = const()[name = string("value_token_17_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_17_squeeze_mask_0 = const()[name = string("value_token_17_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 4, 128]> value_41_cast_fp16 = transpose(perm = value_41_perm_0, x = value_39_cast_fp16)[name = string("transpose_8")];
            tensor<fp16, [1, 16, 128]> value_token_17_cast_fp16 = slice_by_index(begin = value_token_17_begin_0, end = value_token_17_end_0, end_mask = value_token_17_end_mask_0, squeeze_mask = value_token_17_squeeze_mask_0, x = value_41_cast_fp16)[name = string("value_token_17_cast_fp16")];
            tensor<int32, [1]> var_1187_axes_0 = const()[name = string("op_1187_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1187_cast_fp16 = expand_dims(axes = var_1187_axes_0, x = key_token_17_cast_fp16)[name = string("op_1187_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1188_cast_fp16 = mul(x = state_33_cast_fp16, y = var_1187_cast_fp16)[name = string("op_1188_cast_fp16")];
            tensor<int32, [1]> memory_17_axes_0 = const()[name = string("memory_17_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_17_keep_dims_0 = const()[name = string("memory_17_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_17_cast_fp16 = reduce_sum(axes = memory_17_axes_0, keep_dims = memory_17_keep_dims_0, x = var_1188_cast_fp16)[name = string("memory_17_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_1191_cast_fp16 = sub(x = value_token_17_cast_fp16, y = memory_17_cast_fp16)[name = string("op_1191_cast_fp16")];
            tensor<int32, [3]> var_1194_begin_0 = const()[name = string("op_1194_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_1194_end_0 = const()[name = string("op_1194_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_1194_end_mask_0 = const()[name = string("op_1194_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1194_squeeze_mask_0 = const()[name = string("op_1194_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1194_cast_fp16 = slice_by_index(begin = var_1194_begin_0, end = var_1194_end_0, end_mask = var_1194_end_mask_0, squeeze_mask = var_1194_squeeze_mask_0, x = beta_cast_fp16)[name = string("op_1194_cast_fp16")];
            tensor<int32, [1]> var_1195_axes_0 = const()[name = string("op_1195_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1195_cast_fp16 = expand_dims(axes = var_1195_axes_0, x = var_1194_cast_fp16)[name = string("op_1195_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_17_cast_fp16 = mul(x = var_1191_cast_fp16, y = var_1195_cast_fp16)[name = string("delta_17_cast_fp16")];
            tensor<int32, [1]> var_1198_axes_0 = const()[name = string("op_1198_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_1198_cast_fp16 = expand_dims(axes = var_1198_axes_0, x = delta_17_cast_fp16)[name = string("op_1198_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1199_cast_fp16 = mul(x = var_1187_cast_fp16, y = var_1198_cast_fp16)[name = string("op_1199_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_35_cast_fp16 = add(x = state_33_cast_fp16, y = var_1199_cast_fp16)[name = string("state_35_cast_fp16")];
            tensor<int32, [4]> var_1203_begin_0 = const()[name = string("op_1203_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> var_1203_end_0 = const()[name = string("op_1203_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> var_1203_end_mask_0 = const()[name = string("op_1203_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_1203_squeeze_mask_0 = const()[name = string("op_1203_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_1203_cast_fp16 = slice_by_index(begin = var_1203_begin_0, end = var_1203_end_0, end_mask = var_1203_end_mask_0, squeeze_mask = var_1203_squeeze_mask_0, x = _inversed_query_11_cast_fp16)[name = string("op_1203_cast_fp16")];
            tensor<int32, [1]> var_1204_axes_0 = const()[name = string("op_1204_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1204_cast_fp16 = expand_dims(axes = var_1204_axes_0, x = var_1203_cast_fp16)[name = string("op_1204_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1205_cast_fp16 = mul(x = state_35_cast_fp16, y = var_1204_cast_fp16)[name = string("op_1205_cast_fp16")];
            tensor<int32, [1]> var_1207_axes_0 = const()[name = string("op_1207_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_1207_keep_dims_0 = const()[name = string("op_1207_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_1207_cast_fp16 = reduce_sum(axes = var_1207_axes_0, keep_dims = var_1207_keep_dims_0, x = var_1205_cast_fp16)[name = string("op_1207_cast_fp16")];
            tensor<int32, [3]> var_1210_begin_0 = const()[name = string("op_1210_begin_0"), val = tensor<int32, [3]>([0, 0, 1])];
            tensor<int32, [3]> var_1210_end_0 = const()[name = string("op_1210_end_0"), val = tensor<int32, [3]>([1, 16, 2])];
            tensor<bool, [3]> var_1210_end_mask_0 = const()[name = string("op_1210_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1210_squeeze_mask_0 = const()[name = string("op_1210_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1210_cast_fp16 = slice_by_index(begin = var_1210_begin_0, end = var_1210_end_0, end_mask = var_1210_end_mask_0, squeeze_mask = var_1210_squeeze_mask_0, x = decay_cast_fp16)[name = string("op_1210_cast_fp16")];
            tensor<fp16, [1, 16]> var_1211_cast_fp16 = exp(x = var_1210_cast_fp16)[name = string("op_1211_cast_fp16")];
            tensor<int32, [1]> var_1212_axes_0 = const()[name = string("op_1212_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1212_cast_fp16 = expand_dims(axes = var_1212_axes_0, x = var_1211_cast_fp16)[name = string("op_1212_cast_fp16")];
            tensor<int32, [1]> var_1213_axes_0 = const()[name = string("op_1213_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_1213_cast_fp16 = expand_dims(axes = var_1213_axes_0, x = var_1212_cast_fp16)[name = string("op_1213_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_37_cast_fp16 = mul(x = state_35_cast_fp16, y = var_1213_cast_fp16)[name = string("state_37_cast_fp16")];
            tensor<int32, [4]> key_token_19_begin_0 = const()[name = string("key_token_19_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> key_token_19_end_0 = const()[name = string("key_token_19_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> key_token_19_end_mask_0 = const()[name = string("key_token_19_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_19_squeeze_mask_0 = const()[name = string("key_token_19_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_19_cast_fp16 = slice_by_index(begin = key_token_19_begin_0, end = key_token_19_end_0, end_mask = key_token_19_end_mask_0, squeeze_mask = key_token_19_squeeze_mask_0, x = key_11_cast_fp16)[name = string("key_token_19_cast_fp16")];
            tensor<int32, [4]> value_token_19_begin_0 = const()[name = string("value_token_19_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> value_token_19_end_0 = const()[name = string("value_token_19_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> value_token_19_end_mask_0 = const()[name = string("value_token_19_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_19_squeeze_mask_0 = const()[name = string("value_token_19_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_19_cast_fp16 = slice_by_index(begin = value_token_19_begin_0, end = value_token_19_end_0, end_mask = value_token_19_end_mask_0, squeeze_mask = value_token_19_squeeze_mask_0, x = value_41_cast_fp16)[name = string("value_token_19_cast_fp16")];
            tensor<int32, [1]> var_1221_axes_0 = const()[name = string("op_1221_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1221_cast_fp16 = expand_dims(axes = var_1221_axes_0, x = key_token_19_cast_fp16)[name = string("op_1221_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1222_cast_fp16 = mul(x = state_37_cast_fp16, y = var_1221_cast_fp16)[name = string("op_1222_cast_fp16")];
            tensor<int32, [1]> memory_19_axes_0 = const()[name = string("memory_19_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_19_keep_dims_0 = const()[name = string("memory_19_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_19_cast_fp16 = reduce_sum(axes = memory_19_axes_0, keep_dims = memory_19_keep_dims_0, x = var_1222_cast_fp16)[name = string("memory_19_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_1225_cast_fp16 = sub(x = value_token_19_cast_fp16, y = memory_19_cast_fp16)[name = string("op_1225_cast_fp16")];
            tensor<int32, [3]> var_1228_begin_0 = const()[name = string("op_1228_begin_0"), val = tensor<int32, [3]>([0, 0, 1])];
            tensor<int32, [3]> var_1228_end_0 = const()[name = string("op_1228_end_0"), val = tensor<int32, [3]>([1, 16, 2])];
            tensor<bool, [3]> var_1228_end_mask_0 = const()[name = string("op_1228_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1228_squeeze_mask_0 = const()[name = string("op_1228_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1228_cast_fp16 = slice_by_index(begin = var_1228_begin_0, end = var_1228_end_0, end_mask = var_1228_end_mask_0, squeeze_mask = var_1228_squeeze_mask_0, x = beta_cast_fp16)[name = string("op_1228_cast_fp16")];
            tensor<int32, [1]> var_1229_axes_0 = const()[name = string("op_1229_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1229_cast_fp16 = expand_dims(axes = var_1229_axes_0, x = var_1228_cast_fp16)[name = string("op_1229_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_19_cast_fp16 = mul(x = var_1225_cast_fp16, y = var_1229_cast_fp16)[name = string("delta_19_cast_fp16")];
            tensor<int32, [1]> var_1232_axes_0 = const()[name = string("op_1232_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_1232_cast_fp16 = expand_dims(axes = var_1232_axes_0, x = delta_19_cast_fp16)[name = string("op_1232_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1233_cast_fp16 = mul(x = var_1221_cast_fp16, y = var_1232_cast_fp16)[name = string("op_1233_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_39_cast_fp16 = add(x = state_37_cast_fp16, y = var_1233_cast_fp16)[name = string("state_39_cast_fp16")];
            tensor<int32, [4]> var_1237_begin_0 = const()[name = string("op_1237_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
            tensor<int32, [4]> var_1237_end_0 = const()[name = string("op_1237_end_0"), val = tensor<int32, [4]>([1, 16, 2, 128])];
            tensor<bool, [4]> var_1237_end_mask_0 = const()[name = string("op_1237_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_1237_squeeze_mask_0 = const()[name = string("op_1237_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_1237_cast_fp16 = slice_by_index(begin = var_1237_begin_0, end = var_1237_end_0, end_mask = var_1237_end_mask_0, squeeze_mask = var_1237_squeeze_mask_0, x = _inversed_query_11_cast_fp16)[name = string("op_1237_cast_fp16")];
            tensor<int32, [1]> var_1238_axes_0 = const()[name = string("op_1238_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1238_cast_fp16 = expand_dims(axes = var_1238_axes_0, x = var_1237_cast_fp16)[name = string("op_1238_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1239_cast_fp16 = mul(x = state_39_cast_fp16, y = var_1238_cast_fp16)[name = string("op_1239_cast_fp16")];
            tensor<int32, [1]> var_1241_axes_0 = const()[name = string("op_1241_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_1241_keep_dims_0 = const()[name = string("op_1241_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_1241_cast_fp16 = reduce_sum(axes = var_1241_axes_0, keep_dims = var_1241_keep_dims_0, x = var_1239_cast_fp16)[name = string("op_1241_cast_fp16")];
            tensor<int32, [3]> var_1244_begin_0 = const()[name = string("op_1244_begin_0"), val = tensor<int32, [3]>([0, 0, 2])];
            tensor<int32, [3]> var_1244_end_0 = const()[name = string("op_1244_end_0"), val = tensor<int32, [3]>([1, 16, 3])];
            tensor<bool, [3]> var_1244_end_mask_0 = const()[name = string("op_1244_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1244_squeeze_mask_0 = const()[name = string("op_1244_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1244_cast_fp16 = slice_by_index(begin = var_1244_begin_0, end = var_1244_end_0, end_mask = var_1244_end_mask_0, squeeze_mask = var_1244_squeeze_mask_0, x = decay_cast_fp16)[name = string("op_1244_cast_fp16")];
            tensor<fp16, [1, 16]> var_1245_cast_fp16 = exp(x = var_1244_cast_fp16)[name = string("op_1245_cast_fp16")];
            tensor<int32, [1]> var_1246_axes_0 = const()[name = string("op_1246_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1246_cast_fp16 = expand_dims(axes = var_1246_axes_0, x = var_1245_cast_fp16)[name = string("op_1246_cast_fp16")];
            tensor<int32, [1]> var_1247_axes_0 = const()[name = string("op_1247_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_1247_cast_fp16 = expand_dims(axes = var_1247_axes_0, x = var_1246_cast_fp16)[name = string("op_1247_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_41_cast_fp16 = mul(x = state_39_cast_fp16, y = var_1247_cast_fp16)[name = string("state_41_cast_fp16")];
            tensor<int32, [4]> key_token_21_begin_0 = const()[name = string("key_token_21_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> key_token_21_end_0 = const()[name = string("key_token_21_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> key_token_21_end_mask_0 = const()[name = string("key_token_21_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_21_squeeze_mask_0 = const()[name = string("key_token_21_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_21_cast_fp16 = slice_by_index(begin = key_token_21_begin_0, end = key_token_21_end_0, end_mask = key_token_21_end_mask_0, squeeze_mask = key_token_21_squeeze_mask_0, x = key_11_cast_fp16)[name = string("key_token_21_cast_fp16")];
            tensor<int32, [4]> value_token_21_begin_0 = const()[name = string("value_token_21_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> value_token_21_end_0 = const()[name = string("value_token_21_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> value_token_21_end_mask_0 = const()[name = string("value_token_21_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_21_squeeze_mask_0 = const()[name = string("value_token_21_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_21_cast_fp16 = slice_by_index(begin = value_token_21_begin_0, end = value_token_21_end_0, end_mask = value_token_21_end_mask_0, squeeze_mask = value_token_21_squeeze_mask_0, x = value_41_cast_fp16)[name = string("value_token_21_cast_fp16")];
            tensor<int32, [1]> var_1255_axes_0 = const()[name = string("op_1255_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1255_cast_fp16 = expand_dims(axes = var_1255_axes_0, x = key_token_21_cast_fp16)[name = string("op_1255_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1256_cast_fp16 = mul(x = state_41_cast_fp16, y = var_1255_cast_fp16)[name = string("op_1256_cast_fp16")];
            tensor<int32, [1]> memory_21_axes_0 = const()[name = string("memory_21_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_21_keep_dims_0 = const()[name = string("memory_21_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_21_cast_fp16 = reduce_sum(axes = memory_21_axes_0, keep_dims = memory_21_keep_dims_0, x = var_1256_cast_fp16)[name = string("memory_21_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_1259_cast_fp16 = sub(x = value_token_21_cast_fp16, y = memory_21_cast_fp16)[name = string("op_1259_cast_fp16")];
            tensor<int32, [3]> var_1262_begin_0 = const()[name = string("op_1262_begin_0"), val = tensor<int32, [3]>([0, 0, 2])];
            tensor<int32, [3]> var_1262_end_0 = const()[name = string("op_1262_end_0"), val = tensor<int32, [3]>([1, 16, 3])];
            tensor<bool, [3]> var_1262_end_mask_0 = const()[name = string("op_1262_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1262_squeeze_mask_0 = const()[name = string("op_1262_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1262_cast_fp16 = slice_by_index(begin = var_1262_begin_0, end = var_1262_end_0, end_mask = var_1262_end_mask_0, squeeze_mask = var_1262_squeeze_mask_0, x = beta_cast_fp16)[name = string("op_1262_cast_fp16")];
            tensor<int32, [1]> var_1263_axes_0 = const()[name = string("op_1263_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1263_cast_fp16 = expand_dims(axes = var_1263_axes_0, x = var_1262_cast_fp16)[name = string("op_1263_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_21_cast_fp16 = mul(x = var_1259_cast_fp16, y = var_1263_cast_fp16)[name = string("delta_21_cast_fp16")];
            tensor<int32, [1]> var_1266_axes_0 = const()[name = string("op_1266_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_1266_cast_fp16 = expand_dims(axes = var_1266_axes_0, x = delta_21_cast_fp16)[name = string("op_1266_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1267_cast_fp16 = mul(x = var_1255_cast_fp16, y = var_1266_cast_fp16)[name = string("op_1267_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_43_cast_fp16 = add(x = state_41_cast_fp16, y = var_1267_cast_fp16)[name = string("state_43_cast_fp16")];
            tensor<int32, [4]> var_1271_begin_0 = const()[name = string("op_1271_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
            tensor<int32, [4]> var_1271_end_0 = const()[name = string("op_1271_end_0"), val = tensor<int32, [4]>([1, 16, 3, 128])];
            tensor<bool, [4]> var_1271_end_mask_0 = const()[name = string("op_1271_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_1271_squeeze_mask_0 = const()[name = string("op_1271_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_1271_cast_fp16 = slice_by_index(begin = var_1271_begin_0, end = var_1271_end_0, end_mask = var_1271_end_mask_0, squeeze_mask = var_1271_squeeze_mask_0, x = _inversed_query_11_cast_fp16)[name = string("op_1271_cast_fp16")];
            tensor<int32, [1]> var_1272_axes_0 = const()[name = string("op_1272_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1272_cast_fp16 = expand_dims(axes = var_1272_axes_0, x = var_1271_cast_fp16)[name = string("op_1272_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1273_cast_fp16 = mul(x = state_43_cast_fp16, y = var_1272_cast_fp16)[name = string("op_1273_cast_fp16")];
            tensor<int32, [1]> var_1275_axes_0 = const()[name = string("op_1275_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_1275_keep_dims_0 = const()[name = string("op_1275_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_1275_cast_fp16 = reduce_sum(axes = var_1275_axes_0, keep_dims = var_1275_keep_dims_0, x = var_1273_cast_fp16)[name = string("op_1275_cast_fp16")];
            tensor<int32, [3]> var_1278_begin_0 = const()[name = string("op_1278_begin_0"), val = tensor<int32, [3]>([0, 0, 3])];
            tensor<int32, [3]> var_1278_end_0 = const()[name = string("op_1278_end_0"), val = tensor<int32, [3]>([1, 16, 4])];
            tensor<bool, [3]> var_1278_end_mask_0 = const()[name = string("op_1278_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1278_squeeze_mask_0 = const()[name = string("op_1278_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1278_cast_fp16 = slice_by_index(begin = var_1278_begin_0, end = var_1278_end_0, end_mask = var_1278_end_mask_0, squeeze_mask = var_1278_squeeze_mask_0, x = decay_cast_fp16)[name = string("op_1278_cast_fp16")];
            tensor<fp16, [1, 16]> var_1279_cast_fp16 = exp(x = var_1278_cast_fp16)[name = string("op_1279_cast_fp16")];
            tensor<int32, [1]> var_1280_axes_0 = const()[name = string("op_1280_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1280_cast_fp16 = expand_dims(axes = var_1280_axes_0, x = var_1279_cast_fp16)[name = string("op_1280_cast_fp16")];
            tensor<int32, [1]> var_1281_axes_0 = const()[name = string("op_1281_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_1281_cast_fp16 = expand_dims(axes = var_1281_axes_0, x = var_1280_cast_fp16)[name = string("op_1281_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_45_cast_fp16 = mul(x = state_43_cast_fp16, y = var_1281_cast_fp16)[name = string("state_45_cast_fp16")];
            tensor<int32, [4]> key_token_begin_0 = const()[name = string("key_token_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> key_token_end_0 = const()[name = string("key_token_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> key_token_end_mask_0 = const()[name = string("key_token_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_squeeze_mask_0 = const()[name = string("key_token_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> key_token_cast_fp16 = slice_by_index(begin = key_token_begin_0, end = key_token_end_0, end_mask = key_token_end_mask_0, squeeze_mask = key_token_squeeze_mask_0, x = key_11_cast_fp16)[name = string("key_token_cast_fp16")];
            tensor<int32, [4]> value_token_begin_0 = const()[name = string("value_token_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> value_token_end_0 = const()[name = string("value_token_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> value_token_end_mask_0 = const()[name = string("value_token_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_squeeze_mask_0 = const()[name = string("value_token_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> value_token_cast_fp16 = slice_by_index(begin = value_token_begin_0, end = value_token_end_0, end_mask = value_token_end_mask_0, squeeze_mask = value_token_squeeze_mask_0, x = value_41_cast_fp16)[name = string("value_token_cast_fp16")];
            tensor<int32, [1]> var_1289_axes_0 = const()[name = string("op_1289_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1289_cast_fp16 = expand_dims(axes = var_1289_axes_0, x = key_token_cast_fp16)[name = string("op_1289_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1290_cast_fp16 = mul(x = state_45_cast_fp16, y = var_1289_cast_fp16)[name = string("op_1290_cast_fp16")];
            tensor<int32, [1]> memory_axes_0 = const()[name = string("memory_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_keep_dims_0 = const()[name = string("memory_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_cast_fp16 = reduce_sum(axes = memory_axes_0, keep_dims = memory_keep_dims_0, x = var_1290_cast_fp16)[name = string("memory_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_1293_cast_fp16 = sub(x = value_token_cast_fp16, y = memory_cast_fp16)[name = string("op_1293_cast_fp16")];
            tensor<int32, [3]> var_1296_begin_0 = const()[name = string("op_1296_begin_0"), val = tensor<int32, [3]>([0, 0, 3])];
            tensor<int32, [3]> var_1296_end_0 = const()[name = string("op_1296_end_0"), val = tensor<int32, [3]>([1, 16, 4])];
            tensor<bool, [3]> var_1296_end_mask_0 = const()[name = string("op_1296_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_1296_squeeze_mask_0 = const()[name = string("op_1296_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_1296_cast_fp16 = slice_by_index(begin = var_1296_begin_0, end = var_1296_end_0, end_mask = var_1296_end_mask_0, squeeze_mask = var_1296_squeeze_mask_0, x = beta_cast_fp16)[name = string("op_1296_cast_fp16")];
            tensor<int32, [1]> var_1297_axes_0 = const()[name = string("op_1297_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_1297_cast_fp16 = expand_dims(axes = var_1297_axes_0, x = var_1296_cast_fp16)[name = string("op_1297_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_cast_fp16 = mul(x = var_1293_cast_fp16, y = var_1297_cast_fp16)[name = string("delta_cast_fp16")];
            tensor<int32, [1]> var_1300_axes_0 = const()[name = string("op_1300_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_1300_cast_fp16 = expand_dims(axes = var_1300_axes_0, x = delta_cast_fp16)[name = string("op_1300_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1301_cast_fp16 = mul(x = var_1289_cast_fp16, y = var_1300_cast_fp16)[name = string("op_1301_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> rec22_out = add(x = state_45_cast_fp16, y = var_1301_cast_fp16)[name = string("state_cast_fp16")];
            tensor<int32, [4]> var_1305_begin_0 = const()[name = string("op_1305_begin_0"), val = tensor<int32, [4]>([0, 0, 3, 0])];
            tensor<int32, [4]> var_1305_end_0 = const()[name = string("op_1305_end_0"), val = tensor<int32, [4]>([1, 16, 4, 128])];
            tensor<bool, [4]> var_1305_end_mask_0 = const()[name = string("op_1305_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_1305_squeeze_mask_0 = const()[name = string("op_1305_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_1305_cast_fp16 = slice_by_index(begin = var_1305_begin_0, end = var_1305_end_0, end_mask = var_1305_end_mask_0, squeeze_mask = var_1305_squeeze_mask_0, x = _inversed_query_11_cast_fp16)[name = string("op_1305_cast_fp16")];
            tensor<int32, [1]> var_1306_axes_0 = const()[name = string("op_1306_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_1306_cast_fp16 = expand_dims(axes = var_1306_axes_0, x = var_1305_cast_fp16)[name = string("op_1306_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_1307_cast_fp16 = mul(x = rec22_out, y = var_1306_cast_fp16)[name = string("op_1307_cast_fp16")];
            tensor<int32, [1]> var_1309_axes_0 = const()[name = string("op_1309_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_1309_keep_dims_0 = const()[name = string("op_1309_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_1309_cast_fp16 = reduce_sum(axes = var_1309_axes_0, keep_dims = var_1309_keep_dims_0, x = var_1307_cast_fp16)[name = string("op_1309_cast_fp16")];
            int32 attention_21_axis_0 = const()[name = string("attention_21_axis_0"), val = int32(1)];
            tensor<fp16, [1, 4, 16, 128]> attention_21_cast_fp16 = stack(axis = attention_21_axis_0, values = (var_1207_cast_fp16, var_1241_cast_fp16, var_1275_cast_fp16, var_1309_cast_fp16))[name = string("attention_21_cast_fp16")];
            tensor<int32, [2]> var_1312 = const()[name = string("op_1312"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [64, 128]> attention_23_cast_fp16 = reshape(shape = var_1312, x = attention_21_cast_fp16)[name = string("attention_23_cast_fp16")];
            tensor<int32, [2]> var_1314 = const()[name = string("op_1314"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [64, 128]> input_41_cast_fp16 = reshape(shape = var_1314, x = linear_23_cast_fp16)[name = string("input_41_cast_fp16")];
            tensor<fp16, [64, 128]> var_1316_cast_fp16 = mul(x = attention_23_cast_fp16, y = attention_23_cast_fp16)[name = string("op_1316_cast_fp16")];
            tensor<int32, [1]> variance_17_axes_0 = const()[name = string("variance_17_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_17_keep_dims_0 = const()[name = string("variance_17_keep_dims_0"), val = bool(true)];
            tensor<fp16, [64, 1]> variance_17_cast_fp16 = reduce_mean(axes = variance_17_axes_0, keep_dims = variance_17_keep_dims_0, x = var_1316_cast_fp16)[name = string("variance_17_cast_fp16")];
            fp16 var_1319_to_fp16 = const()[name = string("op_1319_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [64, 1]> var_1320_cast_fp16 = add(x = variance_17_cast_fp16, y = var_1319_to_fp16)[name = string("op_1320_cast_fp16")];
            fp32 var_1321_epsilon_0 = const()[name = string("op_1321_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [64, 1]> var_1321_cast_fp16 = rsqrt(epsilon = var_1321_epsilon_0, x = var_1320_cast_fp16)[name = string("op_1321_cast_fp16")];
            tensor<fp16, [64, 128]> attention_25_cast_fp16 = mul(x = attention_23_cast_fp16, y = var_1321_cast_fp16)[name = string("attention_25_cast_fp16")];
            tensor<fp16, [128]> layers20_2_gated_norm_promoted_to_fp16 = const()[name = string("layers20_2_gated_norm_promoted_to_fp16"), val = tensor<fp16, [128]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(242297344)))];
            tensor<fp16, [64, 128]> attention_27_cast_fp16 = mul(x = attention_25_cast_fp16, y = layers20_2_gated_norm_promoted_to_fp16)[name = string("attention_27_cast_fp16")];
            tensor<fp16, [64, 128]> var_1324_cast_fp16 = silu(x = input_41_cast_fp16)[name = string("op_1324_cast_fp16")];
            tensor<fp16, [64, 128]> attention_29_cast_fp16 = mul(x = attention_27_cast_fp16, y = var_1324_cast_fp16)[name = string("attention_29_cast_fp16")];
            tensor<int32, [2]> var_1326 = const()[name = string("op_1326"), val = tensor<int32, [2]>([4, 2048])];
            tensor<fp16, [4, 2048]> input_43_cast_fp16 = reshape(shape = var_1326, x = attention_29_cast_fp16)[name = string("input_43_cast_fp16")];
            tensor<fp16, [1024, 2048]> layers20_2_out_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 2048]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(242297664))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(243870592))))[name = string("layers20_2_out_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 1024]> linear_24_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = layers20_2_out_proj_weight_promoted_to_fp16_palettized, x = input_43_cast_fp16)[name = string("linear_24_cast_fp16")];
            tensor<fp16, [4, 1024]> value_43_cast_fp16 = add(x = value_31_cast_fp16, y = linear_24_cast_fp16)[name = string("value_43_cast_fp16")];
            tensor<fp16, [4, 1024]> var_1331_cast_fp16 = mul(x = value_43_cast_fp16, y = value_43_cast_fp16)[name = string("op_1331_cast_fp16")];
            tensor<int32, [1]> variance_19_axes_0 = const()[name = string("variance_19_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_19_keep_dims_0 = const()[name = string("variance_19_keep_dims_0"), val = bool(true)];
            tensor<fp16, [4, 1]> variance_19_cast_fp16 = reduce_mean(axes = variance_19_axes_0, keep_dims = variance_19_keep_dims_0, x = var_1331_cast_fp16)[name = string("variance_19_cast_fp16")];
            fp16 var_1334_to_fp16 = const()[name = string("op_1334_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [4, 1]> var_1335_cast_fp16 = add(x = variance_19_cast_fp16, y = var_1334_to_fp16)[name = string("op_1335_cast_fp16")];
            fp32 var_1336_epsilon_0 = const()[name = string("op_1336_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [4, 1]> var_1336_cast_fp16 = rsqrt(epsilon = var_1336_epsilon_0, x = var_1335_cast_fp16)[name = string("op_1336_cast_fp16")];
            tensor<fp16, [4, 1024]> var_1337_cast_fp16 = mul(x = value_43_cast_fp16, y = var_1336_cast_fp16)[name = string("op_1337_cast_fp16")];
            tensor<fp16, [1024]> var_1339_to_fp16 = const()[name = string("op_1339_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(243878848)))];
            tensor<fp16, [4, 1024]> input_45_cast_fp16 = mul(x = var_1337_cast_fp16, y = var_1339_to_fp16)[name = string("input_45_cast_fp16")];
            tensor<fp16, [3584, 1024]> layers20_2_gate_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(243880960))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(246633536))))[name = string("layers20_2_gate_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 3584]> linear_25_cast_fp16 = linear(bias = linear_1_bias_0_to_fp16, weight = layers20_2_gate_proj_weight_promoted_to_fp16_palettized, x = input_45_cast_fp16)[name = string("linear_25_cast_fp16")];
            tensor<fp16, [4, 3584]> var_1343_cast_fp16 = silu(x = linear_25_cast_fp16)[name = string("op_1343_cast_fp16")];
            tensor<fp16, [3584, 1024]> layers20_2_up_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(246662272))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(249414848))))[name = string("layers20_2_up_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 3584]> linear_26_cast_fp16 = linear(bias = linear_1_bias_0_to_fp16, weight = layers20_2_up_proj_weight_promoted_to_fp16_palettized, x = input_45_cast_fp16)[name = string("linear_26_cast_fp16")];
            tensor<fp16, [4, 3584]> input_49_cast_fp16 = mul(x = var_1343_cast_fp16, y = linear_26_cast_fp16)[name = string("input_49_cast_fp16")];
            tensor<fp16, [1024, 3584]> layers20_2_down_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 3584]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(249443584))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(252196160))))[name = string("layers20_2_down_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 1024]> linear_27_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = layers20_2_down_proj_weight_promoted_to_fp16_palettized, x = input_49_cast_fp16)[name = string("linear_27_cast_fp16")];
            tensor<fp16, [4, 1024]> value_45_cast_fp16 = add(x = value_43_cast_fp16, y = linear_27_cast_fp16)[name = string("value_45_cast_fp16")];
            int32 var_1367 = const()[name = string("op_1367"), val = int32(-1)];
            tensor<fp16, [4, 1024]> var_1374_cast_fp16 = mul(x = value_45_cast_fp16, y = value_45_cast_fp16)[name = string("op_1374_cast_fp16")];
            tensor<int32, [1]> variance_21_axes_0 = const()[name = string("variance_21_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_21_keep_dims_0 = const()[name = string("variance_21_keep_dims_0"), val = bool(true)];
            tensor<fp16, [4, 1]> variance_21_cast_fp16 = reduce_mean(axes = variance_21_axes_0, keep_dims = variance_21_keep_dims_0, x = var_1374_cast_fp16)[name = string("variance_21_cast_fp16")];
            fp16 var_1377_to_fp16 = const()[name = string("op_1377_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [4, 1]> var_1378_cast_fp16 = add(x = variance_21_cast_fp16, y = var_1377_to_fp16)[name = string("op_1378_cast_fp16")];
            fp32 var_1379_epsilon_0 = const()[name = string("op_1379_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [4, 1]> var_1379_cast_fp16 = rsqrt(epsilon = var_1379_epsilon_0, x = var_1378_cast_fp16)[name = string("op_1379_cast_fp16")];
            tensor<fp16, [4, 1024]> var_1380_cast_fp16 = mul(x = value_45_cast_fp16, y = var_1379_cast_fp16)[name = string("op_1380_cast_fp16")];
            tensor<fp16, [1024]> var_1382_to_fp16 = const()[name = string("op_1382_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(252204416)))];
            tensor<fp16, [4, 1024]> input_51_cast_fp16 = mul(x = var_1380_cast_fp16, y = var_1382_to_fp16)[name = string("input_51_cast_fp16")];
            tensor<fp16, [4096, 1024]> projection23_q_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [4096, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(252206528))), lut = tensor<fp16, [256, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(255352320))))[name = string("projection23_q_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4096]> linear_28_bias_0_to_fp16 = const()[name = string("linear_28_bias_0_to_fp16"), val = tensor<fp16, [4096]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(255385152)))];
            tensor<fp16, [4, 4096]> linear_28_cast_fp16 = linear(bias = linear_28_bias_0_to_fp16, weight = projection23_q_proj_weight_promoted_to_fp16_palettized, x = input_51_cast_fp16)[name = string("linear_28_cast_fp16")];
            tensor<int32, [3]> var_1386 = const()[name = string("op_1386"), val = tensor<int32, [3]>([4, 8, 512])];
            tensor<fp16, [4, 8, 512]> query_and_gate_cast_fp16 = reshape(shape = var_1386, x = linear_28_cast_fp16)[name = string("query_and_gate_cast_fp16")];
            tensor<int32, [2]> var_1388_split_sizes_0 = const()[name = string("op_1388_split_sizes_0"), val = tensor<int32, [2]>([256, 256])];
            int32 var_1388_axis_0 = const()[name = string("op_1388_axis_0"), val = int32(-1)];
            tensor<fp16, [4, 8, 256]> var_1388_cast_fp16_0, tensor<fp16, [4, 8, 256]> var_1388_cast_fp16_1 = split(axis = var_1388_axis_0, split_sizes = var_1388_split_sizes_0, x = query_and_gate_cast_fp16)[name = string("op_1388_cast_fp16")];
            tensor<fp16, [512, 1024]> projection23_k_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [512, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(255393408))), lut = tensor<fp16, [32, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(255786688))))[name = string("projection23_k_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [512]> linear_29_bias_0_to_fp16 = const()[name = string("linear_29_bias_0_to_fp16"), val = tensor<fp16, [512]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(255790848)))];
            tensor<fp16, [4, 512]> linear_29_cast_fp16 = linear(bias = linear_29_bias_0_to_fp16, weight = projection23_k_proj_weight_promoted_to_fp16_palettized, x = input_51_cast_fp16)[name = string("linear_29_cast_fp16")];
            tensor<int32, [3]> var_1392 = const()[name = string("op_1392"), val = tensor<int32, [3]>([4, 2, 256])];
            tensor<fp16, [4, 2, 256]> value_49_cast_fp16 = reshape(shape = var_1392, x = linear_29_cast_fp16)[name = string("value_49_cast_fp16")];
            tensor<fp16, [512, 1024]> projection23_v_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [512, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(255791936))), lut = tensor<fp16, [32, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(256185216))))[name = string("projection23_v_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 512]> linear_30_cast_fp16 = linear(bias = linear_29_bias_0_to_fp16, weight = projection23_v_proj_weight_promoted_to_fp16_palettized, x = input_51_cast_fp16)[name = string("linear_30_cast_fp16")];
            tensor<int32, [3]> var_1396 = const()[name = string("op_1396"), val = tensor<int32, [3]>([4, 2, 256])];
            tensor<fp16, [4, 2, 256]> value_55_cast_fp16 = reshape(shape = var_1396, x = linear_30_cast_fp16)[name = string("value_55_cast_fp16")];
            tensor<fp16, [4, 8, 256]> var_1398_cast_fp16 = mul(x = var_1388_cast_fp16_0, y = var_1388_cast_fp16_0)[name = string("op_1398_cast_fp16")];
            tensor<int32, [1]> variance_23_axes_0 = const()[name = string("variance_23_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_23_keep_dims_0 = const()[name = string("variance_23_keep_dims_0"), val = bool(true)];
            tensor<fp16, [4, 8, 1]> variance_23_cast_fp16 = reduce_mean(axes = variance_23_axes_0, keep_dims = variance_23_keep_dims_0, x = var_1398_cast_fp16)[name = string("variance_23_cast_fp16")];
            fp16 var_1401_to_fp16 = const()[name = string("op_1401_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [4, 8, 1]> var_1402_cast_fp16 = add(x = variance_23_cast_fp16, y = var_1401_to_fp16)[name = string("op_1402_cast_fp16")];
            fp32 var_1403_epsilon_0 = const()[name = string("op_1403_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [4, 8, 1]> var_1403_cast_fp16 = rsqrt(epsilon = var_1403_epsilon_0, x = var_1402_cast_fp16)[name = string("op_1403_cast_fp16")];
            tensor<fp16, [4, 8, 256]> var_1404_cast_fp16 = mul(x = var_1388_cast_fp16_0, y = var_1403_cast_fp16)[name = string("op_1404_cast_fp16")];
            tensor<fp16, [256]> var_1406_to_fp16 = const()[name = string("op_1406_to_fp16"), val = tensor<fp16, [256]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(256189376)))];
            tensor<fp16, [4, 8, 256]> var_1407_cast_fp16 = mul(x = var_1404_cast_fp16, y = var_1406_to_fp16)[name = string("op_1407_cast_fp16")];
            tensor<int32, [3]> var_1408 = const()[name = string("op_1408"), val = tensor<int32, [3]>([1, 0, 2])];
            tensor<int32, [1]> value_51_axes_0 = const()[name = string("value_51_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [8, 4, 256]> var_1409_cast_fp16 = transpose(perm = var_1408, x = var_1407_cast_fp16)[name = string("transpose_7")];
            tensor<fp16, [1, 8, 4, 256]> value_51_cast_fp16 = expand_dims(axes = value_51_axes_0, x = var_1409_cast_fp16)[name = string("value_51_cast_fp16")];
            tensor<fp16, [4, 2, 256]> var_1411_cast_fp16 = mul(x = value_49_cast_fp16, y = value_49_cast_fp16)[name = string("op_1411_cast_fp16")];
            tensor<int32, [1]> variance_25_axes_0 = const()[name = string("variance_25_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_25_keep_dims_0 = const()[name = string("variance_25_keep_dims_0"), val = bool(true)];
            tensor<fp16, [4, 2, 1]> variance_25_cast_fp16 = reduce_mean(axes = variance_25_axes_0, keep_dims = variance_25_keep_dims_0, x = var_1411_cast_fp16)[name = string("variance_25_cast_fp16")];
            fp16 var_1414_to_fp16 = const()[name = string("op_1414_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [4, 2, 1]> var_1415_cast_fp16 = add(x = variance_25_cast_fp16, y = var_1414_to_fp16)[name = string("op_1415_cast_fp16")];
            fp32 var_1416_epsilon_0 = const()[name = string("op_1416_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [4, 2, 1]> var_1416_cast_fp16 = rsqrt(epsilon = var_1416_epsilon_0, x = var_1415_cast_fp16)[name = string("op_1416_cast_fp16")];
            tensor<fp16, [4, 2, 256]> var_1417_cast_fp16 = mul(x = value_49_cast_fp16, y = var_1416_cast_fp16)[name = string("op_1417_cast_fp16")];
            tensor<fp16, [256]> var_1419_to_fp16 = const()[name = string("op_1419_to_fp16"), val = tensor<fp16, [256]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(256189952)))];
            tensor<fp16, [4, 2, 256]> var_1420_cast_fp16 = mul(x = var_1417_cast_fp16, y = var_1419_to_fp16)[name = string("op_1420_cast_fp16")];
            tensor<int32, [3]> var_1421 = const()[name = string("op_1421"), val = tensor<int32, [3]>([1, 0, 2])];
            tensor<int32, [1]> value_53_axes_0 = const()[name = string("value_53_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [2, 4, 256]> var_1422_cast_fp16 = transpose(perm = var_1421, x = var_1420_cast_fp16)[name = string("transpose_6")];
            tensor<fp16, [1, 2, 4, 256]> value_53_cast_fp16 = expand_dims(axes = value_53_axes_0, x = var_1422_cast_fp16)[name = string("value_53_cast_fp16")];
            tensor<int32, [4]> rotated_1_begin_0 = const()[name = string("rotated_1_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> rotated_1_end_0 = const()[name = string("rotated_1_end_0"), val = tensor<int32, [4]>([1, 8, 4, 64])];
            tensor<bool, [4]> rotated_1_end_mask_0 = const()[name = string("rotated_1_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
            tensor<fp16, [1, 8, 4, 64]> rotated_1_cast_fp16 = slice_by_index(begin = rotated_1_begin_0, end = rotated_1_end_0, end_mask = rotated_1_end_mask_0, x = value_51_cast_fp16)[name = string("rotated_1_cast_fp16")];
            tensor<int32, [4]> passthrough_1_begin_0 = const()[name = string("passthrough_1_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 64])];
            tensor<int32, [4]> passthrough_1_end_0 = const()[name = string("passthrough_1_end_0"), val = tensor<int32, [4]>([1, 8, 4, 256])];
            tensor<bool, [4]> passthrough_1_end_mask_0 = const()[name = string("passthrough_1_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
            tensor<fp16, [1, 8, 4, 192]> passthrough_1_cast_fp16 = slice_by_index(begin = passthrough_1_begin_0, end = passthrough_1_end_0, end_mask = passthrough_1_end_mask_0, x = value_51_cast_fp16)[name = string("passthrough_1_cast_fp16")];
            tensor<int32, [2]> var_1426_split_sizes_0 = const()[name = string("op_1426_split_sizes_0"), val = tensor<int32, [2]>([32, 32])];
            int32 var_1426_axis_0 = const()[name = string("op_1426_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 8, 4, 32]> var_1426_cast_fp16_0, tensor<fp16, [1, 8, 4, 32]> var_1426_cast_fp16_1 = split(axis = var_1426_axis_0, split_sizes = var_1426_split_sizes_0, x = rotated_1_cast_fp16)[name = string("op_1426_cast_fp16")];
            fp16 const_41_promoted_to_fp16 = const()[name = string("const_41_promoted_to_fp16"), val = fp16(-0x1p+0)];
            tensor<fp16, [1, 8, 4, 32]> var_1428_cast_fp16 = mul(x = var_1426_cast_fp16_1, y = const_41_promoted_to_fp16)[name = string("op_1428_cast_fp16")];
            bool rotated_half_1_interleave_0 = const()[name = string("rotated_half_1_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 8, 4, 64]> rotated_half_1_cast_fp16 = concat(axis = var_1367, interleave = rotated_half_1_interleave_0, values = (var_1428_cast_fp16, var_1426_cast_fp16_0))[name = string("rotated_half_1_cast_fp16")];
            tensor<fp16, [1, 8, 4, 64]> var_1431_cast_fp16 = mul(x = rotated_1_cast_fp16, y = cos)[name = string("op_1431_cast_fp16")];
            tensor<fp16, [1, 8, 4, 64]> var_1432_cast_fp16 = mul(x = rotated_half_1_cast_fp16, y = sin)[name = string("op_1432_cast_fp16")];
            tensor<fp16, [1, 8, 4, 64]> var_1433_cast_fp16 = add(x = var_1431_cast_fp16, y = var_1432_cast_fp16)[name = string("op_1433_cast_fp16")];
            bool query_13_interleave_0 = const()[name = string("query_13_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 8, 4, 256]> query_13_cast_fp16 = concat(axis = var_1367, interleave = query_13_interleave_0, values = (var_1433_cast_fp16, passthrough_1_cast_fp16))[name = string("query_13_cast_fp16")];
            tensor<int32, [4]> rotated_begin_0 = const()[name = string("rotated_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> rotated_end_0 = const()[name = string("rotated_end_0"), val = tensor<int32, [4]>([1, 2, 4, 64])];
            tensor<bool, [4]> rotated_end_mask_0 = const()[name = string("rotated_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
            tensor<fp16, [1, 2, 4, 64]> rotated_cast_fp16 = slice_by_index(begin = rotated_begin_0, end = rotated_end_0, end_mask = rotated_end_mask_0, x = value_53_cast_fp16)[name = string("rotated_cast_fp16")];
            tensor<int32, [4]> passthrough_begin_0 = const()[name = string("passthrough_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 64])];
            tensor<int32, [4]> passthrough_end_0 = const()[name = string("passthrough_end_0"), val = tensor<int32, [4]>([1, 2, 4, 256])];
            tensor<bool, [4]> passthrough_end_mask_0 = const()[name = string("passthrough_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
            tensor<fp16, [1, 2, 4, 192]> passthrough_cast_fp16 = slice_by_index(begin = passthrough_begin_0, end = passthrough_end_0, end_mask = passthrough_end_mask_0, x = value_53_cast_fp16)[name = string("passthrough_cast_fp16")];
            tensor<int32, [2]> var_1438_split_sizes_0 = const()[name = string("op_1438_split_sizes_0"), val = tensor<int32, [2]>([32, 32])];
            int32 var_1438_axis_0 = const()[name = string("op_1438_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 2, 4, 32]> var_1438_cast_fp16_0, tensor<fp16, [1, 2, 4, 32]> var_1438_cast_fp16_1 = split(axis = var_1438_axis_0, split_sizes = var_1438_split_sizes_0, x = rotated_cast_fp16)[name = string("op_1438_cast_fp16")];
            fp16 const_42_promoted_to_fp16 = const()[name = string("const_42_promoted_to_fp16"), val = fp16(-0x1p+0)];
            tensor<fp16, [1, 2, 4, 32]> var_1440_cast_fp16 = mul(x = var_1438_cast_fp16_1, y = const_42_promoted_to_fp16)[name = string("op_1440_cast_fp16")];
            bool rotated_half_interleave_0 = const()[name = string("rotated_half_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 2, 4, 64]> rotated_half_cast_fp16 = concat(axis = var_1367, interleave = rotated_half_interleave_0, values = (var_1440_cast_fp16, var_1438_cast_fp16_0))[name = string("rotated_half_cast_fp16")];
            tensor<fp16, [1, 2, 4, 64]> var_1443_cast_fp16 = mul(x = rotated_cast_fp16, y = cos)[name = string("op_1443_cast_fp16")];
            tensor<fp16, [1, 2, 4, 64]> var_1444_cast_fp16 = mul(x = rotated_half_cast_fp16, y = sin)[name = string("op_1444_cast_fp16")];
            tensor<fp16, [1, 2, 4, 64]> var_1445_cast_fp16 = add(x = var_1443_cast_fp16, y = var_1444_cast_fp16)[name = string("op_1445_cast_fp16")];
            bool key_13_interleave_0 = const()[name = string("key_13_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 2, 4, 256]> key_13_cast_fp16 = concat(axis = var_1367, interleave = key_13_interleave_0, values = (var_1445_cast_fp16, passthrough_cast_fp16))[name = string("key_13_cast_fp16")];
            tensor<int32, [4]> var_1448 = const()[name = string("op_1448"), val = tensor<int32, [4]>([2, 4, 4, 256])];
            tensor<fp16, [2, 4, 4, 256]> var_1449_cast_fp16 = reshape(shape = var_1448, x = query_13_cast_fp16)[name = string("op_1449_cast_fp16")];
            tensor<int32, [4]> transpose_1_perm_0 = const()[name = string("transpose_1_perm_0"), val = tensor<int32, [4]>([0, 1, 3, 2])];
            tensor<int32, [3]> var_1452 = const()[name = string("op_1452"), val = tensor<int32, [3]>([2, 4, 256])];
            tensor<fp16, [2, 4, 256]> key_cast_fp16 = reshape(shape = var_1452, x = key_13_cast_fp16)[name = string("key_cast_fp16")];
            tensor<int32, [3]> var_1454 = const()[name = string("op_1454"), val = tensor<int32, [3]>([1, 0, 2])];
            tensor<int32, [2]> var_1456 = const()[name = string("op_1456"), val = tensor<int32, [2]>([4, 2048])];
            tensor<fp16, [4, 2048]> gate_cast_fp16 = reshape(shape = var_1456, x = var_1388_cast_fp16_1)[name = string("gate_cast_fp16")];
            tensor<int32, [1]> var_1464_axes_0 = const()[name = string("op_1464_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [1, 2, 4, 256]> var_1464_cast_fp16 = expand_dims(axes = var_1464_axes_0, x = key_cast_fp16)[name = string("op_1464_cast_fp16")];
            tensor<int32, [1]> var_1466_axes_0 = const()[name = string("op_1466_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [1, 1, 2, 4, 256]> var_1466_cast_fp16 = expand_dims(axes = var_1466_axes_0, x = var_1464_cast_fp16)[name = string("op_1466_cast_fp16")];
            tensor<int32, [1]> expand_dims_16 = const()[name = string("expand_dims_16"), val = tensor<int32, [1]>([1])];
            tensor<int32, [1]> expand_dims_17 = const()[name = string("expand_dims_17"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_18 = const()[name = string("expand_dims_18"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_20 = const()[name = string("expand_dims_20"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_21 = const()[name = string("expand_dims_21"), val = tensor<int32, [1]>([2])];
            tensor<int32, [1]> expand_dims_22 = const()[name = string("expand_dims_22"), val = tensor<int32, [1]>([1])];
            int32 concat_10_axis_0 = const()[name = string("concat_10_axis_0"), val = int32(0)];
            bool concat_10_interleave_0 = const()[name = string("concat_10_interleave_0"), val = bool(false)];
            tensor<int32, [5]> concat_10 = concat(axis = concat_10_axis_0, interleave = concat_10_interleave_0, values = (expand_dims_16, expand_dims_17, expand_dims_18, current_pos, expand_dims_20))[name = string("concat_10")];
            tensor<int32, [1]> concat_11_values2_0 = const()[name = string("concat_11_values2_0"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> concat_11_values4_0 = const()[name = string("concat_11_values4_0"), val = tensor<int32, [1]>([0])];
            int32 concat_11_axis_0 = const()[name = string("concat_11_axis_0"), val = int32(0)];
            bool concat_11_interleave_0 = const()[name = string("concat_11_interleave_0"), val = bool(false)];
            tensor<int32, [5]> concat_11 = concat(axis = concat_11_axis_0, interleave = concat_11_interleave_0, values = (expand_dims_21, expand_dims_22, concat_11_values2_0, var_38, concat_11_values4_0))[name = string("concat_11")];
            tensor<int32, [5]> kv_cache_internal_tensor_assign_3_stride_0 = const()[name = string("kv_cache_internal_tensor_assign_3_stride_0"), val = tensor<int32, [5]>([1, 1, 1, 1, 1])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_3_begin_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_3_begin_mask_0"), val = tensor<bool, [5]>([false, false, false, false, false])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_3_end_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_3_end_mask_0"), val = tensor<bool, [5]>([false, false, true, false, true])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_3_squeeze_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_3_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, false, false, false])];
            tensor<fp16, [2, 2, 2, 2048, 256]> kv_cache_internal_tensor_assign_3_cast_fp16 = slice_update(begin = concat_10, begin_mask = kv_cache_internal_tensor_assign_3_begin_mask_0, end = concat_11, end_mask = kv_cache_internal_tensor_assign_3_end_mask_0, squeeze_mask = kv_cache_internal_tensor_assign_3_squeeze_mask_0, stride = kv_cache_internal_tensor_assign_3_stride_0, update = var_1466_cast_fp16, x = coreml_update_state_5)[name = string("kv_cache_internal_tensor_assign_3_cast_fp16")];
            write_state(data = kv_cache_internal_tensor_assign_3_cast_fp16, input = kv_cache)[name = string("coreml_update_state_10_write_state")];
            tensor<fp16, [2, 2, 2, 2048, 256]> coreml_update_state_6 = read_state(input = kv_cache)[name = string("coreml_update_state_10")];
            tensor<int32, [1]> var_1497_axes_0 = const()[name = string("op_1497_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [2, 4, 256]> value_57_cast_fp16 = transpose(perm = var_1454, x = value_55_cast_fp16)[name = string("transpose_5")];
            tensor<fp16, [1, 2, 4, 256]> var_1497_cast_fp16 = expand_dims(axes = var_1497_axes_0, x = value_57_cast_fp16)[name = string("op_1497_cast_fp16")];
            tensor<int32, [1]> var_1499_axes_0 = const()[name = string("op_1499_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [1, 1, 2, 4, 256]> var_1499_cast_fp16 = expand_dims(axes = var_1499_axes_0, x = var_1497_cast_fp16)[name = string("op_1499_cast_fp16")];
            tensor<int32, [1]> expand_dims_24 = const()[name = string("expand_dims_24"), val = tensor<int32, [1]>([1])];
            tensor<int32, [1]> expand_dims_25 = const()[name = string("expand_dims_25"), val = tensor<int32, [1]>([1])];
            tensor<int32, [1]> expand_dims_26 = const()[name = string("expand_dims_26"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_28 = const()[name = string("expand_dims_28"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_29 = const()[name = string("expand_dims_29"), val = tensor<int32, [1]>([2])];
            tensor<int32, [1]> expand_dims_30 = const()[name = string("expand_dims_30"), val = tensor<int32, [1]>([2])];
            int32 concat_14_axis_0 = const()[name = string("concat_14_axis_0"), val = int32(0)];
            bool concat_14_interleave_0 = const()[name = string("concat_14_interleave_0"), val = bool(false)];
            tensor<int32, [5]> concat_14 = concat(axis = concat_14_axis_0, interleave = concat_14_interleave_0, values = (expand_dims_24, expand_dims_25, expand_dims_26, current_pos, expand_dims_28))[name = string("concat_14")];
            tensor<int32, [1]> concat_15_values2_0 = const()[name = string("concat_15_values2_0"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> concat_15_values4_0 = const()[name = string("concat_15_values4_0"), val = tensor<int32, [1]>([0])];
            int32 concat_15_axis_0 = const()[name = string("concat_15_axis_0"), val = int32(0)];
            bool concat_15_interleave_0 = const()[name = string("concat_15_interleave_0"), val = bool(false)];
            tensor<int32, [5]> concat_15 = concat(axis = concat_15_axis_0, interleave = concat_15_interleave_0, values = (expand_dims_29, expand_dims_30, concat_15_values2_0, var_38, concat_15_values4_0))[name = string("concat_15")];
            tensor<int32, [5]> kv_cache_internal_tensor_assign_4_stride_0 = const()[name = string("kv_cache_internal_tensor_assign_4_stride_0"), val = tensor<int32, [5]>([1, 1, 1, 1, 1])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_4_begin_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_4_begin_mask_0"), val = tensor<bool, [5]>([false, false, false, false, false])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_4_end_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_4_end_mask_0"), val = tensor<bool, [5]>([false, false, true, false, true])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_4_squeeze_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_4_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, false, false, false])];
            tensor<fp16, [2, 2, 2, 2048, 256]> kv_cache_internal_tensor_assign_4_cast_fp16 = slice_update(begin = concat_14, begin_mask = kv_cache_internal_tensor_assign_4_begin_mask_0, end = concat_15, end_mask = kv_cache_internal_tensor_assign_4_end_mask_0, squeeze_mask = kv_cache_internal_tensor_assign_4_squeeze_mask_0, stride = kv_cache_internal_tensor_assign_4_stride_0, update = var_1499_cast_fp16, x = coreml_update_state_6)[name = string("kv_cache_internal_tensor_assign_4_cast_fp16")];
            write_state(data = kv_cache_internal_tensor_assign_4_cast_fp16, input = kv_cache)[name = string("coreml_update_state_11_write_state")];
            tensor<fp16, [2, 2, 2, 2048, 256]> coreml_update_state_7 = read_state(input = kv_cache)[name = string("coreml_update_state_11")];
            tensor<int32, [5]> var_1531_begin_0 = const()[name = string("op_1531_begin_0"), val = tensor<int32, [5]>([1, 0, 0, 0, 0])];
            tensor<int32, [5]> var_1531_end_0 = const()[name = string("op_1531_end_0"), val = tensor<int32, [5]>([2, 2, 2, 2048, 256])];
            tensor<bool, [5]> var_1531_end_mask_0 = const()[name = string("op_1531_end_mask_0"), val = tensor<bool, [5]>([false, true, true, true, true])];
            tensor<bool, [5]> var_1531_squeeze_mask_0 = const()[name = string("op_1531_squeeze_mask_0"), val = tensor<bool, [5]>([true, false, false, false, false])];
            tensor<fp16, [2, 2, 2048, 256]> var_1531_cast_fp16 = slice_by_index(begin = var_1531_begin_0, end = var_1531_end_0, end_mask = var_1531_end_mask_0, squeeze_mask = var_1531_squeeze_mask_0, x = coreml_update_state_7)[name = string("op_1531_cast_fp16")];
            tensor<int32, [4]> keys_begin_0 = const()[name = string("keys_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> keys_end_0 = const()[name = string("keys_end_0"), val = tensor<int32, [4]>([1, 2, 2048, 256])];
            tensor<bool, [4]> keys_end_mask_0 = const()[name = string("keys_end_mask_0"), val = tensor<bool, [4]>([false, true, true, true])];
            tensor<bool, [4]> keys_squeeze_mask_0 = const()[name = string("keys_squeeze_mask_0"), val = tensor<bool, [4]>([true, false, false, false])];
            tensor<fp16, [2, 2048, 256]> keys_cast_fp16 = slice_by_index(begin = keys_begin_0, end = keys_end_0, end_mask = keys_end_mask_0, squeeze_mask = keys_squeeze_mask_0, x = var_1531_cast_fp16)[name = string("keys_cast_fp16")];
            tensor<int32, [4]> values_begin_0 = const()[name = string("values_begin_0"), val = tensor<int32, [4]>([1, 0, 0, 0])];
            tensor<int32, [4]> values_end_0 = const()[name = string("values_end_0"), val = tensor<int32, [4]>([2, 2, 2048, 256])];
            tensor<bool, [4]> values_end_mask_0 = const()[name = string("values_end_mask_0"), val = tensor<bool, [4]>([false, true, true, true])];
            tensor<bool, [4]> values_squeeze_mask_0 = const()[name = string("values_squeeze_mask_0"), val = tensor<bool, [4]>([true, false, false, false])];
            tensor<fp16, [2, 2048, 256]> values_cast_fp16 = slice_by_index(begin = values_begin_0, end = values_end_0, end_mask = values_end_mask_0, squeeze_mask = values_squeeze_mask_0, x = var_1531_cast_fp16)[name = string("values_cast_fp16")];
            int32 var_1548 = const()[name = string("op_1548"), val = int32(1)];
            tensor<int32, [4]> var_1555 = const()[name = string("op_1555"), val = tensor<int32, [4]>([1, 2048, 1, 4])];
            tensor<fp16, [2, 4, 256, 4]> transpose_1_cast_fp16 = transpose(perm = transpose_1_perm_0, x = var_1449_cast_fp16)[name = string("transpose_4")];
            tensor<fp16, [1, 2048, 1, 4]> var_1556_cast_fp16 = reshape(shape = var_1555, x = transpose_1_cast_fp16)[name = string("op_1556_cast_fp16")];
            tensor<int32, [3]> var_1557 = const()[name = string("op_1557"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [4]> var_1560 = const()[name = string("op_1560"), val = tensor<int32, [4]>([1, 512, 1, 2048])];
            tensor<fp16, [2, 256, 2048]> var_1558_cast_fp16 = transpose(perm = var_1557, x = keys_cast_fp16)[name = string("transpose_3")];
            tensor<fp16, [1, 512, 1, 2048]> key_native_cast_fp16 = reshape(shape = var_1560, x = var_1558_cast_fp16)[name = string("key_native_cast_fp16")];
            tensor<int32, [3]> var_1562 = const()[name = string("op_1562"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [4]> var_1565 = const()[name = string("op_1565"), val = tensor<int32, [4]>([1, 512, 1, 2048])];
            tensor<fp16, [2, 256, 2048]> var_1563_cast_fp16 = transpose(perm = var_1562, x = values_cast_fp16)[name = string("transpose_2")];
            tensor<fp16, [1, 512, 1, 2048]> var_1566_cast_fp16 = reshape(shape = var_1565, x = var_1563_cast_fp16)[name = string("op_1566_cast_fp16")];
            tensor<int32, [8]> tile_19 = const()[name = string("tile_19"), val = tensor<int32, [8]>([256, 256, 256, 256, 256, 256, 256, 256])];
            int32 var_1570_axis_0 = const()[name = string("op_1570_axis_0"), val = int32(1)];
            tensor<fp16, [1, 256, 1, 4]> var_1570_cast_fp16_0, tensor<fp16, [1, 256, 1, 4]> var_1570_cast_fp16_1, tensor<fp16, [1, 256, 1, 4]> var_1570_cast_fp16_2, tensor<fp16, [1, 256, 1, 4]> var_1570_cast_fp16_3, tensor<fp16, [1, 256, 1, 4]> var_1570_cast_fp16_4, tensor<fp16, [1, 256, 1, 4]> var_1570_cast_fp16_5, tensor<fp16, [1, 256, 1, 4]> var_1570_cast_fp16_6, tensor<fp16, [1, 256, 1, 4]> var_1570_cast_fp16_7 = split(axis = var_1570_axis_0, split_sizes = tile_19, x = var_1556_cast_fp16)[name = string("op_1570_cast_fp16")];
            tensor<int32, [4]> var_1579_perm_0 = const()[name = string("op_1579_perm_0"), val = tensor<int32, [4]>([0, 3, 2, 1])];
            tensor<int32, [2]> tile_20 = const()[name = string("tile_20"), val = tensor<int32, [2]>([256, 256])];
            int32 var_1580_axis_0 = const()[name = string("op_1580_axis_0"), val = int32(3)];
            tensor<fp16, [1, 2048, 1, 512]> var_1579_cast_fp16 = transpose(perm = var_1579_perm_0, x = key_native_cast_fp16)[name = string("transpose_1")];
            tensor<fp16, [1, 2048, 1, 256]> var_1580_cast_fp16_0, tensor<fp16, [1, 2048, 1, 256]> var_1580_cast_fp16_1 = split(axis = var_1580_axis_0, split_sizes = tile_20, x = var_1579_cast_fp16)[name = string("op_1580_cast_fp16")];
            tensor<int32, [2]> tile_21 = const()[name = string("tile_21"), val = tensor<int32, [2]>([256, 256])];
            int32 var_1583_axis_0 = const()[name = string("op_1583_axis_0"), val = int32(1)];
            tensor<fp16, [1, 256, 1, 2048]> var_1583_cast_fp16_0, tensor<fp16, [1, 256, 1, 2048]> var_1583_cast_fp16_1 = split(axis = var_1583_axis_0, split_sizes = tile_21, x = var_1566_cast_fp16)[name = string("op_1583_cast_fp16")];
            string scores_17_equation_0 = const()[name = string("scores_17_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 4]> scores_17_cast_fp16 = einsum(equation = scores_17_equation_0, values = (var_1580_cast_fp16_0, var_1570_cast_fp16_0))[name = string("scores_17_cast_fp16")];
            fp16 var_1588_to_fp16 = const()[name = string("op_1588_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 4]> var_1589_cast_fp16 = mul(x = scores_17_cast_fp16, y = var_1588_to_fp16)[name = string("op_1589_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_1590_cast_fp16 = add(x = var_1589_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_1590_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_1591_cast_fp16 = softmax(axis = var_1548, x = var_1590_cast_fp16)[name = string("op_1591_cast_fp16")];
            tensor<int32, [4]> tile_22 = const()[name = string("tile_22"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_1592_axis_0 = const()[name = string("op_1592_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 4]> var_1592_cast_fp16_0, tensor<fp16, [1, 512, 1, 4]> var_1592_cast_fp16_1, tensor<fp16, [1, 512, 1, 4]> var_1592_cast_fp16_2, tensor<fp16, [1, 512, 1, 4]> var_1592_cast_fp16_3 = split(axis = var_1592_axis_0, split_sizes = tile_22, x = var_1591_cast_fp16)[name = string("op_1592_cast_fp16")];
            tensor<int32, [4]> tile_23 = const()[name = string("tile_23"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_1597_axis_0 = const()[name = string("op_1597_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_1597_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_1597_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_1597_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_1597_cast_fp16_3 = split(axis = var_1597_axis_0, split_sizes = tile_23, x = var_1583_cast_fp16_0)[name = string("op_1597_cast_fp16")];
            fp16 var_1602_to_fp16 = const()[name = string("op_1602_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_1603_cast_fp16 = mul(x = var_1592_cast_fp16_0, y = var_1602_to_fp16)[name = string("op_1603_cast_fp16")];
            string var_1605_equation_0 = const()[name = string("op_1605_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_1605_cast_fp16 = einsum(equation = var_1605_equation_0, values = (var_1597_cast_fp16_0, var_1603_cast_fp16))[name = string("op_1605_cast_fp16")];
            fp16 var_1606_to_fp16 = const()[name = string("op_1606_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_1607_cast_fp16 = mul(x = var_1592_cast_fp16_1, y = var_1606_to_fp16)[name = string("op_1607_cast_fp16")];
            string var_1609_equation_0 = const()[name = string("op_1609_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_1609_cast_fp16 = einsum(equation = var_1609_equation_0, values = (var_1597_cast_fp16_1, var_1607_cast_fp16))[name = string("op_1609_cast_fp16")];
            fp16 var_1610_to_fp16 = const()[name = string("op_1610_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_1611_cast_fp16 = mul(x = var_1592_cast_fp16_2, y = var_1610_to_fp16)[name = string("op_1611_cast_fp16")];
            string var_1613_equation_0 = const()[name = string("op_1613_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_1613_cast_fp16 = einsum(equation = var_1613_equation_0, values = (var_1597_cast_fp16_2, var_1611_cast_fp16))[name = string("op_1613_cast_fp16")];
            fp16 var_1614_to_fp16 = const()[name = string("op_1614_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_1615_cast_fp16 = mul(x = var_1592_cast_fp16_3, y = var_1614_to_fp16)[name = string("op_1615_cast_fp16")];
            string var_1617_equation_0 = const()[name = string("op_1617_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_1617_cast_fp16 = einsum(equation = var_1617_equation_0, values = (var_1597_cast_fp16_3, var_1615_cast_fp16))[name = string("op_1617_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_1618_cast_fp16 = add(x = var_1605_cast_fp16, y = var_1609_cast_fp16)[name = string("op_1618_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_1619_cast_fp16 = add(x = var_1613_cast_fp16, y = var_1617_cast_fp16)[name = string("op_1619_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_1620_cast_fp16 = add(x = var_1618_cast_fp16, y = var_1619_cast_fp16)[name = string("op_1620_cast_fp16")];
            fp16 _inversed_1622_y_0_to_fp16 = const()[name = string("_inversed_1622_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 4]> _inversed_1622_cast_fp16 = mul(x = var_1620_cast_fp16, y = _inversed_1622_y_0_to_fp16)[name = string("_inversed_1622_cast_fp16")];
            string scores_19_equation_0 = const()[name = string("scores_19_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 4]> scores_19_cast_fp16 = einsum(equation = scores_19_equation_0, values = (var_1580_cast_fp16_0, var_1570_cast_fp16_1))[name = string("scores_19_cast_fp16")];
            fp16 var_1625_to_fp16 = const()[name = string("op_1625_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 4]> var_1626_cast_fp16 = mul(x = scores_19_cast_fp16, y = var_1625_to_fp16)[name = string("op_1626_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_1627_cast_fp16 = add(x = var_1626_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_1627_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_1628_cast_fp16 = softmax(axis = var_1548, x = var_1627_cast_fp16)[name = string("op_1628_cast_fp16")];
            tensor<int32, [4]> tile_24 = const()[name = string("tile_24"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_1629_axis_0 = const()[name = string("op_1629_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 4]> var_1629_cast_fp16_0, tensor<fp16, [1, 512, 1, 4]> var_1629_cast_fp16_1, tensor<fp16, [1, 512, 1, 4]> var_1629_cast_fp16_2, tensor<fp16, [1, 512, 1, 4]> var_1629_cast_fp16_3 = split(axis = var_1629_axis_0, split_sizes = tile_24, x = var_1628_cast_fp16)[name = string("op_1629_cast_fp16")];
            tensor<int32, [4]> tile_25 = const()[name = string("tile_25"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_1634_axis_0 = const()[name = string("op_1634_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_1634_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_1634_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_1634_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_1634_cast_fp16_3 = split(axis = var_1634_axis_0, split_sizes = tile_25, x = var_1583_cast_fp16_0)[name = string("op_1634_cast_fp16")];
            fp16 var_1639_to_fp16 = const()[name = string("op_1639_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_1640_cast_fp16 = mul(x = var_1629_cast_fp16_0, y = var_1639_to_fp16)[name = string("op_1640_cast_fp16")];
            string var_1642_equation_0 = const()[name = string("op_1642_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_1642_cast_fp16 = einsum(equation = var_1642_equation_0, values = (var_1634_cast_fp16_0, var_1640_cast_fp16))[name = string("op_1642_cast_fp16")];
            fp16 var_1643_to_fp16 = const()[name = string("op_1643_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_1644_cast_fp16 = mul(x = var_1629_cast_fp16_1, y = var_1643_to_fp16)[name = string("op_1644_cast_fp16")];
            string var_1646_equation_0 = const()[name = string("op_1646_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_1646_cast_fp16 = einsum(equation = var_1646_equation_0, values = (var_1634_cast_fp16_1, var_1644_cast_fp16))[name = string("op_1646_cast_fp16")];
            fp16 var_1647_to_fp16 = const()[name = string("op_1647_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_1648_cast_fp16 = mul(x = var_1629_cast_fp16_2, y = var_1647_to_fp16)[name = string("op_1648_cast_fp16")];
            string var_1650_equation_0 = const()[name = string("op_1650_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_1650_cast_fp16 = einsum(equation = var_1650_equation_0, values = (var_1634_cast_fp16_2, var_1648_cast_fp16))[name = string("op_1650_cast_fp16")];
            fp16 var_1651_to_fp16 = const()[name = string("op_1651_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_1652_cast_fp16 = mul(x = var_1629_cast_fp16_3, y = var_1651_to_fp16)[name = string("op_1652_cast_fp16")];
            string var_1654_equation_0 = const()[name = string("op_1654_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_1654_cast_fp16 = einsum(equation = var_1654_equation_0, values = (var_1634_cast_fp16_3, var_1652_cast_fp16))[name = string("op_1654_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_1655_cast_fp16 = add(x = var_1642_cast_fp16, y = var_1646_cast_fp16)[name = string("op_1655_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_1656_cast_fp16 = add(x = var_1650_cast_fp16, y = var_1654_cast_fp16)[name = string("op_1656_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_1657_cast_fp16 = add(x = var_1655_cast_fp16, y = var_1656_cast_fp16)[name = string("op_1657_cast_fp16")];
            fp16 _inversed_1659_y_0_to_fp16 = const()[name = string("_inversed_1659_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 4]> _inversed_1659_cast_fp16 = mul(x = var_1657_cast_fp16, y = _inversed_1659_y_0_to_fp16)[name = string("_inversed_1659_cast_fp16")];
            string scores_21_equation_0 = const()[name = string("scores_21_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 4]> scores_21_cast_fp16 = einsum(equation = scores_21_equation_0, values = (var_1580_cast_fp16_0, var_1570_cast_fp16_2))[name = string("scores_21_cast_fp16")];
            fp16 var_1662_to_fp16 = const()[name = string("op_1662_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 4]> var_1663_cast_fp16 = mul(x = scores_21_cast_fp16, y = var_1662_to_fp16)[name = string("op_1663_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_1664_cast_fp16 = add(x = var_1663_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_1664_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_1665_cast_fp16 = softmax(axis = var_1548, x = var_1664_cast_fp16)[name = string("op_1665_cast_fp16")];
            tensor<int32, [4]> tile_26 = const()[name = string("tile_26"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_1666_axis_0 = const()[name = string("op_1666_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 4]> var_1666_cast_fp16_0, tensor<fp16, [1, 512, 1, 4]> var_1666_cast_fp16_1, tensor<fp16, [1, 512, 1, 4]> var_1666_cast_fp16_2, tensor<fp16, [1, 512, 1, 4]> var_1666_cast_fp16_3 = split(axis = var_1666_axis_0, split_sizes = tile_26, x = var_1665_cast_fp16)[name = string("op_1666_cast_fp16")];
            tensor<int32, [4]> tile_27 = const()[name = string("tile_27"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_1671_axis_0 = const()[name = string("op_1671_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_1671_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_1671_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_1671_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_1671_cast_fp16_3 = split(axis = var_1671_axis_0, split_sizes = tile_27, x = var_1583_cast_fp16_0)[name = string("op_1671_cast_fp16")];
            fp16 var_1676_to_fp16 = const()[name = string("op_1676_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_1677_cast_fp16 = mul(x = var_1666_cast_fp16_0, y = var_1676_to_fp16)[name = string("op_1677_cast_fp16")];
            string var_1679_equation_0 = const()[name = string("op_1679_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_1679_cast_fp16 = einsum(equation = var_1679_equation_0, values = (var_1671_cast_fp16_0, var_1677_cast_fp16))[name = string("op_1679_cast_fp16")];
            fp16 var_1680_to_fp16 = const()[name = string("op_1680_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_1681_cast_fp16 = mul(x = var_1666_cast_fp16_1, y = var_1680_to_fp16)[name = string("op_1681_cast_fp16")];
            string var_1683_equation_0 = const()[name = string("op_1683_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_1683_cast_fp16 = einsum(equation = var_1683_equation_0, values = (var_1671_cast_fp16_1, var_1681_cast_fp16))[name = string("op_1683_cast_fp16")];
            fp16 var_1684_to_fp16 = const()[name = string("op_1684_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_1685_cast_fp16 = mul(x = var_1666_cast_fp16_2, y = var_1684_to_fp16)[name = string("op_1685_cast_fp16")];
            string var_1687_equation_0 = const()[name = string("op_1687_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_1687_cast_fp16 = einsum(equation = var_1687_equation_0, values = (var_1671_cast_fp16_2, var_1685_cast_fp16))[name = string("op_1687_cast_fp16")];
            fp16 var_1688_to_fp16 = const()[name = string("op_1688_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_1689_cast_fp16 = mul(x = var_1666_cast_fp16_3, y = var_1688_to_fp16)[name = string("op_1689_cast_fp16")];
            string var_1691_equation_0 = const()[name = string("op_1691_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_1691_cast_fp16 = einsum(equation = var_1691_equation_0, values = (var_1671_cast_fp16_3, var_1689_cast_fp16))[name = string("op_1691_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_1692_cast_fp16 = add(x = var_1679_cast_fp16, y = var_1683_cast_fp16)[name = string("op_1692_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_1693_cast_fp16 = add(x = var_1687_cast_fp16, y = var_1691_cast_fp16)[name = string("op_1693_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_1694_cast_fp16 = add(x = var_1692_cast_fp16, y = var_1693_cast_fp16)[name = string("op_1694_cast_fp16")];
            fp16 _inversed_1696_y_0_to_fp16 = const()[name = string("_inversed_1696_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 4]> _inversed_1696_cast_fp16 = mul(x = var_1694_cast_fp16, y = _inversed_1696_y_0_to_fp16)[name = string("_inversed_1696_cast_fp16")];
            string scores_23_equation_0 = const()[name = string("scores_23_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 4]> scores_23_cast_fp16 = einsum(equation = scores_23_equation_0, values = (var_1580_cast_fp16_0, var_1570_cast_fp16_3))[name = string("scores_23_cast_fp16")];
            fp16 var_1699_to_fp16 = const()[name = string("op_1699_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 4]> var_1700_cast_fp16 = mul(x = scores_23_cast_fp16, y = var_1699_to_fp16)[name = string("op_1700_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_1701_cast_fp16 = add(x = var_1700_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_1701_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_1702_cast_fp16 = softmax(axis = var_1548, x = var_1701_cast_fp16)[name = string("op_1702_cast_fp16")];
            tensor<int32, [4]> tile_28 = const()[name = string("tile_28"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_1703_axis_0 = const()[name = string("op_1703_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 4]> var_1703_cast_fp16_0, tensor<fp16, [1, 512, 1, 4]> var_1703_cast_fp16_1, tensor<fp16, [1, 512, 1, 4]> var_1703_cast_fp16_2, tensor<fp16, [1, 512, 1, 4]> var_1703_cast_fp16_3 = split(axis = var_1703_axis_0, split_sizes = tile_28, x = var_1702_cast_fp16)[name = string("op_1703_cast_fp16")];
            tensor<int32, [4]> tile_29 = const()[name = string("tile_29"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_1708_axis_0 = const()[name = string("op_1708_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_1708_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_1708_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_1708_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_1708_cast_fp16_3 = split(axis = var_1708_axis_0, split_sizes = tile_29, x = var_1583_cast_fp16_0)[name = string("op_1708_cast_fp16")];
            fp16 var_1713_to_fp16 = const()[name = string("op_1713_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_1714_cast_fp16 = mul(x = var_1703_cast_fp16_0, y = var_1713_to_fp16)[name = string("op_1714_cast_fp16")];
            string var_1716_equation_0 = const()[name = string("op_1716_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_1716_cast_fp16 = einsum(equation = var_1716_equation_0, values = (var_1708_cast_fp16_0, var_1714_cast_fp16))[name = string("op_1716_cast_fp16")];
            fp16 var_1717_to_fp16 = const()[name = string("op_1717_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_1718_cast_fp16 = mul(x = var_1703_cast_fp16_1, y = var_1717_to_fp16)[name = string("op_1718_cast_fp16")];
            string var_1720_equation_0 = const()[name = string("op_1720_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_1720_cast_fp16 = einsum(equation = var_1720_equation_0, values = (var_1708_cast_fp16_1, var_1718_cast_fp16))[name = string("op_1720_cast_fp16")];
            fp16 var_1721_to_fp16 = const()[name = string("op_1721_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_1722_cast_fp16 = mul(x = var_1703_cast_fp16_2, y = var_1721_to_fp16)[name = string("op_1722_cast_fp16")];
            string var_1724_equation_0 = const()[name = string("op_1724_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_1724_cast_fp16 = einsum(equation = var_1724_equation_0, values = (var_1708_cast_fp16_2, var_1722_cast_fp16))[name = string("op_1724_cast_fp16")];
            fp16 var_1725_to_fp16 = const()[name = string("op_1725_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_1726_cast_fp16 = mul(x = var_1703_cast_fp16_3, y = var_1725_to_fp16)[name = string("op_1726_cast_fp16")];
            string var_1728_equation_0 = const()[name = string("op_1728_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_1728_cast_fp16 = einsum(equation = var_1728_equation_0, values = (var_1708_cast_fp16_3, var_1726_cast_fp16))[name = string("op_1728_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_1729_cast_fp16 = add(x = var_1716_cast_fp16, y = var_1720_cast_fp16)[name = string("op_1729_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_1730_cast_fp16 = add(x = var_1724_cast_fp16, y = var_1728_cast_fp16)[name = string("op_1730_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_1731_cast_fp16 = add(x = var_1729_cast_fp16, y = var_1730_cast_fp16)[name = string("op_1731_cast_fp16")];
            fp16 _inversed_1733_y_0_to_fp16 = const()[name = string("_inversed_1733_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 4]> _inversed_1733_cast_fp16 = mul(x = var_1731_cast_fp16, y = _inversed_1733_y_0_to_fp16)[name = string("_inversed_1733_cast_fp16")];
            string scores_25_equation_0 = const()[name = string("scores_25_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 4]> scores_25_cast_fp16 = einsum(equation = scores_25_equation_0, values = (var_1580_cast_fp16_1, var_1570_cast_fp16_4))[name = string("scores_25_cast_fp16")];
            fp16 var_1736_to_fp16 = const()[name = string("op_1736_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 4]> var_1737_cast_fp16 = mul(x = scores_25_cast_fp16, y = var_1736_to_fp16)[name = string("op_1737_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_1738_cast_fp16 = add(x = var_1737_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_1738_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_1739_cast_fp16 = softmax(axis = var_1548, x = var_1738_cast_fp16)[name = string("op_1739_cast_fp16")];
            tensor<int32, [4]> tile_30 = const()[name = string("tile_30"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_1740_axis_0 = const()[name = string("op_1740_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 4]> var_1740_cast_fp16_0, tensor<fp16, [1, 512, 1, 4]> var_1740_cast_fp16_1, tensor<fp16, [1, 512, 1, 4]> var_1740_cast_fp16_2, tensor<fp16, [1, 512, 1, 4]> var_1740_cast_fp16_3 = split(axis = var_1740_axis_0, split_sizes = tile_30, x = var_1739_cast_fp16)[name = string("op_1740_cast_fp16")];
            tensor<int32, [4]> tile_31 = const()[name = string("tile_31"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_1745_axis_0 = const()[name = string("op_1745_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_1745_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_1745_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_1745_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_1745_cast_fp16_3 = split(axis = var_1745_axis_0, split_sizes = tile_31, x = var_1583_cast_fp16_1)[name = string("op_1745_cast_fp16")];
            fp16 var_1750_to_fp16 = const()[name = string("op_1750_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_1751_cast_fp16 = mul(x = var_1740_cast_fp16_0, y = var_1750_to_fp16)[name = string("op_1751_cast_fp16")];
            string var_1753_equation_0 = const()[name = string("op_1753_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_1753_cast_fp16 = einsum(equation = var_1753_equation_0, values = (var_1745_cast_fp16_0, var_1751_cast_fp16))[name = string("op_1753_cast_fp16")];
            fp16 var_1754_to_fp16 = const()[name = string("op_1754_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_1755_cast_fp16 = mul(x = var_1740_cast_fp16_1, y = var_1754_to_fp16)[name = string("op_1755_cast_fp16")];
            string var_1757_equation_0 = const()[name = string("op_1757_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_1757_cast_fp16 = einsum(equation = var_1757_equation_0, values = (var_1745_cast_fp16_1, var_1755_cast_fp16))[name = string("op_1757_cast_fp16")];
            fp16 var_1758_to_fp16 = const()[name = string("op_1758_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_1759_cast_fp16 = mul(x = var_1740_cast_fp16_2, y = var_1758_to_fp16)[name = string("op_1759_cast_fp16")];
            string var_1761_equation_0 = const()[name = string("op_1761_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_1761_cast_fp16 = einsum(equation = var_1761_equation_0, values = (var_1745_cast_fp16_2, var_1759_cast_fp16))[name = string("op_1761_cast_fp16")];
            fp16 var_1762_to_fp16 = const()[name = string("op_1762_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_1763_cast_fp16 = mul(x = var_1740_cast_fp16_3, y = var_1762_to_fp16)[name = string("op_1763_cast_fp16")];
            string var_1765_equation_0 = const()[name = string("op_1765_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_1765_cast_fp16 = einsum(equation = var_1765_equation_0, values = (var_1745_cast_fp16_3, var_1763_cast_fp16))[name = string("op_1765_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_1766_cast_fp16 = add(x = var_1753_cast_fp16, y = var_1757_cast_fp16)[name = string("op_1766_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_1767_cast_fp16 = add(x = var_1761_cast_fp16, y = var_1765_cast_fp16)[name = string("op_1767_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_1768_cast_fp16 = add(x = var_1766_cast_fp16, y = var_1767_cast_fp16)[name = string("op_1768_cast_fp16")];
            fp16 _inversed_1770_y_0_to_fp16 = const()[name = string("_inversed_1770_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 4]> _inversed_1770_cast_fp16 = mul(x = var_1768_cast_fp16, y = _inversed_1770_y_0_to_fp16)[name = string("_inversed_1770_cast_fp16")];
            string scores_27_equation_0 = const()[name = string("scores_27_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 4]> scores_27_cast_fp16 = einsum(equation = scores_27_equation_0, values = (var_1580_cast_fp16_1, var_1570_cast_fp16_5))[name = string("scores_27_cast_fp16")];
            fp16 var_1773_to_fp16 = const()[name = string("op_1773_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 4]> var_1774_cast_fp16 = mul(x = scores_27_cast_fp16, y = var_1773_to_fp16)[name = string("op_1774_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_1775_cast_fp16 = add(x = var_1774_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_1775_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_1776_cast_fp16 = softmax(axis = var_1548, x = var_1775_cast_fp16)[name = string("op_1776_cast_fp16")];
            tensor<int32, [4]> tile_32 = const()[name = string("tile_32"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_1777_axis_0 = const()[name = string("op_1777_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 4]> var_1777_cast_fp16_0, tensor<fp16, [1, 512, 1, 4]> var_1777_cast_fp16_1, tensor<fp16, [1, 512, 1, 4]> var_1777_cast_fp16_2, tensor<fp16, [1, 512, 1, 4]> var_1777_cast_fp16_3 = split(axis = var_1777_axis_0, split_sizes = tile_32, x = var_1776_cast_fp16)[name = string("op_1777_cast_fp16")];
            tensor<int32, [4]> tile_33 = const()[name = string("tile_33"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_1782_axis_0 = const()[name = string("op_1782_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_1782_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_1782_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_1782_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_1782_cast_fp16_3 = split(axis = var_1782_axis_0, split_sizes = tile_33, x = var_1583_cast_fp16_1)[name = string("op_1782_cast_fp16")];
            fp16 var_1787_to_fp16 = const()[name = string("op_1787_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_1788_cast_fp16 = mul(x = var_1777_cast_fp16_0, y = var_1787_to_fp16)[name = string("op_1788_cast_fp16")];
            string var_1790_equation_0 = const()[name = string("op_1790_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_1790_cast_fp16 = einsum(equation = var_1790_equation_0, values = (var_1782_cast_fp16_0, var_1788_cast_fp16))[name = string("op_1790_cast_fp16")];
            fp16 var_1791_to_fp16 = const()[name = string("op_1791_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_1792_cast_fp16 = mul(x = var_1777_cast_fp16_1, y = var_1791_to_fp16)[name = string("op_1792_cast_fp16")];
            string var_1794_equation_0 = const()[name = string("op_1794_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_1794_cast_fp16 = einsum(equation = var_1794_equation_0, values = (var_1782_cast_fp16_1, var_1792_cast_fp16))[name = string("op_1794_cast_fp16")];
            fp16 var_1795_to_fp16 = const()[name = string("op_1795_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_1796_cast_fp16 = mul(x = var_1777_cast_fp16_2, y = var_1795_to_fp16)[name = string("op_1796_cast_fp16")];
            string var_1798_equation_0 = const()[name = string("op_1798_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_1798_cast_fp16 = einsum(equation = var_1798_equation_0, values = (var_1782_cast_fp16_2, var_1796_cast_fp16))[name = string("op_1798_cast_fp16")];
            fp16 var_1799_to_fp16 = const()[name = string("op_1799_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_1800_cast_fp16 = mul(x = var_1777_cast_fp16_3, y = var_1799_to_fp16)[name = string("op_1800_cast_fp16")];
            string var_1802_equation_0 = const()[name = string("op_1802_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_1802_cast_fp16 = einsum(equation = var_1802_equation_0, values = (var_1782_cast_fp16_3, var_1800_cast_fp16))[name = string("op_1802_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_1803_cast_fp16 = add(x = var_1790_cast_fp16, y = var_1794_cast_fp16)[name = string("op_1803_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_1804_cast_fp16 = add(x = var_1798_cast_fp16, y = var_1802_cast_fp16)[name = string("op_1804_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_1805_cast_fp16 = add(x = var_1803_cast_fp16, y = var_1804_cast_fp16)[name = string("op_1805_cast_fp16")];
            fp16 _inversed_1807_y_0_to_fp16 = const()[name = string("_inversed_1807_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 4]> _inversed_1807_cast_fp16 = mul(x = var_1805_cast_fp16, y = _inversed_1807_y_0_to_fp16)[name = string("_inversed_1807_cast_fp16")];
            string scores_29_equation_0 = const()[name = string("scores_29_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 4]> scores_29_cast_fp16 = einsum(equation = scores_29_equation_0, values = (var_1580_cast_fp16_1, var_1570_cast_fp16_6))[name = string("scores_29_cast_fp16")];
            fp16 var_1810_to_fp16 = const()[name = string("op_1810_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 4]> var_1811_cast_fp16 = mul(x = scores_29_cast_fp16, y = var_1810_to_fp16)[name = string("op_1811_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_1812_cast_fp16 = add(x = var_1811_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_1812_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_1813_cast_fp16 = softmax(axis = var_1548, x = var_1812_cast_fp16)[name = string("op_1813_cast_fp16")];
            tensor<int32, [4]> tile_34 = const()[name = string("tile_34"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_1814_axis_0 = const()[name = string("op_1814_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 4]> var_1814_cast_fp16_0, tensor<fp16, [1, 512, 1, 4]> var_1814_cast_fp16_1, tensor<fp16, [1, 512, 1, 4]> var_1814_cast_fp16_2, tensor<fp16, [1, 512, 1, 4]> var_1814_cast_fp16_3 = split(axis = var_1814_axis_0, split_sizes = tile_34, x = var_1813_cast_fp16)[name = string("op_1814_cast_fp16")];
            tensor<int32, [4]> tile_35 = const()[name = string("tile_35"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_1819_axis_0 = const()[name = string("op_1819_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_1819_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_1819_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_1819_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_1819_cast_fp16_3 = split(axis = var_1819_axis_0, split_sizes = tile_35, x = var_1583_cast_fp16_1)[name = string("op_1819_cast_fp16")];
            fp16 var_1824_to_fp16 = const()[name = string("op_1824_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_1825_cast_fp16 = mul(x = var_1814_cast_fp16_0, y = var_1824_to_fp16)[name = string("op_1825_cast_fp16")];
            string var_1827_equation_0 = const()[name = string("op_1827_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_1827_cast_fp16 = einsum(equation = var_1827_equation_0, values = (var_1819_cast_fp16_0, var_1825_cast_fp16))[name = string("op_1827_cast_fp16")];
            fp16 var_1828_to_fp16 = const()[name = string("op_1828_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_1829_cast_fp16 = mul(x = var_1814_cast_fp16_1, y = var_1828_to_fp16)[name = string("op_1829_cast_fp16")];
            string var_1831_equation_0 = const()[name = string("op_1831_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_1831_cast_fp16 = einsum(equation = var_1831_equation_0, values = (var_1819_cast_fp16_1, var_1829_cast_fp16))[name = string("op_1831_cast_fp16")];
            fp16 var_1832_to_fp16 = const()[name = string("op_1832_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_1833_cast_fp16 = mul(x = var_1814_cast_fp16_2, y = var_1832_to_fp16)[name = string("op_1833_cast_fp16")];
            string var_1835_equation_0 = const()[name = string("op_1835_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_1835_cast_fp16 = einsum(equation = var_1835_equation_0, values = (var_1819_cast_fp16_2, var_1833_cast_fp16))[name = string("op_1835_cast_fp16")];
            fp16 var_1836_to_fp16 = const()[name = string("op_1836_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_1837_cast_fp16 = mul(x = var_1814_cast_fp16_3, y = var_1836_to_fp16)[name = string("op_1837_cast_fp16")];
            string var_1839_equation_0 = const()[name = string("op_1839_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_1839_cast_fp16 = einsum(equation = var_1839_equation_0, values = (var_1819_cast_fp16_3, var_1837_cast_fp16))[name = string("op_1839_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_1840_cast_fp16 = add(x = var_1827_cast_fp16, y = var_1831_cast_fp16)[name = string("op_1840_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_1841_cast_fp16 = add(x = var_1835_cast_fp16, y = var_1839_cast_fp16)[name = string("op_1841_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_1842_cast_fp16 = add(x = var_1840_cast_fp16, y = var_1841_cast_fp16)[name = string("op_1842_cast_fp16")];
            fp16 _inversed_1844_y_0_to_fp16 = const()[name = string("_inversed_1844_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 4]> _inversed_1844_cast_fp16 = mul(x = var_1842_cast_fp16, y = _inversed_1844_y_0_to_fp16)[name = string("_inversed_1844_cast_fp16")];
            string scores_equation_0 = const()[name = string("scores_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 4]> scores_cast_fp16 = einsum(equation = scores_equation_0, values = (var_1580_cast_fp16_1, var_1570_cast_fp16_7))[name = string("scores_cast_fp16")];
            fp16 var_1847_to_fp16 = const()[name = string("op_1847_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 4]> var_1848_cast_fp16 = mul(x = scores_cast_fp16, y = var_1847_to_fp16)[name = string("op_1848_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_1849_cast_fp16 = add(x = var_1848_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_1849_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 4]> var_1850_cast_fp16 = softmax(axis = var_1548, x = var_1849_cast_fp16)[name = string("op_1850_cast_fp16")];
            tensor<int32, [4]> tile_36 = const()[name = string("tile_36"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_1851_axis_0 = const()[name = string("op_1851_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 4]> var_1851_cast_fp16_0, tensor<fp16, [1, 512, 1, 4]> var_1851_cast_fp16_1, tensor<fp16, [1, 512, 1, 4]> var_1851_cast_fp16_2, tensor<fp16, [1, 512, 1, 4]> var_1851_cast_fp16_3 = split(axis = var_1851_axis_0, split_sizes = tile_36, x = var_1850_cast_fp16)[name = string("op_1851_cast_fp16")];
            tensor<int32, [4]> tile_37 = const()[name = string("tile_37"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_1856_axis_0 = const()[name = string("op_1856_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_1856_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_1856_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_1856_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_1856_cast_fp16_3 = split(axis = var_1856_axis_0, split_sizes = tile_37, x = var_1583_cast_fp16_1)[name = string("op_1856_cast_fp16")];
            fp16 var_1861_to_fp16 = const()[name = string("op_1861_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_1862_cast_fp16 = mul(x = var_1851_cast_fp16_0, y = var_1861_to_fp16)[name = string("op_1862_cast_fp16")];
            string var_1864_equation_0 = const()[name = string("op_1864_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_1864_cast_fp16 = einsum(equation = var_1864_equation_0, values = (var_1856_cast_fp16_0, var_1862_cast_fp16))[name = string("op_1864_cast_fp16")];
            fp16 var_1865_to_fp16 = const()[name = string("op_1865_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_1866_cast_fp16 = mul(x = var_1851_cast_fp16_1, y = var_1865_to_fp16)[name = string("op_1866_cast_fp16")];
            string var_1868_equation_0 = const()[name = string("op_1868_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_1868_cast_fp16 = einsum(equation = var_1868_equation_0, values = (var_1856_cast_fp16_1, var_1866_cast_fp16))[name = string("op_1868_cast_fp16")];
            fp16 var_1869_to_fp16 = const()[name = string("op_1869_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_1870_cast_fp16 = mul(x = var_1851_cast_fp16_2, y = var_1869_to_fp16)[name = string("op_1870_cast_fp16")];
            string var_1872_equation_0 = const()[name = string("op_1872_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_1872_cast_fp16 = einsum(equation = var_1872_equation_0, values = (var_1856_cast_fp16_2, var_1870_cast_fp16))[name = string("op_1872_cast_fp16")];
            fp16 var_1873_to_fp16 = const()[name = string("op_1873_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 4]> var_1874_cast_fp16 = mul(x = var_1851_cast_fp16_3, y = var_1873_to_fp16)[name = string("op_1874_cast_fp16")];
            string var_1876_equation_0 = const()[name = string("op_1876_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 4]> var_1876_cast_fp16 = einsum(equation = var_1876_equation_0, values = (var_1856_cast_fp16_3, var_1874_cast_fp16))[name = string("op_1876_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_1877_cast_fp16 = add(x = var_1864_cast_fp16, y = var_1868_cast_fp16)[name = string("op_1877_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_1878_cast_fp16 = add(x = var_1872_cast_fp16, y = var_1876_cast_fp16)[name = string("op_1878_cast_fp16")];
            tensor<fp16, [1, 256, 1, 4]> var_1879_cast_fp16 = add(x = var_1877_cast_fp16, y = var_1878_cast_fp16)[name = string("op_1879_cast_fp16")];
            fp16 _inversed_1881_y_0_to_fp16 = const()[name = string("_inversed_1881_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 4]> _inversed_1881_cast_fp16 = mul(x = var_1879_cast_fp16, y = _inversed_1881_y_0_to_fp16)[name = string("_inversed_1881_cast_fp16")];
            bool var_1883_interleave_0 = const()[name = string("op_1883_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 2048, 1, 4]> var_1883_cast_fp16 = concat(axis = var_1548, interleave = var_1883_interleave_0, values = (_inversed_1622_cast_fp16, _inversed_1659_cast_fp16, _inversed_1696_cast_fp16, _inversed_1733_cast_fp16, _inversed_1770_cast_fp16, _inversed_1807_cast_fp16, _inversed_1844_cast_fp16, _inversed_1881_cast_fp16))[name = string("op_1883_cast_fp16")];
            tensor<int32, [4]> var_1884 = const()[name = string("op_1884"), val = tensor<int32, [4]>([2, 4, 256, 4])];
            tensor<fp16, [2, 4, 256, 4]> var_1885_cast_fp16 = reshape(shape = var_1884, x = var_1883_cast_fp16)[name = string("op_1885_cast_fp16")];
            tensor<int32, [4]> transpose_2_perm_0 = const()[name = string("transpose_2_perm_0"), val = tensor<int32, [4]>([3, 0, 1, 2])];
            tensor<int32, [2]> var_1904 = const()[name = string("op_1904"), val = tensor<int32, [2]>([4, 2048])];
            tensor<fp16, [4, 2, 4, 256]> transpose_2_cast_fp16 = transpose(perm = transpose_2_perm_0, x = var_1885_cast_fp16)[name = string("transpose_0")];
            tensor<fp16, [4, 2048]> attention_output_cast_fp16 = reshape(shape = var_1904, x = transpose_2_cast_fp16)[name = string("attention_output_cast_fp16")];
            tensor<fp16, [4, 2048]> var_1906_cast_fp16 = sigmoid(x = gate_cast_fp16)[name = string("op_1906_cast_fp16")];
            tensor<fp16, [4, 2048]> input_53_cast_fp16 = mul(x = attention_output_cast_fp16, y = var_1906_cast_fp16)[name = string("input_53_cast_fp16")];
            tensor<fp16, [1024, 2048]> completion23_o_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 2048]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(256190528))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(257763456))))[name = string("completion23_o_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 1024]> linear_31_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = completion23_o_proj_weight_promoted_to_fp16_palettized, x = input_53_cast_fp16)[name = string("linear_31_cast_fp16")];
            tensor<fp16, [4, 1024]> value_59_cast_fp16 = add(x = value_45_cast_fp16, y = linear_31_cast_fp16)[name = string("value_59_cast_fp16")];
            tensor<fp16, [4, 1024]> var_1911_cast_fp16 = mul(x = value_59_cast_fp16, y = value_59_cast_fp16)[name = string("op_1911_cast_fp16")];
            tensor<int32, [1]> variance_27_axes_0 = const()[name = string("variance_27_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_27_keep_dims_0 = const()[name = string("variance_27_keep_dims_0"), val = bool(true)];
            tensor<fp16, [4, 1]> variance_27_cast_fp16 = reduce_mean(axes = variance_27_axes_0, keep_dims = variance_27_keep_dims_0, x = var_1911_cast_fp16)[name = string("variance_27_cast_fp16")];
            fp16 var_1914_to_fp16 = const()[name = string("op_1914_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [4, 1]> var_1915_cast_fp16 = add(x = variance_27_cast_fp16, y = var_1914_to_fp16)[name = string("op_1915_cast_fp16")];
            fp32 var_1916_epsilon_0 = const()[name = string("op_1916_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [4, 1]> var_1916_cast_fp16 = rsqrt(epsilon = var_1916_epsilon_0, x = var_1915_cast_fp16)[name = string("op_1916_cast_fp16")];
            tensor<fp16, [4, 1024]> var_1917_cast_fp16 = mul(x = value_59_cast_fp16, y = var_1916_cast_fp16)[name = string("op_1917_cast_fp16")];
            tensor<fp16, [1024]> var_1919_to_fp16 = const()[name = string("op_1919_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(257771712)))];
            tensor<fp16, [4, 1024]> input_55_cast_fp16 = mul(x = var_1917_cast_fp16, y = var_1919_to_fp16)[name = string("input_55_cast_fp16")];
            tensor<fp16, [3584, 1024]> completion23_gate_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(257773824))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(260526400))))[name = string("completion23_gate_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 3584]> linear_32_cast_fp16 = linear(bias = linear_1_bias_0_to_fp16, weight = completion23_gate_proj_weight_promoted_to_fp16_palettized, x = input_55_cast_fp16)[name = string("linear_32_cast_fp16")];
            tensor<fp16, [4, 3584]> var_1923_cast_fp16 = silu(x = linear_32_cast_fp16)[name = string("op_1923_cast_fp16")];
            tensor<fp16, [3584, 1024]> completion23_up_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(260555136))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(263307712))))[name = string("completion23_up_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 3584]> linear_33_cast_fp16 = linear(bias = linear_1_bias_0_to_fp16, weight = completion23_up_proj_weight_promoted_to_fp16_palettized, x = input_55_cast_fp16)[name = string("linear_33_cast_fp16")];
            tensor<fp16, [4, 3584]> input_cast_fp16 = mul(x = var_1923_cast_fp16, y = linear_33_cast_fp16)[name = string("input_cast_fp16")];
            tensor<fp16, [1024, 3584]> completion23_down_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 3584]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(263336448))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(266089024))))[name = string("completion23_down_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4, 1024]> linear_34_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = completion23_down_proj_weight_promoted_to_fp16_palettized, x = input_cast_fp16)[name = string("linear_34_cast_fp16")];
            tensor<fp16, [4, 1024]> value_cast_fp16 = add(x = value_59_cast_fp16, y = linear_34_cast_fp16)[name = string("value_cast_fp16")];
            tensor<fp16, [4, 1024]> var_1930_cast_fp16 = mul(x = value_cast_fp16, y = value_cast_fp16)[name = string("op_1930_cast_fp16")];
            tensor<int32, [1]> variance_axes_0 = const()[name = string("variance_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_keep_dims_0 = const()[name = string("variance_keep_dims_0"), val = bool(true)];
            tensor<fp16, [4, 1]> variance_cast_fp16 = reduce_mean(axes = variance_axes_0, keep_dims = variance_keep_dims_0, x = var_1930_cast_fp16)[name = string("variance_cast_fp16")];
            fp16 var_1937_to_fp16 = const()[name = string("op_1937_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [4, 1]> var_1938_cast_fp16 = add(x = variance_cast_fp16, y = var_1937_to_fp16)[name = string("op_1938_cast_fp16")];
            fp32 var_1939_epsilon_0 = const()[name = string("op_1939_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [4, 1]> var_1939_cast_fp16 = rsqrt(epsilon = var_1939_epsilon_0, x = var_1938_cast_fp16)[name = string("op_1939_cast_fp16")];
            tensor<fp16, [4, 1024]> var_1940_cast_fp16 = mul(x = value_cast_fp16, y = var_1939_cast_fp16)[name = string("op_1940_cast_fp16")];
            tensor<fp16, [1024]> var_1943_to_fp16 = const()[name = string("op_1943_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(266097280)))];
            tensor<fp16, [4, 1024]> hidden_out = mul(x = var_1940_cast_fp16, y = var_1943_to_fp16)[name = string("hidden_cast_fp16")];
            tensor<int32, [2]> var_1949_begin_0 = const()[name = string("op_1949_begin_0"), val = tensor<int32, [2]>([-1, 0])];
            tensor<int32, [2]> var_1949_end_0 = const()[name = string("op_1949_end_0"), val = tensor<int32, [2]>([4, 1024])];
            tensor<bool, [2]> var_1949_end_mask_0 = const()[name = string("op_1949_end_mask_0"), val = tensor<bool, [2]>([true, true])];
            tensor<fp16, [1, 1024]> var_1949_cast_fp16 = slice_by_index(begin = var_1949_begin_0, end = var_1949_end_0, end_mask = var_1949_end_mask_0, x = hidden_out)[name = string("op_1949_cast_fp16")];
            tensor<fp16, [248320, 1024]> head_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [248320, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(64))), lut = tensor<fp16, [15520, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(190709888))))[name = string("head_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [248320]> linear_35_bias_0_to_fp16 = const()[name = string("linear_35_bias_0_to_fp16"), val = tensor<fp16, [248320]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(192696512)))];
            tensor<fp16, [1, 248320]> linear_35_cast_fp16 = linear(bias = linear_35_bias_0_to_fp16, weight = head_weight_promoted_to_fp16_palettized, x = var_1949_cast_fp16)[name = string("linear_35_cast_fp16")];
            int32 var_1954_axis_0 = const()[name = string("op_1954_axis_0"), val = int32(-1)];
            bool var_1954_keep_dims_0 = const()[name = string("op_1954_keep_dims_0"), val = bool(false)];
            string var_1954_output_dtype_0 = const()[name = string("op_1954_output_dtype_0"), val = string("int32")];
            tensor<int32, [1]> next_token = reduce_argmax(axis = var_1954_axis_0, keep_dims = var_1954_keep_dims_0, output_dtype = var_1954_output_dtype_0, x = linear_35_cast_fp16)[name = string("op_1954_cast_fp16")];
        } -> (hidden_out, conv20_out, rec20_out, conv21_out, rec21_out, conv22_out, rec22_out, next_token);
    func stateful_stage5_final_head<ios18>(tensor<fp16, [1, 6144, 3]> conv20, tensor<fp16, [1, 6144, 3]> conv21, tensor<fp16, [1, 6144, 3]> conv22, tensor<fp16, [1, 1, 1, 64]> cos, tensor<int32, [1]> current_pos, tensor<fp16, [1, 2048]> gate19, tensor<fp16, [1, 1024]> hidden, tensor<fp16, [2, 1, 256]> k19, state<tensor<fp16, [2, 2, 2, 2048, 256]>> kv_cache, tensor<fp16, [1, 1, 2048]> mask19, tensor<fp16, [2, 1, 4, 256]> q19, tensor<fp16, [1, 16, 128, 128]> rec20, tensor<fp16, [1, 16, 128, 128]> rec21, tensor<fp16, [1, 16, 128, 128]> rec22, tensor<fp16, [1, 1, 1, 64]> sin, tensor<fp16, [2, 1, 256]> v19) {
            tensor<int32, [1]> var_33_axes_0 = const()[name = string("op_33_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [1, 2, 1, 256]> var_33_cast_fp16 = expand_dims(axes = var_33_axes_0, x = k19)[name = string("op_33_cast_fp16")];
            tensor<int32, [1]> var_35_axes_0 = const()[name = string("op_35_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [1, 1, 2, 1, 256]> var_35_cast_fp16 = expand_dims(axes = var_35_axes_0, x = var_33_cast_fp16)[name = string("op_35_cast_fp16")];
            int32 var_37 = const()[name = string("op_37"), val = int32(1)];
            tensor<int32, [1]> var_38 = add(x = current_pos, y = var_37)[name = string("op_38")];
            tensor<fp16, [2, 2, 2, 2048, 256]> read_state_0 = read_state(input = kv_cache)[name = string("read_state_0")];
            tensor<int32, [1]> expand_dims_0 = const()[name = string("expand_dims_0"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_1 = const()[name = string("expand_dims_1"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_2 = const()[name = string("expand_dims_2"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_4 = const()[name = string("expand_dims_4"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_5 = const()[name = string("expand_dims_5"), val = tensor<int32, [1]>([1])];
            tensor<int32, [1]> expand_dims_6 = const()[name = string("expand_dims_6"), val = tensor<int32, [1]>([1])];
            int32 concat_2_axis_0 = const()[name = string("concat_2_axis_0"), val = int32(0)];
            bool concat_2_interleave_0 = const()[name = string("concat_2_interleave_0"), val = bool(false)];
            tensor<int32, [5]> concat_2 = concat(axis = concat_2_axis_0, interleave = concat_2_interleave_0, values = (expand_dims_0, expand_dims_1, expand_dims_2, current_pos, expand_dims_4))[name = string("concat_2")];
            tensor<int32, [1]> concat_3_values2_0 = const()[name = string("concat_3_values2_0"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> concat_3_values4_0 = const()[name = string("concat_3_values4_0"), val = tensor<int32, [1]>([0])];
            int32 concat_3_axis_0 = const()[name = string("concat_3_axis_0"), val = int32(0)];
            bool concat_3_interleave_0 = const()[name = string("concat_3_interleave_0"), val = bool(false)];
            tensor<int32, [5]> concat_3 = concat(axis = concat_3_axis_0, interleave = concat_3_interleave_0, values = (expand_dims_5, expand_dims_6, concat_3_values2_0, var_38, concat_3_values4_0))[name = string("concat_3")];
            tensor<int32, [5]> kv_cache_internal_tensor_assign_1_stride_0 = const()[name = string("kv_cache_internal_tensor_assign_1_stride_0"), val = tensor<int32, [5]>([1, 1, 1, 1, 1])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_1_begin_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_1_begin_mask_0"), val = tensor<bool, [5]>([false, false, false, false, false])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_1_end_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_1_end_mask_0"), val = tensor<bool, [5]>([false, false, true, false, true])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_1_squeeze_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_1_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, false, false, false])];
            tensor<fp16, [2, 2, 2, 2048, 256]> kv_cache_internal_tensor_assign_1_cast_fp16 = slice_update(begin = concat_2, begin_mask = kv_cache_internal_tensor_assign_1_begin_mask_0, end = concat_3, end_mask = kv_cache_internal_tensor_assign_1_end_mask_0, squeeze_mask = kv_cache_internal_tensor_assign_1_squeeze_mask_0, stride = kv_cache_internal_tensor_assign_1_stride_0, update = var_35_cast_fp16, x = read_state_0)[name = string("kv_cache_internal_tensor_assign_1_cast_fp16")];
            write_state(data = kv_cache_internal_tensor_assign_1_cast_fp16, input = kv_cache)[name = string("coreml_update_state_0_write_state")];
            tensor<fp16, [2, 2, 2, 2048, 256]> coreml_update_state_4 = read_state(input = kv_cache)[name = string("coreml_update_state_0")];
            tensor<int32, [1]> var_66_axes_0 = const()[name = string("op_66_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [1, 2, 1, 256]> var_66_cast_fp16 = expand_dims(axes = var_66_axes_0, x = v19)[name = string("op_66_cast_fp16")];
            tensor<int32, [1]> var_68_axes_0 = const()[name = string("op_68_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [1, 1, 2, 1, 256]> var_68_cast_fp16 = expand_dims(axes = var_68_axes_0, x = var_66_cast_fp16)[name = string("op_68_cast_fp16")];
            tensor<int32, [1]> expand_dims_8 = const()[name = string("expand_dims_8"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_9 = const()[name = string("expand_dims_9"), val = tensor<int32, [1]>([1])];
            tensor<int32, [1]> expand_dims_10 = const()[name = string("expand_dims_10"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_12 = const()[name = string("expand_dims_12"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_13 = const()[name = string("expand_dims_13"), val = tensor<int32, [1]>([1])];
            tensor<int32, [1]> expand_dims_14 = const()[name = string("expand_dims_14"), val = tensor<int32, [1]>([2])];
            int32 concat_6_axis_0 = const()[name = string("concat_6_axis_0"), val = int32(0)];
            bool concat_6_interleave_0 = const()[name = string("concat_6_interleave_0"), val = bool(false)];
            tensor<int32, [5]> concat_6 = concat(axis = concat_6_axis_0, interleave = concat_6_interleave_0, values = (expand_dims_8, expand_dims_9, expand_dims_10, current_pos, expand_dims_12))[name = string("concat_6")];
            tensor<int32, [1]> concat_7_values2_0 = const()[name = string("concat_7_values2_0"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> concat_7_values4_0 = const()[name = string("concat_7_values4_0"), val = tensor<int32, [1]>([0])];
            int32 concat_7_axis_0 = const()[name = string("concat_7_axis_0"), val = int32(0)];
            bool concat_7_interleave_0 = const()[name = string("concat_7_interleave_0"), val = bool(false)];
            tensor<int32, [5]> concat_7 = concat(axis = concat_7_axis_0, interleave = concat_7_interleave_0, values = (expand_dims_13, expand_dims_14, concat_7_values2_0, var_38, concat_7_values4_0))[name = string("concat_7")];
            tensor<int32, [5]> kv_cache_internal_tensor_assign_2_stride_0 = const()[name = string("kv_cache_internal_tensor_assign_2_stride_0"), val = tensor<int32, [5]>([1, 1, 1, 1, 1])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_2_begin_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_2_begin_mask_0"), val = tensor<bool, [5]>([false, false, false, false, false])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_2_end_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_2_end_mask_0"), val = tensor<bool, [5]>([false, false, true, false, true])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_2_squeeze_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_2_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, false, false, false])];
            tensor<fp16, [2, 2, 2, 2048, 256]> kv_cache_internal_tensor_assign_2_cast_fp16 = slice_update(begin = concat_6, begin_mask = kv_cache_internal_tensor_assign_2_begin_mask_0, end = concat_7, end_mask = kv_cache_internal_tensor_assign_2_end_mask_0, squeeze_mask = kv_cache_internal_tensor_assign_2_squeeze_mask_0, stride = kv_cache_internal_tensor_assign_2_stride_0, update = var_68_cast_fp16, x = coreml_update_state_4)[name = string("kv_cache_internal_tensor_assign_2_cast_fp16")];
            write_state(data = kv_cache_internal_tensor_assign_2_cast_fp16, input = kv_cache)[name = string("coreml_update_state_1_write_state")];
            tensor<fp16, [2, 2, 2, 2048, 256]> coreml_update_state_5 = read_state(input = kv_cache)[name = string("coreml_update_state_1")];
            tensor<int32, [5]> var_100_begin_0 = const()[name = string("op_100_begin_0"), val = tensor<int32, [5]>([0, 0, 0, 0, 0])];
            tensor<int32, [5]> var_100_end_0 = const()[name = string("op_100_end_0"), val = tensor<int32, [5]>([1, 2, 2, 2048, 256])];
            tensor<bool, [5]> var_100_end_mask_0 = const()[name = string("op_100_end_mask_0"), val = tensor<bool, [5]>([false, true, true, true, true])];
            tensor<bool, [5]> var_100_squeeze_mask_0 = const()[name = string("op_100_squeeze_mask_0"), val = tensor<bool, [5]>([true, false, false, false, false])];
            tensor<fp16, [2, 2, 2048, 256]> var_100_cast_fp16 = slice_by_index(begin = var_100_begin_0, end = var_100_end_0, end_mask = var_100_end_mask_0, squeeze_mask = var_100_squeeze_mask_0, x = coreml_update_state_5)[name = string("op_100_cast_fp16")];
            tensor<int32, [4]> keys_1_begin_0 = const()[name = string("keys_1_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> keys_1_end_0 = const()[name = string("keys_1_end_0"), val = tensor<int32, [4]>([1, 2, 2048, 256])];
            tensor<bool, [4]> keys_1_end_mask_0 = const()[name = string("keys_1_end_mask_0"), val = tensor<bool, [4]>([false, true, true, true])];
            tensor<bool, [4]> keys_1_squeeze_mask_0 = const()[name = string("keys_1_squeeze_mask_0"), val = tensor<bool, [4]>([true, false, false, false])];
            tensor<fp16, [2, 2048, 256]> keys_1_cast_fp16 = slice_by_index(begin = keys_1_begin_0, end = keys_1_end_0, end_mask = keys_1_end_mask_0, squeeze_mask = keys_1_squeeze_mask_0, x = var_100_cast_fp16)[name = string("keys_1_cast_fp16")];
            tensor<int32, [4]> values_1_begin_0 = const()[name = string("values_1_begin_0"), val = tensor<int32, [4]>([1, 0, 0, 0])];
            tensor<int32, [4]> values_1_end_0 = const()[name = string("values_1_end_0"), val = tensor<int32, [4]>([2, 2, 2048, 256])];
            tensor<bool, [4]> values_1_end_mask_0 = const()[name = string("values_1_end_mask_0"), val = tensor<bool, [4]>([false, true, true, true])];
            tensor<bool, [4]> values_1_squeeze_mask_0 = const()[name = string("values_1_squeeze_mask_0"), val = tensor<bool, [4]>([true, false, false, false])];
            tensor<fp16, [2, 2048, 256]> values_1_cast_fp16 = slice_by_index(begin = values_1_begin_0, end = values_1_end_0, end_mask = values_1_end_mask_0, squeeze_mask = values_1_squeeze_mask_0, x = var_100_cast_fp16)[name = string("values_1_cast_fp16")];
            int32 var_117 = const()[name = string("op_117"), val = int32(1)];
            tensor<int32, [4]> var_121 = const()[name = string("op_121"), val = tensor<int32, [4]>([0, 2, 3, 1])];
            tensor<int32, [4]> var_124 = const()[name = string("op_124"), val = tensor<int32, [4]>([1, 2048, 1, 1])];
            tensor<fp16, [2, 4, 256, 1]> var_122_cast_fp16 = transpose(perm = var_121, x = q19)[name = string("transpose_34")];
            tensor<fp16, [1, 2048, 1, 1]> var_125_cast_fp16 = reshape(shape = var_124, x = var_122_cast_fp16)[name = string("op_125_cast_fp16")];
            tensor<int32, [3]> var_126 = const()[name = string("op_126"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [4]> var_129 = const()[name = string("op_129"), val = tensor<int32, [4]>([1, 512, 1, 2048])];
            tensor<fp16, [2, 256, 2048]> var_127_cast_fp16 = transpose(perm = var_126, x = keys_1_cast_fp16)[name = string("transpose_33")];
            tensor<fp16, [1, 512, 1, 2048]> key_native_1_cast_fp16 = reshape(shape = var_129, x = var_127_cast_fp16)[name = string("key_native_1_cast_fp16")];
            tensor<int32, [3]> var_131 = const()[name = string("op_131"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [4]> var_134 = const()[name = string("op_134"), val = tensor<int32, [4]>([1, 512, 1, 2048])];
            tensor<fp16, [2, 256, 2048]> var_132_cast_fp16 = transpose(perm = var_131, x = values_1_cast_fp16)[name = string("transpose_32")];
            tensor<fp16, [1, 512, 1, 2048]> var_135_cast_fp16 = reshape(shape = var_134, x = var_132_cast_fp16)[name = string("op_135_cast_fp16")];
            tensor<int32, [3]> var_136 = const()[name = string("op_136"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [1]> mask_native_1_axes_0 = const()[name = string("mask_native_1_axes_0"), val = tensor<int32, [1]>([2])];
            tensor<fp16, [1, 2048, 1]> var_137_cast_fp16 = transpose(perm = var_136, x = mask19)[name = string("transpose_31")];
            tensor<fp16, [1, 2048, 1, 1]> mask_native_1_cast_fp16 = expand_dims(axes = mask_native_1_axes_0, x = var_137_cast_fp16)[name = string("mask_native_1_cast_fp16")];
            tensor<int32, [8]> tile_0 = const()[name = string("tile_0"), val = tensor<int32, [8]>([256, 256, 256, 256, 256, 256, 256, 256])];
            int32 var_139_axis_0 = const()[name = string("op_139_axis_0"), val = int32(1)];
            tensor<fp16, [1, 256, 1, 1]> var_139_cast_fp16_0, tensor<fp16, [1, 256, 1, 1]> var_139_cast_fp16_1, tensor<fp16, [1, 256, 1, 1]> var_139_cast_fp16_2, tensor<fp16, [1, 256, 1, 1]> var_139_cast_fp16_3, tensor<fp16, [1, 256, 1, 1]> var_139_cast_fp16_4, tensor<fp16, [1, 256, 1, 1]> var_139_cast_fp16_5, tensor<fp16, [1, 256, 1, 1]> var_139_cast_fp16_6, tensor<fp16, [1, 256, 1, 1]> var_139_cast_fp16_7 = split(axis = var_139_axis_0, split_sizes = tile_0, x = var_125_cast_fp16)[name = string("op_139_cast_fp16")];
            tensor<int32, [4]> var_148_perm_0 = const()[name = string("op_148_perm_0"), val = tensor<int32, [4]>([0, 3, 2, 1])];
            tensor<int32, [2]> tile_1 = const()[name = string("tile_1"), val = tensor<int32, [2]>([256, 256])];
            int32 var_149_axis_0 = const()[name = string("op_149_axis_0"), val = int32(3)];
            tensor<fp16, [1, 2048, 1, 512]> var_148_cast_fp16 = transpose(perm = var_148_perm_0, x = key_native_1_cast_fp16)[name = string("transpose_30")];
            tensor<fp16, [1, 2048, 1, 256]> var_149_cast_fp16_0, tensor<fp16, [1, 2048, 1, 256]> var_149_cast_fp16_1 = split(axis = var_149_axis_0, split_sizes = tile_1, x = var_148_cast_fp16)[name = string("op_149_cast_fp16")];
            tensor<int32, [2]> tile_2 = const()[name = string("tile_2"), val = tensor<int32, [2]>([256, 256])];
            int32 var_152_axis_0 = const()[name = string("op_152_axis_0"), val = int32(1)];
            tensor<fp16, [1, 256, 1, 2048]> var_152_cast_fp16_0, tensor<fp16, [1, 256, 1, 2048]> var_152_cast_fp16_1 = split(axis = var_152_axis_0, split_sizes = tile_2, x = var_135_cast_fp16)[name = string("op_152_cast_fp16")];
            string scores_1_equation_0 = const()[name = string("scores_1_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 1]> scores_1_cast_fp16 = einsum(equation = scores_1_equation_0, values = (var_149_cast_fp16_0, var_139_cast_fp16_0))[name = string("scores_1_cast_fp16")];
            fp16 var_157_to_fp16 = const()[name = string("op_157_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 1]> var_158_cast_fp16 = mul(x = scores_1_cast_fp16, y = var_157_to_fp16)[name = string("op_158_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_159_cast_fp16 = add(x = var_158_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_159_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_160_cast_fp16 = softmax(axis = var_117, x = var_159_cast_fp16)[name = string("op_160_cast_fp16")];
            tensor<int32, [4]> tile_3 = const()[name = string("tile_3"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_161_axis_0 = const()[name = string("op_161_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 1]> var_161_cast_fp16_0, tensor<fp16, [1, 512, 1, 1]> var_161_cast_fp16_1, tensor<fp16, [1, 512, 1, 1]> var_161_cast_fp16_2, tensor<fp16, [1, 512, 1, 1]> var_161_cast_fp16_3 = split(axis = var_161_axis_0, split_sizes = tile_3, x = var_160_cast_fp16)[name = string("op_161_cast_fp16")];
            tensor<int32, [4]> tile_4 = const()[name = string("tile_4"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_166_axis_0 = const()[name = string("op_166_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_166_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_166_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_166_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_166_cast_fp16_3 = split(axis = var_166_axis_0, split_sizes = tile_4, x = var_152_cast_fp16_0)[name = string("op_166_cast_fp16")];
            fp16 var_171_to_fp16 = const()[name = string("op_171_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_172_cast_fp16 = mul(x = var_161_cast_fp16_0, y = var_171_to_fp16)[name = string("op_172_cast_fp16")];
            string var_174_equation_0 = const()[name = string("op_174_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_174_cast_fp16 = einsum(equation = var_174_equation_0, values = (var_166_cast_fp16_0, var_172_cast_fp16))[name = string("op_174_cast_fp16")];
            fp16 var_175_to_fp16 = const()[name = string("op_175_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_176_cast_fp16 = mul(x = var_161_cast_fp16_1, y = var_175_to_fp16)[name = string("op_176_cast_fp16")];
            string var_178_equation_0 = const()[name = string("op_178_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_178_cast_fp16 = einsum(equation = var_178_equation_0, values = (var_166_cast_fp16_1, var_176_cast_fp16))[name = string("op_178_cast_fp16")];
            fp16 var_179_to_fp16 = const()[name = string("op_179_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_180_cast_fp16 = mul(x = var_161_cast_fp16_2, y = var_179_to_fp16)[name = string("op_180_cast_fp16")];
            string var_182_equation_0 = const()[name = string("op_182_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_182_cast_fp16 = einsum(equation = var_182_equation_0, values = (var_166_cast_fp16_2, var_180_cast_fp16))[name = string("op_182_cast_fp16")];
            fp16 var_183_to_fp16 = const()[name = string("op_183_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_184_cast_fp16 = mul(x = var_161_cast_fp16_3, y = var_183_to_fp16)[name = string("op_184_cast_fp16")];
            string var_186_equation_0 = const()[name = string("op_186_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_186_cast_fp16 = einsum(equation = var_186_equation_0, values = (var_166_cast_fp16_3, var_184_cast_fp16))[name = string("op_186_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_187_cast_fp16 = add(x = var_174_cast_fp16, y = var_178_cast_fp16)[name = string("op_187_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_188_cast_fp16 = add(x = var_182_cast_fp16, y = var_186_cast_fp16)[name = string("op_188_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_189_cast_fp16 = add(x = var_187_cast_fp16, y = var_188_cast_fp16)[name = string("op_189_cast_fp16")];
            fp16 _inversed_191_y_0_to_fp16 = const()[name = string("_inversed_191_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 1]> _inversed_191_cast_fp16 = mul(x = var_189_cast_fp16, y = _inversed_191_y_0_to_fp16)[name = string("_inversed_191_cast_fp16")];
            string scores_3_equation_0 = const()[name = string("scores_3_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 1]> scores_3_cast_fp16 = einsum(equation = scores_3_equation_0, values = (var_149_cast_fp16_0, var_139_cast_fp16_1))[name = string("scores_3_cast_fp16")];
            fp16 var_194_to_fp16 = const()[name = string("op_194_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 1]> var_195_cast_fp16 = mul(x = scores_3_cast_fp16, y = var_194_to_fp16)[name = string("op_195_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_196_cast_fp16 = add(x = var_195_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_196_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_197_cast_fp16 = softmax(axis = var_117, x = var_196_cast_fp16)[name = string("op_197_cast_fp16")];
            tensor<int32, [4]> tile_5 = const()[name = string("tile_5"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_198_axis_0 = const()[name = string("op_198_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 1]> var_198_cast_fp16_0, tensor<fp16, [1, 512, 1, 1]> var_198_cast_fp16_1, tensor<fp16, [1, 512, 1, 1]> var_198_cast_fp16_2, tensor<fp16, [1, 512, 1, 1]> var_198_cast_fp16_3 = split(axis = var_198_axis_0, split_sizes = tile_5, x = var_197_cast_fp16)[name = string("op_198_cast_fp16")];
            tensor<int32, [4]> tile_6 = const()[name = string("tile_6"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_203_axis_0 = const()[name = string("op_203_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_203_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_203_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_203_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_203_cast_fp16_3 = split(axis = var_203_axis_0, split_sizes = tile_6, x = var_152_cast_fp16_0)[name = string("op_203_cast_fp16")];
            fp16 var_208_to_fp16 = const()[name = string("op_208_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_209_cast_fp16 = mul(x = var_198_cast_fp16_0, y = var_208_to_fp16)[name = string("op_209_cast_fp16")];
            string var_211_equation_0 = const()[name = string("op_211_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_211_cast_fp16 = einsum(equation = var_211_equation_0, values = (var_203_cast_fp16_0, var_209_cast_fp16))[name = string("op_211_cast_fp16")];
            fp16 var_212_to_fp16 = const()[name = string("op_212_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_213_cast_fp16 = mul(x = var_198_cast_fp16_1, y = var_212_to_fp16)[name = string("op_213_cast_fp16")];
            string var_215_equation_0 = const()[name = string("op_215_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_215_cast_fp16 = einsum(equation = var_215_equation_0, values = (var_203_cast_fp16_1, var_213_cast_fp16))[name = string("op_215_cast_fp16")];
            fp16 var_216_to_fp16 = const()[name = string("op_216_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_217_cast_fp16 = mul(x = var_198_cast_fp16_2, y = var_216_to_fp16)[name = string("op_217_cast_fp16")];
            string var_219_equation_0 = const()[name = string("op_219_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_219_cast_fp16 = einsum(equation = var_219_equation_0, values = (var_203_cast_fp16_2, var_217_cast_fp16))[name = string("op_219_cast_fp16")];
            fp16 var_220_to_fp16 = const()[name = string("op_220_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_221_cast_fp16 = mul(x = var_198_cast_fp16_3, y = var_220_to_fp16)[name = string("op_221_cast_fp16")];
            string var_223_equation_0 = const()[name = string("op_223_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_223_cast_fp16 = einsum(equation = var_223_equation_0, values = (var_203_cast_fp16_3, var_221_cast_fp16))[name = string("op_223_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_224_cast_fp16 = add(x = var_211_cast_fp16, y = var_215_cast_fp16)[name = string("op_224_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_225_cast_fp16 = add(x = var_219_cast_fp16, y = var_223_cast_fp16)[name = string("op_225_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_226_cast_fp16 = add(x = var_224_cast_fp16, y = var_225_cast_fp16)[name = string("op_226_cast_fp16")];
            fp16 _inversed_228_y_0_to_fp16 = const()[name = string("_inversed_228_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 1]> _inversed_228_cast_fp16 = mul(x = var_226_cast_fp16, y = _inversed_228_y_0_to_fp16)[name = string("_inversed_228_cast_fp16")];
            string scores_5_equation_0 = const()[name = string("scores_5_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 1]> scores_5_cast_fp16 = einsum(equation = scores_5_equation_0, values = (var_149_cast_fp16_0, var_139_cast_fp16_2))[name = string("scores_5_cast_fp16")];
            fp16 var_231_to_fp16 = const()[name = string("op_231_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 1]> var_232_cast_fp16 = mul(x = scores_5_cast_fp16, y = var_231_to_fp16)[name = string("op_232_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_233_cast_fp16 = add(x = var_232_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_233_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_234_cast_fp16 = softmax(axis = var_117, x = var_233_cast_fp16)[name = string("op_234_cast_fp16")];
            tensor<int32, [4]> tile_7 = const()[name = string("tile_7"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_235_axis_0 = const()[name = string("op_235_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 1]> var_235_cast_fp16_0, tensor<fp16, [1, 512, 1, 1]> var_235_cast_fp16_1, tensor<fp16, [1, 512, 1, 1]> var_235_cast_fp16_2, tensor<fp16, [1, 512, 1, 1]> var_235_cast_fp16_3 = split(axis = var_235_axis_0, split_sizes = tile_7, x = var_234_cast_fp16)[name = string("op_235_cast_fp16")];
            tensor<int32, [4]> tile_8 = const()[name = string("tile_8"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_240_axis_0 = const()[name = string("op_240_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_240_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_240_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_240_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_240_cast_fp16_3 = split(axis = var_240_axis_0, split_sizes = tile_8, x = var_152_cast_fp16_0)[name = string("op_240_cast_fp16")];
            fp16 var_245_to_fp16 = const()[name = string("op_245_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_246_cast_fp16 = mul(x = var_235_cast_fp16_0, y = var_245_to_fp16)[name = string("op_246_cast_fp16")];
            string var_248_equation_0 = const()[name = string("op_248_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_248_cast_fp16 = einsum(equation = var_248_equation_0, values = (var_240_cast_fp16_0, var_246_cast_fp16))[name = string("op_248_cast_fp16")];
            fp16 var_249_to_fp16 = const()[name = string("op_249_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_250_cast_fp16 = mul(x = var_235_cast_fp16_1, y = var_249_to_fp16)[name = string("op_250_cast_fp16")];
            string var_252_equation_0 = const()[name = string("op_252_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_252_cast_fp16 = einsum(equation = var_252_equation_0, values = (var_240_cast_fp16_1, var_250_cast_fp16))[name = string("op_252_cast_fp16")];
            fp16 var_253_to_fp16 = const()[name = string("op_253_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_254_cast_fp16 = mul(x = var_235_cast_fp16_2, y = var_253_to_fp16)[name = string("op_254_cast_fp16")];
            string var_256_equation_0 = const()[name = string("op_256_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_256_cast_fp16 = einsum(equation = var_256_equation_0, values = (var_240_cast_fp16_2, var_254_cast_fp16))[name = string("op_256_cast_fp16")];
            fp16 var_257_to_fp16 = const()[name = string("op_257_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_258_cast_fp16 = mul(x = var_235_cast_fp16_3, y = var_257_to_fp16)[name = string("op_258_cast_fp16")];
            string var_260_equation_0 = const()[name = string("op_260_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_260_cast_fp16 = einsum(equation = var_260_equation_0, values = (var_240_cast_fp16_3, var_258_cast_fp16))[name = string("op_260_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_261_cast_fp16 = add(x = var_248_cast_fp16, y = var_252_cast_fp16)[name = string("op_261_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_262_cast_fp16 = add(x = var_256_cast_fp16, y = var_260_cast_fp16)[name = string("op_262_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_263_cast_fp16 = add(x = var_261_cast_fp16, y = var_262_cast_fp16)[name = string("op_263_cast_fp16")];
            fp16 _inversed_265_y_0_to_fp16 = const()[name = string("_inversed_265_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 1]> _inversed_265_cast_fp16 = mul(x = var_263_cast_fp16, y = _inversed_265_y_0_to_fp16)[name = string("_inversed_265_cast_fp16")];
            string scores_7_equation_0 = const()[name = string("scores_7_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 1]> scores_7_cast_fp16 = einsum(equation = scores_7_equation_0, values = (var_149_cast_fp16_0, var_139_cast_fp16_3))[name = string("scores_7_cast_fp16")];
            fp16 var_268_to_fp16 = const()[name = string("op_268_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 1]> var_269_cast_fp16 = mul(x = scores_7_cast_fp16, y = var_268_to_fp16)[name = string("op_269_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_270_cast_fp16 = add(x = var_269_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_270_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_271_cast_fp16 = softmax(axis = var_117, x = var_270_cast_fp16)[name = string("op_271_cast_fp16")];
            tensor<int32, [4]> tile_9 = const()[name = string("tile_9"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_272_axis_0 = const()[name = string("op_272_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 1]> var_272_cast_fp16_0, tensor<fp16, [1, 512, 1, 1]> var_272_cast_fp16_1, tensor<fp16, [1, 512, 1, 1]> var_272_cast_fp16_2, tensor<fp16, [1, 512, 1, 1]> var_272_cast_fp16_3 = split(axis = var_272_axis_0, split_sizes = tile_9, x = var_271_cast_fp16)[name = string("op_272_cast_fp16")];
            tensor<int32, [4]> tile_10 = const()[name = string("tile_10"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_277_axis_0 = const()[name = string("op_277_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_277_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_277_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_277_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_277_cast_fp16_3 = split(axis = var_277_axis_0, split_sizes = tile_10, x = var_152_cast_fp16_0)[name = string("op_277_cast_fp16")];
            fp16 var_282_to_fp16 = const()[name = string("op_282_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_283_cast_fp16 = mul(x = var_272_cast_fp16_0, y = var_282_to_fp16)[name = string("op_283_cast_fp16")];
            string var_285_equation_0 = const()[name = string("op_285_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_285_cast_fp16 = einsum(equation = var_285_equation_0, values = (var_277_cast_fp16_0, var_283_cast_fp16))[name = string("op_285_cast_fp16")];
            fp16 var_286_to_fp16 = const()[name = string("op_286_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_287_cast_fp16 = mul(x = var_272_cast_fp16_1, y = var_286_to_fp16)[name = string("op_287_cast_fp16")];
            string var_289_equation_0 = const()[name = string("op_289_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_289_cast_fp16 = einsum(equation = var_289_equation_0, values = (var_277_cast_fp16_1, var_287_cast_fp16))[name = string("op_289_cast_fp16")];
            fp16 var_290_to_fp16 = const()[name = string("op_290_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_291_cast_fp16 = mul(x = var_272_cast_fp16_2, y = var_290_to_fp16)[name = string("op_291_cast_fp16")];
            string var_293_equation_0 = const()[name = string("op_293_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_293_cast_fp16 = einsum(equation = var_293_equation_0, values = (var_277_cast_fp16_2, var_291_cast_fp16))[name = string("op_293_cast_fp16")];
            fp16 var_294_to_fp16 = const()[name = string("op_294_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_295_cast_fp16 = mul(x = var_272_cast_fp16_3, y = var_294_to_fp16)[name = string("op_295_cast_fp16")];
            string var_297_equation_0 = const()[name = string("op_297_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_297_cast_fp16 = einsum(equation = var_297_equation_0, values = (var_277_cast_fp16_3, var_295_cast_fp16))[name = string("op_297_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_298_cast_fp16 = add(x = var_285_cast_fp16, y = var_289_cast_fp16)[name = string("op_298_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_299_cast_fp16 = add(x = var_293_cast_fp16, y = var_297_cast_fp16)[name = string("op_299_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_300_cast_fp16 = add(x = var_298_cast_fp16, y = var_299_cast_fp16)[name = string("op_300_cast_fp16")];
            fp16 _inversed_302_y_0_to_fp16 = const()[name = string("_inversed_302_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 1]> _inversed_302_cast_fp16 = mul(x = var_300_cast_fp16, y = _inversed_302_y_0_to_fp16)[name = string("_inversed_302_cast_fp16")];
            string scores_9_equation_0 = const()[name = string("scores_9_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 1]> scores_9_cast_fp16 = einsum(equation = scores_9_equation_0, values = (var_149_cast_fp16_1, var_139_cast_fp16_4))[name = string("scores_9_cast_fp16")];
            fp16 var_305_to_fp16 = const()[name = string("op_305_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 1]> var_306_cast_fp16 = mul(x = scores_9_cast_fp16, y = var_305_to_fp16)[name = string("op_306_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_307_cast_fp16 = add(x = var_306_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_307_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_308_cast_fp16 = softmax(axis = var_117, x = var_307_cast_fp16)[name = string("op_308_cast_fp16")];
            tensor<int32, [4]> tile_11 = const()[name = string("tile_11"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_309_axis_0 = const()[name = string("op_309_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 1]> var_309_cast_fp16_0, tensor<fp16, [1, 512, 1, 1]> var_309_cast_fp16_1, tensor<fp16, [1, 512, 1, 1]> var_309_cast_fp16_2, tensor<fp16, [1, 512, 1, 1]> var_309_cast_fp16_3 = split(axis = var_309_axis_0, split_sizes = tile_11, x = var_308_cast_fp16)[name = string("op_309_cast_fp16")];
            tensor<int32, [4]> tile_12 = const()[name = string("tile_12"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_314_axis_0 = const()[name = string("op_314_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_314_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_314_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_314_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_314_cast_fp16_3 = split(axis = var_314_axis_0, split_sizes = tile_12, x = var_152_cast_fp16_1)[name = string("op_314_cast_fp16")];
            fp16 var_319_to_fp16 = const()[name = string("op_319_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_320_cast_fp16 = mul(x = var_309_cast_fp16_0, y = var_319_to_fp16)[name = string("op_320_cast_fp16")];
            string var_322_equation_0 = const()[name = string("op_322_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_322_cast_fp16 = einsum(equation = var_322_equation_0, values = (var_314_cast_fp16_0, var_320_cast_fp16))[name = string("op_322_cast_fp16")];
            fp16 var_323_to_fp16 = const()[name = string("op_323_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_324_cast_fp16 = mul(x = var_309_cast_fp16_1, y = var_323_to_fp16)[name = string("op_324_cast_fp16")];
            string var_326_equation_0 = const()[name = string("op_326_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_326_cast_fp16 = einsum(equation = var_326_equation_0, values = (var_314_cast_fp16_1, var_324_cast_fp16))[name = string("op_326_cast_fp16")];
            fp16 var_327_to_fp16 = const()[name = string("op_327_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_328_cast_fp16 = mul(x = var_309_cast_fp16_2, y = var_327_to_fp16)[name = string("op_328_cast_fp16")];
            string var_330_equation_0 = const()[name = string("op_330_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_330_cast_fp16 = einsum(equation = var_330_equation_0, values = (var_314_cast_fp16_2, var_328_cast_fp16))[name = string("op_330_cast_fp16")];
            fp16 var_331_to_fp16 = const()[name = string("op_331_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_332_cast_fp16 = mul(x = var_309_cast_fp16_3, y = var_331_to_fp16)[name = string("op_332_cast_fp16")];
            string var_334_equation_0 = const()[name = string("op_334_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_334_cast_fp16 = einsum(equation = var_334_equation_0, values = (var_314_cast_fp16_3, var_332_cast_fp16))[name = string("op_334_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_335_cast_fp16 = add(x = var_322_cast_fp16, y = var_326_cast_fp16)[name = string("op_335_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_336_cast_fp16 = add(x = var_330_cast_fp16, y = var_334_cast_fp16)[name = string("op_336_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_337_cast_fp16 = add(x = var_335_cast_fp16, y = var_336_cast_fp16)[name = string("op_337_cast_fp16")];
            fp16 _inversed_339_y_0_to_fp16 = const()[name = string("_inversed_339_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 1]> _inversed_339_cast_fp16 = mul(x = var_337_cast_fp16, y = _inversed_339_y_0_to_fp16)[name = string("_inversed_339_cast_fp16")];
            string scores_11_equation_0 = const()[name = string("scores_11_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 1]> scores_11_cast_fp16 = einsum(equation = scores_11_equation_0, values = (var_149_cast_fp16_1, var_139_cast_fp16_5))[name = string("scores_11_cast_fp16")];
            fp16 var_342_to_fp16 = const()[name = string("op_342_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 1]> var_343_cast_fp16 = mul(x = scores_11_cast_fp16, y = var_342_to_fp16)[name = string("op_343_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_344_cast_fp16 = add(x = var_343_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_344_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_345_cast_fp16 = softmax(axis = var_117, x = var_344_cast_fp16)[name = string("op_345_cast_fp16")];
            tensor<int32, [4]> tile_13 = const()[name = string("tile_13"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_346_axis_0 = const()[name = string("op_346_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 1]> var_346_cast_fp16_0, tensor<fp16, [1, 512, 1, 1]> var_346_cast_fp16_1, tensor<fp16, [1, 512, 1, 1]> var_346_cast_fp16_2, tensor<fp16, [1, 512, 1, 1]> var_346_cast_fp16_3 = split(axis = var_346_axis_0, split_sizes = tile_13, x = var_345_cast_fp16)[name = string("op_346_cast_fp16")];
            tensor<int32, [4]> tile_14 = const()[name = string("tile_14"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_351_axis_0 = const()[name = string("op_351_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_351_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_351_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_351_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_351_cast_fp16_3 = split(axis = var_351_axis_0, split_sizes = tile_14, x = var_152_cast_fp16_1)[name = string("op_351_cast_fp16")];
            fp16 var_356_to_fp16 = const()[name = string("op_356_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_357_cast_fp16 = mul(x = var_346_cast_fp16_0, y = var_356_to_fp16)[name = string("op_357_cast_fp16")];
            string var_359_equation_0 = const()[name = string("op_359_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_359_cast_fp16 = einsum(equation = var_359_equation_0, values = (var_351_cast_fp16_0, var_357_cast_fp16))[name = string("op_359_cast_fp16")];
            fp16 var_360_to_fp16 = const()[name = string("op_360_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_361_cast_fp16 = mul(x = var_346_cast_fp16_1, y = var_360_to_fp16)[name = string("op_361_cast_fp16")];
            string var_363_equation_0 = const()[name = string("op_363_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_363_cast_fp16 = einsum(equation = var_363_equation_0, values = (var_351_cast_fp16_1, var_361_cast_fp16))[name = string("op_363_cast_fp16")];
            fp16 var_364_to_fp16 = const()[name = string("op_364_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_365_cast_fp16 = mul(x = var_346_cast_fp16_2, y = var_364_to_fp16)[name = string("op_365_cast_fp16")];
            string var_367_equation_0 = const()[name = string("op_367_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_367_cast_fp16 = einsum(equation = var_367_equation_0, values = (var_351_cast_fp16_2, var_365_cast_fp16))[name = string("op_367_cast_fp16")];
            fp16 var_368_to_fp16 = const()[name = string("op_368_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_369_cast_fp16 = mul(x = var_346_cast_fp16_3, y = var_368_to_fp16)[name = string("op_369_cast_fp16")];
            string var_371_equation_0 = const()[name = string("op_371_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_371_cast_fp16 = einsum(equation = var_371_equation_0, values = (var_351_cast_fp16_3, var_369_cast_fp16))[name = string("op_371_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_372_cast_fp16 = add(x = var_359_cast_fp16, y = var_363_cast_fp16)[name = string("op_372_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_373_cast_fp16 = add(x = var_367_cast_fp16, y = var_371_cast_fp16)[name = string("op_373_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_374_cast_fp16 = add(x = var_372_cast_fp16, y = var_373_cast_fp16)[name = string("op_374_cast_fp16")];
            fp16 _inversed_376_y_0_to_fp16 = const()[name = string("_inversed_376_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 1]> _inversed_376_cast_fp16 = mul(x = var_374_cast_fp16, y = _inversed_376_y_0_to_fp16)[name = string("_inversed_376_cast_fp16")];
            string scores_13_equation_0 = const()[name = string("scores_13_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 1]> scores_13_cast_fp16 = einsum(equation = scores_13_equation_0, values = (var_149_cast_fp16_1, var_139_cast_fp16_6))[name = string("scores_13_cast_fp16")];
            fp16 var_379_to_fp16 = const()[name = string("op_379_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 1]> var_380_cast_fp16 = mul(x = scores_13_cast_fp16, y = var_379_to_fp16)[name = string("op_380_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_381_cast_fp16 = add(x = var_380_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_381_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_382_cast_fp16 = softmax(axis = var_117, x = var_381_cast_fp16)[name = string("op_382_cast_fp16")];
            tensor<int32, [4]> tile_15 = const()[name = string("tile_15"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_383_axis_0 = const()[name = string("op_383_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 1]> var_383_cast_fp16_0, tensor<fp16, [1, 512, 1, 1]> var_383_cast_fp16_1, tensor<fp16, [1, 512, 1, 1]> var_383_cast_fp16_2, tensor<fp16, [1, 512, 1, 1]> var_383_cast_fp16_3 = split(axis = var_383_axis_0, split_sizes = tile_15, x = var_382_cast_fp16)[name = string("op_383_cast_fp16")];
            tensor<int32, [4]> tile_16 = const()[name = string("tile_16"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_388_axis_0 = const()[name = string("op_388_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_388_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_388_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_388_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_388_cast_fp16_3 = split(axis = var_388_axis_0, split_sizes = tile_16, x = var_152_cast_fp16_1)[name = string("op_388_cast_fp16")];
            fp16 var_393_to_fp16 = const()[name = string("op_393_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_394_cast_fp16 = mul(x = var_383_cast_fp16_0, y = var_393_to_fp16)[name = string("op_394_cast_fp16")];
            string var_396_equation_0 = const()[name = string("op_396_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_396_cast_fp16 = einsum(equation = var_396_equation_0, values = (var_388_cast_fp16_0, var_394_cast_fp16))[name = string("op_396_cast_fp16")];
            fp16 var_397_to_fp16 = const()[name = string("op_397_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_398_cast_fp16 = mul(x = var_383_cast_fp16_1, y = var_397_to_fp16)[name = string("op_398_cast_fp16")];
            string var_400_equation_0 = const()[name = string("op_400_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_400_cast_fp16 = einsum(equation = var_400_equation_0, values = (var_388_cast_fp16_1, var_398_cast_fp16))[name = string("op_400_cast_fp16")];
            fp16 var_401_to_fp16 = const()[name = string("op_401_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_402_cast_fp16 = mul(x = var_383_cast_fp16_2, y = var_401_to_fp16)[name = string("op_402_cast_fp16")];
            string var_404_equation_0 = const()[name = string("op_404_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_404_cast_fp16 = einsum(equation = var_404_equation_0, values = (var_388_cast_fp16_2, var_402_cast_fp16))[name = string("op_404_cast_fp16")];
            fp16 var_405_to_fp16 = const()[name = string("op_405_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_406_cast_fp16 = mul(x = var_383_cast_fp16_3, y = var_405_to_fp16)[name = string("op_406_cast_fp16")];
            string var_408_equation_0 = const()[name = string("op_408_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_408_cast_fp16 = einsum(equation = var_408_equation_0, values = (var_388_cast_fp16_3, var_406_cast_fp16))[name = string("op_408_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_409_cast_fp16 = add(x = var_396_cast_fp16, y = var_400_cast_fp16)[name = string("op_409_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_410_cast_fp16 = add(x = var_404_cast_fp16, y = var_408_cast_fp16)[name = string("op_410_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_411_cast_fp16 = add(x = var_409_cast_fp16, y = var_410_cast_fp16)[name = string("op_411_cast_fp16")];
            fp16 _inversed_413_y_0_to_fp16 = const()[name = string("_inversed_413_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 1]> _inversed_413_cast_fp16 = mul(x = var_411_cast_fp16, y = _inversed_413_y_0_to_fp16)[name = string("_inversed_413_cast_fp16")];
            string scores_15_equation_0 = const()[name = string("scores_15_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 1]> scores_15_cast_fp16 = einsum(equation = scores_15_equation_0, values = (var_149_cast_fp16_1, var_139_cast_fp16_7))[name = string("scores_15_cast_fp16")];
            fp16 var_416_to_fp16 = const()[name = string("op_416_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 1]> var_417_cast_fp16 = mul(x = scores_15_cast_fp16, y = var_416_to_fp16)[name = string("op_417_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_418_cast_fp16 = add(x = var_417_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_418_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_419_cast_fp16 = softmax(axis = var_117, x = var_418_cast_fp16)[name = string("op_419_cast_fp16")];
            tensor<int32, [4]> tile_17 = const()[name = string("tile_17"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_420_axis_0 = const()[name = string("op_420_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 1]> var_420_cast_fp16_0, tensor<fp16, [1, 512, 1, 1]> var_420_cast_fp16_1, tensor<fp16, [1, 512, 1, 1]> var_420_cast_fp16_2, tensor<fp16, [1, 512, 1, 1]> var_420_cast_fp16_3 = split(axis = var_420_axis_0, split_sizes = tile_17, x = var_419_cast_fp16)[name = string("op_420_cast_fp16")];
            tensor<int32, [4]> tile_18 = const()[name = string("tile_18"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_425_axis_0 = const()[name = string("op_425_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_425_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_425_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_425_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_425_cast_fp16_3 = split(axis = var_425_axis_0, split_sizes = tile_18, x = var_152_cast_fp16_1)[name = string("op_425_cast_fp16")];
            fp16 var_430_to_fp16 = const()[name = string("op_430_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_431_cast_fp16 = mul(x = var_420_cast_fp16_0, y = var_430_to_fp16)[name = string("op_431_cast_fp16")];
            string var_433_equation_0 = const()[name = string("op_433_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_433_cast_fp16 = einsum(equation = var_433_equation_0, values = (var_425_cast_fp16_0, var_431_cast_fp16))[name = string("op_433_cast_fp16")];
            fp16 var_434_to_fp16 = const()[name = string("op_434_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_435_cast_fp16 = mul(x = var_420_cast_fp16_1, y = var_434_to_fp16)[name = string("op_435_cast_fp16")];
            string var_437_equation_0 = const()[name = string("op_437_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_437_cast_fp16 = einsum(equation = var_437_equation_0, values = (var_425_cast_fp16_1, var_435_cast_fp16))[name = string("op_437_cast_fp16")];
            fp16 var_438_to_fp16 = const()[name = string("op_438_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_439_cast_fp16 = mul(x = var_420_cast_fp16_2, y = var_438_to_fp16)[name = string("op_439_cast_fp16")];
            string var_441_equation_0 = const()[name = string("op_441_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_441_cast_fp16 = einsum(equation = var_441_equation_0, values = (var_425_cast_fp16_2, var_439_cast_fp16))[name = string("op_441_cast_fp16")];
            fp16 var_442_to_fp16 = const()[name = string("op_442_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_443_cast_fp16 = mul(x = var_420_cast_fp16_3, y = var_442_to_fp16)[name = string("op_443_cast_fp16")];
            string var_445_equation_0 = const()[name = string("op_445_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_445_cast_fp16 = einsum(equation = var_445_equation_0, values = (var_425_cast_fp16_3, var_443_cast_fp16))[name = string("op_445_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_446_cast_fp16 = add(x = var_433_cast_fp16, y = var_437_cast_fp16)[name = string("op_446_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_447_cast_fp16 = add(x = var_441_cast_fp16, y = var_445_cast_fp16)[name = string("op_447_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_448_cast_fp16 = add(x = var_446_cast_fp16, y = var_447_cast_fp16)[name = string("op_448_cast_fp16")];
            fp16 _inversed_450_y_0_to_fp16 = const()[name = string("_inversed_450_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 1]> _inversed_450_cast_fp16 = mul(x = var_448_cast_fp16, y = _inversed_450_y_0_to_fp16)[name = string("_inversed_450_cast_fp16")];
            bool var_452_interleave_0 = const()[name = string("op_452_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 2048, 1, 1]> var_452_cast_fp16 = concat(axis = var_117, interleave = var_452_interleave_0, values = (_inversed_191_cast_fp16, _inversed_228_cast_fp16, _inversed_265_cast_fp16, _inversed_302_cast_fp16, _inversed_339_cast_fp16, _inversed_376_cast_fp16, _inversed_413_cast_fp16, _inversed_450_cast_fp16))[name = string("op_452_cast_fp16")];
            tensor<int32, [4]> var_453 = const()[name = string("op_453"), val = tensor<int32, [4]>([2, 4, 256, 1])];
            tensor<fp16, [2, 4, 256, 1]> var_454_cast_fp16 = reshape(shape = var_453, x = var_452_cast_fp16)[name = string("op_454_cast_fp16")];
            tensor<int32, [4]> transpose_0_perm_0 = const()[name = string("transpose_0_perm_0"), val = tensor<int32, [4]>([3, 0, 1, 2])];
            tensor<int32, [2]> var_472 = const()[name = string("op_472"), val = tensor<int32, [2]>([1, 2048])];
            tensor<fp16, [1, 2, 4, 256]> transpose_0_cast_fp16 = transpose(perm = transpose_0_perm_0, x = var_454_cast_fp16)[name = string("transpose_29")];
            tensor<fp16, [1, 2048]> attention_output_3_cast_fp16 = reshape(shape = var_472, x = transpose_0_cast_fp16)[name = string("attention_output_3_cast_fp16")];
            tensor<fp16, [1, 2048]> var_474_cast_fp16 = sigmoid(x = gate19)[name = string("op_474_cast_fp16")];
            tensor<fp16, [1, 2048]> input_1_cast_fp16 = mul(x = attention_output_3_cast_fp16, y = var_474_cast_fp16)[name = string("input_1_cast_fp16")];
            tensor<fp16, [1024, 2048]> completion19_o_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 2048]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(193193216))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(194766144))))[name = string("completion19_o_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1024]> linear_0_bias_0_to_fp16 = const()[name = string("linear_0_bias_0_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(194774400)))];
            tensor<fp16, [1, 1024]> linear_0_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = completion19_o_proj_weight_promoted_to_fp16_palettized, x = input_1_cast_fp16)[name = string("linear_0_cast_fp16")];
            tensor<fp16, [1, 1024]> value_1_cast_fp16 = add(x = hidden, y = linear_0_cast_fp16)[name = string("value_1_cast_fp16")];
            tensor<fp16, [1, 1024]> var_479_cast_fp16 = mul(x = value_1_cast_fp16, y = value_1_cast_fp16)[name = string("op_479_cast_fp16")];
            tensor<int32, [1]> variance_1_axes_0 = const()[name = string("variance_1_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_1_keep_dims_0 = const()[name = string("variance_1_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 1]> variance_1_cast_fp16 = reduce_mean(axes = variance_1_axes_0, keep_dims = variance_1_keep_dims_0, x = var_479_cast_fp16)[name = string("variance_1_cast_fp16")];
            fp16 var_482_to_fp16 = const()[name = string("op_482_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 1]> var_483_cast_fp16 = add(x = variance_1_cast_fp16, y = var_482_to_fp16)[name = string("op_483_cast_fp16")];
            fp32 var_484_epsilon_0 = const()[name = string("op_484_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 1]> var_484_cast_fp16 = rsqrt(epsilon = var_484_epsilon_0, x = var_483_cast_fp16)[name = string("op_484_cast_fp16")];
            tensor<fp16, [1, 1024]> var_485_cast_fp16 = mul(x = value_1_cast_fp16, y = var_484_cast_fp16)[name = string("op_485_cast_fp16")];
            tensor<fp16, [1024]> var_487_to_fp16 = const()[name = string("op_487_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(194776512)))];
            tensor<fp16, [1, 1024]> input_3_cast_fp16 = mul(x = var_485_cast_fp16, y = var_487_to_fp16)[name = string("input_3_cast_fp16")];
            tensor<fp16, [3584, 1024]> completion19_gate_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(194778624))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(197531200))))[name = string("completion19_gate_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [3584]> linear_1_bias_0_to_fp16 = const()[name = string("linear_1_bias_0_to_fp16"), val = tensor<fp16, [3584]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(197559936)))];
            tensor<fp16, [1, 3584]> linear_1_cast_fp16 = linear(bias = linear_1_bias_0_to_fp16, weight = completion19_gate_proj_weight_promoted_to_fp16_palettized, x = input_3_cast_fp16)[name = string("linear_1_cast_fp16")];
            tensor<fp16, [1, 3584]> var_491_cast_fp16 = silu(x = linear_1_cast_fp16)[name = string("op_491_cast_fp16")];
            tensor<fp16, [3584, 1024]> completion19_up_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(197567168))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(200319744))))[name = string("completion19_up_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 3584]> linear_2_cast_fp16 = linear(bias = linear_1_bias_0_to_fp16, weight = completion19_up_proj_weight_promoted_to_fp16_palettized, x = input_3_cast_fp16)[name = string("linear_2_cast_fp16")];
            tensor<fp16, [1, 3584]> input_7_cast_fp16 = mul(x = var_491_cast_fp16, y = linear_2_cast_fp16)[name = string("input_7_cast_fp16")];
            tensor<fp16, [1024, 3584]> completion19_down_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 3584]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(200348480))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(203101056))))[name = string("completion19_down_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 1024]> linear_3_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = completion19_down_proj_weight_promoted_to_fp16_palettized, x = input_7_cast_fp16)[name = string("linear_3_cast_fp16")];
            tensor<fp16, [1, 1024]> value_3_cast_fp16 = add(x = value_1_cast_fp16, y = linear_3_cast_fp16)[name = string("value_3_cast_fp16")];
            int32 var_512 = const()[name = string("op_512"), val = int32(-1)];
            tensor<fp16, [1, 1024]> var_527_cast_fp16 = mul(x = value_3_cast_fp16, y = value_3_cast_fp16)[name = string("op_527_cast_fp16")];
            tensor<int32, [1]> variance_3_axes_0 = const()[name = string("variance_3_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_3_keep_dims_0 = const()[name = string("variance_3_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 1]> variance_3_cast_fp16 = reduce_mean(axes = variance_3_axes_0, keep_dims = variance_3_keep_dims_0, x = var_527_cast_fp16)[name = string("variance_3_cast_fp16")];
            fp16 var_530_to_fp16 = const()[name = string("op_530_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 1]> var_531_cast_fp16 = add(x = variance_3_cast_fp16, y = var_530_to_fp16)[name = string("op_531_cast_fp16")];
            fp32 var_532_epsilon_0 = const()[name = string("op_532_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 1]> var_532_cast_fp16 = rsqrt(epsilon = var_532_epsilon_0, x = var_531_cast_fp16)[name = string("op_532_cast_fp16")];
            tensor<fp16, [1, 1024]> var_533_cast_fp16 = mul(x = value_3_cast_fp16, y = var_532_cast_fp16)[name = string("op_533_cast_fp16")];
            tensor<fp16, [1024]> var_535_to_fp16 = const()[name = string("op_535_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(203109312)))];
            tensor<fp16, [1, 1024]> input_9_cast_fp16 = mul(x = var_533_cast_fp16, y = var_535_to_fp16)[name = string("input_9_cast_fp16")];
            tensor<fp16, [6144, 1024]> layers20_0_in_proj_qkv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(203111424))), lut = tensor<fp16, [384, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(207830080))))[name = string("layers20_0_in_proj_qkv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [6144]> linear_4_bias_0_to_fp16 = const()[name = string("linear_4_bias_0_to_fp16"), val = tensor<fp16, [6144]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(207879296)))];
            tensor<fp16, [1, 6144]> linear_4_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = layers20_0_in_proj_qkv_weight_promoted_to_fp16_palettized, x = input_9_cast_fp16)[name = string("linear_4_cast_fp16")];
            tensor<int32, [2]> var_539_perm_0 = const()[name = string("op_539_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> mixed_1_axes_0 = const()[name = string("mixed_1_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [6144, 1]> var_539_cast_fp16 = transpose(perm = var_539_perm_0, x = linear_4_cast_fp16)[name = string("transpose_28")];
            tensor<fp16, [1, 6144, 1]> mixed_1_cast_fp16 = expand_dims(axes = mixed_1_axes_0, x = var_539_cast_fp16)[name = string("mixed_1_cast_fp16")];
            bool convolution_input_1_interleave_0 = const()[name = string("convolution_input_1_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 6144, 4]> convolution_input_1_cast_fp16 = concat(axis = var_512, interleave = convolution_input_1_interleave_0, values = (conv20, mixed_1_cast_fp16))[name = string("convolution_input_1_cast_fp16")];
            string input_11_pad_type_0 = const()[name = string("input_11_pad_type_0"), val = string("valid")];
            int32 input_11_groups_0 = const()[name = string("input_11_groups_0"), val = int32(6144)];
            tensor<int32, [1]> input_11_strides_0 = const()[name = string("input_11_strides_0"), val = tensor<int32, [1]>([1])];
            tensor<int32, [2]> input_11_pad_0 = const()[name = string("input_11_pad_0"), val = tensor<int32, [2]>([0, 0])];
            tensor<int32, [1]> input_11_dilations_0 = const()[name = string("input_11_dilations_0"), val = tensor<int32, [1]>([1])];
            tensor<fp16, [6144, 1, 4]> layers20_0_conv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1, 4]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(207891648))), lut = tensor<fp16, [384, 1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(207910144))))[name = string("layers20_0_conv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 6144, 1]> input_11_cast_fp16 = conv(dilations = input_11_dilations_0, groups = input_11_groups_0, pad = input_11_pad_0, pad_type = input_11_pad_type_0, strides = input_11_strides_0, weight = layers20_0_conv_weight_promoted_to_fp16_palettized, x = convolution_input_1_cast_fp16)[name = string("input_11_cast_fp16")];
            tensor<fp16, [1, 6144, 1]> var_548_cast_fp16 = silu(x = input_11_cast_fp16)[name = string("op_548_cast_fp16")];
            tensor<int32, [3]> var_549_perm_0 = const()[name = string("op_549_perm_0"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [3]> var_552_begin_0 = const()[name = string("op_552_begin_0"), val = tensor<int32, [3]>([0, 0, 1])];
            tensor<int32, [3]> var_552_end_0 = const()[name = string("op_552_end_0"), val = tensor<int32, [3]>([1, 6144, 4])];
            tensor<bool, [3]> var_552_end_mask_0 = const()[name = string("op_552_end_mask_0"), val = tensor<bool, [3]>([true, true, true])];
            tensor<fp16, [1, 6144, 3]> conv20_out = slice_by_index(begin = var_552_begin_0, end = var_552_end_0, end_mask = var_552_end_mask_0, x = convolution_input_1_cast_fp16)[name = string("op_552_cast_fp16")];
            tensor<int32, [3]> var_553 = const()[name = string("op_553"), val = tensor<int32, [3]>([2048, 2048, 2048])];
            int32 var_554_axis_0 = const()[name = string("op_554_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 1, 6144]> var_549_cast_fp16 = transpose(perm = var_549_perm_0, x = var_548_cast_fp16)[name = string("transpose_27")];
            tensor<fp16, [1, 1, 2048]> var_554_cast_fp16_0, tensor<fp16, [1, 1, 2048]> var_554_cast_fp16_1, tensor<fp16, [1, 1, 2048]> var_554_cast_fp16_2 = split(axis = var_554_axis_0, split_sizes = var_553, x = var_549_cast_fp16)[name = string("op_554_cast_fp16")];
            tensor<int32, [4]> var_558 = const()[name = string("op_558"), val = tensor<int32, [4]>([1, 1, 16, 128])];
            tensor<fp16, [1, 1, 16, 128]> value_7_cast_fp16 = reshape(shape = var_558, x = var_554_cast_fp16_0)[name = string("value_7_cast_fp16")];
            tensor<int32, [4]> var_560 = const()[name = string("op_560"), val = tensor<int32, [4]>([1, 1, 16, 128])];
            tensor<fp16, [1, 1, 16, 128]> value_9_cast_fp16 = reshape(shape = var_560, x = var_554_cast_fp16_1)[name = string("value_9_cast_fp16")];
            tensor<int32, [4]> var_562 = const()[name = string("op_562"), val = tensor<int32, [4]>([1, 1, 16, 128])];
            tensor<fp16, [1, 1, 16, 128]> value_11_cast_fp16 = reshape(shape = var_562, x = var_554_cast_fp16_2)[name = string("value_11_cast_fp16")];
            tensor<fp16, [1, 1, 16, 128]> var_564_cast_fp16 = mul(x = value_7_cast_fp16, y = value_7_cast_fp16)[name = string("op_564_cast_fp16")];
            tensor<int32, [1]> var_566_axes_0 = const()[name = string("op_566_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_566_keep_dims_0 = const()[name = string("op_566_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 1, 16, 1]> var_566_cast_fp16 = reduce_sum(axes = var_566_axes_0, keep_dims = var_566_keep_dims_0, x = var_564_cast_fp16)[name = string("op_566_cast_fp16")];
            fp16 var_567_to_fp16 = const()[name = string("op_567_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 1, 16, 1]> var_568_cast_fp16 = add(x = var_566_cast_fp16, y = var_567_to_fp16)[name = string("op_568_cast_fp16")];
            fp32 var_569_epsilon_0 = const()[name = string("op_569_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 1, 16, 1]> var_569_cast_fp16 = rsqrt(epsilon = var_569_epsilon_0, x = var_568_cast_fp16)[name = string("op_569_cast_fp16")];
            tensor<fp16, [1, 1, 16, 128]> var_570_cast_fp16 = mul(x = value_7_cast_fp16, y = var_569_cast_fp16)[name = string("op_570_cast_fp16")];
            tensor<int32, [4]> var_571_perm_0 = const()[name = string("op_571_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            fp16 _inversed_query_3_y_0_to_fp16 = const()[name = string("_inversed_query_3_y_0_to_fp16"), val = fp16(0x1.6ap-4)];
            tensor<fp16, [1, 16, 1, 128]> var_571_cast_fp16 = transpose(perm = var_571_perm_0, x = var_570_cast_fp16)[name = string("transpose_26")];
            tensor<fp16, [1, 16, 1, 128]> _inversed_query_3_cast_fp16 = mul(x = var_571_cast_fp16, y = _inversed_query_3_y_0_to_fp16)[name = string("_inversed_query_3_cast_fp16")];
            tensor<fp16, [1, 1, 16, 128]> var_574_cast_fp16 = mul(x = value_9_cast_fp16, y = value_9_cast_fp16)[name = string("op_574_cast_fp16")];
            tensor<int32, [1]> var_576_axes_0 = const()[name = string("op_576_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_576_keep_dims_0 = const()[name = string("op_576_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 1, 16, 1]> var_576_cast_fp16 = reduce_sum(axes = var_576_axes_0, keep_dims = var_576_keep_dims_0, x = var_574_cast_fp16)[name = string("op_576_cast_fp16")];
            fp16 var_577_to_fp16 = const()[name = string("op_577_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 1, 16, 1]> var_578_cast_fp16 = add(x = var_576_cast_fp16, y = var_577_to_fp16)[name = string("op_578_cast_fp16")];
            fp32 var_579_epsilon_0 = const()[name = string("op_579_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 1, 16, 1]> var_579_cast_fp16 = rsqrt(epsilon = var_579_epsilon_0, x = var_578_cast_fp16)[name = string("op_579_cast_fp16")];
            tensor<fp16, [1, 1, 16, 128]> var_580_cast_fp16 = mul(x = value_9_cast_fp16, y = var_579_cast_fp16)[name = string("op_580_cast_fp16")];
            tensor<int32, [4]> key_3_perm_0 = const()[name = string("key_3_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<int32, [4]> value_13_perm_0 = const()[name = string("value_13_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<fp16, [16, 1024]> layers20_0_in_proj_b_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(207959360))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(207971712))))[name = string("layers20_0_in_proj_b_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16]> linear_5_bias_0_to_fp16 = const()[name = string("linear_5_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(207971904)))];
            tensor<fp16, [1, 16]> linear_5_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = layers20_0_in_proj_b_weight_promoted_to_fp16_palettized, x = input_9_cast_fp16)[name = string("linear_5_cast_fp16")];
            tensor<fp16, [1, 16]> var_585_cast_fp16 = sigmoid(x = linear_5_cast_fp16)[name = string("op_585_cast_fp16")];
            tensor<int32, [2]> var_586_perm_0 = const()[name = string("op_586_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> beta_1_axes_0 = const()[name = string("beta_1_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 1]> var_586_cast_fp16 = transpose(perm = var_586_perm_0, x = var_585_cast_fp16)[name = string("transpose_25")];
            tensor<fp16, [1, 16, 1]> beta_1_cast_fp16 = expand_dims(axes = beta_1_axes_0, x = var_586_cast_fp16)[name = string("beta_1_cast_fp16")];
            tensor<fp16, [16, 1024]> op_592_weight_0_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(207972032))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(207984384))))[name = string("op_592_weight_0_to_fp16_palettized")];
            tensor<fp16, [16]> var_592_bias_0_to_fp16 = const()[name = string("op_592_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(207984576)))];
            tensor<fp16, [1, 16]> var_592_cast_fp16 = linear(bias = var_592_bias_0_to_fp16, weight = op_592_weight_0_to_fp16_palettized, x = input_9_cast_fp16)[name = string("op_592_cast_fp16")];
            tensor<fp16, [1, 16]> var_593_cast_fp16 = softplus(x = var_592_cast_fp16)[name = string("op_593_cast_fp16")];
            tensor<fp16, [16]> var_589_promoted_to_fp16 = const()[name = string("op_589_promoted_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(207984704)))];
            tensor<fp16, [1, 16]> var_594_cast_fp16 = mul(x = var_589_promoted_to_fp16, y = var_593_cast_fp16)[name = string("op_594_cast_fp16")];
            tensor<int32, [2]> var_595_perm_0 = const()[name = string("op_595_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> decay_1_axes_0 = const()[name = string("decay_1_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 1]> var_595_cast_fp16 = transpose(perm = var_595_perm_0, x = var_594_cast_fp16)[name = string("transpose_24")];
            tensor<fp16, [1, 16, 1]> decay_1_cast_fp16 = expand_dims(axes = decay_1_axes_0, x = var_595_cast_fp16)[name = string("decay_1_cast_fp16")];
            tensor<fp16, [2048, 1024]> layers20_0_in_proj_z_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [2048, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(207984832))), lut = tensor<fp16, [128, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(209557760))))[name = string("layers20_0_in_proj_z_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [2048]> linear_7_bias_0_to_fp16 = const()[name = string("linear_7_bias_0_to_fp16"), val = tensor<fp16, [2048]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(209574208)))];
            tensor<fp16, [1, 2048]> linear_7_cast_fp16 = linear(bias = linear_7_bias_0_to_fp16, weight = layers20_0_in_proj_z_weight_promoted_to_fp16_palettized, x = input_9_cast_fp16)[name = string("linear_7_cast_fp16")];
            tensor<int32, [3]> var_603_begin_0 = const()[name = string("op_603_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_603_end_0 = const()[name = string("op_603_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_603_end_mask_0 = const()[name = string("op_603_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_603_squeeze_mask_0 = const()[name = string("op_603_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_603_cast_fp16 = slice_by_index(begin = var_603_begin_0, end = var_603_end_0, end_mask = var_603_end_mask_0, squeeze_mask = var_603_squeeze_mask_0, x = decay_1_cast_fp16)[name = string("op_603_cast_fp16")];
            tensor<fp16, [1, 16]> var_604_cast_fp16 = exp(x = var_603_cast_fp16)[name = string("op_604_cast_fp16")];
            tensor<int32, [1]> var_605_axes_0 = const()[name = string("op_605_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_605_cast_fp16 = expand_dims(axes = var_605_axes_0, x = var_604_cast_fp16)[name = string("op_605_cast_fp16")];
            tensor<int32, [1]> var_606_axes_0 = const()[name = string("op_606_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_606_cast_fp16 = expand_dims(axes = var_606_axes_0, x = var_605_cast_fp16)[name = string("op_606_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_1_cast_fp16 = mul(x = rec20, y = var_606_cast_fp16)[name = string("state_1_cast_fp16")];
            tensor<int32, [4]> key_token_1_begin_0 = const()[name = string("key_token_1_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> key_token_1_end_0 = const()[name = string("key_token_1_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> key_token_1_end_mask_0 = const()[name = string("key_token_1_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_1_squeeze_mask_0 = const()[name = string("key_token_1_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 1, 128]> key_3_cast_fp16 = transpose(perm = key_3_perm_0, x = var_580_cast_fp16)[name = string("transpose_23")];
            tensor<fp16, [1, 16, 128]> key_token_1_cast_fp16 = slice_by_index(begin = key_token_1_begin_0, end = key_token_1_end_0, end_mask = key_token_1_end_mask_0, squeeze_mask = key_token_1_squeeze_mask_0, x = key_3_cast_fp16)[name = string("key_token_1_cast_fp16")];
            tensor<int32, [4]> value_token_1_begin_0 = const()[name = string("value_token_1_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> value_token_1_end_0 = const()[name = string("value_token_1_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> value_token_1_end_mask_0 = const()[name = string("value_token_1_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_1_squeeze_mask_0 = const()[name = string("value_token_1_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 1, 128]> value_13_cast_fp16 = transpose(perm = value_13_perm_0, x = value_11_cast_fp16)[name = string("transpose_22")];
            tensor<fp16, [1, 16, 128]> value_token_1_cast_fp16 = slice_by_index(begin = value_token_1_begin_0, end = value_token_1_end_0, end_mask = value_token_1_end_mask_0, squeeze_mask = value_token_1_squeeze_mask_0, x = value_13_cast_fp16)[name = string("value_token_1_cast_fp16")];
            tensor<int32, [1]> var_614_axes_0 = const()[name = string("op_614_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_614_cast_fp16 = expand_dims(axes = var_614_axes_0, x = key_token_1_cast_fp16)[name = string("op_614_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_615_cast_fp16 = mul(x = state_1_cast_fp16, y = var_614_cast_fp16)[name = string("op_615_cast_fp16")];
            tensor<int32, [1]> memory_1_axes_0 = const()[name = string("memory_1_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_1_keep_dims_0 = const()[name = string("memory_1_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_1_cast_fp16 = reduce_sum(axes = memory_1_axes_0, keep_dims = memory_1_keep_dims_0, x = var_615_cast_fp16)[name = string("memory_1_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_618_cast_fp16 = sub(x = value_token_1_cast_fp16, y = memory_1_cast_fp16)[name = string("op_618_cast_fp16")];
            tensor<int32, [3]> var_621_begin_0 = const()[name = string("op_621_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_621_end_0 = const()[name = string("op_621_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_621_end_mask_0 = const()[name = string("op_621_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_621_squeeze_mask_0 = const()[name = string("op_621_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_621_cast_fp16 = slice_by_index(begin = var_621_begin_0, end = var_621_end_0, end_mask = var_621_end_mask_0, squeeze_mask = var_621_squeeze_mask_0, x = beta_1_cast_fp16)[name = string("op_621_cast_fp16")];
            tensor<int32, [1]> var_622_axes_0 = const()[name = string("op_622_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_622_cast_fp16 = expand_dims(axes = var_622_axes_0, x = var_621_cast_fp16)[name = string("op_622_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_1_cast_fp16 = mul(x = var_618_cast_fp16, y = var_622_cast_fp16)[name = string("delta_1_cast_fp16")];
            tensor<int32, [1]> var_625_axes_0 = const()[name = string("op_625_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_625_cast_fp16 = expand_dims(axes = var_625_axes_0, x = delta_1_cast_fp16)[name = string("op_625_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_626_cast_fp16 = mul(x = var_614_cast_fp16, y = var_625_cast_fp16)[name = string("op_626_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> rec20_out = add(x = state_1_cast_fp16, y = var_626_cast_fp16)[name = string("state_3_cast_fp16")];
            tensor<int32, [4]> var_630_begin_0 = const()[name = string("op_630_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> var_630_end_0 = const()[name = string("op_630_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> var_630_end_mask_0 = const()[name = string("op_630_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_630_squeeze_mask_0 = const()[name = string("op_630_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_630_cast_fp16 = slice_by_index(begin = var_630_begin_0, end = var_630_end_0, end_mask = var_630_end_mask_0, squeeze_mask = var_630_squeeze_mask_0, x = _inversed_query_3_cast_fp16)[name = string("op_630_cast_fp16")];
            tensor<int32, [1]> var_631_axes_0 = const()[name = string("op_631_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_631_cast_fp16 = expand_dims(axes = var_631_axes_0, x = var_630_cast_fp16)[name = string("op_631_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_632_cast_fp16 = mul(x = rec20_out, y = var_631_cast_fp16)[name = string("op_632_cast_fp16")];
            tensor<int32, [1]> var_634_axes_0 = const()[name = string("op_634_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_634_keep_dims_0 = const()[name = string("op_634_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_634_cast_fp16 = reduce_sum(axes = var_634_axes_0, keep_dims = var_634_keep_dims_0, x = var_632_cast_fp16)[name = string("op_634_cast_fp16")];
            tensor<int32, [1]> attention_1_axes_0 = const()[name = string("attention_1_axes_0"), val = tensor<int32, [1]>([1])];
            tensor<fp16, [1, 1, 16, 128]> attention_1_cast_fp16 = expand_dims(axes = attention_1_axes_0, x = var_634_cast_fp16)[name = string("attention_1_cast_fp16")];
            tensor<int32, [2]> var_637 = const()[name = string("op_637"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [16, 128]> attention_3_cast_fp16 = reshape(shape = var_637, x = attention_1_cast_fp16)[name = string("attention_3_cast_fp16")];
            tensor<int32, [2]> var_639 = const()[name = string("op_639"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [16, 128]> input_13_cast_fp16 = reshape(shape = var_639, x = linear_7_cast_fp16)[name = string("input_13_cast_fp16")];
            tensor<fp16, [16, 128]> var_641_cast_fp16 = mul(x = attention_3_cast_fp16, y = attention_3_cast_fp16)[name = string("op_641_cast_fp16")];
            tensor<int32, [1]> variance_5_axes_0 = const()[name = string("variance_5_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_5_keep_dims_0 = const()[name = string("variance_5_keep_dims_0"), val = bool(true)];
            tensor<fp16, [16, 1]> variance_5_cast_fp16 = reduce_mean(axes = variance_5_axes_0, keep_dims = variance_5_keep_dims_0, x = var_641_cast_fp16)[name = string("variance_5_cast_fp16")];
            fp16 var_644_to_fp16 = const()[name = string("op_644_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [16, 1]> var_645_cast_fp16 = add(x = variance_5_cast_fp16, y = var_644_to_fp16)[name = string("op_645_cast_fp16")];
            fp32 var_646_epsilon_0 = const()[name = string("op_646_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [16, 1]> var_646_cast_fp16 = rsqrt(epsilon = var_646_epsilon_0, x = var_645_cast_fp16)[name = string("op_646_cast_fp16")];
            tensor<fp16, [16, 128]> attention_5_cast_fp16 = mul(x = attention_3_cast_fp16, y = var_646_cast_fp16)[name = string("attention_5_cast_fp16")];
            tensor<fp16, [128]> layers20_0_gated_norm_promoted_to_fp16 = const()[name = string("layers20_0_gated_norm_promoted_to_fp16"), val = tensor<fp16, [128]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(209578368)))];
            tensor<fp16, [16, 128]> attention_7_cast_fp16 = mul(x = attention_5_cast_fp16, y = layers20_0_gated_norm_promoted_to_fp16)[name = string("attention_7_cast_fp16")];
            tensor<fp16, [16, 128]> var_649_cast_fp16 = silu(x = input_13_cast_fp16)[name = string("op_649_cast_fp16")];
            tensor<fp16, [16, 128]> attention_9_cast_fp16 = mul(x = attention_7_cast_fp16, y = var_649_cast_fp16)[name = string("attention_9_cast_fp16")];
            tensor<int32, [2]> var_651 = const()[name = string("op_651"), val = tensor<int32, [2]>([1, 2048])];
            tensor<fp16, [1, 2048]> input_15_cast_fp16 = reshape(shape = var_651, x = attention_9_cast_fp16)[name = string("input_15_cast_fp16")];
            tensor<fp16, [1024, 2048]> layers20_0_out_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 2048]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(209578688))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(211151616))))[name = string("layers20_0_out_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 1024]> linear_8_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = layers20_0_out_proj_weight_promoted_to_fp16_palettized, x = input_15_cast_fp16)[name = string("linear_8_cast_fp16")];
            tensor<fp16, [1, 1024]> value_15_cast_fp16 = add(x = value_3_cast_fp16, y = linear_8_cast_fp16)[name = string("value_15_cast_fp16")];
            tensor<fp16, [1, 1024]> var_656_cast_fp16 = mul(x = value_15_cast_fp16, y = value_15_cast_fp16)[name = string("op_656_cast_fp16")];
            tensor<int32, [1]> variance_7_axes_0 = const()[name = string("variance_7_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_7_keep_dims_0 = const()[name = string("variance_7_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 1]> variance_7_cast_fp16 = reduce_mean(axes = variance_7_axes_0, keep_dims = variance_7_keep_dims_0, x = var_656_cast_fp16)[name = string("variance_7_cast_fp16")];
            fp16 var_659_to_fp16 = const()[name = string("op_659_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 1]> var_660_cast_fp16 = add(x = variance_7_cast_fp16, y = var_659_to_fp16)[name = string("op_660_cast_fp16")];
            fp32 var_661_epsilon_0 = const()[name = string("op_661_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 1]> var_661_cast_fp16 = rsqrt(epsilon = var_661_epsilon_0, x = var_660_cast_fp16)[name = string("op_661_cast_fp16")];
            tensor<fp16, [1, 1024]> var_662_cast_fp16 = mul(x = value_15_cast_fp16, y = var_661_cast_fp16)[name = string("op_662_cast_fp16")];
            tensor<fp16, [1024]> var_664_to_fp16 = const()[name = string("op_664_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(211159872)))];
            tensor<fp16, [1, 1024]> input_17_cast_fp16 = mul(x = var_662_cast_fp16, y = var_664_to_fp16)[name = string("input_17_cast_fp16")];
            tensor<fp16, [3584, 1024]> layers20_0_gate_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(211161984))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(213914560))))[name = string("layers20_0_gate_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 3584]> linear_9_cast_fp16 = linear(bias = linear_1_bias_0_to_fp16, weight = layers20_0_gate_proj_weight_promoted_to_fp16_palettized, x = input_17_cast_fp16)[name = string("linear_9_cast_fp16")];
            tensor<fp16, [1, 3584]> var_668_cast_fp16 = silu(x = linear_9_cast_fp16)[name = string("op_668_cast_fp16")];
            tensor<fp16, [3584, 1024]> layers20_0_up_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(213943296))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(216695872))))[name = string("layers20_0_up_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 3584]> linear_10_cast_fp16 = linear(bias = linear_1_bias_0_to_fp16, weight = layers20_0_up_proj_weight_promoted_to_fp16_palettized, x = input_17_cast_fp16)[name = string("linear_10_cast_fp16")];
            tensor<fp16, [1, 3584]> input_21_cast_fp16 = mul(x = var_668_cast_fp16, y = linear_10_cast_fp16)[name = string("input_21_cast_fp16")];
            tensor<fp16, [1024, 3584]> layers20_0_down_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 3584]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(216724608))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(219477184))))[name = string("layers20_0_down_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 1024]> linear_11_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = layers20_0_down_proj_weight_promoted_to_fp16_palettized, x = input_21_cast_fp16)[name = string("linear_11_cast_fp16")];
            tensor<fp16, [1, 1024]> value_17_cast_fp16 = add(x = value_15_cast_fp16, y = linear_11_cast_fp16)[name = string("value_17_cast_fp16")];
            int32 var_693 = const()[name = string("op_693"), val = int32(-1)];
            tensor<fp16, [1, 1024]> var_708_cast_fp16 = mul(x = value_17_cast_fp16, y = value_17_cast_fp16)[name = string("op_708_cast_fp16")];
            tensor<int32, [1]> variance_9_axes_0 = const()[name = string("variance_9_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_9_keep_dims_0 = const()[name = string("variance_9_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 1]> variance_9_cast_fp16 = reduce_mean(axes = variance_9_axes_0, keep_dims = variance_9_keep_dims_0, x = var_708_cast_fp16)[name = string("variance_9_cast_fp16")];
            fp16 var_711_to_fp16 = const()[name = string("op_711_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 1]> var_712_cast_fp16 = add(x = variance_9_cast_fp16, y = var_711_to_fp16)[name = string("op_712_cast_fp16")];
            fp32 var_713_epsilon_0 = const()[name = string("op_713_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 1]> var_713_cast_fp16 = rsqrt(epsilon = var_713_epsilon_0, x = var_712_cast_fp16)[name = string("op_713_cast_fp16")];
            tensor<fp16, [1, 1024]> var_714_cast_fp16 = mul(x = value_17_cast_fp16, y = var_713_cast_fp16)[name = string("op_714_cast_fp16")];
            tensor<fp16, [1024]> var_716_to_fp16 = const()[name = string("op_716_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(219485440)))];
            tensor<fp16, [1, 1024]> input_23_cast_fp16 = mul(x = var_714_cast_fp16, y = var_716_to_fp16)[name = string("input_23_cast_fp16")];
            tensor<fp16, [6144, 1024]> layers20_1_in_proj_qkv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(219487552))), lut = tensor<fp16, [384, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(224206208))))[name = string("layers20_1_in_proj_qkv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 6144]> linear_12_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = layers20_1_in_proj_qkv_weight_promoted_to_fp16_palettized, x = input_23_cast_fp16)[name = string("linear_12_cast_fp16")];
            tensor<int32, [2]> var_720_perm_0 = const()[name = string("op_720_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> mixed_3_axes_0 = const()[name = string("mixed_3_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [6144, 1]> var_720_cast_fp16 = transpose(perm = var_720_perm_0, x = linear_12_cast_fp16)[name = string("transpose_21")];
            tensor<fp16, [1, 6144, 1]> mixed_3_cast_fp16 = expand_dims(axes = mixed_3_axes_0, x = var_720_cast_fp16)[name = string("mixed_3_cast_fp16")];
            bool convolution_input_3_interleave_0 = const()[name = string("convolution_input_3_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 6144, 4]> convolution_input_3_cast_fp16 = concat(axis = var_693, interleave = convolution_input_3_interleave_0, values = (conv21, mixed_3_cast_fp16))[name = string("convolution_input_3_cast_fp16")];
            string input_25_pad_type_0 = const()[name = string("input_25_pad_type_0"), val = string("valid")];
            int32 input_25_groups_0 = const()[name = string("input_25_groups_0"), val = int32(6144)];
            tensor<int32, [1]> input_25_strides_0 = const()[name = string("input_25_strides_0"), val = tensor<int32, [1]>([1])];
            tensor<int32, [2]> input_25_pad_0 = const()[name = string("input_25_pad_0"), val = tensor<int32, [2]>([0, 0])];
            tensor<int32, [1]> input_25_dilations_0 = const()[name = string("input_25_dilations_0"), val = tensor<int32, [1]>([1])];
            tensor<fp16, [6144, 1, 4]> layers20_1_conv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1, 4]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(224255424))), lut = tensor<fp16, [384, 1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(224273920))))[name = string("layers20_1_conv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 6144, 1]> input_25_cast_fp16 = conv(dilations = input_25_dilations_0, groups = input_25_groups_0, pad = input_25_pad_0, pad_type = input_25_pad_type_0, strides = input_25_strides_0, weight = layers20_1_conv_weight_promoted_to_fp16_palettized, x = convolution_input_3_cast_fp16)[name = string("input_25_cast_fp16")];
            tensor<fp16, [1, 6144, 1]> var_729_cast_fp16 = silu(x = input_25_cast_fp16)[name = string("op_729_cast_fp16")];
            tensor<int32, [3]> var_730_perm_0 = const()[name = string("op_730_perm_0"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [3]> var_733_begin_0 = const()[name = string("op_733_begin_0"), val = tensor<int32, [3]>([0, 0, 1])];
            tensor<int32, [3]> var_733_end_0 = const()[name = string("op_733_end_0"), val = tensor<int32, [3]>([1, 6144, 4])];
            tensor<bool, [3]> var_733_end_mask_0 = const()[name = string("op_733_end_mask_0"), val = tensor<bool, [3]>([true, true, true])];
            tensor<fp16, [1, 6144, 3]> conv21_out = slice_by_index(begin = var_733_begin_0, end = var_733_end_0, end_mask = var_733_end_mask_0, x = convolution_input_3_cast_fp16)[name = string("op_733_cast_fp16")];
            tensor<int32, [3]> var_734 = const()[name = string("op_734"), val = tensor<int32, [3]>([2048, 2048, 2048])];
            int32 var_735_axis_0 = const()[name = string("op_735_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 1, 6144]> var_730_cast_fp16 = transpose(perm = var_730_perm_0, x = var_729_cast_fp16)[name = string("transpose_20")];
            tensor<fp16, [1, 1, 2048]> var_735_cast_fp16_0, tensor<fp16, [1, 1, 2048]> var_735_cast_fp16_1, tensor<fp16, [1, 1, 2048]> var_735_cast_fp16_2 = split(axis = var_735_axis_0, split_sizes = var_734, x = var_730_cast_fp16)[name = string("op_735_cast_fp16")];
            tensor<int32, [4]> var_739 = const()[name = string("op_739"), val = tensor<int32, [4]>([1, 1, 16, 128])];
            tensor<fp16, [1, 1, 16, 128]> value_21_cast_fp16 = reshape(shape = var_739, x = var_735_cast_fp16_0)[name = string("value_21_cast_fp16")];
            tensor<int32, [4]> var_741 = const()[name = string("op_741"), val = tensor<int32, [4]>([1, 1, 16, 128])];
            tensor<fp16, [1, 1, 16, 128]> value_23_cast_fp16 = reshape(shape = var_741, x = var_735_cast_fp16_1)[name = string("value_23_cast_fp16")];
            tensor<int32, [4]> var_743 = const()[name = string("op_743"), val = tensor<int32, [4]>([1, 1, 16, 128])];
            tensor<fp16, [1, 1, 16, 128]> value_25_cast_fp16 = reshape(shape = var_743, x = var_735_cast_fp16_2)[name = string("value_25_cast_fp16")];
            tensor<fp16, [1, 1, 16, 128]> var_745_cast_fp16 = mul(x = value_21_cast_fp16, y = value_21_cast_fp16)[name = string("op_745_cast_fp16")];
            tensor<int32, [1]> var_747_axes_0 = const()[name = string("op_747_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_747_keep_dims_0 = const()[name = string("op_747_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 1, 16, 1]> var_747_cast_fp16 = reduce_sum(axes = var_747_axes_0, keep_dims = var_747_keep_dims_0, x = var_745_cast_fp16)[name = string("op_747_cast_fp16")];
            fp16 var_748_to_fp16 = const()[name = string("op_748_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 1, 16, 1]> var_749_cast_fp16 = add(x = var_747_cast_fp16, y = var_748_to_fp16)[name = string("op_749_cast_fp16")];
            fp32 var_750_epsilon_0 = const()[name = string("op_750_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 1, 16, 1]> var_750_cast_fp16 = rsqrt(epsilon = var_750_epsilon_0, x = var_749_cast_fp16)[name = string("op_750_cast_fp16")];
            tensor<fp16, [1, 1, 16, 128]> var_751_cast_fp16 = mul(x = value_21_cast_fp16, y = var_750_cast_fp16)[name = string("op_751_cast_fp16")];
            tensor<int32, [4]> var_752_perm_0 = const()[name = string("op_752_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            fp16 _inversed_query_7_y_0_to_fp16 = const()[name = string("_inversed_query_7_y_0_to_fp16"), val = fp16(0x1.6ap-4)];
            tensor<fp16, [1, 16, 1, 128]> var_752_cast_fp16 = transpose(perm = var_752_perm_0, x = var_751_cast_fp16)[name = string("transpose_19")];
            tensor<fp16, [1, 16, 1, 128]> _inversed_query_7_cast_fp16 = mul(x = var_752_cast_fp16, y = _inversed_query_7_y_0_to_fp16)[name = string("_inversed_query_7_cast_fp16")];
            tensor<fp16, [1, 1, 16, 128]> var_755_cast_fp16 = mul(x = value_23_cast_fp16, y = value_23_cast_fp16)[name = string("op_755_cast_fp16")];
            tensor<int32, [1]> var_757_axes_0 = const()[name = string("op_757_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_757_keep_dims_0 = const()[name = string("op_757_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 1, 16, 1]> var_757_cast_fp16 = reduce_sum(axes = var_757_axes_0, keep_dims = var_757_keep_dims_0, x = var_755_cast_fp16)[name = string("op_757_cast_fp16")];
            fp16 var_758_to_fp16 = const()[name = string("op_758_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 1, 16, 1]> var_759_cast_fp16 = add(x = var_757_cast_fp16, y = var_758_to_fp16)[name = string("op_759_cast_fp16")];
            fp32 var_760_epsilon_0 = const()[name = string("op_760_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 1, 16, 1]> var_760_cast_fp16 = rsqrt(epsilon = var_760_epsilon_0, x = var_759_cast_fp16)[name = string("op_760_cast_fp16")];
            tensor<fp16, [1, 1, 16, 128]> var_761_cast_fp16 = mul(x = value_23_cast_fp16, y = var_760_cast_fp16)[name = string("op_761_cast_fp16")];
            tensor<int32, [4]> key_7_perm_0 = const()[name = string("key_7_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<int32, [4]> value_27_perm_0 = const()[name = string("value_27_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<fp16, [16, 1024]> layers20_1_in_proj_b_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(224323136))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(224335488))))[name = string("layers20_1_in_proj_b_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16]> linear_13_bias_0_to_fp16 = const()[name = string("linear_13_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(207971904)))];
            tensor<fp16, [1, 16]> linear_13_cast_fp16 = linear(bias = linear_13_bias_0_to_fp16, weight = layers20_1_in_proj_b_weight_promoted_to_fp16_palettized, x = input_23_cast_fp16)[name = string("linear_13_cast_fp16")];
            tensor<fp16, [1, 16]> var_766_cast_fp16 = sigmoid(x = linear_13_cast_fp16)[name = string("op_766_cast_fp16")];
            tensor<int32, [2]> var_767_perm_0 = const()[name = string("op_767_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> beta_3_axes_0 = const()[name = string("beta_3_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 1]> var_767_cast_fp16 = transpose(perm = var_767_perm_0, x = var_766_cast_fp16)[name = string("transpose_18")];
            tensor<fp16, [1, 16, 1]> beta_3_cast_fp16 = expand_dims(axes = beta_3_axes_0, x = var_767_cast_fp16)[name = string("beta_3_cast_fp16")];
            tensor<fp16, [16, 1024]> op_773_weight_0_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(224335680))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(224348032))))[name = string("op_773_weight_0_to_fp16_palettized")];
            tensor<fp16, [16]> var_773_bias_0_to_fp16 = const()[name = string("op_773_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(224348224)))];
            tensor<fp16, [1, 16]> var_773_cast_fp16 = linear(bias = var_773_bias_0_to_fp16, weight = op_773_weight_0_to_fp16_palettized, x = input_23_cast_fp16)[name = string("op_773_cast_fp16")];
            tensor<fp16, [1, 16]> var_774_cast_fp16 = softplus(x = var_773_cast_fp16)[name = string("op_774_cast_fp16")];
            tensor<fp16, [16]> var_770_promoted_to_fp16 = const()[name = string("op_770_promoted_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(224348352)))];
            tensor<fp16, [1, 16]> var_775_cast_fp16 = mul(x = var_770_promoted_to_fp16, y = var_774_cast_fp16)[name = string("op_775_cast_fp16")];
            tensor<int32, [2]> var_776_perm_0 = const()[name = string("op_776_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> decay_3_axes_0 = const()[name = string("decay_3_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 1]> var_776_cast_fp16 = transpose(perm = var_776_perm_0, x = var_775_cast_fp16)[name = string("transpose_17")];
            tensor<fp16, [1, 16, 1]> decay_3_cast_fp16 = expand_dims(axes = decay_3_axes_0, x = var_776_cast_fp16)[name = string("decay_3_cast_fp16")];
            tensor<fp16, [2048, 1024]> layers20_1_in_proj_z_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [2048, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(224348480))), lut = tensor<fp16, [128, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(225921408))))[name = string("layers20_1_in_proj_z_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 2048]> linear_15_cast_fp16 = linear(bias = linear_7_bias_0_to_fp16, weight = layers20_1_in_proj_z_weight_promoted_to_fp16_palettized, x = input_23_cast_fp16)[name = string("linear_15_cast_fp16")];
            tensor<int32, [3]> var_784_begin_0 = const()[name = string("op_784_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_784_end_0 = const()[name = string("op_784_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_784_end_mask_0 = const()[name = string("op_784_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_784_squeeze_mask_0 = const()[name = string("op_784_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_784_cast_fp16 = slice_by_index(begin = var_784_begin_0, end = var_784_end_0, end_mask = var_784_end_mask_0, squeeze_mask = var_784_squeeze_mask_0, x = decay_3_cast_fp16)[name = string("op_784_cast_fp16")];
            tensor<fp16, [1, 16]> var_785_cast_fp16 = exp(x = var_784_cast_fp16)[name = string("op_785_cast_fp16")];
            tensor<int32, [1]> var_786_axes_0 = const()[name = string("op_786_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_786_cast_fp16 = expand_dims(axes = var_786_axes_0, x = var_785_cast_fp16)[name = string("op_786_cast_fp16")];
            tensor<int32, [1]> var_787_axes_0 = const()[name = string("op_787_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_787_cast_fp16 = expand_dims(axes = var_787_axes_0, x = var_786_cast_fp16)[name = string("op_787_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_5_cast_fp16 = mul(x = rec21, y = var_787_cast_fp16)[name = string("state_5_cast_fp16")];
            tensor<int32, [4]> key_token_3_begin_0 = const()[name = string("key_token_3_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> key_token_3_end_0 = const()[name = string("key_token_3_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> key_token_3_end_mask_0 = const()[name = string("key_token_3_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_3_squeeze_mask_0 = const()[name = string("key_token_3_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 1, 128]> key_7_cast_fp16 = transpose(perm = key_7_perm_0, x = var_761_cast_fp16)[name = string("transpose_16")];
            tensor<fp16, [1, 16, 128]> key_token_3_cast_fp16 = slice_by_index(begin = key_token_3_begin_0, end = key_token_3_end_0, end_mask = key_token_3_end_mask_0, squeeze_mask = key_token_3_squeeze_mask_0, x = key_7_cast_fp16)[name = string("key_token_3_cast_fp16")];
            tensor<int32, [4]> value_token_3_begin_0 = const()[name = string("value_token_3_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> value_token_3_end_0 = const()[name = string("value_token_3_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> value_token_3_end_mask_0 = const()[name = string("value_token_3_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_3_squeeze_mask_0 = const()[name = string("value_token_3_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 1, 128]> value_27_cast_fp16 = transpose(perm = value_27_perm_0, x = value_25_cast_fp16)[name = string("transpose_15")];
            tensor<fp16, [1, 16, 128]> value_token_3_cast_fp16 = slice_by_index(begin = value_token_3_begin_0, end = value_token_3_end_0, end_mask = value_token_3_end_mask_0, squeeze_mask = value_token_3_squeeze_mask_0, x = value_27_cast_fp16)[name = string("value_token_3_cast_fp16")];
            tensor<int32, [1]> var_795_axes_0 = const()[name = string("op_795_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_795_cast_fp16 = expand_dims(axes = var_795_axes_0, x = key_token_3_cast_fp16)[name = string("op_795_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_796_cast_fp16 = mul(x = state_5_cast_fp16, y = var_795_cast_fp16)[name = string("op_796_cast_fp16")];
            tensor<int32, [1]> memory_3_axes_0 = const()[name = string("memory_3_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_3_keep_dims_0 = const()[name = string("memory_3_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_3_cast_fp16 = reduce_sum(axes = memory_3_axes_0, keep_dims = memory_3_keep_dims_0, x = var_796_cast_fp16)[name = string("memory_3_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_799_cast_fp16 = sub(x = value_token_3_cast_fp16, y = memory_3_cast_fp16)[name = string("op_799_cast_fp16")];
            tensor<int32, [3]> var_802_begin_0 = const()[name = string("op_802_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_802_end_0 = const()[name = string("op_802_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_802_end_mask_0 = const()[name = string("op_802_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_802_squeeze_mask_0 = const()[name = string("op_802_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_802_cast_fp16 = slice_by_index(begin = var_802_begin_0, end = var_802_end_0, end_mask = var_802_end_mask_0, squeeze_mask = var_802_squeeze_mask_0, x = beta_3_cast_fp16)[name = string("op_802_cast_fp16")];
            tensor<int32, [1]> var_803_axes_0 = const()[name = string("op_803_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_803_cast_fp16 = expand_dims(axes = var_803_axes_0, x = var_802_cast_fp16)[name = string("op_803_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_3_cast_fp16 = mul(x = var_799_cast_fp16, y = var_803_cast_fp16)[name = string("delta_3_cast_fp16")];
            tensor<int32, [1]> var_806_axes_0 = const()[name = string("op_806_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_806_cast_fp16 = expand_dims(axes = var_806_axes_0, x = delta_3_cast_fp16)[name = string("op_806_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_807_cast_fp16 = mul(x = var_795_cast_fp16, y = var_806_cast_fp16)[name = string("op_807_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> rec21_out = add(x = state_5_cast_fp16, y = var_807_cast_fp16)[name = string("state_7_cast_fp16")];
            tensor<int32, [4]> var_811_begin_0 = const()[name = string("op_811_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> var_811_end_0 = const()[name = string("op_811_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> var_811_end_mask_0 = const()[name = string("op_811_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_811_squeeze_mask_0 = const()[name = string("op_811_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_811_cast_fp16 = slice_by_index(begin = var_811_begin_0, end = var_811_end_0, end_mask = var_811_end_mask_0, squeeze_mask = var_811_squeeze_mask_0, x = _inversed_query_7_cast_fp16)[name = string("op_811_cast_fp16")];
            tensor<int32, [1]> var_812_axes_0 = const()[name = string("op_812_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_812_cast_fp16 = expand_dims(axes = var_812_axes_0, x = var_811_cast_fp16)[name = string("op_812_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_813_cast_fp16 = mul(x = rec21_out, y = var_812_cast_fp16)[name = string("op_813_cast_fp16")];
            tensor<int32, [1]> var_815_axes_0 = const()[name = string("op_815_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_815_keep_dims_0 = const()[name = string("op_815_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_815_cast_fp16 = reduce_sum(axes = var_815_axes_0, keep_dims = var_815_keep_dims_0, x = var_813_cast_fp16)[name = string("op_815_cast_fp16")];
            tensor<int32, [1]> attention_11_axes_0 = const()[name = string("attention_11_axes_0"), val = tensor<int32, [1]>([1])];
            tensor<fp16, [1, 1, 16, 128]> attention_11_cast_fp16 = expand_dims(axes = attention_11_axes_0, x = var_815_cast_fp16)[name = string("attention_11_cast_fp16")];
            tensor<int32, [2]> var_818 = const()[name = string("op_818"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [16, 128]> attention_13_cast_fp16 = reshape(shape = var_818, x = attention_11_cast_fp16)[name = string("attention_13_cast_fp16")];
            tensor<int32, [2]> var_820 = const()[name = string("op_820"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [16, 128]> input_27_cast_fp16 = reshape(shape = var_820, x = linear_15_cast_fp16)[name = string("input_27_cast_fp16")];
            tensor<fp16, [16, 128]> var_822_cast_fp16 = mul(x = attention_13_cast_fp16, y = attention_13_cast_fp16)[name = string("op_822_cast_fp16")];
            tensor<int32, [1]> variance_11_axes_0 = const()[name = string("variance_11_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_11_keep_dims_0 = const()[name = string("variance_11_keep_dims_0"), val = bool(true)];
            tensor<fp16, [16, 1]> variance_11_cast_fp16 = reduce_mean(axes = variance_11_axes_0, keep_dims = variance_11_keep_dims_0, x = var_822_cast_fp16)[name = string("variance_11_cast_fp16")];
            fp16 var_825_to_fp16 = const()[name = string("op_825_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [16, 1]> var_826_cast_fp16 = add(x = variance_11_cast_fp16, y = var_825_to_fp16)[name = string("op_826_cast_fp16")];
            fp32 var_827_epsilon_0 = const()[name = string("op_827_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [16, 1]> var_827_cast_fp16 = rsqrt(epsilon = var_827_epsilon_0, x = var_826_cast_fp16)[name = string("op_827_cast_fp16")];
            tensor<fp16, [16, 128]> attention_15_cast_fp16 = mul(x = attention_13_cast_fp16, y = var_827_cast_fp16)[name = string("attention_15_cast_fp16")];
            tensor<fp16, [128]> layers20_1_gated_norm_promoted_to_fp16 = const()[name = string("layers20_1_gated_norm_promoted_to_fp16"), val = tensor<fp16, [128]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(225937856)))];
            tensor<fp16, [16, 128]> attention_17_cast_fp16 = mul(x = attention_15_cast_fp16, y = layers20_1_gated_norm_promoted_to_fp16)[name = string("attention_17_cast_fp16")];
            tensor<fp16, [16, 128]> var_830_cast_fp16 = silu(x = input_27_cast_fp16)[name = string("op_830_cast_fp16")];
            tensor<fp16, [16, 128]> attention_19_cast_fp16 = mul(x = attention_17_cast_fp16, y = var_830_cast_fp16)[name = string("attention_19_cast_fp16")];
            tensor<int32, [2]> var_832 = const()[name = string("op_832"), val = tensor<int32, [2]>([1, 2048])];
            tensor<fp16, [1, 2048]> input_29_cast_fp16 = reshape(shape = var_832, x = attention_19_cast_fp16)[name = string("input_29_cast_fp16")];
            tensor<fp16, [1024, 2048]> layers20_1_out_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 2048]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(225938176))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(227511104))))[name = string("layers20_1_out_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 1024]> linear_16_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = layers20_1_out_proj_weight_promoted_to_fp16_palettized, x = input_29_cast_fp16)[name = string("linear_16_cast_fp16")];
            tensor<fp16, [1, 1024]> value_29_cast_fp16 = add(x = value_17_cast_fp16, y = linear_16_cast_fp16)[name = string("value_29_cast_fp16")];
            tensor<fp16, [1, 1024]> var_837_cast_fp16 = mul(x = value_29_cast_fp16, y = value_29_cast_fp16)[name = string("op_837_cast_fp16")];
            tensor<int32, [1]> variance_13_axes_0 = const()[name = string("variance_13_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_13_keep_dims_0 = const()[name = string("variance_13_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 1]> variance_13_cast_fp16 = reduce_mean(axes = variance_13_axes_0, keep_dims = variance_13_keep_dims_0, x = var_837_cast_fp16)[name = string("variance_13_cast_fp16")];
            fp16 var_840_to_fp16 = const()[name = string("op_840_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 1]> var_841_cast_fp16 = add(x = variance_13_cast_fp16, y = var_840_to_fp16)[name = string("op_841_cast_fp16")];
            fp32 var_842_epsilon_0 = const()[name = string("op_842_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 1]> var_842_cast_fp16 = rsqrt(epsilon = var_842_epsilon_0, x = var_841_cast_fp16)[name = string("op_842_cast_fp16")];
            tensor<fp16, [1, 1024]> var_843_cast_fp16 = mul(x = value_29_cast_fp16, y = var_842_cast_fp16)[name = string("op_843_cast_fp16")];
            tensor<fp16, [1024]> var_845_to_fp16 = const()[name = string("op_845_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(227519360)))];
            tensor<fp16, [1, 1024]> input_31_cast_fp16 = mul(x = var_843_cast_fp16, y = var_845_to_fp16)[name = string("input_31_cast_fp16")];
            tensor<fp16, [3584, 1024]> layers20_1_gate_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(227521472))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(230274048))))[name = string("layers20_1_gate_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 3584]> linear_17_cast_fp16 = linear(bias = linear_1_bias_0_to_fp16, weight = layers20_1_gate_proj_weight_promoted_to_fp16_palettized, x = input_31_cast_fp16)[name = string("linear_17_cast_fp16")];
            tensor<fp16, [1, 3584]> var_849_cast_fp16 = silu(x = linear_17_cast_fp16)[name = string("op_849_cast_fp16")];
            tensor<fp16, [3584, 1024]> layers20_1_up_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(230302784))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(233055360))))[name = string("layers20_1_up_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 3584]> linear_18_cast_fp16 = linear(bias = linear_1_bias_0_to_fp16, weight = layers20_1_up_proj_weight_promoted_to_fp16_palettized, x = input_31_cast_fp16)[name = string("linear_18_cast_fp16")];
            tensor<fp16, [1, 3584]> input_35_cast_fp16 = mul(x = var_849_cast_fp16, y = linear_18_cast_fp16)[name = string("input_35_cast_fp16")];
            tensor<fp16, [1024, 3584]> layers20_1_down_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 3584]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(233084096))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(235836672))))[name = string("layers20_1_down_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 1024]> linear_19_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = layers20_1_down_proj_weight_promoted_to_fp16_palettized, x = input_35_cast_fp16)[name = string("linear_19_cast_fp16")];
            tensor<fp16, [1, 1024]> value_31_cast_fp16 = add(x = value_29_cast_fp16, y = linear_19_cast_fp16)[name = string("value_31_cast_fp16")];
            int32 var_874 = const()[name = string("op_874"), val = int32(-1)];
            tensor<fp16, [1, 1024]> var_889_cast_fp16 = mul(x = value_31_cast_fp16, y = value_31_cast_fp16)[name = string("op_889_cast_fp16")];
            tensor<int32, [1]> variance_15_axes_0 = const()[name = string("variance_15_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_15_keep_dims_0 = const()[name = string("variance_15_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 1]> variance_15_cast_fp16 = reduce_mean(axes = variance_15_axes_0, keep_dims = variance_15_keep_dims_0, x = var_889_cast_fp16)[name = string("variance_15_cast_fp16")];
            fp16 var_892_to_fp16 = const()[name = string("op_892_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 1]> var_893_cast_fp16 = add(x = variance_15_cast_fp16, y = var_892_to_fp16)[name = string("op_893_cast_fp16")];
            fp32 var_894_epsilon_0 = const()[name = string("op_894_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 1]> var_894_cast_fp16 = rsqrt(epsilon = var_894_epsilon_0, x = var_893_cast_fp16)[name = string("op_894_cast_fp16")];
            tensor<fp16, [1, 1024]> var_895_cast_fp16 = mul(x = value_31_cast_fp16, y = var_894_cast_fp16)[name = string("op_895_cast_fp16")];
            tensor<fp16, [1024]> var_897_to_fp16 = const()[name = string("op_897_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(235844928)))];
            tensor<fp16, [1, 1024]> input_37_cast_fp16 = mul(x = var_895_cast_fp16, y = var_897_to_fp16)[name = string("input_37_cast_fp16")];
            tensor<fp16, [6144, 1024]> layers20_2_in_proj_qkv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(235847040))), lut = tensor<fp16, [384, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(240565696))))[name = string("layers20_2_in_proj_qkv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 6144]> linear_20_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = layers20_2_in_proj_qkv_weight_promoted_to_fp16_palettized, x = input_37_cast_fp16)[name = string("linear_20_cast_fp16")];
            tensor<int32, [2]> var_901_perm_0 = const()[name = string("op_901_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> mixed_axes_0 = const()[name = string("mixed_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [6144, 1]> var_901_cast_fp16 = transpose(perm = var_901_perm_0, x = linear_20_cast_fp16)[name = string("transpose_14")];
            tensor<fp16, [1, 6144, 1]> mixed_cast_fp16 = expand_dims(axes = mixed_axes_0, x = var_901_cast_fp16)[name = string("mixed_cast_fp16")];
            bool convolution_input_interleave_0 = const()[name = string("convolution_input_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 6144, 4]> convolution_input_cast_fp16 = concat(axis = var_874, interleave = convolution_input_interleave_0, values = (conv22, mixed_cast_fp16))[name = string("convolution_input_cast_fp16")];
            string input_39_pad_type_0 = const()[name = string("input_39_pad_type_0"), val = string("valid")];
            int32 input_39_groups_0 = const()[name = string("input_39_groups_0"), val = int32(6144)];
            tensor<int32, [1]> input_39_strides_0 = const()[name = string("input_39_strides_0"), val = tensor<int32, [1]>([1])];
            tensor<int32, [2]> input_39_pad_0 = const()[name = string("input_39_pad_0"), val = tensor<int32, [2]>([0, 0])];
            tensor<int32, [1]> input_39_dilations_0 = const()[name = string("input_39_dilations_0"), val = tensor<int32, [1]>([1])];
            tensor<fp16, [6144, 1, 4]> layers20_2_conv_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [6144, 1, 4]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(240614912))), lut = tensor<fp16, [384, 1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(240633408))))[name = string("layers20_2_conv_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 6144, 1]> input_39_cast_fp16 = conv(dilations = input_39_dilations_0, groups = input_39_groups_0, pad = input_39_pad_0, pad_type = input_39_pad_type_0, strides = input_39_strides_0, weight = layers20_2_conv_weight_promoted_to_fp16_palettized, x = convolution_input_cast_fp16)[name = string("input_39_cast_fp16")];
            tensor<fp16, [1, 6144, 1]> var_910_cast_fp16 = silu(x = input_39_cast_fp16)[name = string("op_910_cast_fp16")];
            tensor<int32, [3]> var_911_perm_0 = const()[name = string("op_911_perm_0"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [3]> var_914_begin_0 = const()[name = string("op_914_begin_0"), val = tensor<int32, [3]>([0, 0, 1])];
            tensor<int32, [3]> var_914_end_0 = const()[name = string("op_914_end_0"), val = tensor<int32, [3]>([1, 6144, 4])];
            tensor<bool, [3]> var_914_end_mask_0 = const()[name = string("op_914_end_mask_0"), val = tensor<bool, [3]>([true, true, true])];
            tensor<fp16, [1, 6144, 3]> conv22_out = slice_by_index(begin = var_914_begin_0, end = var_914_end_0, end_mask = var_914_end_mask_0, x = convolution_input_cast_fp16)[name = string("op_914_cast_fp16")];
            tensor<int32, [3]> var_915 = const()[name = string("op_915"), val = tensor<int32, [3]>([2048, 2048, 2048])];
            int32 var_916_axis_0 = const()[name = string("op_916_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 1, 6144]> var_911_cast_fp16 = transpose(perm = var_911_perm_0, x = var_910_cast_fp16)[name = string("transpose_13")];
            tensor<fp16, [1, 1, 2048]> var_916_cast_fp16_0, tensor<fp16, [1, 1, 2048]> var_916_cast_fp16_1, tensor<fp16, [1, 1, 2048]> var_916_cast_fp16_2 = split(axis = var_916_axis_0, split_sizes = var_915, x = var_911_cast_fp16)[name = string("op_916_cast_fp16")];
            tensor<int32, [4]> var_920 = const()[name = string("op_920"), val = tensor<int32, [4]>([1, 1, 16, 128])];
            tensor<fp16, [1, 1, 16, 128]> value_35_cast_fp16 = reshape(shape = var_920, x = var_916_cast_fp16_0)[name = string("value_35_cast_fp16")];
            tensor<int32, [4]> var_922 = const()[name = string("op_922"), val = tensor<int32, [4]>([1, 1, 16, 128])];
            tensor<fp16, [1, 1, 16, 128]> value_37_cast_fp16 = reshape(shape = var_922, x = var_916_cast_fp16_1)[name = string("value_37_cast_fp16")];
            tensor<int32, [4]> var_924 = const()[name = string("op_924"), val = tensor<int32, [4]>([1, 1, 16, 128])];
            tensor<fp16, [1, 1, 16, 128]> value_39_cast_fp16 = reshape(shape = var_924, x = var_916_cast_fp16_2)[name = string("value_39_cast_fp16")];
            tensor<fp16, [1, 1, 16, 128]> var_926_cast_fp16 = mul(x = value_35_cast_fp16, y = value_35_cast_fp16)[name = string("op_926_cast_fp16")];
            tensor<int32, [1]> var_928_axes_0 = const()[name = string("op_928_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_928_keep_dims_0 = const()[name = string("op_928_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 1, 16, 1]> var_928_cast_fp16 = reduce_sum(axes = var_928_axes_0, keep_dims = var_928_keep_dims_0, x = var_926_cast_fp16)[name = string("op_928_cast_fp16")];
            fp16 var_929_to_fp16 = const()[name = string("op_929_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 1, 16, 1]> var_930_cast_fp16 = add(x = var_928_cast_fp16, y = var_929_to_fp16)[name = string("op_930_cast_fp16")];
            fp32 var_931_epsilon_0 = const()[name = string("op_931_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 1, 16, 1]> var_931_cast_fp16 = rsqrt(epsilon = var_931_epsilon_0, x = var_930_cast_fp16)[name = string("op_931_cast_fp16")];
            tensor<fp16, [1, 1, 16, 128]> var_932_cast_fp16 = mul(x = value_35_cast_fp16, y = var_931_cast_fp16)[name = string("op_932_cast_fp16")];
            tensor<int32, [4]> var_933_perm_0 = const()[name = string("op_933_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            fp16 _inversed_query_11_y_0_to_fp16 = const()[name = string("_inversed_query_11_y_0_to_fp16"), val = fp16(0x1.6ap-4)];
            tensor<fp16, [1, 16, 1, 128]> var_933_cast_fp16 = transpose(perm = var_933_perm_0, x = var_932_cast_fp16)[name = string("transpose_12")];
            tensor<fp16, [1, 16, 1, 128]> _inversed_query_11_cast_fp16 = mul(x = var_933_cast_fp16, y = _inversed_query_11_y_0_to_fp16)[name = string("_inversed_query_11_cast_fp16")];
            tensor<fp16, [1, 1, 16, 128]> var_936_cast_fp16 = mul(x = value_37_cast_fp16, y = value_37_cast_fp16)[name = string("op_936_cast_fp16")];
            tensor<int32, [1]> var_938_axes_0 = const()[name = string("op_938_axes_0"), val = tensor<int32, [1]>([-1])];
            bool var_938_keep_dims_0 = const()[name = string("op_938_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 1, 16, 1]> var_938_cast_fp16 = reduce_sum(axes = var_938_axes_0, keep_dims = var_938_keep_dims_0, x = var_936_cast_fp16)[name = string("op_938_cast_fp16")];
            fp16 var_939_to_fp16 = const()[name = string("op_939_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 1, 16, 1]> var_940_cast_fp16 = add(x = var_938_cast_fp16, y = var_939_to_fp16)[name = string("op_940_cast_fp16")];
            fp32 var_941_epsilon_0 = const()[name = string("op_941_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 1, 16, 1]> var_941_cast_fp16 = rsqrt(epsilon = var_941_epsilon_0, x = var_940_cast_fp16)[name = string("op_941_cast_fp16")];
            tensor<fp16, [1, 1, 16, 128]> var_942_cast_fp16 = mul(x = value_37_cast_fp16, y = var_941_cast_fp16)[name = string("op_942_cast_fp16")];
            tensor<int32, [4]> key_11_perm_0 = const()[name = string("key_11_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<int32, [4]> value_41_perm_0 = const()[name = string("value_41_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
            tensor<fp16, [16, 1024]> layers20_2_in_proj_b_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(240682624))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(240694976))))[name = string("layers20_2_in_proj_b_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [16]> linear_21_bias_0_to_fp16 = const()[name = string("linear_21_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(207971904)))];
            tensor<fp16, [1, 16]> linear_21_cast_fp16 = linear(bias = linear_21_bias_0_to_fp16, weight = layers20_2_in_proj_b_weight_promoted_to_fp16_palettized, x = input_37_cast_fp16)[name = string("linear_21_cast_fp16")];
            tensor<fp16, [1, 16]> var_947_cast_fp16 = sigmoid(x = linear_21_cast_fp16)[name = string("op_947_cast_fp16")];
            tensor<int32, [2]> var_948_perm_0 = const()[name = string("op_948_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> beta_axes_0 = const()[name = string("beta_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 1]> var_948_cast_fp16 = transpose(perm = var_948_perm_0, x = var_947_cast_fp16)[name = string("transpose_11")];
            tensor<fp16, [1, 16, 1]> beta_cast_fp16 = expand_dims(axes = beta_axes_0, x = var_948_cast_fp16)[name = string("beta_cast_fp16")];
            tensor<fp16, [16, 1024]> op_954_weight_0_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [16, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(240695168))), lut = tensor<fp16, [1, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(240707520))))[name = string("op_954_weight_0_to_fp16_palettized")];
            tensor<fp16, [16]> var_954_bias_0_to_fp16 = const()[name = string("op_954_bias_0_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(240707712)))];
            tensor<fp16, [1, 16]> var_954_cast_fp16 = linear(bias = var_954_bias_0_to_fp16, weight = op_954_weight_0_to_fp16_palettized, x = input_37_cast_fp16)[name = string("op_954_cast_fp16")];
            tensor<fp16, [1, 16]> var_955_cast_fp16 = softplus(x = var_954_cast_fp16)[name = string("op_955_cast_fp16")];
            tensor<fp16, [16]> var_951_promoted_to_fp16 = const()[name = string("op_951_promoted_to_fp16"), val = tensor<fp16, [16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(240707840)))];
            tensor<fp16, [1, 16]> var_956_cast_fp16 = mul(x = var_951_promoted_to_fp16, y = var_955_cast_fp16)[name = string("op_956_cast_fp16")];
            tensor<int32, [2]> var_957_perm_0 = const()[name = string("op_957_perm_0"), val = tensor<int32, [2]>([1, 0])];
            tensor<int32, [1]> decay_axes_0 = const()[name = string("decay_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [16, 1]> var_957_cast_fp16 = transpose(perm = var_957_perm_0, x = var_956_cast_fp16)[name = string("transpose_10")];
            tensor<fp16, [1, 16, 1]> decay_cast_fp16 = expand_dims(axes = decay_axes_0, x = var_957_cast_fp16)[name = string("decay_cast_fp16")];
            tensor<fp16, [2048, 1024]> layers20_2_in_proj_z_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [2048, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(240707968))), lut = tensor<fp16, [128, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(242280896))))[name = string("layers20_2_in_proj_z_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 2048]> linear_23_cast_fp16 = linear(bias = linear_7_bias_0_to_fp16, weight = layers20_2_in_proj_z_weight_promoted_to_fp16_palettized, x = input_37_cast_fp16)[name = string("linear_23_cast_fp16")];
            tensor<int32, [3]> var_965_begin_0 = const()[name = string("op_965_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_965_end_0 = const()[name = string("op_965_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_965_end_mask_0 = const()[name = string("op_965_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_965_squeeze_mask_0 = const()[name = string("op_965_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_965_cast_fp16 = slice_by_index(begin = var_965_begin_0, end = var_965_end_0, end_mask = var_965_end_mask_0, squeeze_mask = var_965_squeeze_mask_0, x = decay_cast_fp16)[name = string("op_965_cast_fp16")];
            tensor<fp16, [1, 16]> var_966_cast_fp16 = exp(x = var_965_cast_fp16)[name = string("op_966_cast_fp16")];
            tensor<int32, [1]> var_967_axes_0 = const()[name = string("op_967_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_967_cast_fp16 = expand_dims(axes = var_967_axes_0, x = var_966_cast_fp16)[name = string("op_967_cast_fp16")];
            tensor<int32, [1]> var_968_axes_0 = const()[name = string("op_968_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1, 1]> var_968_cast_fp16 = expand_dims(axes = var_968_axes_0, x = var_967_cast_fp16)[name = string("op_968_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> state_9_cast_fp16 = mul(x = rec22, y = var_968_cast_fp16)[name = string("state_9_cast_fp16")];
            tensor<int32, [4]> key_token_begin_0 = const()[name = string("key_token_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> key_token_end_0 = const()[name = string("key_token_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> key_token_end_mask_0 = const()[name = string("key_token_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> key_token_squeeze_mask_0 = const()[name = string("key_token_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 1, 128]> key_11_cast_fp16 = transpose(perm = key_11_perm_0, x = var_942_cast_fp16)[name = string("transpose_9")];
            tensor<fp16, [1, 16, 128]> key_token_cast_fp16 = slice_by_index(begin = key_token_begin_0, end = key_token_end_0, end_mask = key_token_end_mask_0, squeeze_mask = key_token_squeeze_mask_0, x = key_11_cast_fp16)[name = string("key_token_cast_fp16")];
            tensor<int32, [4]> value_token_begin_0 = const()[name = string("value_token_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> value_token_end_0 = const()[name = string("value_token_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> value_token_end_mask_0 = const()[name = string("value_token_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> value_token_squeeze_mask_0 = const()[name = string("value_token_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 1, 128]> value_41_cast_fp16 = transpose(perm = value_41_perm_0, x = value_39_cast_fp16)[name = string("transpose_8")];
            tensor<fp16, [1, 16, 128]> value_token_cast_fp16 = slice_by_index(begin = value_token_begin_0, end = value_token_end_0, end_mask = value_token_end_mask_0, squeeze_mask = value_token_squeeze_mask_0, x = value_41_cast_fp16)[name = string("value_token_cast_fp16")];
            tensor<int32, [1]> var_976_axes_0 = const()[name = string("op_976_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_976_cast_fp16 = expand_dims(axes = var_976_axes_0, x = key_token_cast_fp16)[name = string("op_976_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_977_cast_fp16 = mul(x = state_9_cast_fp16, y = var_976_cast_fp16)[name = string("op_977_cast_fp16")];
            tensor<int32, [1]> memory_axes_0 = const()[name = string("memory_axes_0"), val = tensor<int32, [1]>([-2])];
            bool memory_keep_dims_0 = const()[name = string("memory_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> memory_cast_fp16 = reduce_sum(axes = memory_axes_0, keep_dims = memory_keep_dims_0, x = var_977_cast_fp16)[name = string("memory_cast_fp16")];
            tensor<fp16, [1, 16, 128]> var_980_cast_fp16 = sub(x = value_token_cast_fp16, y = memory_cast_fp16)[name = string("op_980_cast_fp16")];
            tensor<int32, [3]> var_983_begin_0 = const()[name = string("op_983_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
            tensor<int32, [3]> var_983_end_0 = const()[name = string("op_983_end_0"), val = tensor<int32, [3]>([1, 16, 1])];
            tensor<bool, [3]> var_983_end_mask_0 = const()[name = string("op_983_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
            tensor<bool, [3]> var_983_squeeze_mask_0 = const()[name = string("op_983_squeeze_mask_0"), val = tensor<bool, [3]>([false, false, true])];
            tensor<fp16, [1, 16]> var_983_cast_fp16 = slice_by_index(begin = var_983_begin_0, end = var_983_end_0, end_mask = var_983_end_mask_0, squeeze_mask = var_983_squeeze_mask_0, x = beta_cast_fp16)[name = string("op_983_cast_fp16")];
            tensor<int32, [1]> var_984_axes_0 = const()[name = string("op_984_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 1]> var_984_cast_fp16 = expand_dims(axes = var_984_axes_0, x = var_983_cast_fp16)[name = string("op_984_cast_fp16")];
            tensor<fp16, [1, 16, 128]> delta_cast_fp16 = mul(x = var_980_cast_fp16, y = var_984_cast_fp16)[name = string("delta_cast_fp16")];
            tensor<int32, [1]> var_987_axes_0 = const()[name = string("op_987_axes_0"), val = tensor<int32, [1]>([-2])];
            tensor<fp16, [1, 16, 1, 128]> var_987_cast_fp16 = expand_dims(axes = var_987_axes_0, x = delta_cast_fp16)[name = string("op_987_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_988_cast_fp16 = mul(x = var_976_cast_fp16, y = var_987_cast_fp16)[name = string("op_988_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> rec22_out = add(x = state_9_cast_fp16, y = var_988_cast_fp16)[name = string("state_cast_fp16")];
            tensor<int32, [4]> var_992_begin_0 = const()[name = string("op_992_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> var_992_end_0 = const()[name = string("op_992_end_0"), val = tensor<int32, [4]>([1, 16, 1, 128])];
            tensor<bool, [4]> var_992_end_mask_0 = const()[name = string("op_992_end_mask_0"), val = tensor<bool, [4]>([true, true, false, true])];
            tensor<bool, [4]> var_992_squeeze_mask_0 = const()[name = string("op_992_squeeze_mask_0"), val = tensor<bool, [4]>([false, false, true, false])];
            tensor<fp16, [1, 16, 128]> var_992_cast_fp16 = slice_by_index(begin = var_992_begin_0, end = var_992_end_0, end_mask = var_992_end_mask_0, squeeze_mask = var_992_squeeze_mask_0, x = _inversed_query_11_cast_fp16)[name = string("op_992_cast_fp16")];
            tensor<int32, [1]> var_993_axes_0 = const()[name = string("op_993_axes_0"), val = tensor<int32, [1]>([-1])];
            tensor<fp16, [1, 16, 128, 1]> var_993_cast_fp16 = expand_dims(axes = var_993_axes_0, x = var_992_cast_fp16)[name = string("op_993_cast_fp16")];
            tensor<fp16, [1, 16, 128, 128]> var_994_cast_fp16 = mul(x = rec22_out, y = var_993_cast_fp16)[name = string("op_994_cast_fp16")];
            tensor<int32, [1]> var_996_axes_0 = const()[name = string("op_996_axes_0"), val = tensor<int32, [1]>([-2])];
            bool var_996_keep_dims_0 = const()[name = string("op_996_keep_dims_0"), val = bool(false)];
            tensor<fp16, [1, 16, 128]> var_996_cast_fp16 = reduce_sum(axes = var_996_axes_0, keep_dims = var_996_keep_dims_0, x = var_994_cast_fp16)[name = string("op_996_cast_fp16")];
            tensor<int32, [1]> attention_21_axes_0 = const()[name = string("attention_21_axes_0"), val = tensor<int32, [1]>([1])];
            tensor<fp16, [1, 1, 16, 128]> attention_21_cast_fp16 = expand_dims(axes = attention_21_axes_0, x = var_996_cast_fp16)[name = string("attention_21_cast_fp16")];
            tensor<int32, [2]> var_999 = const()[name = string("op_999"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [16, 128]> attention_23_cast_fp16 = reshape(shape = var_999, x = attention_21_cast_fp16)[name = string("attention_23_cast_fp16")];
            tensor<int32, [2]> var_1001 = const()[name = string("op_1001"), val = tensor<int32, [2]>([-1, 128])];
            tensor<fp16, [16, 128]> input_41_cast_fp16 = reshape(shape = var_1001, x = linear_23_cast_fp16)[name = string("input_41_cast_fp16")];
            tensor<fp16, [16, 128]> var_1003_cast_fp16 = mul(x = attention_23_cast_fp16, y = attention_23_cast_fp16)[name = string("op_1003_cast_fp16")];
            tensor<int32, [1]> variance_17_axes_0 = const()[name = string("variance_17_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_17_keep_dims_0 = const()[name = string("variance_17_keep_dims_0"), val = bool(true)];
            tensor<fp16, [16, 1]> variance_17_cast_fp16 = reduce_mean(axes = variance_17_axes_0, keep_dims = variance_17_keep_dims_0, x = var_1003_cast_fp16)[name = string("variance_17_cast_fp16")];
            fp16 var_1006_to_fp16 = const()[name = string("op_1006_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [16, 1]> var_1007_cast_fp16 = add(x = variance_17_cast_fp16, y = var_1006_to_fp16)[name = string("op_1007_cast_fp16")];
            fp32 var_1008_epsilon_0 = const()[name = string("op_1008_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [16, 1]> var_1008_cast_fp16 = rsqrt(epsilon = var_1008_epsilon_0, x = var_1007_cast_fp16)[name = string("op_1008_cast_fp16")];
            tensor<fp16, [16, 128]> attention_25_cast_fp16 = mul(x = attention_23_cast_fp16, y = var_1008_cast_fp16)[name = string("attention_25_cast_fp16")];
            tensor<fp16, [128]> layers20_2_gated_norm_promoted_to_fp16 = const()[name = string("layers20_2_gated_norm_promoted_to_fp16"), val = tensor<fp16, [128]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(242297344)))];
            tensor<fp16, [16, 128]> attention_27_cast_fp16 = mul(x = attention_25_cast_fp16, y = layers20_2_gated_norm_promoted_to_fp16)[name = string("attention_27_cast_fp16")];
            tensor<fp16, [16, 128]> var_1011_cast_fp16 = silu(x = input_41_cast_fp16)[name = string("op_1011_cast_fp16")];
            tensor<fp16, [16, 128]> attention_29_cast_fp16 = mul(x = attention_27_cast_fp16, y = var_1011_cast_fp16)[name = string("attention_29_cast_fp16")];
            tensor<int32, [2]> var_1013 = const()[name = string("op_1013"), val = tensor<int32, [2]>([1, 2048])];
            tensor<fp16, [1, 2048]> input_43_cast_fp16 = reshape(shape = var_1013, x = attention_29_cast_fp16)[name = string("input_43_cast_fp16")];
            tensor<fp16, [1024, 2048]> layers20_2_out_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 2048]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(242297664))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(243870592))))[name = string("layers20_2_out_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 1024]> linear_24_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = layers20_2_out_proj_weight_promoted_to_fp16_palettized, x = input_43_cast_fp16)[name = string("linear_24_cast_fp16")];
            tensor<fp16, [1, 1024]> value_43_cast_fp16 = add(x = value_31_cast_fp16, y = linear_24_cast_fp16)[name = string("value_43_cast_fp16")];
            tensor<fp16, [1, 1024]> var_1018_cast_fp16 = mul(x = value_43_cast_fp16, y = value_43_cast_fp16)[name = string("op_1018_cast_fp16")];
            tensor<int32, [1]> variance_19_axes_0 = const()[name = string("variance_19_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_19_keep_dims_0 = const()[name = string("variance_19_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 1]> variance_19_cast_fp16 = reduce_mean(axes = variance_19_axes_0, keep_dims = variance_19_keep_dims_0, x = var_1018_cast_fp16)[name = string("variance_19_cast_fp16")];
            fp16 var_1021_to_fp16 = const()[name = string("op_1021_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 1]> var_1022_cast_fp16 = add(x = variance_19_cast_fp16, y = var_1021_to_fp16)[name = string("op_1022_cast_fp16")];
            fp32 var_1023_epsilon_0 = const()[name = string("op_1023_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 1]> var_1023_cast_fp16 = rsqrt(epsilon = var_1023_epsilon_0, x = var_1022_cast_fp16)[name = string("op_1023_cast_fp16")];
            tensor<fp16, [1, 1024]> var_1024_cast_fp16 = mul(x = value_43_cast_fp16, y = var_1023_cast_fp16)[name = string("op_1024_cast_fp16")];
            tensor<fp16, [1024]> var_1026_to_fp16 = const()[name = string("op_1026_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(243878848)))];
            tensor<fp16, [1, 1024]> input_45_cast_fp16 = mul(x = var_1024_cast_fp16, y = var_1026_to_fp16)[name = string("input_45_cast_fp16")];
            tensor<fp16, [3584, 1024]> layers20_2_gate_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(243880960))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(246633536))))[name = string("layers20_2_gate_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 3584]> linear_25_cast_fp16 = linear(bias = linear_1_bias_0_to_fp16, weight = layers20_2_gate_proj_weight_promoted_to_fp16_palettized, x = input_45_cast_fp16)[name = string("linear_25_cast_fp16")];
            tensor<fp16, [1, 3584]> var_1030_cast_fp16 = silu(x = linear_25_cast_fp16)[name = string("op_1030_cast_fp16")];
            tensor<fp16, [3584, 1024]> layers20_2_up_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(246662272))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(249414848))))[name = string("layers20_2_up_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 3584]> linear_26_cast_fp16 = linear(bias = linear_1_bias_0_to_fp16, weight = layers20_2_up_proj_weight_promoted_to_fp16_palettized, x = input_45_cast_fp16)[name = string("linear_26_cast_fp16")];
            tensor<fp16, [1, 3584]> input_49_cast_fp16 = mul(x = var_1030_cast_fp16, y = linear_26_cast_fp16)[name = string("input_49_cast_fp16")];
            tensor<fp16, [1024, 3584]> layers20_2_down_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 3584]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(249443584))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(252196160))))[name = string("layers20_2_down_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 1024]> linear_27_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = layers20_2_down_proj_weight_promoted_to_fp16_palettized, x = input_49_cast_fp16)[name = string("linear_27_cast_fp16")];
            tensor<fp16, [1, 1024]> value_45_cast_fp16 = add(x = value_43_cast_fp16, y = linear_27_cast_fp16)[name = string("value_45_cast_fp16")];
            int32 var_1054 = const()[name = string("op_1054"), val = int32(-1)];
            tensor<fp16, [1, 1024]> var_1061_cast_fp16 = mul(x = value_45_cast_fp16, y = value_45_cast_fp16)[name = string("op_1061_cast_fp16")];
            tensor<int32, [1]> variance_21_axes_0 = const()[name = string("variance_21_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_21_keep_dims_0 = const()[name = string("variance_21_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 1]> variance_21_cast_fp16 = reduce_mean(axes = variance_21_axes_0, keep_dims = variance_21_keep_dims_0, x = var_1061_cast_fp16)[name = string("variance_21_cast_fp16")];
            fp16 var_1064_to_fp16 = const()[name = string("op_1064_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 1]> var_1065_cast_fp16 = add(x = variance_21_cast_fp16, y = var_1064_to_fp16)[name = string("op_1065_cast_fp16")];
            fp32 var_1066_epsilon_0 = const()[name = string("op_1066_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 1]> var_1066_cast_fp16 = rsqrt(epsilon = var_1066_epsilon_0, x = var_1065_cast_fp16)[name = string("op_1066_cast_fp16")];
            tensor<fp16, [1, 1024]> var_1067_cast_fp16 = mul(x = value_45_cast_fp16, y = var_1066_cast_fp16)[name = string("op_1067_cast_fp16")];
            tensor<fp16, [1024]> var_1069_to_fp16 = const()[name = string("op_1069_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(252204416)))];
            tensor<fp16, [1, 1024]> input_51_cast_fp16 = mul(x = var_1067_cast_fp16, y = var_1069_to_fp16)[name = string("input_51_cast_fp16")];
            tensor<fp16, [4096, 1024]> projection23_q_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [4096, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(252206528))), lut = tensor<fp16, [256, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(255352320))))[name = string("projection23_q_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [4096]> linear_28_bias_0_to_fp16 = const()[name = string("linear_28_bias_0_to_fp16"), val = tensor<fp16, [4096]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(255385152)))];
            tensor<fp16, [1, 4096]> linear_28_cast_fp16 = linear(bias = linear_28_bias_0_to_fp16, weight = projection23_q_proj_weight_promoted_to_fp16_palettized, x = input_51_cast_fp16)[name = string("linear_28_cast_fp16")];
            tensor<int32, [3]> var_1073 = const()[name = string("op_1073"), val = tensor<int32, [3]>([1, 8, 512])];
            tensor<fp16, [1, 8, 512]> query_and_gate_cast_fp16 = reshape(shape = var_1073, x = linear_28_cast_fp16)[name = string("query_and_gate_cast_fp16")];
            tensor<int32, [2]> var_1075_split_sizes_0 = const()[name = string("op_1075_split_sizes_0"), val = tensor<int32, [2]>([256, 256])];
            int32 var_1075_axis_0 = const()[name = string("op_1075_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 8, 256]> var_1075_cast_fp16_0, tensor<fp16, [1, 8, 256]> var_1075_cast_fp16_1 = split(axis = var_1075_axis_0, split_sizes = var_1075_split_sizes_0, x = query_and_gate_cast_fp16)[name = string("op_1075_cast_fp16")];
            tensor<fp16, [512, 1024]> projection23_k_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [512, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(255393408))), lut = tensor<fp16, [32, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(255786688))))[name = string("projection23_k_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [512]> linear_29_bias_0_to_fp16 = const()[name = string("linear_29_bias_0_to_fp16"), val = tensor<fp16, [512]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(255790848)))];
            tensor<fp16, [1, 512]> linear_29_cast_fp16 = linear(bias = linear_29_bias_0_to_fp16, weight = projection23_k_proj_weight_promoted_to_fp16_palettized, x = input_51_cast_fp16)[name = string("linear_29_cast_fp16")];
            tensor<int32, [3]> var_1079 = const()[name = string("op_1079"), val = tensor<int32, [3]>([1, 2, 256])];
            tensor<fp16, [1, 2, 256]> value_49_cast_fp16 = reshape(shape = var_1079, x = linear_29_cast_fp16)[name = string("value_49_cast_fp16")];
            tensor<fp16, [512, 1024]> projection23_v_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [512, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(255791936))), lut = tensor<fp16, [32, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(256185216))))[name = string("projection23_v_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 512]> linear_30_cast_fp16 = linear(bias = linear_29_bias_0_to_fp16, weight = projection23_v_proj_weight_promoted_to_fp16_palettized, x = input_51_cast_fp16)[name = string("linear_30_cast_fp16")];
            tensor<int32, [3]> var_1083 = const()[name = string("op_1083"), val = tensor<int32, [3]>([1, 2, 256])];
            tensor<fp16, [1, 2, 256]> value_55_cast_fp16 = reshape(shape = var_1083, x = linear_30_cast_fp16)[name = string("value_55_cast_fp16")];
            tensor<fp16, [1, 8, 256]> var_1085_cast_fp16 = mul(x = var_1075_cast_fp16_0, y = var_1075_cast_fp16_0)[name = string("op_1085_cast_fp16")];
            tensor<int32, [1]> variance_23_axes_0 = const()[name = string("variance_23_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_23_keep_dims_0 = const()[name = string("variance_23_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 8, 1]> variance_23_cast_fp16 = reduce_mean(axes = variance_23_axes_0, keep_dims = variance_23_keep_dims_0, x = var_1085_cast_fp16)[name = string("variance_23_cast_fp16")];
            fp16 var_1088_to_fp16 = const()[name = string("op_1088_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 8, 1]> var_1089_cast_fp16 = add(x = variance_23_cast_fp16, y = var_1088_to_fp16)[name = string("op_1089_cast_fp16")];
            fp32 var_1090_epsilon_0 = const()[name = string("op_1090_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 8, 1]> var_1090_cast_fp16 = rsqrt(epsilon = var_1090_epsilon_0, x = var_1089_cast_fp16)[name = string("op_1090_cast_fp16")];
            tensor<fp16, [1, 8, 256]> var_1091_cast_fp16 = mul(x = var_1075_cast_fp16_0, y = var_1090_cast_fp16)[name = string("op_1091_cast_fp16")];
            tensor<fp16, [256]> var_1093_to_fp16 = const()[name = string("op_1093_to_fp16"), val = tensor<fp16, [256]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(256189376)))];
            tensor<fp16, [1, 8, 256]> var_1094_cast_fp16 = mul(x = var_1091_cast_fp16, y = var_1093_to_fp16)[name = string("op_1094_cast_fp16")];
            tensor<int32, [3]> var_1095 = const()[name = string("op_1095"), val = tensor<int32, [3]>([1, 0, 2])];
            tensor<int32, [1]> value_51_axes_0 = const()[name = string("value_51_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [8, 1, 256]> var_1096_cast_fp16 = transpose(perm = var_1095, x = var_1094_cast_fp16)[name = string("transpose_7")];
            tensor<fp16, [1, 8, 1, 256]> value_51_cast_fp16 = expand_dims(axes = value_51_axes_0, x = var_1096_cast_fp16)[name = string("value_51_cast_fp16")];
            tensor<fp16, [1, 2, 256]> var_1098_cast_fp16 = mul(x = value_49_cast_fp16, y = value_49_cast_fp16)[name = string("op_1098_cast_fp16")];
            tensor<int32, [1]> variance_25_axes_0 = const()[name = string("variance_25_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_25_keep_dims_0 = const()[name = string("variance_25_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 2, 1]> variance_25_cast_fp16 = reduce_mean(axes = variance_25_axes_0, keep_dims = variance_25_keep_dims_0, x = var_1098_cast_fp16)[name = string("variance_25_cast_fp16")];
            fp16 var_1101_to_fp16 = const()[name = string("op_1101_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 2, 1]> var_1102_cast_fp16 = add(x = variance_25_cast_fp16, y = var_1101_to_fp16)[name = string("op_1102_cast_fp16")];
            fp32 var_1103_epsilon_0 = const()[name = string("op_1103_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 2, 1]> var_1103_cast_fp16 = rsqrt(epsilon = var_1103_epsilon_0, x = var_1102_cast_fp16)[name = string("op_1103_cast_fp16")];
            tensor<fp16, [1, 2, 256]> var_1104_cast_fp16 = mul(x = value_49_cast_fp16, y = var_1103_cast_fp16)[name = string("op_1104_cast_fp16")];
            tensor<fp16, [256]> var_1106_to_fp16 = const()[name = string("op_1106_to_fp16"), val = tensor<fp16, [256]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(256189952)))];
            tensor<fp16, [1, 2, 256]> var_1107_cast_fp16 = mul(x = var_1104_cast_fp16, y = var_1106_to_fp16)[name = string("op_1107_cast_fp16")];
            tensor<int32, [3]> var_1108 = const()[name = string("op_1108"), val = tensor<int32, [3]>([1, 0, 2])];
            tensor<int32, [1]> value_53_axes_0 = const()[name = string("value_53_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [2, 1, 256]> var_1109_cast_fp16 = transpose(perm = var_1108, x = var_1107_cast_fp16)[name = string("transpose_6")];
            tensor<fp16, [1, 2, 1, 256]> value_53_cast_fp16 = expand_dims(axes = value_53_axes_0, x = var_1109_cast_fp16)[name = string("value_53_cast_fp16")];
            tensor<int32, [4]> rotated_1_begin_0 = const()[name = string("rotated_1_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> rotated_1_end_0 = const()[name = string("rotated_1_end_0"), val = tensor<int32, [4]>([1, 8, 1, 64])];
            tensor<bool, [4]> rotated_1_end_mask_0 = const()[name = string("rotated_1_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
            tensor<fp16, [1, 8, 1, 64]> rotated_1_cast_fp16 = slice_by_index(begin = rotated_1_begin_0, end = rotated_1_end_0, end_mask = rotated_1_end_mask_0, x = value_51_cast_fp16)[name = string("rotated_1_cast_fp16")];
            tensor<int32, [4]> passthrough_1_begin_0 = const()[name = string("passthrough_1_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 64])];
            tensor<int32, [4]> passthrough_1_end_0 = const()[name = string("passthrough_1_end_0"), val = tensor<int32, [4]>([1, 8, 1, 256])];
            tensor<bool, [4]> passthrough_1_end_mask_0 = const()[name = string("passthrough_1_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
            tensor<fp16, [1, 8, 1, 192]> passthrough_1_cast_fp16 = slice_by_index(begin = passthrough_1_begin_0, end = passthrough_1_end_0, end_mask = passthrough_1_end_mask_0, x = value_51_cast_fp16)[name = string("passthrough_1_cast_fp16")];
            tensor<int32, [2]> var_1113_split_sizes_0 = const()[name = string("op_1113_split_sizes_0"), val = tensor<int32, [2]>([32, 32])];
            int32 var_1113_axis_0 = const()[name = string("op_1113_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 8, 1, 32]> var_1113_cast_fp16_0, tensor<fp16, [1, 8, 1, 32]> var_1113_cast_fp16_1 = split(axis = var_1113_axis_0, split_sizes = var_1113_split_sizes_0, x = rotated_1_cast_fp16)[name = string("op_1113_cast_fp16")];
            fp16 const_41_promoted_to_fp16 = const()[name = string("const_41_promoted_to_fp16"), val = fp16(-0x1p+0)];
            tensor<fp16, [1, 8, 1, 32]> var_1115_cast_fp16 = mul(x = var_1113_cast_fp16_1, y = const_41_promoted_to_fp16)[name = string("op_1115_cast_fp16")];
            bool rotated_half_1_interleave_0 = const()[name = string("rotated_half_1_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 8, 1, 64]> rotated_half_1_cast_fp16 = concat(axis = var_1054, interleave = rotated_half_1_interleave_0, values = (var_1115_cast_fp16, var_1113_cast_fp16_0))[name = string("rotated_half_1_cast_fp16")];
            tensor<fp16, [1, 8, 1, 64]> var_1118_cast_fp16 = mul(x = rotated_1_cast_fp16, y = cos)[name = string("op_1118_cast_fp16")];
            tensor<fp16, [1, 8, 1, 64]> var_1119_cast_fp16 = mul(x = rotated_half_1_cast_fp16, y = sin)[name = string("op_1119_cast_fp16")];
            tensor<fp16, [1, 8, 1, 64]> var_1120_cast_fp16 = add(x = var_1118_cast_fp16, y = var_1119_cast_fp16)[name = string("op_1120_cast_fp16")];
            bool query_13_interleave_0 = const()[name = string("query_13_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 8, 1, 256]> query_13_cast_fp16 = concat(axis = var_1054, interleave = query_13_interleave_0, values = (var_1120_cast_fp16, passthrough_1_cast_fp16))[name = string("query_13_cast_fp16")];
            tensor<int32, [4]> rotated_begin_0 = const()[name = string("rotated_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> rotated_end_0 = const()[name = string("rotated_end_0"), val = tensor<int32, [4]>([1, 2, 1, 64])];
            tensor<bool, [4]> rotated_end_mask_0 = const()[name = string("rotated_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
            tensor<fp16, [1, 2, 1, 64]> rotated_cast_fp16 = slice_by_index(begin = rotated_begin_0, end = rotated_end_0, end_mask = rotated_end_mask_0, x = value_53_cast_fp16)[name = string("rotated_cast_fp16")];
            tensor<int32, [4]> passthrough_begin_0 = const()[name = string("passthrough_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 64])];
            tensor<int32, [4]> passthrough_end_0 = const()[name = string("passthrough_end_0"), val = tensor<int32, [4]>([1, 2, 1, 256])];
            tensor<bool, [4]> passthrough_end_mask_0 = const()[name = string("passthrough_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
            tensor<fp16, [1, 2, 1, 192]> passthrough_cast_fp16 = slice_by_index(begin = passthrough_begin_0, end = passthrough_end_0, end_mask = passthrough_end_mask_0, x = value_53_cast_fp16)[name = string("passthrough_cast_fp16")];
            tensor<int32, [2]> var_1125_split_sizes_0 = const()[name = string("op_1125_split_sizes_0"), val = tensor<int32, [2]>([32, 32])];
            int32 var_1125_axis_0 = const()[name = string("op_1125_axis_0"), val = int32(-1)];
            tensor<fp16, [1, 2, 1, 32]> var_1125_cast_fp16_0, tensor<fp16, [1, 2, 1, 32]> var_1125_cast_fp16_1 = split(axis = var_1125_axis_0, split_sizes = var_1125_split_sizes_0, x = rotated_cast_fp16)[name = string("op_1125_cast_fp16")];
            fp16 const_42_promoted_to_fp16 = const()[name = string("const_42_promoted_to_fp16"), val = fp16(-0x1p+0)];
            tensor<fp16, [1, 2, 1, 32]> var_1127_cast_fp16 = mul(x = var_1125_cast_fp16_1, y = const_42_promoted_to_fp16)[name = string("op_1127_cast_fp16")];
            bool rotated_half_interleave_0 = const()[name = string("rotated_half_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 2, 1, 64]> rotated_half_cast_fp16 = concat(axis = var_1054, interleave = rotated_half_interleave_0, values = (var_1127_cast_fp16, var_1125_cast_fp16_0))[name = string("rotated_half_cast_fp16")];
            tensor<fp16, [1, 2, 1, 64]> var_1130_cast_fp16 = mul(x = rotated_cast_fp16, y = cos)[name = string("op_1130_cast_fp16")];
            tensor<fp16, [1, 2, 1, 64]> var_1131_cast_fp16 = mul(x = rotated_half_cast_fp16, y = sin)[name = string("op_1131_cast_fp16")];
            tensor<fp16, [1, 2, 1, 64]> var_1132_cast_fp16 = add(x = var_1130_cast_fp16, y = var_1131_cast_fp16)[name = string("op_1132_cast_fp16")];
            bool key_13_interleave_0 = const()[name = string("key_13_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 2, 1, 256]> key_13_cast_fp16 = concat(axis = var_1054, interleave = key_13_interleave_0, values = (var_1132_cast_fp16, passthrough_cast_fp16))[name = string("key_13_cast_fp16")];
            tensor<int32, [4]> var_1135 = const()[name = string("op_1135"), val = tensor<int32, [4]>([2, 4, 1, 256])];
            tensor<fp16, [2, 4, 1, 256]> var_1136_cast_fp16 = reshape(shape = var_1135, x = query_13_cast_fp16)[name = string("op_1136_cast_fp16")];
            tensor<int32, [4]> transpose_1_perm_0 = const()[name = string("transpose_1_perm_0"), val = tensor<int32, [4]>([0, 1, 3, 2])];
            tensor<int32, [3]> var_1139 = const()[name = string("op_1139"), val = tensor<int32, [3]>([2, 1, 256])];
            tensor<fp16, [2, 1, 256]> key_cast_fp16 = reshape(shape = var_1139, x = key_13_cast_fp16)[name = string("key_cast_fp16")];
            tensor<int32, [3]> var_1141 = const()[name = string("op_1141"), val = tensor<int32, [3]>([1, 0, 2])];
            tensor<int32, [2]> var_1143 = const()[name = string("op_1143"), val = tensor<int32, [2]>([1, 2048])];
            tensor<fp16, [1, 2048]> gate_cast_fp16 = reshape(shape = var_1143, x = var_1075_cast_fp16_1)[name = string("gate_cast_fp16")];
            tensor<int32, [1]> var_1151_axes_0 = const()[name = string("op_1151_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [1, 2, 1, 256]> var_1151_cast_fp16 = expand_dims(axes = var_1151_axes_0, x = key_cast_fp16)[name = string("op_1151_cast_fp16")];
            tensor<int32, [1]> var_1153_axes_0 = const()[name = string("op_1153_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [1, 1, 2, 1, 256]> var_1153_cast_fp16 = expand_dims(axes = var_1153_axes_0, x = var_1151_cast_fp16)[name = string("op_1153_cast_fp16")];
            tensor<int32, [1]> expand_dims_16 = const()[name = string("expand_dims_16"), val = tensor<int32, [1]>([1])];
            tensor<int32, [1]> expand_dims_17 = const()[name = string("expand_dims_17"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_18 = const()[name = string("expand_dims_18"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_20 = const()[name = string("expand_dims_20"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_21 = const()[name = string("expand_dims_21"), val = tensor<int32, [1]>([2])];
            tensor<int32, [1]> expand_dims_22 = const()[name = string("expand_dims_22"), val = tensor<int32, [1]>([1])];
            int32 concat_10_axis_0 = const()[name = string("concat_10_axis_0"), val = int32(0)];
            bool concat_10_interleave_0 = const()[name = string("concat_10_interleave_0"), val = bool(false)];
            tensor<int32, [5]> concat_10 = concat(axis = concat_10_axis_0, interleave = concat_10_interleave_0, values = (expand_dims_16, expand_dims_17, expand_dims_18, current_pos, expand_dims_20))[name = string("concat_10")];
            tensor<int32, [1]> concat_11_values2_0 = const()[name = string("concat_11_values2_0"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> concat_11_values4_0 = const()[name = string("concat_11_values4_0"), val = tensor<int32, [1]>([0])];
            int32 concat_11_axis_0 = const()[name = string("concat_11_axis_0"), val = int32(0)];
            bool concat_11_interleave_0 = const()[name = string("concat_11_interleave_0"), val = bool(false)];
            tensor<int32, [5]> concat_11 = concat(axis = concat_11_axis_0, interleave = concat_11_interleave_0, values = (expand_dims_21, expand_dims_22, concat_11_values2_0, var_38, concat_11_values4_0))[name = string("concat_11")];
            tensor<int32, [5]> kv_cache_internal_tensor_assign_3_stride_0 = const()[name = string("kv_cache_internal_tensor_assign_3_stride_0"), val = tensor<int32, [5]>([1, 1, 1, 1, 1])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_3_begin_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_3_begin_mask_0"), val = tensor<bool, [5]>([false, false, false, false, false])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_3_end_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_3_end_mask_0"), val = tensor<bool, [5]>([false, false, true, false, true])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_3_squeeze_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_3_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, false, false, false])];
            tensor<fp16, [2, 2, 2, 2048, 256]> kv_cache_internal_tensor_assign_3_cast_fp16 = slice_update(begin = concat_10, begin_mask = kv_cache_internal_tensor_assign_3_begin_mask_0, end = concat_11, end_mask = kv_cache_internal_tensor_assign_3_end_mask_0, squeeze_mask = kv_cache_internal_tensor_assign_3_squeeze_mask_0, stride = kv_cache_internal_tensor_assign_3_stride_0, update = var_1153_cast_fp16, x = coreml_update_state_5)[name = string("kv_cache_internal_tensor_assign_3_cast_fp16")];
            write_state(data = kv_cache_internal_tensor_assign_3_cast_fp16, input = kv_cache)[name = string("coreml_update_state_2_write_state")];
            tensor<fp16, [2, 2, 2, 2048, 256]> coreml_update_state_6 = read_state(input = kv_cache)[name = string("coreml_update_state_2")];
            tensor<int32, [1]> var_1184_axes_0 = const()[name = string("op_1184_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [2, 1, 256]> value_57_cast_fp16 = transpose(perm = var_1141, x = value_55_cast_fp16)[name = string("transpose_5")];
            tensor<fp16, [1, 2, 1, 256]> var_1184_cast_fp16 = expand_dims(axes = var_1184_axes_0, x = value_57_cast_fp16)[name = string("op_1184_cast_fp16")];
            tensor<int32, [1]> var_1186_axes_0 = const()[name = string("op_1186_axes_0"), val = tensor<int32, [1]>([0])];
            tensor<fp16, [1, 1, 2, 1, 256]> var_1186_cast_fp16 = expand_dims(axes = var_1186_axes_0, x = var_1184_cast_fp16)[name = string("op_1186_cast_fp16")];
            tensor<int32, [1]> expand_dims_24 = const()[name = string("expand_dims_24"), val = tensor<int32, [1]>([1])];
            tensor<int32, [1]> expand_dims_25 = const()[name = string("expand_dims_25"), val = tensor<int32, [1]>([1])];
            tensor<int32, [1]> expand_dims_26 = const()[name = string("expand_dims_26"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_28 = const()[name = string("expand_dims_28"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> expand_dims_29 = const()[name = string("expand_dims_29"), val = tensor<int32, [1]>([2])];
            tensor<int32, [1]> expand_dims_30 = const()[name = string("expand_dims_30"), val = tensor<int32, [1]>([2])];
            int32 concat_14_axis_0 = const()[name = string("concat_14_axis_0"), val = int32(0)];
            bool concat_14_interleave_0 = const()[name = string("concat_14_interleave_0"), val = bool(false)];
            tensor<int32, [5]> concat_14 = concat(axis = concat_14_axis_0, interleave = concat_14_interleave_0, values = (expand_dims_24, expand_dims_25, expand_dims_26, current_pos, expand_dims_28))[name = string("concat_14")];
            tensor<int32, [1]> concat_15_values2_0 = const()[name = string("concat_15_values2_0"), val = tensor<int32, [1]>([0])];
            tensor<int32, [1]> concat_15_values4_0 = const()[name = string("concat_15_values4_0"), val = tensor<int32, [1]>([0])];
            int32 concat_15_axis_0 = const()[name = string("concat_15_axis_0"), val = int32(0)];
            bool concat_15_interleave_0 = const()[name = string("concat_15_interleave_0"), val = bool(false)];
            tensor<int32, [5]> concat_15 = concat(axis = concat_15_axis_0, interleave = concat_15_interleave_0, values = (expand_dims_29, expand_dims_30, concat_15_values2_0, var_38, concat_15_values4_0))[name = string("concat_15")];
            tensor<int32, [5]> kv_cache_internal_tensor_assign_4_stride_0 = const()[name = string("kv_cache_internal_tensor_assign_4_stride_0"), val = tensor<int32, [5]>([1, 1, 1, 1, 1])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_4_begin_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_4_begin_mask_0"), val = tensor<bool, [5]>([false, false, false, false, false])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_4_end_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_4_end_mask_0"), val = tensor<bool, [5]>([false, false, true, false, true])];
            tensor<bool, [5]> kv_cache_internal_tensor_assign_4_squeeze_mask_0 = const()[name = string("kv_cache_internal_tensor_assign_4_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, false, false, false])];
            tensor<fp16, [2, 2, 2, 2048, 256]> kv_cache_internal_tensor_assign_4_cast_fp16 = slice_update(begin = concat_14, begin_mask = kv_cache_internal_tensor_assign_4_begin_mask_0, end = concat_15, end_mask = kv_cache_internal_tensor_assign_4_end_mask_0, squeeze_mask = kv_cache_internal_tensor_assign_4_squeeze_mask_0, stride = kv_cache_internal_tensor_assign_4_stride_0, update = var_1186_cast_fp16, x = coreml_update_state_6)[name = string("kv_cache_internal_tensor_assign_4_cast_fp16")];
            write_state(data = kv_cache_internal_tensor_assign_4_cast_fp16, input = kv_cache)[name = string("coreml_update_state_3_write_state")];
            tensor<fp16, [2, 2, 2, 2048, 256]> coreml_update_state_7 = read_state(input = kv_cache)[name = string("coreml_update_state_3")];
            tensor<int32, [5]> var_1218_begin_0 = const()[name = string("op_1218_begin_0"), val = tensor<int32, [5]>([1, 0, 0, 0, 0])];
            tensor<int32, [5]> var_1218_end_0 = const()[name = string("op_1218_end_0"), val = tensor<int32, [5]>([2, 2, 2, 2048, 256])];
            tensor<bool, [5]> var_1218_end_mask_0 = const()[name = string("op_1218_end_mask_0"), val = tensor<bool, [5]>([false, true, true, true, true])];
            tensor<bool, [5]> var_1218_squeeze_mask_0 = const()[name = string("op_1218_squeeze_mask_0"), val = tensor<bool, [5]>([true, false, false, false, false])];
            tensor<fp16, [2, 2, 2048, 256]> var_1218_cast_fp16 = slice_by_index(begin = var_1218_begin_0, end = var_1218_end_0, end_mask = var_1218_end_mask_0, squeeze_mask = var_1218_squeeze_mask_0, x = coreml_update_state_7)[name = string("op_1218_cast_fp16")];
            tensor<int32, [4]> keys_begin_0 = const()[name = string("keys_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
            tensor<int32, [4]> keys_end_0 = const()[name = string("keys_end_0"), val = tensor<int32, [4]>([1, 2, 2048, 256])];
            tensor<bool, [4]> keys_end_mask_0 = const()[name = string("keys_end_mask_0"), val = tensor<bool, [4]>([false, true, true, true])];
            tensor<bool, [4]> keys_squeeze_mask_0 = const()[name = string("keys_squeeze_mask_0"), val = tensor<bool, [4]>([true, false, false, false])];
            tensor<fp16, [2, 2048, 256]> keys_cast_fp16 = slice_by_index(begin = keys_begin_0, end = keys_end_0, end_mask = keys_end_mask_0, squeeze_mask = keys_squeeze_mask_0, x = var_1218_cast_fp16)[name = string("keys_cast_fp16")];
            tensor<int32, [4]> values_begin_0 = const()[name = string("values_begin_0"), val = tensor<int32, [4]>([1, 0, 0, 0])];
            tensor<int32, [4]> values_end_0 = const()[name = string("values_end_0"), val = tensor<int32, [4]>([2, 2, 2048, 256])];
            tensor<bool, [4]> values_end_mask_0 = const()[name = string("values_end_mask_0"), val = tensor<bool, [4]>([false, true, true, true])];
            tensor<bool, [4]> values_squeeze_mask_0 = const()[name = string("values_squeeze_mask_0"), val = tensor<bool, [4]>([true, false, false, false])];
            tensor<fp16, [2, 2048, 256]> values_cast_fp16 = slice_by_index(begin = values_begin_0, end = values_end_0, end_mask = values_end_mask_0, squeeze_mask = values_squeeze_mask_0, x = var_1218_cast_fp16)[name = string("values_cast_fp16")];
            int32 var_1235 = const()[name = string("op_1235"), val = int32(1)];
            tensor<int32, [4]> var_1242 = const()[name = string("op_1242"), val = tensor<int32, [4]>([1, 2048, 1, 1])];
            tensor<fp16, [2, 4, 256, 1]> transpose_1_cast_fp16 = transpose(perm = transpose_1_perm_0, x = var_1136_cast_fp16)[name = string("transpose_4")];
            tensor<fp16, [1, 2048, 1, 1]> var_1243_cast_fp16 = reshape(shape = var_1242, x = transpose_1_cast_fp16)[name = string("op_1243_cast_fp16")];
            tensor<int32, [3]> var_1244 = const()[name = string("op_1244"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [4]> var_1247 = const()[name = string("op_1247"), val = tensor<int32, [4]>([1, 512, 1, 2048])];
            tensor<fp16, [2, 256, 2048]> var_1245_cast_fp16 = transpose(perm = var_1244, x = keys_cast_fp16)[name = string("transpose_3")];
            tensor<fp16, [1, 512, 1, 2048]> key_native_cast_fp16 = reshape(shape = var_1247, x = var_1245_cast_fp16)[name = string("key_native_cast_fp16")];
            tensor<int32, [3]> var_1249 = const()[name = string("op_1249"), val = tensor<int32, [3]>([0, 2, 1])];
            tensor<int32, [4]> var_1252 = const()[name = string("op_1252"), val = tensor<int32, [4]>([1, 512, 1, 2048])];
            tensor<fp16, [2, 256, 2048]> var_1250_cast_fp16 = transpose(perm = var_1249, x = values_cast_fp16)[name = string("transpose_2")];
            tensor<fp16, [1, 512, 1, 2048]> var_1253_cast_fp16 = reshape(shape = var_1252, x = var_1250_cast_fp16)[name = string("op_1253_cast_fp16")];
            tensor<int32, [8]> tile_19 = const()[name = string("tile_19"), val = tensor<int32, [8]>([256, 256, 256, 256, 256, 256, 256, 256])];
            int32 var_1257_axis_0 = const()[name = string("op_1257_axis_0"), val = int32(1)];
            tensor<fp16, [1, 256, 1, 1]> var_1257_cast_fp16_0, tensor<fp16, [1, 256, 1, 1]> var_1257_cast_fp16_1, tensor<fp16, [1, 256, 1, 1]> var_1257_cast_fp16_2, tensor<fp16, [1, 256, 1, 1]> var_1257_cast_fp16_3, tensor<fp16, [1, 256, 1, 1]> var_1257_cast_fp16_4, tensor<fp16, [1, 256, 1, 1]> var_1257_cast_fp16_5, tensor<fp16, [1, 256, 1, 1]> var_1257_cast_fp16_6, tensor<fp16, [1, 256, 1, 1]> var_1257_cast_fp16_7 = split(axis = var_1257_axis_0, split_sizes = tile_19, x = var_1243_cast_fp16)[name = string("op_1257_cast_fp16")];
            tensor<int32, [4]> var_1266_perm_0 = const()[name = string("op_1266_perm_0"), val = tensor<int32, [4]>([0, 3, 2, 1])];
            tensor<int32, [2]> tile_20 = const()[name = string("tile_20"), val = tensor<int32, [2]>([256, 256])];
            int32 var_1267_axis_0 = const()[name = string("op_1267_axis_0"), val = int32(3)];
            tensor<fp16, [1, 2048, 1, 512]> var_1266_cast_fp16 = transpose(perm = var_1266_perm_0, x = key_native_cast_fp16)[name = string("transpose_1")];
            tensor<fp16, [1, 2048, 1, 256]> var_1267_cast_fp16_0, tensor<fp16, [1, 2048, 1, 256]> var_1267_cast_fp16_1 = split(axis = var_1267_axis_0, split_sizes = tile_20, x = var_1266_cast_fp16)[name = string("op_1267_cast_fp16")];
            tensor<int32, [2]> tile_21 = const()[name = string("tile_21"), val = tensor<int32, [2]>([256, 256])];
            int32 var_1270_axis_0 = const()[name = string("op_1270_axis_0"), val = int32(1)];
            tensor<fp16, [1, 256, 1, 2048]> var_1270_cast_fp16_0, tensor<fp16, [1, 256, 1, 2048]> var_1270_cast_fp16_1 = split(axis = var_1270_axis_0, split_sizes = tile_21, x = var_1253_cast_fp16)[name = string("op_1270_cast_fp16")];
            string scores_17_equation_0 = const()[name = string("scores_17_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 1]> scores_17_cast_fp16 = einsum(equation = scores_17_equation_0, values = (var_1267_cast_fp16_0, var_1257_cast_fp16_0))[name = string("scores_17_cast_fp16")];
            fp16 var_1275_to_fp16 = const()[name = string("op_1275_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 1]> var_1276_cast_fp16 = mul(x = scores_17_cast_fp16, y = var_1275_to_fp16)[name = string("op_1276_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_1277_cast_fp16 = add(x = var_1276_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_1277_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_1278_cast_fp16 = softmax(axis = var_1235, x = var_1277_cast_fp16)[name = string("op_1278_cast_fp16")];
            tensor<int32, [4]> tile_22 = const()[name = string("tile_22"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_1279_axis_0 = const()[name = string("op_1279_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 1]> var_1279_cast_fp16_0, tensor<fp16, [1, 512, 1, 1]> var_1279_cast_fp16_1, tensor<fp16, [1, 512, 1, 1]> var_1279_cast_fp16_2, tensor<fp16, [1, 512, 1, 1]> var_1279_cast_fp16_3 = split(axis = var_1279_axis_0, split_sizes = tile_22, x = var_1278_cast_fp16)[name = string("op_1279_cast_fp16")];
            tensor<int32, [4]> tile_23 = const()[name = string("tile_23"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_1284_axis_0 = const()[name = string("op_1284_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_1284_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_1284_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_1284_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_1284_cast_fp16_3 = split(axis = var_1284_axis_0, split_sizes = tile_23, x = var_1270_cast_fp16_0)[name = string("op_1284_cast_fp16")];
            fp16 var_1289_to_fp16 = const()[name = string("op_1289_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_1290_cast_fp16 = mul(x = var_1279_cast_fp16_0, y = var_1289_to_fp16)[name = string("op_1290_cast_fp16")];
            string var_1292_equation_0 = const()[name = string("op_1292_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_1292_cast_fp16 = einsum(equation = var_1292_equation_0, values = (var_1284_cast_fp16_0, var_1290_cast_fp16))[name = string("op_1292_cast_fp16")];
            fp16 var_1293_to_fp16 = const()[name = string("op_1293_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_1294_cast_fp16 = mul(x = var_1279_cast_fp16_1, y = var_1293_to_fp16)[name = string("op_1294_cast_fp16")];
            string var_1296_equation_0 = const()[name = string("op_1296_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_1296_cast_fp16 = einsum(equation = var_1296_equation_0, values = (var_1284_cast_fp16_1, var_1294_cast_fp16))[name = string("op_1296_cast_fp16")];
            fp16 var_1297_to_fp16 = const()[name = string("op_1297_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_1298_cast_fp16 = mul(x = var_1279_cast_fp16_2, y = var_1297_to_fp16)[name = string("op_1298_cast_fp16")];
            string var_1300_equation_0 = const()[name = string("op_1300_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_1300_cast_fp16 = einsum(equation = var_1300_equation_0, values = (var_1284_cast_fp16_2, var_1298_cast_fp16))[name = string("op_1300_cast_fp16")];
            fp16 var_1301_to_fp16 = const()[name = string("op_1301_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_1302_cast_fp16 = mul(x = var_1279_cast_fp16_3, y = var_1301_to_fp16)[name = string("op_1302_cast_fp16")];
            string var_1304_equation_0 = const()[name = string("op_1304_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_1304_cast_fp16 = einsum(equation = var_1304_equation_0, values = (var_1284_cast_fp16_3, var_1302_cast_fp16))[name = string("op_1304_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_1305_cast_fp16 = add(x = var_1292_cast_fp16, y = var_1296_cast_fp16)[name = string("op_1305_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_1306_cast_fp16 = add(x = var_1300_cast_fp16, y = var_1304_cast_fp16)[name = string("op_1306_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_1307_cast_fp16 = add(x = var_1305_cast_fp16, y = var_1306_cast_fp16)[name = string("op_1307_cast_fp16")];
            fp16 _inversed_1309_y_0_to_fp16 = const()[name = string("_inversed_1309_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 1]> _inversed_1309_cast_fp16 = mul(x = var_1307_cast_fp16, y = _inversed_1309_y_0_to_fp16)[name = string("_inversed_1309_cast_fp16")];
            string scores_19_equation_0 = const()[name = string("scores_19_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 1]> scores_19_cast_fp16 = einsum(equation = scores_19_equation_0, values = (var_1267_cast_fp16_0, var_1257_cast_fp16_1))[name = string("scores_19_cast_fp16")];
            fp16 var_1312_to_fp16 = const()[name = string("op_1312_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 1]> var_1313_cast_fp16 = mul(x = scores_19_cast_fp16, y = var_1312_to_fp16)[name = string("op_1313_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_1314_cast_fp16 = add(x = var_1313_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_1314_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_1315_cast_fp16 = softmax(axis = var_1235, x = var_1314_cast_fp16)[name = string("op_1315_cast_fp16")];
            tensor<int32, [4]> tile_24 = const()[name = string("tile_24"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_1316_axis_0 = const()[name = string("op_1316_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 1]> var_1316_cast_fp16_0, tensor<fp16, [1, 512, 1, 1]> var_1316_cast_fp16_1, tensor<fp16, [1, 512, 1, 1]> var_1316_cast_fp16_2, tensor<fp16, [1, 512, 1, 1]> var_1316_cast_fp16_3 = split(axis = var_1316_axis_0, split_sizes = tile_24, x = var_1315_cast_fp16)[name = string("op_1316_cast_fp16")];
            tensor<int32, [4]> tile_25 = const()[name = string("tile_25"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_1321_axis_0 = const()[name = string("op_1321_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_1321_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_1321_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_1321_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_1321_cast_fp16_3 = split(axis = var_1321_axis_0, split_sizes = tile_25, x = var_1270_cast_fp16_0)[name = string("op_1321_cast_fp16")];
            fp16 var_1326_to_fp16 = const()[name = string("op_1326_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_1327_cast_fp16 = mul(x = var_1316_cast_fp16_0, y = var_1326_to_fp16)[name = string("op_1327_cast_fp16")];
            string var_1329_equation_0 = const()[name = string("op_1329_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_1329_cast_fp16 = einsum(equation = var_1329_equation_0, values = (var_1321_cast_fp16_0, var_1327_cast_fp16))[name = string("op_1329_cast_fp16")];
            fp16 var_1330_to_fp16 = const()[name = string("op_1330_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_1331_cast_fp16 = mul(x = var_1316_cast_fp16_1, y = var_1330_to_fp16)[name = string("op_1331_cast_fp16")];
            string var_1333_equation_0 = const()[name = string("op_1333_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_1333_cast_fp16 = einsum(equation = var_1333_equation_0, values = (var_1321_cast_fp16_1, var_1331_cast_fp16))[name = string("op_1333_cast_fp16")];
            fp16 var_1334_to_fp16 = const()[name = string("op_1334_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_1335_cast_fp16 = mul(x = var_1316_cast_fp16_2, y = var_1334_to_fp16)[name = string("op_1335_cast_fp16")];
            string var_1337_equation_0 = const()[name = string("op_1337_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_1337_cast_fp16 = einsum(equation = var_1337_equation_0, values = (var_1321_cast_fp16_2, var_1335_cast_fp16))[name = string("op_1337_cast_fp16")];
            fp16 var_1338_to_fp16 = const()[name = string("op_1338_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_1339_cast_fp16 = mul(x = var_1316_cast_fp16_3, y = var_1338_to_fp16)[name = string("op_1339_cast_fp16")];
            string var_1341_equation_0 = const()[name = string("op_1341_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_1341_cast_fp16 = einsum(equation = var_1341_equation_0, values = (var_1321_cast_fp16_3, var_1339_cast_fp16))[name = string("op_1341_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_1342_cast_fp16 = add(x = var_1329_cast_fp16, y = var_1333_cast_fp16)[name = string("op_1342_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_1343_cast_fp16 = add(x = var_1337_cast_fp16, y = var_1341_cast_fp16)[name = string("op_1343_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_1344_cast_fp16 = add(x = var_1342_cast_fp16, y = var_1343_cast_fp16)[name = string("op_1344_cast_fp16")];
            fp16 _inversed_1346_y_0_to_fp16 = const()[name = string("_inversed_1346_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 1]> _inversed_1346_cast_fp16 = mul(x = var_1344_cast_fp16, y = _inversed_1346_y_0_to_fp16)[name = string("_inversed_1346_cast_fp16")];
            string scores_21_equation_0 = const()[name = string("scores_21_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 1]> scores_21_cast_fp16 = einsum(equation = scores_21_equation_0, values = (var_1267_cast_fp16_0, var_1257_cast_fp16_2))[name = string("scores_21_cast_fp16")];
            fp16 var_1349_to_fp16 = const()[name = string("op_1349_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 1]> var_1350_cast_fp16 = mul(x = scores_21_cast_fp16, y = var_1349_to_fp16)[name = string("op_1350_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_1351_cast_fp16 = add(x = var_1350_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_1351_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_1352_cast_fp16 = softmax(axis = var_1235, x = var_1351_cast_fp16)[name = string("op_1352_cast_fp16")];
            tensor<int32, [4]> tile_26 = const()[name = string("tile_26"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_1353_axis_0 = const()[name = string("op_1353_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 1]> var_1353_cast_fp16_0, tensor<fp16, [1, 512, 1, 1]> var_1353_cast_fp16_1, tensor<fp16, [1, 512, 1, 1]> var_1353_cast_fp16_2, tensor<fp16, [1, 512, 1, 1]> var_1353_cast_fp16_3 = split(axis = var_1353_axis_0, split_sizes = tile_26, x = var_1352_cast_fp16)[name = string("op_1353_cast_fp16")];
            tensor<int32, [4]> tile_27 = const()[name = string("tile_27"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_1358_axis_0 = const()[name = string("op_1358_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_1358_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_1358_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_1358_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_1358_cast_fp16_3 = split(axis = var_1358_axis_0, split_sizes = tile_27, x = var_1270_cast_fp16_0)[name = string("op_1358_cast_fp16")];
            fp16 var_1363_to_fp16 = const()[name = string("op_1363_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_1364_cast_fp16 = mul(x = var_1353_cast_fp16_0, y = var_1363_to_fp16)[name = string("op_1364_cast_fp16")];
            string var_1366_equation_0 = const()[name = string("op_1366_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_1366_cast_fp16 = einsum(equation = var_1366_equation_0, values = (var_1358_cast_fp16_0, var_1364_cast_fp16))[name = string("op_1366_cast_fp16")];
            fp16 var_1367_to_fp16 = const()[name = string("op_1367_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_1368_cast_fp16 = mul(x = var_1353_cast_fp16_1, y = var_1367_to_fp16)[name = string("op_1368_cast_fp16")];
            string var_1370_equation_0 = const()[name = string("op_1370_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_1370_cast_fp16 = einsum(equation = var_1370_equation_0, values = (var_1358_cast_fp16_1, var_1368_cast_fp16))[name = string("op_1370_cast_fp16")];
            fp16 var_1371_to_fp16 = const()[name = string("op_1371_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_1372_cast_fp16 = mul(x = var_1353_cast_fp16_2, y = var_1371_to_fp16)[name = string("op_1372_cast_fp16")];
            string var_1374_equation_0 = const()[name = string("op_1374_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_1374_cast_fp16 = einsum(equation = var_1374_equation_0, values = (var_1358_cast_fp16_2, var_1372_cast_fp16))[name = string("op_1374_cast_fp16")];
            fp16 var_1375_to_fp16 = const()[name = string("op_1375_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_1376_cast_fp16 = mul(x = var_1353_cast_fp16_3, y = var_1375_to_fp16)[name = string("op_1376_cast_fp16")];
            string var_1378_equation_0 = const()[name = string("op_1378_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_1378_cast_fp16 = einsum(equation = var_1378_equation_0, values = (var_1358_cast_fp16_3, var_1376_cast_fp16))[name = string("op_1378_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_1379_cast_fp16 = add(x = var_1366_cast_fp16, y = var_1370_cast_fp16)[name = string("op_1379_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_1380_cast_fp16 = add(x = var_1374_cast_fp16, y = var_1378_cast_fp16)[name = string("op_1380_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_1381_cast_fp16 = add(x = var_1379_cast_fp16, y = var_1380_cast_fp16)[name = string("op_1381_cast_fp16")];
            fp16 _inversed_1383_y_0_to_fp16 = const()[name = string("_inversed_1383_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 1]> _inversed_1383_cast_fp16 = mul(x = var_1381_cast_fp16, y = _inversed_1383_y_0_to_fp16)[name = string("_inversed_1383_cast_fp16")];
            string scores_23_equation_0 = const()[name = string("scores_23_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 1]> scores_23_cast_fp16 = einsum(equation = scores_23_equation_0, values = (var_1267_cast_fp16_0, var_1257_cast_fp16_3))[name = string("scores_23_cast_fp16")];
            fp16 var_1386_to_fp16 = const()[name = string("op_1386_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 1]> var_1387_cast_fp16 = mul(x = scores_23_cast_fp16, y = var_1386_to_fp16)[name = string("op_1387_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_1388_cast_fp16 = add(x = var_1387_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_1388_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_1389_cast_fp16 = softmax(axis = var_1235, x = var_1388_cast_fp16)[name = string("op_1389_cast_fp16")];
            tensor<int32, [4]> tile_28 = const()[name = string("tile_28"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_1390_axis_0 = const()[name = string("op_1390_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 1]> var_1390_cast_fp16_0, tensor<fp16, [1, 512, 1, 1]> var_1390_cast_fp16_1, tensor<fp16, [1, 512, 1, 1]> var_1390_cast_fp16_2, tensor<fp16, [1, 512, 1, 1]> var_1390_cast_fp16_3 = split(axis = var_1390_axis_0, split_sizes = tile_28, x = var_1389_cast_fp16)[name = string("op_1390_cast_fp16")];
            tensor<int32, [4]> tile_29 = const()[name = string("tile_29"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_1395_axis_0 = const()[name = string("op_1395_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_1395_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_1395_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_1395_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_1395_cast_fp16_3 = split(axis = var_1395_axis_0, split_sizes = tile_29, x = var_1270_cast_fp16_0)[name = string("op_1395_cast_fp16")];
            fp16 var_1400_to_fp16 = const()[name = string("op_1400_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_1401_cast_fp16 = mul(x = var_1390_cast_fp16_0, y = var_1400_to_fp16)[name = string("op_1401_cast_fp16")];
            string var_1403_equation_0 = const()[name = string("op_1403_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_1403_cast_fp16 = einsum(equation = var_1403_equation_0, values = (var_1395_cast_fp16_0, var_1401_cast_fp16))[name = string("op_1403_cast_fp16")];
            fp16 var_1404_to_fp16 = const()[name = string("op_1404_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_1405_cast_fp16 = mul(x = var_1390_cast_fp16_1, y = var_1404_to_fp16)[name = string("op_1405_cast_fp16")];
            string var_1407_equation_0 = const()[name = string("op_1407_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_1407_cast_fp16 = einsum(equation = var_1407_equation_0, values = (var_1395_cast_fp16_1, var_1405_cast_fp16))[name = string("op_1407_cast_fp16")];
            fp16 var_1408_to_fp16 = const()[name = string("op_1408_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_1409_cast_fp16 = mul(x = var_1390_cast_fp16_2, y = var_1408_to_fp16)[name = string("op_1409_cast_fp16")];
            string var_1411_equation_0 = const()[name = string("op_1411_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_1411_cast_fp16 = einsum(equation = var_1411_equation_0, values = (var_1395_cast_fp16_2, var_1409_cast_fp16))[name = string("op_1411_cast_fp16")];
            fp16 var_1412_to_fp16 = const()[name = string("op_1412_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_1413_cast_fp16 = mul(x = var_1390_cast_fp16_3, y = var_1412_to_fp16)[name = string("op_1413_cast_fp16")];
            string var_1415_equation_0 = const()[name = string("op_1415_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_1415_cast_fp16 = einsum(equation = var_1415_equation_0, values = (var_1395_cast_fp16_3, var_1413_cast_fp16))[name = string("op_1415_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_1416_cast_fp16 = add(x = var_1403_cast_fp16, y = var_1407_cast_fp16)[name = string("op_1416_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_1417_cast_fp16 = add(x = var_1411_cast_fp16, y = var_1415_cast_fp16)[name = string("op_1417_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_1418_cast_fp16 = add(x = var_1416_cast_fp16, y = var_1417_cast_fp16)[name = string("op_1418_cast_fp16")];
            fp16 _inversed_1420_y_0_to_fp16 = const()[name = string("_inversed_1420_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 1]> _inversed_1420_cast_fp16 = mul(x = var_1418_cast_fp16, y = _inversed_1420_y_0_to_fp16)[name = string("_inversed_1420_cast_fp16")];
            string scores_25_equation_0 = const()[name = string("scores_25_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 1]> scores_25_cast_fp16 = einsum(equation = scores_25_equation_0, values = (var_1267_cast_fp16_1, var_1257_cast_fp16_4))[name = string("scores_25_cast_fp16")];
            fp16 var_1423_to_fp16 = const()[name = string("op_1423_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 1]> var_1424_cast_fp16 = mul(x = scores_25_cast_fp16, y = var_1423_to_fp16)[name = string("op_1424_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_1425_cast_fp16 = add(x = var_1424_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_1425_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_1426_cast_fp16 = softmax(axis = var_1235, x = var_1425_cast_fp16)[name = string("op_1426_cast_fp16")];
            tensor<int32, [4]> tile_30 = const()[name = string("tile_30"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_1427_axis_0 = const()[name = string("op_1427_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 1]> var_1427_cast_fp16_0, tensor<fp16, [1, 512, 1, 1]> var_1427_cast_fp16_1, tensor<fp16, [1, 512, 1, 1]> var_1427_cast_fp16_2, tensor<fp16, [1, 512, 1, 1]> var_1427_cast_fp16_3 = split(axis = var_1427_axis_0, split_sizes = tile_30, x = var_1426_cast_fp16)[name = string("op_1427_cast_fp16")];
            tensor<int32, [4]> tile_31 = const()[name = string("tile_31"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_1432_axis_0 = const()[name = string("op_1432_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_1432_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_1432_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_1432_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_1432_cast_fp16_3 = split(axis = var_1432_axis_0, split_sizes = tile_31, x = var_1270_cast_fp16_1)[name = string("op_1432_cast_fp16")];
            fp16 var_1437_to_fp16 = const()[name = string("op_1437_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_1438_cast_fp16 = mul(x = var_1427_cast_fp16_0, y = var_1437_to_fp16)[name = string("op_1438_cast_fp16")];
            string var_1440_equation_0 = const()[name = string("op_1440_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_1440_cast_fp16 = einsum(equation = var_1440_equation_0, values = (var_1432_cast_fp16_0, var_1438_cast_fp16))[name = string("op_1440_cast_fp16")];
            fp16 var_1441_to_fp16 = const()[name = string("op_1441_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_1442_cast_fp16 = mul(x = var_1427_cast_fp16_1, y = var_1441_to_fp16)[name = string("op_1442_cast_fp16")];
            string var_1444_equation_0 = const()[name = string("op_1444_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_1444_cast_fp16 = einsum(equation = var_1444_equation_0, values = (var_1432_cast_fp16_1, var_1442_cast_fp16))[name = string("op_1444_cast_fp16")];
            fp16 var_1445_to_fp16 = const()[name = string("op_1445_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_1446_cast_fp16 = mul(x = var_1427_cast_fp16_2, y = var_1445_to_fp16)[name = string("op_1446_cast_fp16")];
            string var_1448_equation_0 = const()[name = string("op_1448_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_1448_cast_fp16 = einsum(equation = var_1448_equation_0, values = (var_1432_cast_fp16_2, var_1446_cast_fp16))[name = string("op_1448_cast_fp16")];
            fp16 var_1449_to_fp16 = const()[name = string("op_1449_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_1450_cast_fp16 = mul(x = var_1427_cast_fp16_3, y = var_1449_to_fp16)[name = string("op_1450_cast_fp16")];
            string var_1452_equation_0 = const()[name = string("op_1452_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_1452_cast_fp16 = einsum(equation = var_1452_equation_0, values = (var_1432_cast_fp16_3, var_1450_cast_fp16))[name = string("op_1452_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_1453_cast_fp16 = add(x = var_1440_cast_fp16, y = var_1444_cast_fp16)[name = string("op_1453_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_1454_cast_fp16 = add(x = var_1448_cast_fp16, y = var_1452_cast_fp16)[name = string("op_1454_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_1455_cast_fp16 = add(x = var_1453_cast_fp16, y = var_1454_cast_fp16)[name = string("op_1455_cast_fp16")];
            fp16 _inversed_1457_y_0_to_fp16 = const()[name = string("_inversed_1457_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 1]> _inversed_1457_cast_fp16 = mul(x = var_1455_cast_fp16, y = _inversed_1457_y_0_to_fp16)[name = string("_inversed_1457_cast_fp16")];
            string scores_27_equation_0 = const()[name = string("scores_27_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 1]> scores_27_cast_fp16 = einsum(equation = scores_27_equation_0, values = (var_1267_cast_fp16_1, var_1257_cast_fp16_5))[name = string("scores_27_cast_fp16")];
            fp16 var_1460_to_fp16 = const()[name = string("op_1460_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 1]> var_1461_cast_fp16 = mul(x = scores_27_cast_fp16, y = var_1460_to_fp16)[name = string("op_1461_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_1462_cast_fp16 = add(x = var_1461_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_1462_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_1463_cast_fp16 = softmax(axis = var_1235, x = var_1462_cast_fp16)[name = string("op_1463_cast_fp16")];
            tensor<int32, [4]> tile_32 = const()[name = string("tile_32"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_1464_axis_0 = const()[name = string("op_1464_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 1]> var_1464_cast_fp16_0, tensor<fp16, [1, 512, 1, 1]> var_1464_cast_fp16_1, tensor<fp16, [1, 512, 1, 1]> var_1464_cast_fp16_2, tensor<fp16, [1, 512, 1, 1]> var_1464_cast_fp16_3 = split(axis = var_1464_axis_0, split_sizes = tile_32, x = var_1463_cast_fp16)[name = string("op_1464_cast_fp16")];
            tensor<int32, [4]> tile_33 = const()[name = string("tile_33"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_1469_axis_0 = const()[name = string("op_1469_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_1469_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_1469_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_1469_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_1469_cast_fp16_3 = split(axis = var_1469_axis_0, split_sizes = tile_33, x = var_1270_cast_fp16_1)[name = string("op_1469_cast_fp16")];
            fp16 var_1474_to_fp16 = const()[name = string("op_1474_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_1475_cast_fp16 = mul(x = var_1464_cast_fp16_0, y = var_1474_to_fp16)[name = string("op_1475_cast_fp16")];
            string var_1477_equation_0 = const()[name = string("op_1477_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_1477_cast_fp16 = einsum(equation = var_1477_equation_0, values = (var_1469_cast_fp16_0, var_1475_cast_fp16))[name = string("op_1477_cast_fp16")];
            fp16 var_1478_to_fp16 = const()[name = string("op_1478_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_1479_cast_fp16 = mul(x = var_1464_cast_fp16_1, y = var_1478_to_fp16)[name = string("op_1479_cast_fp16")];
            string var_1481_equation_0 = const()[name = string("op_1481_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_1481_cast_fp16 = einsum(equation = var_1481_equation_0, values = (var_1469_cast_fp16_1, var_1479_cast_fp16))[name = string("op_1481_cast_fp16")];
            fp16 var_1482_to_fp16 = const()[name = string("op_1482_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_1483_cast_fp16 = mul(x = var_1464_cast_fp16_2, y = var_1482_to_fp16)[name = string("op_1483_cast_fp16")];
            string var_1485_equation_0 = const()[name = string("op_1485_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_1485_cast_fp16 = einsum(equation = var_1485_equation_0, values = (var_1469_cast_fp16_2, var_1483_cast_fp16))[name = string("op_1485_cast_fp16")];
            fp16 var_1486_to_fp16 = const()[name = string("op_1486_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_1487_cast_fp16 = mul(x = var_1464_cast_fp16_3, y = var_1486_to_fp16)[name = string("op_1487_cast_fp16")];
            string var_1489_equation_0 = const()[name = string("op_1489_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_1489_cast_fp16 = einsum(equation = var_1489_equation_0, values = (var_1469_cast_fp16_3, var_1487_cast_fp16))[name = string("op_1489_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_1490_cast_fp16 = add(x = var_1477_cast_fp16, y = var_1481_cast_fp16)[name = string("op_1490_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_1491_cast_fp16 = add(x = var_1485_cast_fp16, y = var_1489_cast_fp16)[name = string("op_1491_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_1492_cast_fp16 = add(x = var_1490_cast_fp16, y = var_1491_cast_fp16)[name = string("op_1492_cast_fp16")];
            fp16 _inversed_1494_y_0_to_fp16 = const()[name = string("_inversed_1494_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 1]> _inversed_1494_cast_fp16 = mul(x = var_1492_cast_fp16, y = _inversed_1494_y_0_to_fp16)[name = string("_inversed_1494_cast_fp16")];
            string scores_29_equation_0 = const()[name = string("scores_29_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 1]> scores_29_cast_fp16 = einsum(equation = scores_29_equation_0, values = (var_1267_cast_fp16_1, var_1257_cast_fp16_6))[name = string("scores_29_cast_fp16")];
            fp16 var_1497_to_fp16 = const()[name = string("op_1497_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 1]> var_1498_cast_fp16 = mul(x = scores_29_cast_fp16, y = var_1497_to_fp16)[name = string("op_1498_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_1499_cast_fp16 = add(x = var_1498_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_1499_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_1500_cast_fp16 = softmax(axis = var_1235, x = var_1499_cast_fp16)[name = string("op_1500_cast_fp16")];
            tensor<int32, [4]> tile_34 = const()[name = string("tile_34"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_1501_axis_0 = const()[name = string("op_1501_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 1]> var_1501_cast_fp16_0, tensor<fp16, [1, 512, 1, 1]> var_1501_cast_fp16_1, tensor<fp16, [1, 512, 1, 1]> var_1501_cast_fp16_2, tensor<fp16, [1, 512, 1, 1]> var_1501_cast_fp16_3 = split(axis = var_1501_axis_0, split_sizes = tile_34, x = var_1500_cast_fp16)[name = string("op_1501_cast_fp16")];
            tensor<int32, [4]> tile_35 = const()[name = string("tile_35"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_1506_axis_0 = const()[name = string("op_1506_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_1506_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_1506_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_1506_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_1506_cast_fp16_3 = split(axis = var_1506_axis_0, split_sizes = tile_35, x = var_1270_cast_fp16_1)[name = string("op_1506_cast_fp16")];
            fp16 var_1511_to_fp16 = const()[name = string("op_1511_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_1512_cast_fp16 = mul(x = var_1501_cast_fp16_0, y = var_1511_to_fp16)[name = string("op_1512_cast_fp16")];
            string var_1514_equation_0 = const()[name = string("op_1514_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_1514_cast_fp16 = einsum(equation = var_1514_equation_0, values = (var_1506_cast_fp16_0, var_1512_cast_fp16))[name = string("op_1514_cast_fp16")];
            fp16 var_1515_to_fp16 = const()[name = string("op_1515_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_1516_cast_fp16 = mul(x = var_1501_cast_fp16_1, y = var_1515_to_fp16)[name = string("op_1516_cast_fp16")];
            string var_1518_equation_0 = const()[name = string("op_1518_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_1518_cast_fp16 = einsum(equation = var_1518_equation_0, values = (var_1506_cast_fp16_1, var_1516_cast_fp16))[name = string("op_1518_cast_fp16")];
            fp16 var_1519_to_fp16 = const()[name = string("op_1519_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_1520_cast_fp16 = mul(x = var_1501_cast_fp16_2, y = var_1519_to_fp16)[name = string("op_1520_cast_fp16")];
            string var_1522_equation_0 = const()[name = string("op_1522_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_1522_cast_fp16 = einsum(equation = var_1522_equation_0, values = (var_1506_cast_fp16_2, var_1520_cast_fp16))[name = string("op_1522_cast_fp16")];
            fp16 var_1523_to_fp16 = const()[name = string("op_1523_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_1524_cast_fp16 = mul(x = var_1501_cast_fp16_3, y = var_1523_to_fp16)[name = string("op_1524_cast_fp16")];
            string var_1526_equation_0 = const()[name = string("op_1526_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_1526_cast_fp16 = einsum(equation = var_1526_equation_0, values = (var_1506_cast_fp16_3, var_1524_cast_fp16))[name = string("op_1526_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_1527_cast_fp16 = add(x = var_1514_cast_fp16, y = var_1518_cast_fp16)[name = string("op_1527_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_1528_cast_fp16 = add(x = var_1522_cast_fp16, y = var_1526_cast_fp16)[name = string("op_1528_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_1529_cast_fp16 = add(x = var_1527_cast_fp16, y = var_1528_cast_fp16)[name = string("op_1529_cast_fp16")];
            fp16 _inversed_1531_y_0_to_fp16 = const()[name = string("_inversed_1531_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 1]> _inversed_1531_cast_fp16 = mul(x = var_1529_cast_fp16, y = _inversed_1531_y_0_to_fp16)[name = string("_inversed_1531_cast_fp16")];
            string scores_equation_0 = const()[name = string("scores_equation_0"), val = string("bkhc,bchq->bkhq")];
            tensor<fp16, [1, 2048, 1, 1]> scores_cast_fp16 = einsum(equation = scores_equation_0, values = (var_1267_cast_fp16_1, var_1257_cast_fp16_7))[name = string("scores_cast_fp16")];
            fp16 var_1534_to_fp16 = const()[name = string("op_1534_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 2048, 1, 1]> var_1535_cast_fp16 = mul(x = scores_cast_fp16, y = var_1534_to_fp16)[name = string("op_1535_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_1536_cast_fp16 = add(x = var_1535_cast_fp16, y = mask_native_1_cast_fp16)[name = string("op_1536_cast_fp16")];
            tensor<fp16, [1, 2048, 1, 1]> var_1537_cast_fp16 = softmax(axis = var_1235, x = var_1536_cast_fp16)[name = string("op_1537_cast_fp16")];
            tensor<int32, [4]> tile_36 = const()[name = string("tile_36"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_1538_axis_0 = const()[name = string("op_1538_axis_0"), val = int32(1)];
            tensor<fp16, [1, 512, 1, 1]> var_1538_cast_fp16_0, tensor<fp16, [1, 512, 1, 1]> var_1538_cast_fp16_1, tensor<fp16, [1, 512, 1, 1]> var_1538_cast_fp16_2, tensor<fp16, [1, 512, 1, 1]> var_1538_cast_fp16_3 = split(axis = var_1538_axis_0, split_sizes = tile_36, x = var_1537_cast_fp16)[name = string("op_1538_cast_fp16")];
            tensor<int32, [4]> tile_37 = const()[name = string("tile_37"), val = tensor<int32, [4]>([512, 512, 512, 512])];
            int32 var_1543_axis_0 = const()[name = string("op_1543_axis_0"), val = int32(3)];
            tensor<fp16, [1, 256, 1, 512]> var_1543_cast_fp16_0, tensor<fp16, [1, 256, 1, 512]> var_1543_cast_fp16_1, tensor<fp16, [1, 256, 1, 512]> var_1543_cast_fp16_2, tensor<fp16, [1, 256, 1, 512]> var_1543_cast_fp16_3 = split(axis = var_1543_axis_0, split_sizes = tile_37, x = var_1270_cast_fp16_1)[name = string("op_1543_cast_fp16")];
            fp16 var_1548_to_fp16 = const()[name = string("op_1548_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_1549_cast_fp16 = mul(x = var_1538_cast_fp16_0, y = var_1548_to_fp16)[name = string("op_1549_cast_fp16")];
            string var_1551_equation_0 = const()[name = string("op_1551_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_1551_cast_fp16 = einsum(equation = var_1551_equation_0, values = (var_1543_cast_fp16_0, var_1549_cast_fp16))[name = string("op_1551_cast_fp16")];
            fp16 var_1552_to_fp16 = const()[name = string("op_1552_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_1553_cast_fp16 = mul(x = var_1538_cast_fp16_1, y = var_1552_to_fp16)[name = string("op_1553_cast_fp16")];
            string var_1555_equation_0 = const()[name = string("op_1555_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_1555_cast_fp16 = einsum(equation = var_1555_equation_0, values = (var_1543_cast_fp16_1, var_1553_cast_fp16))[name = string("op_1555_cast_fp16")];
            fp16 var_1556_to_fp16 = const()[name = string("op_1556_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_1557_cast_fp16 = mul(x = var_1538_cast_fp16_2, y = var_1556_to_fp16)[name = string("op_1557_cast_fp16")];
            string var_1559_equation_0 = const()[name = string("op_1559_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_1559_cast_fp16 = einsum(equation = var_1559_equation_0, values = (var_1543_cast_fp16_2, var_1557_cast_fp16))[name = string("op_1559_cast_fp16")];
            fp16 var_1560_to_fp16 = const()[name = string("op_1560_to_fp16"), val = fp16(0x1p+4)];
            tensor<fp16, [1, 512, 1, 1]> var_1561_cast_fp16 = mul(x = var_1538_cast_fp16_3, y = var_1560_to_fp16)[name = string("op_1561_cast_fp16")];
            string var_1563_equation_0 = const()[name = string("op_1563_equation_0"), val = string("bchk,bkhq->bchq")];
            tensor<fp16, [1, 256, 1, 1]> var_1563_cast_fp16 = einsum(equation = var_1563_equation_0, values = (var_1543_cast_fp16_3, var_1561_cast_fp16))[name = string("op_1563_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_1564_cast_fp16 = add(x = var_1551_cast_fp16, y = var_1555_cast_fp16)[name = string("op_1564_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_1565_cast_fp16 = add(x = var_1559_cast_fp16, y = var_1563_cast_fp16)[name = string("op_1565_cast_fp16")];
            tensor<fp16, [1, 256, 1, 1]> var_1566_cast_fp16 = add(x = var_1564_cast_fp16, y = var_1565_cast_fp16)[name = string("op_1566_cast_fp16")];
            fp16 _inversed_1568_y_0_to_fp16 = const()[name = string("_inversed_1568_y_0_to_fp16"), val = fp16(0x1p-4)];
            tensor<fp16, [1, 256, 1, 1]> _inversed_1568_cast_fp16 = mul(x = var_1566_cast_fp16, y = _inversed_1568_y_0_to_fp16)[name = string("_inversed_1568_cast_fp16")];
            bool var_1570_interleave_0 = const()[name = string("op_1570_interleave_0"), val = bool(false)];
            tensor<fp16, [1, 2048, 1, 1]> var_1570_cast_fp16 = concat(axis = var_1235, interleave = var_1570_interleave_0, values = (_inversed_1309_cast_fp16, _inversed_1346_cast_fp16, _inversed_1383_cast_fp16, _inversed_1420_cast_fp16, _inversed_1457_cast_fp16, _inversed_1494_cast_fp16, _inversed_1531_cast_fp16, _inversed_1568_cast_fp16))[name = string("op_1570_cast_fp16")];
            tensor<int32, [4]> var_1571 = const()[name = string("op_1571"), val = tensor<int32, [4]>([2, 4, 256, 1])];
            tensor<fp16, [2, 4, 256, 1]> var_1572_cast_fp16 = reshape(shape = var_1571, x = var_1570_cast_fp16)[name = string("op_1572_cast_fp16")];
            tensor<int32, [4]> transpose_2_perm_0 = const()[name = string("transpose_2_perm_0"), val = tensor<int32, [4]>([3, 0, 1, 2])];
            tensor<int32, [2]> var_1590 = const()[name = string("op_1590"), val = tensor<int32, [2]>([1, 2048])];
            tensor<fp16, [1, 2, 4, 256]> transpose_2_cast_fp16 = transpose(perm = transpose_2_perm_0, x = var_1572_cast_fp16)[name = string("transpose_0")];
            tensor<fp16, [1, 2048]> attention_output_cast_fp16 = reshape(shape = var_1590, x = transpose_2_cast_fp16)[name = string("attention_output_cast_fp16")];
            tensor<fp16, [1, 2048]> var_1592_cast_fp16 = sigmoid(x = gate_cast_fp16)[name = string("op_1592_cast_fp16")];
            tensor<fp16, [1, 2048]> input_53_cast_fp16 = mul(x = attention_output_cast_fp16, y = var_1592_cast_fp16)[name = string("input_53_cast_fp16")];
            tensor<fp16, [1024, 2048]> completion23_o_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 2048]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(256190528))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(257763456))))[name = string("completion23_o_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 1024]> linear_31_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = completion23_o_proj_weight_promoted_to_fp16_palettized, x = input_53_cast_fp16)[name = string("linear_31_cast_fp16")];
            tensor<fp16, [1, 1024]> value_59_cast_fp16 = add(x = value_45_cast_fp16, y = linear_31_cast_fp16)[name = string("value_59_cast_fp16")];
            tensor<fp16, [1, 1024]> var_1597_cast_fp16 = mul(x = value_59_cast_fp16, y = value_59_cast_fp16)[name = string("op_1597_cast_fp16")];
            tensor<int32, [1]> variance_27_axes_0 = const()[name = string("variance_27_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_27_keep_dims_0 = const()[name = string("variance_27_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 1]> variance_27_cast_fp16 = reduce_mean(axes = variance_27_axes_0, keep_dims = variance_27_keep_dims_0, x = var_1597_cast_fp16)[name = string("variance_27_cast_fp16")];
            fp16 var_1600_to_fp16 = const()[name = string("op_1600_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 1]> var_1601_cast_fp16 = add(x = variance_27_cast_fp16, y = var_1600_to_fp16)[name = string("op_1601_cast_fp16")];
            fp32 var_1602_epsilon_0 = const()[name = string("op_1602_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 1]> var_1602_cast_fp16 = rsqrt(epsilon = var_1602_epsilon_0, x = var_1601_cast_fp16)[name = string("op_1602_cast_fp16")];
            tensor<fp16, [1, 1024]> var_1603_cast_fp16 = mul(x = value_59_cast_fp16, y = var_1602_cast_fp16)[name = string("op_1603_cast_fp16")];
            tensor<fp16, [1024]> var_1605_to_fp16 = const()[name = string("op_1605_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(257771712)))];
            tensor<fp16, [1, 1024]> input_55_cast_fp16 = mul(x = var_1603_cast_fp16, y = var_1605_to_fp16)[name = string("input_55_cast_fp16")];
            tensor<fp16, [3584, 1024]> completion23_gate_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(257773824))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(260526400))))[name = string("completion23_gate_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 3584]> linear_32_cast_fp16 = linear(bias = linear_1_bias_0_to_fp16, weight = completion23_gate_proj_weight_promoted_to_fp16_palettized, x = input_55_cast_fp16)[name = string("linear_32_cast_fp16")];
            tensor<fp16, [1, 3584]> var_1609_cast_fp16 = silu(x = linear_32_cast_fp16)[name = string("op_1609_cast_fp16")];
            tensor<fp16, [3584, 1024]> completion23_up_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [3584, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(260555136))), lut = tensor<fp16, [224, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(263307712))))[name = string("completion23_up_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 3584]> linear_33_cast_fp16 = linear(bias = linear_1_bias_0_to_fp16, weight = completion23_up_proj_weight_promoted_to_fp16_palettized, x = input_55_cast_fp16)[name = string("linear_33_cast_fp16")];
            tensor<fp16, [1, 3584]> input_cast_fp16 = mul(x = var_1609_cast_fp16, y = linear_33_cast_fp16)[name = string("input_cast_fp16")];
            tensor<fp16, [1024, 3584]> completion23_down_proj_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [1024, 3584]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(263336448))), lut = tensor<fp16, [64, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(266089024))))[name = string("completion23_down_proj_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [1, 1024]> linear_34_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = completion23_down_proj_weight_promoted_to_fp16_palettized, x = input_cast_fp16)[name = string("linear_34_cast_fp16")];
            tensor<fp16, [1, 1024]> value_cast_fp16 = add(x = value_59_cast_fp16, y = linear_34_cast_fp16)[name = string("value_cast_fp16")];
            tensor<fp16, [1, 1024]> var_1616_cast_fp16 = mul(x = value_cast_fp16, y = value_cast_fp16)[name = string("op_1616_cast_fp16")];
            tensor<int32, [1]> variance_axes_0 = const()[name = string("variance_axes_0"), val = tensor<int32, [1]>([-1])];
            bool variance_keep_dims_0 = const()[name = string("variance_keep_dims_0"), val = bool(true)];
            tensor<fp16, [1, 1]> variance_cast_fp16 = reduce_mean(axes = variance_axes_0, keep_dims = variance_keep_dims_0, x = var_1616_cast_fp16)[name = string("variance_cast_fp16")];
            fp16 var_1623_to_fp16 = const()[name = string("op_1623_to_fp16"), val = fp16(0x1.1p-20)];
            tensor<fp16, [1, 1]> var_1624_cast_fp16 = add(x = variance_cast_fp16, y = var_1623_to_fp16)[name = string("op_1624_cast_fp16")];
            fp32 var_1625_epsilon_0 = const()[name = string("op_1625_epsilon_0"), val = fp32(0x1.197998p-40)];
            tensor<fp16, [1, 1]> var_1625_cast_fp16 = rsqrt(epsilon = var_1625_epsilon_0, x = var_1624_cast_fp16)[name = string("op_1625_cast_fp16")];
            tensor<fp16, [1, 1024]> var_1626_cast_fp16 = mul(x = value_cast_fp16, y = var_1625_cast_fp16)[name = string("op_1626_cast_fp16")];
            tensor<fp16, [1024]> var_1629_to_fp16 = const()[name = string("op_1629_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(266097280)))];
            tensor<fp16, [1, 1024]> hidden_out = mul(x = var_1626_cast_fp16, y = var_1629_to_fp16)[name = string("hidden_cast_fp16")];
            tensor<fp16, [248320, 1024]> head_weight_promoted_to_fp16_palettized = constexpr_lut_to_dense(indices = tensor<uint6, [248320, 1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(64))), lut = tensor<fp16, [15520, 1, 64, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(190709888))))[name = string("head_weight_promoted_to_fp16_palettized")];
            tensor<fp16, [248320]> linear_35_bias_0_to_fp16 = const()[name = string("linear_35_bias_0_to_fp16"), val = tensor<fp16, [248320]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(192696512)))];
            tensor<fp16, [1, 248320]> linear_35_cast_fp16 = linear(bias = linear_35_bias_0_to_fp16, weight = head_weight_promoted_to_fp16_palettized, x = hidden_out)[name = string("linear_35_cast_fp16")];
            int32 var_1640_axis_0 = const()[name = string("op_1640_axis_0"), val = int32(-1)];
            bool var_1640_keep_dims_0 = const()[name = string("op_1640_keep_dims_0"), val = bool(false)];
            string var_1640_output_dtype_0 = const()[name = string("op_1640_output_dtype_0"), val = string("int32")];
            tensor<int32, [1]> next_token = reduce_argmax(axis = var_1640_axis_0, keep_dims = var_1640_keep_dims_0, output_dtype = var_1640_output_dtype_0, x = linear_35_cast_fp16)[name = string("op_1640_cast_fp16")];
        } -> (hidden_out, conv20_out, rec20_out, conv21_out, rec21_out, conv22_out, rec22_out, next_token);
}