File size: 64,830 Bytes
5dd41aa | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 | program(1.0)
[buildInfo = dict<tensor<string, []>, tensor<string, []>>({{"coremlc-component-MIL", "3510.2.1"}, {"coremlc-version", "3500.32.1"}, {"coremltools-component-torch", "2.5.0"}, {"coremltools-source-dialect", "TorchScript"}, {"coremltools-version", "8.3.0"}})]
{
func main<ios17>(tensor<int32, [1, 512]> input_ids) {
tensor<int32, []> var_3 = const()[name = tensor<string, []>("op_3"), val = tensor<int32, []>(0)];
tensor<bool, [1, 512]> var_4 = not_equal(x = input_ids, y = var_3)[name = tensor<string, []>("op_4")];
tensor<string, []> cast_0_dtype_0 = const()[name = tensor<string, []>("cast_0_dtype_0"), val = tensor<string, []>("int32")];
tensor<int32, []> var_16 = const()[name = tensor<string, []>("op_16"), val = tensor<int32, []>(-1)];
tensor<int32, [1]> var_41_axes_0 = const()[name = tensor<string, []>("op_41_axes_0"), val = tensor<int32, [1]>([1])];
tensor<int32, [1, 512]> cast_0 = cast(dtype = cast_0_dtype_0, x = var_4)[name = tensor<string, []>("cast_37")];
tensor<int32, [1, 1, 512]> var_41 = expand_dims(axes = var_41_axes_0, x = cast_0)[name = tensor<string, []>("op_41")];
tensor<int32, [1]> var_42_axes_0 = const()[name = tensor<string, []>("op_42_axes_0"), val = tensor<int32, [1]>([2])];
tensor<int32, [1, 1, 1, 512]> var_42 = expand_dims(axes = var_42_axes_0, x = var_41)[name = tensor<string, []>("op_42")];
tensor<fp16, []> var_21_to_fp16 = const()[name = tensor<string, []>("op_21_to_fp16"), val = tensor<fp16, []>(0x1p+0)];
tensor<string, []> cast_1_to_fp16_dtype_0 = const()[name = tensor<string, []>("cast_1_to_fp16_dtype_0"), val = tensor<string, []>("fp16")];
tensor<fp16, [1, 1, 1, 512]> var_42_to_fp16 = cast(dtype = cast_1_to_fp16_dtype_0, x = var_42)[name = tensor<string, []>("cast_36")];
tensor<fp16, [1, 1, 1, 512]> var_45_cast_fp16 = sub(x = var_21_to_fp16, y = var_42_to_fp16)[name = tensor<string, []>("op_45_cast_fp16")];
tensor<fp16, []> var_46_to_fp16 = const()[name = tensor<string, []>("op_46_to_fp16"), val = tensor<fp16, []>(-0x1.d4cp+14)];
tensor<fp16, [1, 1, 1, 512]> attention_mask_cast_fp16 = mul(x = var_45_cast_fp16, y = var_46_to_fp16)[name = tensor<string, []>("attention_mask_cast_fp16")];
tensor<int32, []> inputs_embeds_axis_0 = const()[name = tensor<string, []>("inputs_embeds_axis_0"), val = tensor<int32, []>(0)];
tensor<int32, []> inputs_embeds_batch_dims_0 = const()[name = tensor<string, []>("inputs_embeds_batch_dims_0"), val = tensor<int32, []>(0)];
tensor<bool, []> inputs_embeds_validate_indices_0 = const()[name = tensor<string, []>("inputs_embeds_validate_indices_0"), val = tensor<bool, []>(false)];
tensor<fp16, [30528, 512]> m_embeddings_word_embeddings_weight_to_fp16 = const()[name = tensor<string, []>("m_embeddings_word_embeddings_weight_to_fp16"), val = tensor<fp16, [30528, 512]>(BLOBFILE(path = tensor<string, []>("@model_path/weights/weight.bin"), offset = tensor<uint64, []>(64)))];
tensor<string, []> input_ids_to_int16_dtype_0 = const()[name = tensor<string, []>("input_ids_to_int16_dtype_0"), val = tensor<string, []>("int16")];
tensor<int16, [1, 512]> input_ids_to_int16 = cast(dtype = input_ids_to_int16_dtype_0, x = input_ids)[name = tensor<string, []>("cast_35")];
tensor<fp16, [1, 512, 512]> inputs_embeds_cast_fp16_cast_uint16 = gather(axis = inputs_embeds_axis_0, batch_dims = inputs_embeds_batch_dims_0, indices = input_ids_to_int16, validate_indices = inputs_embeds_validate_indices_0, x = m_embeddings_word_embeddings_weight_to_fp16)[name = tensor<string, []>("inputs_embeds_cast_fp16_cast_uint16")];
tensor<fp16, [1, 512, 512]> token_type_embeddings_1_to_fp16 = const()[name = tensor<string, []>("token_type_embeddings_1_to_fp16"), val = tensor<fp16, [1, 512, 512]>(BLOBFILE(path = tensor<string, []>("@model_path/weights/weight.bin"), offset = tensor<uint64, []>(31260800)))];
tensor<fp16, [1, 512, 512]> input_3_cast_fp16 = add(x = inputs_embeds_cast_fp16_cast_uint16, y = token_type_embeddings_1_to_fp16)[name = tensor<string, []>("input_3_cast_fp16")];
tensor<int32, [1]> input_5_axes_0 = const()[name = tensor<string, []>("input_5_axes_0"), val = tensor<int32, [1]>([-1])];
tensor<fp16, [512]> m_embeddings_LayerNorm_weight_to_fp16 = const()[name = tensor<string, []>("m_embeddings_LayerNorm_weight_to_fp16"), val = tensor<fp16, [512]>(BLOBFILE(path = tensor<string, []>("@model_path/weights/weight.bin"), offset = tensor<uint64, []>(31785152)))];
tensor<fp16, [512]> m_embeddings_LayerNorm_bias_to_fp16 = const()[name = tensor<string, []>("m_embeddings_LayerNorm_bias_to_fp16"), val = tensor<fp16, [512]>(BLOBFILE(path = tensor<string, []>("@model_path/weights/weight.bin"), offset = tensor<uint64, []>(31786240)))];
tensor<fp16, []> var_18_to_fp16 = const()[name = tensor<string, []>("op_18_to_fp16"), val = tensor<fp16, []>(0x1p-24)];
tensor<fp16, [1, 512, 512]> input_5_cast_fp16 = layer_norm(axes = input_5_axes_0, beta = m_embeddings_LayerNorm_bias_to_fp16, epsilon = var_18_to_fp16, gamma = m_embeddings_LayerNorm_weight_to_fp16, x = input_3_cast_fp16)[name = tensor<string, []>("input_5_cast_fp16")];
tensor<fp16, [512, 512]> m_encoder_layer_0_attention_self_query_weight_to_fp16 = const()[name = tensor<string, []>("m_encoder_layer_0_attention_self_query_weight_to_fp16"), val = tensor<fp16, [512, 512]>(BLOBFILE(path = tensor<string, []>("@model_path/weights/weight.bin"), offset = tensor<uint64, []>(31787328)))];
tensor<fp16, [512]> m_encoder_layer_0_attention_self_query_bias_to_fp16 = const()[name = tensor<string, []>("m_encoder_layer_0_attention_self_query_bias_to_fp16"), val = tensor<fp16, [512]>(BLOBFILE(path = tensor<string, []>("@model_path/weights/weight.bin"), offset = tensor<uint64, []>(32311680)))];
tensor<fp16, [1, 512, 512]> linear_0_cast_fp16 = linear(bias = m_encoder_layer_0_attention_self_query_bias_to_fp16, weight = m_encoder_layer_0_attention_self_query_weight_to_fp16, x = input_5_cast_fp16)[name = tensor<string, []>("linear_0_cast_fp16")];
tensor<fp16, [512, 512]> m_encoder_layer_0_attention_self_key_weight_to_fp16 = const()[name = tensor<string, []>("m_encoder_layer_0_attention_self_key_weight_to_fp16"), val = tensor<fp16, [512, 512]>(BLOBFILE(path = tensor<string, []>("@model_path/weights/weight.bin"), offset = tensor<uint64, []>(32312768)))];
tensor<fp16, [512]> m_encoder_layer_0_attention_self_key_bias_to_fp16 = const()[name = tensor<string, []>("m_encoder_layer_0_attention_self_key_bias_to_fp16"), val = tensor<fp16, [512]>(BLOBFILE(path = tensor<string, []>("@model_path/weights/weight.bin"), offset = tensor<uint64, []>(32837120)))];
tensor<fp16, [1, 512, 512]> linear_1_cast_fp16 = linear(bias = m_encoder_layer_0_attention_self_key_bias_to_fp16, weight = m_encoder_layer_0_attention_self_key_weight_to_fp16, x = input_5_cast_fp16)[name = tensor<string, []>("linear_1_cast_fp16")];
tensor<int32, [4]> var_90 = const()[name = tensor<string, []>("op_90"), val = tensor<int32, [4]>([1, 512, 8, 64])];
tensor<fp16, [1, 512, 8, 64]> x_3_cast_fp16 = reshape(shape = var_90, x = linear_1_cast_fp16)[name = tensor<string, []>("x_3_cast_fp16")];
tensor<fp16, [512, 512]> m_encoder_layer_0_attention_self_value_weight_to_fp16 = const()[name = tensor<string, []>("m_encoder_layer_0_attention_self_value_weight_to_fp16"), val = tensor<fp16, [512, 512]>(BLOBFILE(path = tensor<string, []>("@model_path/weights/weight.bin"), offset = tensor<uint64, []>(32838208)))];
tensor<fp16, [512]> m_encoder_layer_0_attention_self_value_bias_to_fp16 = const()[name = tensor<string, []>("m_encoder_layer_0_attention_self_value_bias_to_fp16"), val = tensor<fp16, [512]>(BLOBFILE(path = tensor<string, []>("@model_path/weights/weight.bin"), offset = tensor<uint64, []>(33362560)))];
tensor<fp16, [1, 512, 512]> linear_2_cast_fp16 = linear(bias = m_encoder_layer_0_attention_self_value_bias_to_fp16, weight = m_encoder_layer_0_attention_self_value_weight_to_fp16, x = input_5_cast_fp16)[name = tensor<string, []>("linear_2_cast_fp16")];
tensor<int32, [4]> var_99 = const()[name = tensor<string, []>("op_99"), val = tensor<int32, [4]>([1, 512, 8, 64])];
tensor<fp16, [1, 512, 8, 64]> x_7_cast_fp16 = reshape(shape = var_99, x = linear_2_cast_fp16)[name = tensor<string, []>("x_7_cast_fp16")];
tensor<int32, [4]> var_101 = const()[name = tensor<string, []>("op_101"), val = tensor<int32, [4]>([0, 2, 1, 3])];
tensor<int32, [4]> var_105 = const()[name = tensor<string, []>("op_105"), val = tensor<int32, [4]>([1, 512, 8, 64])];
tensor<fp16, [1, 512, 8, 64]> x_11_cast_fp16 = reshape(shape = var_105, x = linear_0_cast_fp16)[name = tensor<string, []>("x_11_cast_fp16")];
tensor<bool, []> attention_scores_1_transpose_x_0 = const()[name = tensor<string, []>("attention_scores_1_transpose_x_0"), val = tensor<bool, []>(false)];
tensor<bool, []> attention_scores_1_transpose_y_0 = const()[name = tensor<string, []>("attention_scores_1_transpose_y_0"), val = tensor<bool, []>(false)];
tensor<int32, [4]> transpose_12_perm_0 = const()[name = tensor<string, []>("transpose_12_perm_0"), val = tensor<int32, [4]>([0, 2, -3, -1])];
tensor<int32, [4]> transpose_13_perm_0 = const()[name = tensor<string, []>("transpose_13_perm_0"), val = tensor<int32, [4]>([0, 2, -1, -3])];
tensor<fp16, [1, 8, 64, 512]> transpose_13 = transpose(perm = transpose_13_perm_0, x = x_3_cast_fp16)[name = tensor<string, []>("transpose_33")];
tensor<fp16, [1, 8, 512, 64]> transpose_12 = transpose(perm = transpose_12_perm_0, x = x_11_cast_fp16)[name = tensor<string, []>("transpose_34")];
tensor<fp16, [1, 8, 512, 512]> attention_scores_1_cast_fp16 = matmul(transpose_x = attention_scores_1_transpose_x_0, transpose_y = attention_scores_1_transpose_y_0, x = transpose_12, y = transpose_13)[name = tensor<string, []>("attention_scores_1_cast_fp16")];
tensor<fp16, []> _inversed_attention_scores_3_y_0_to_fp16 = const()[name = tensor<string, []>("_inversed_attention_scores_3_y_0_to_fp16"), val = tensor<fp16, []>(0x1p-3)];
tensor<fp16, [1, 8, 512, 512]> _inversed_attention_scores_3_cast_fp16 = mul(x = attention_scores_1_cast_fp16, y = _inversed_attention_scores_3_y_0_to_fp16)[name = tensor<string, []>("_inversed_attention_scores_3_cast_fp16")];
tensor<fp16, [1, 8, 512, 512]> attention_scores_5_cast_fp16 = add(x = _inversed_attention_scores_3_cast_fp16, y = attention_mask_cast_fp16)[name = tensor<string, []>("attention_scores_5_cast_fp16")];
tensor<fp16, [1, 8, 512, 512]> bias_3_to_fp16 = const()[name = tensor<string, []>("bias_3_to_fp16"), val = tensor<fp16, [1, 8, 512, 512]>(BLOBFILE(path = tensor<string, []>("@model_path/weights/weight.bin"), offset = tensor<uint64, []>(33363648)))];
tensor<fp16, [1, 8, 512, 512]> input_7_cast_fp16 = add(x = attention_scores_5_cast_fp16, y = bias_3_to_fp16)[name = tensor<string, []>("input_7_cast_fp16")];
tensor<fp16, [1, 8, 512, 512]> input_9_cast_fp16 = softmax(axis = var_16, x = input_7_cast_fp16)[name = tensor<string, []>("input_9_cast_fp16")];
tensor<bool, []> context_layer_1_transpose_x_0 = const()[name = tensor<string, []>("context_layer_1_transpose_x_0"), val = tensor<bool, []>(false)];
tensor<bool, []> context_layer_1_transpose_y_0 = const()[name = tensor<string, []>("context_layer_1_transpose_y_0"), val = tensor<bool, []>(false)];
tensor<fp16, [1, 8, 512, 64]> value_layer_1_cast_fp16 = transpose(perm = var_101, x = x_7_cast_fp16)[name = tensor<string, []>("transpose_35")];
tensor<fp16, [1, 8, 512, 64]> context_layer_1_cast_fp16 = matmul(transpose_x = context_layer_1_transpose_x_0, transpose_y = context_layer_1_transpose_y_0, x = input_9_cast_fp16, y = value_layer_1_cast_fp16)[name = tensor<string, []>("context_layer_1_cast_fp16")];
tensor<int32, [4]> var_118 = const()[name = tensor<string, []>("op_118"), val = tensor<int32, [4]>([0, 2, 1, 3])];
tensor<int32, [3]> var_123 = const()[name = tensor<string, []>("op_123"), val = tensor<int32, [3]>([1, 512, 512])];
tensor<fp16, [1, 512, 8, 64]> var_119_cast_fp16 = transpose(perm = var_118, x = context_layer_1_cast_fp16)[name = tensor<string, []>("transpose_32")];
tensor<fp16, [1, 512, 512]> input_11_cast_fp16 = reshape(shape = var_123, x = var_119_cast_fp16)[name = tensor<string, []>("input_11_cast_fp16")];
tensor<fp16, [512, 512]> m_encoder_layer_0_attention_output_dense_weight_to_fp16 = const()[name = tensor<string, []>("m_encoder_layer_0_attention_output_dense_weight_to_fp16"), val = tensor<fp16, [512, 512]>(BLOBFILE(path = tensor<string, []>("@model_path/weights/weight.bin"), offset = tensor<uint64, []>(37558016)))];
tensor<fp16, [512]> m_encoder_layer_0_attention_output_dense_bias_to_fp16 = const()[name = tensor<string, []>("m_encoder_layer_0_attention_output_dense_bias_to_fp16"), val = tensor<fp16, [512]>(BLOBFILE(path = tensor<string, []>("@model_path/weights/weight.bin"), offset = tensor<uint64, []>(38082368)))];
tensor<fp16, [1, 512, 512]> linear_3_cast_fp16 = linear(bias = m_encoder_layer_0_attention_output_dense_bias_to_fp16, weight = m_encoder_layer_0_attention_output_dense_weight_to_fp16, x = input_11_cast_fp16)[name = tensor<string, []>("linear_3_cast_fp16")];
tensor<fp16, [1, 512, 512]> input_15_cast_fp16 = add(x = linear_3_cast_fp16, y = input_5_cast_fp16)[name = tensor<string, []>("input_15_cast_fp16")];
tensor<int32, [1]> input_17_axes_0 = const()[name = tensor<string, []>("input_17_axes_0"), val = tensor<int32, [1]>([-1])];
tensor<fp16, [512]> m_encoder_layer_0_attention_output_LayerNorm_weight_to_fp16 = const()[name = tensor<string, []>("m_encoder_layer_0_attention_output_LayerNorm_weight_to_fp16"), val = tensor<fp16, [512]>(BLOBFILE(path = tensor<string, []>("@model_path/weights/weight.bin"), offset = tensor<uint64, []>(38083456)))];
tensor<fp16, [512]> m_encoder_layer_0_attention_output_LayerNorm_bias_to_fp16 = const()[name = tensor<string, []>("m_encoder_layer_0_attention_output_LayerNorm_bias_to_fp16"), val = tensor<fp16, [512]>(BLOBFILE(path = tensor<string, []>("@model_path/weights/weight.bin"), offset = tensor<uint64, []>(38084544)))];
tensor<fp16, [1, 512, 512]> input_17_cast_fp16 = layer_norm(axes = input_17_axes_0, beta = m_encoder_layer_0_attention_output_LayerNorm_bias_to_fp16, epsilon = var_18_to_fp16, gamma = m_encoder_layer_0_attention_output_LayerNorm_weight_to_fp16, x = input_15_cast_fp16)[name = tensor<string, []>("input_17_cast_fp16")];
tensor<fp16, [4096, 512]> m_encoder_layer_0_mlp_gated_layers_weight_to_fp16 = const()[name = tensor<string, []>("m_encoder_layer_0_mlp_gated_layers_weight_to_fp16"), val = tensor<fp16, [4096, 512]>(BLOBFILE(path = tensor<string, []>("@model_path/weights/weight.bin"), offset = tensor<uint64, []>(38085632)))];
tensor<fp16, [4096]> linear_4_bias_0_to_fp16 = const()[name = tensor<string, []>("linear_4_bias_0_to_fp16"), val = tensor<fp16, [4096]>(BLOBFILE(path = tensor<string, []>("@model_path/weights/weight.bin"), offset = tensor<uint64, []>(42280000)))];
tensor<fp16, [1, 512, 4096]> linear_4_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = m_encoder_layer_0_mlp_gated_layers_weight_to_fp16, x = input_17_cast_fp16)[name = tensor<string, []>("linear_4_cast_fp16")];
tensor<int32, [3]> input_19_begin_0 = const()[name = tensor<string, []>("input_19_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
tensor<int32, [3]> input_19_end_0 = const()[name = tensor<string, []>("input_19_end_0"), val = tensor<int32, [3]>([1, 512, 2048])];
tensor<bool, [3]> input_19_end_mask_0 = const()[name = tensor<string, []>("input_19_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
tensor<fp16, [1, 512, 2048]> input_19_cast_fp16 = slice_by_index(begin = input_19_begin_0, end = input_19_end_0, end_mask = input_19_end_mask_0, x = linear_4_cast_fp16)[name = tensor<string, []>("input_19_cast_fp16")];
tensor<int32, [3]> non_gated_1_begin_0 = const()[name = tensor<string, []>("non_gated_1_begin_0"), val = tensor<int32, [3]>([0, 0, 2048])];
tensor<int32, [3]> non_gated_1_end_0 = const()[name = tensor<string, []>("non_gated_1_end_0"), val = tensor<int32, [3]>([1, 512, 4096])];
tensor<bool, [3]> non_gated_1_end_mask_0 = const()[name = tensor<string, []>("non_gated_1_end_mask_0"), val = tensor<bool, [3]>([true, true, true])];
tensor<fp16, [1, 512, 2048]> non_gated_1_cast_fp16 = slice_by_index(begin = non_gated_1_begin_0, end = non_gated_1_end_0, end_mask = non_gated_1_end_mask_0, x = linear_4_cast_fp16)[name = tensor<string, []>("non_gated_1_cast_fp16")];
tensor<string, []> var_147_mode_0 = const()[name = tensor<string, []>("op_147_mode_0"), val = tensor<string, []>("EXACT")];
tensor<fp16, [1, 512, 2048]> var_147_cast_fp16 = gelu(mode = var_147_mode_0, x = input_19_cast_fp16)[name = tensor<string, []>("op_147_cast_fp16")];
tensor<fp16, [1, 512, 2048]> input_21_cast_fp16 = mul(x = var_147_cast_fp16, y = non_gated_1_cast_fp16)[name = tensor<string, []>("input_21_cast_fp16")];
tensor<fp16, [512, 2048]> m_encoder_layer_0_mlp_wo_weight_to_fp16 = const()[name = tensor<string, []>("m_encoder_layer_0_mlp_wo_weight_to_fp16"), val = tensor<fp16, [512, 2048]>(BLOBFILE(path = tensor<string, []>("@model_path/weights/weight.bin"), offset = tensor<uint64, []>(42288256)))];
tensor<fp16, [512]> m_encoder_layer_0_mlp_wo_bias_to_fp16 = const()[name = tensor<string, []>("m_encoder_layer_0_mlp_wo_bias_to_fp16"), val = tensor<fp16, [512]>(BLOBFILE(path = tensor<string, []>("@model_path/weights/weight.bin"), offset = tensor<uint64, []>(44385472)))];
tensor<fp16, [1, 512, 512]> linear_5_cast_fp16 = linear(bias = m_encoder_layer_0_mlp_wo_bias_to_fp16, weight = m_encoder_layer_0_mlp_wo_weight_to_fp16, x = input_21_cast_fp16)[name = tensor<string, []>("linear_5_cast_fp16")];
tensor<fp16, [1, 512, 512]> input_25_cast_fp16 = add(x = linear_5_cast_fp16, y = input_17_cast_fp16)[name = tensor<string, []>("input_25_cast_fp16")];
tensor<int32, [1]> input_27_axes_0 = const()[name = tensor<string, []>("input_27_axes_0"), val = tensor<int32, [1]>([-1])];
tensor<fp16, [512]> m_encoder_layer_0_mlp_layernorm_weight_to_fp16 = const()[name = tensor<string, []>("m_encoder_layer_0_mlp_layernorm_weight_to_fp16"), val = tensor<fp16, [512]>(BLOBFILE(path = tensor<string, []>("@model_path/weights/weight.bin"), offset = tensor<uint64, []>(44386560)))];
tensor<fp16, [512]> m_encoder_layer_0_mlp_layernorm_bias_to_fp16 = const()[name = tensor<string, []>("m_encoder_layer_0_mlp_layernorm_bias_to_fp16"), val = tensor<fp16, [512]>(BLOBFILE(path = tensor<string, []>("@model_path/weights/weight.bin"), offset = tensor<uint64, []>(44387648)))];
tensor<fp16, [1, 512, 512]> input_27_cast_fp16 = layer_norm(axes = input_27_axes_0, beta = m_encoder_layer_0_mlp_layernorm_bias_to_fp16, epsilon = var_18_to_fp16, gamma = m_encoder_layer_0_mlp_layernorm_weight_to_fp16, x = input_25_cast_fp16)[name = tensor<string, []>("input_27_cast_fp16")];
tensor<fp16, [512, 512]> m_encoder_layer_1_attention_self_query_weight_to_fp16 = const()[name = tensor<string, []>("m_encoder_layer_1_attention_self_query_weight_to_fp16"), val = tensor<fp16, [512, 512]>(BLOBFILE(path = tensor<string, []>("@model_path/weights/weight.bin"), offset = tensor<uint64, []>(44388736)))];
tensor<fp16, [512]> m_encoder_layer_1_attention_self_query_bias_to_fp16 = const()[name = tensor<string, []>("m_encoder_layer_1_attention_self_query_bias_to_fp16"), val = tensor<fp16, [512]>(BLOBFILE(path = tensor<string, []>("@model_path/weights/weight.bin"), offset = tensor<uint64, []>(44913088)))];
tensor<fp16, [1, 512, 512]> linear_6_cast_fp16 = linear(bias = m_encoder_layer_1_attention_self_query_bias_to_fp16, weight = m_encoder_layer_1_attention_self_query_weight_to_fp16, x = input_27_cast_fp16)[name = tensor<string, []>("linear_6_cast_fp16")];
tensor<fp16, [512, 512]> m_encoder_layer_1_attention_self_key_weight_to_fp16 = const()[name = tensor<string, []>("m_encoder_layer_1_attention_self_key_weight_to_fp16"), val = tensor<fp16, [512, 512]>(BLOBFILE(path = tensor<string, []>("@model_path/weights/weight.bin"), offset = tensor<uint64, []>(44914176)))];
tensor<fp16, [512]> m_encoder_layer_1_attention_self_key_bias_to_fp16 = const()[name = tensor<string, []>("m_encoder_layer_1_attention_self_key_bias_to_fp16"), val = tensor<fp16, [512]>(BLOBFILE(path = tensor<string, []>("@model_path/weights/weight.bin"), offset = tensor<uint64, []>(45438528)))];
tensor<fp16, [1, 512, 512]> linear_7_cast_fp16 = linear(bias = m_encoder_layer_1_attention_self_key_bias_to_fp16, weight = m_encoder_layer_1_attention_self_key_weight_to_fp16, x = input_27_cast_fp16)[name = tensor<string, []>("linear_7_cast_fp16")];
tensor<int32, [4]> var_173 = const()[name = tensor<string, []>("op_173"), val = tensor<int32, [4]>([1, 512, 8, 64])];
tensor<fp16, [1, 512, 8, 64]> x_15_cast_fp16 = reshape(shape = var_173, x = linear_7_cast_fp16)[name = tensor<string, []>("x_15_cast_fp16")];
tensor<fp16, [512, 512]> m_encoder_layer_1_attention_self_value_weight_to_fp16 = const()[name = tensor<string, []>("m_encoder_layer_1_attention_self_value_weight_to_fp16"), val = tensor<fp16, [512, 512]>(BLOBFILE(path = tensor<string, []>("@model_path/weights/weight.bin"), offset = tensor<uint64, []>(45439616)))];
tensor<fp16, [512]> m_encoder_layer_1_attention_self_value_bias_to_fp16 = const()[name = tensor<string, []>("m_encoder_layer_1_attention_self_value_bias_to_fp16"), val = tensor<fp16, [512]>(BLOBFILE(path = tensor<string, []>("@model_path/weights/weight.bin"), offset = tensor<uint64, []>(45963968)))];
tensor<fp16, [1, 512, 512]> linear_8_cast_fp16 = linear(bias = m_encoder_layer_1_attention_self_value_bias_to_fp16, weight = m_encoder_layer_1_attention_self_value_weight_to_fp16, x = input_27_cast_fp16)[name = tensor<string, []>("linear_8_cast_fp16")];
tensor<int32, [4]> var_182 = const()[name = tensor<string, []>("op_182"), val = tensor<int32, [4]>([1, 512, 8, 64])];
tensor<fp16, [1, 512, 8, 64]> x_19_cast_fp16 = reshape(shape = var_182, x = linear_8_cast_fp16)[name = tensor<string, []>("x_19_cast_fp16")];
tensor<int32, [4]> var_184 = const()[name = tensor<string, []>("op_184"), val = tensor<int32, [4]>([0, 2, 1, 3])];
tensor<int32, [4]> var_188 = const()[name = tensor<string, []>("op_188"), val = tensor<int32, [4]>([1, 512, 8, 64])];
tensor<fp16, [1, 512, 8, 64]> x_23_cast_fp16 = reshape(shape = var_188, x = linear_6_cast_fp16)[name = tensor<string, []>("x_23_cast_fp16")];
tensor<bool, []> attention_scores_7_transpose_x_0 = const()[name = tensor<string, []>("attention_scores_7_transpose_x_0"), val = tensor<bool, []>(false)];
tensor<bool, []> attention_scores_7_transpose_y_0 = const()[name = tensor<string, []>("attention_scores_7_transpose_y_0"), val = tensor<bool, []>(false)];
tensor<int32, [4]> transpose_14_perm_0 = const()[name = tensor<string, []>("transpose_14_perm_0"), val = tensor<int32, [4]>([0, 2, -3, -1])];
tensor<int32, [4]> transpose_15_perm_0 = const()[name = tensor<string, []>("transpose_15_perm_0"), val = tensor<int32, [4]>([0, 2, -1, -3])];
tensor<fp16, [1, 8, 64, 512]> transpose_15 = transpose(perm = transpose_15_perm_0, x = x_15_cast_fp16)[name = tensor<string, []>("transpose_29")];
tensor<fp16, [1, 8, 512, 64]> transpose_14 = transpose(perm = transpose_14_perm_0, x = x_23_cast_fp16)[name = tensor<string, []>("transpose_30")];
tensor<fp16, [1, 8, 512, 512]> attention_scores_7_cast_fp16 = matmul(transpose_x = attention_scores_7_transpose_x_0, transpose_y = attention_scores_7_transpose_y_0, x = transpose_14, y = transpose_15)[name = tensor<string, []>("attention_scores_7_cast_fp16")];
tensor<fp16, []> _inversed_attention_scores_9_y_0_to_fp16 = const()[name = tensor<string, []>("_inversed_attention_scores_9_y_0_to_fp16"), val = tensor<fp16, []>(0x1p-3)];
tensor<fp16, [1, 8, 512, 512]> _inversed_attention_scores_9_cast_fp16 = mul(x = attention_scores_7_cast_fp16, y = _inversed_attention_scores_9_y_0_to_fp16)[name = tensor<string, []>("_inversed_attention_scores_9_cast_fp16")];
tensor<fp16, [1, 8, 512, 512]> attention_scores_11_cast_fp16 = add(x = _inversed_attention_scores_9_cast_fp16, y = attention_mask_cast_fp16)[name = tensor<string, []>("attention_scores_11_cast_fp16")];
tensor<fp16, [1, 8, 512, 512]> input_29_cast_fp16 = add(x = attention_scores_11_cast_fp16, y = bias_3_to_fp16)[name = tensor<string, []>("input_29_cast_fp16")];
tensor<fp16, [1, 8, 512, 512]> input_31_cast_fp16 = softmax(axis = var_16, x = input_29_cast_fp16)[name = tensor<string, []>("input_31_cast_fp16")];
tensor<bool, []> context_layer_5_transpose_x_0 = const()[name = tensor<string, []>("context_layer_5_transpose_x_0"), val = tensor<bool, []>(false)];
tensor<bool, []> context_layer_5_transpose_y_0 = const()[name = tensor<string, []>("context_layer_5_transpose_y_0"), val = tensor<bool, []>(false)];
tensor<fp16, [1, 8, 512, 64]> value_layer_3_cast_fp16 = transpose(perm = var_184, x = x_19_cast_fp16)[name = tensor<string, []>("transpose_31")];
tensor<fp16, [1, 8, 512, 64]> context_layer_5_cast_fp16 = matmul(transpose_x = context_layer_5_transpose_x_0, transpose_y = context_layer_5_transpose_y_0, x = input_31_cast_fp16, y = value_layer_3_cast_fp16)[name = tensor<string, []>("context_layer_5_cast_fp16")];
tensor<int32, [4]> var_201 = const()[name = tensor<string, []>("op_201"), val = tensor<int32, [4]>([0, 2, 1, 3])];
tensor<int32, [3]> var_206 = const()[name = tensor<string, []>("op_206"), val = tensor<int32, [3]>([1, 512, 512])];
tensor<fp16, [1, 512, 8, 64]> var_202_cast_fp16 = transpose(perm = var_201, x = context_layer_5_cast_fp16)[name = tensor<string, []>("transpose_28")];
tensor<fp16, [1, 512, 512]> input_33_cast_fp16 = reshape(shape = var_206, x = var_202_cast_fp16)[name = tensor<string, []>("input_33_cast_fp16")];
tensor<fp16, [512, 512]> m_encoder_layer_1_attention_output_dense_weight_to_fp16 = const()[name = tensor<string, []>("m_encoder_layer_1_attention_output_dense_weight_to_fp16"), val = tensor<fp16, [512, 512]>(BLOBFILE(path = tensor<string, []>("@model_path/weights/weight.bin"), offset = tensor<uint64, []>(45965056)))];
tensor<fp16, [512]> m_encoder_layer_1_attention_output_dense_bias_to_fp16 = const()[name = tensor<string, []>("m_encoder_layer_1_attention_output_dense_bias_to_fp16"), val = tensor<fp16, [512]>(BLOBFILE(path = tensor<string, []>("@model_path/weights/weight.bin"), offset = tensor<uint64, []>(46489408)))];
tensor<fp16, [1, 512, 512]> linear_9_cast_fp16 = linear(bias = m_encoder_layer_1_attention_output_dense_bias_to_fp16, weight = m_encoder_layer_1_attention_output_dense_weight_to_fp16, x = input_33_cast_fp16)[name = tensor<string, []>("linear_9_cast_fp16")];
tensor<fp16, [1, 512, 512]> input_37_cast_fp16 = add(x = linear_9_cast_fp16, y = input_27_cast_fp16)[name = tensor<string, []>("input_37_cast_fp16")];
tensor<int32, [1]> input_39_axes_0 = const()[name = tensor<string, []>("input_39_axes_0"), val = tensor<int32, [1]>([-1])];
tensor<fp16, [512]> m_encoder_layer_1_attention_output_LayerNorm_weight_to_fp16 = const()[name = tensor<string, []>("m_encoder_layer_1_attention_output_LayerNorm_weight_to_fp16"), val = tensor<fp16, [512]>(BLOBFILE(path = tensor<string, []>("@model_path/weights/weight.bin"), offset = tensor<uint64, []>(46490496)))];
tensor<fp16, [512]> m_encoder_layer_1_attention_output_LayerNorm_bias_to_fp16 = const()[name = tensor<string, []>("m_encoder_layer_1_attention_output_LayerNorm_bias_to_fp16"), val = tensor<fp16, [512]>(BLOBFILE(path = tensor<string, []>("@model_path/weights/weight.bin"), offset = tensor<uint64, []>(46491584)))];
tensor<fp16, [1, 512, 512]> input_39_cast_fp16 = layer_norm(axes = input_39_axes_0, beta = m_encoder_layer_1_attention_output_LayerNorm_bias_to_fp16, epsilon = var_18_to_fp16, gamma = m_encoder_layer_1_attention_output_LayerNorm_weight_to_fp16, x = input_37_cast_fp16)[name = tensor<string, []>("input_39_cast_fp16")];
tensor<fp16, [4096, 512]> m_encoder_layer_1_mlp_gated_layers_weight_to_fp16 = const()[name = tensor<string, []>("m_encoder_layer_1_mlp_gated_layers_weight_to_fp16"), val = tensor<fp16, [4096, 512]>(BLOBFILE(path = tensor<string, []>("@model_path/weights/weight.bin"), offset = tensor<uint64, []>(46492672)))];
tensor<fp16, [1, 512, 4096]> linear_10_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = m_encoder_layer_1_mlp_gated_layers_weight_to_fp16, x = input_39_cast_fp16)[name = tensor<string, []>("linear_10_cast_fp16")];
tensor<int32, [3]> input_41_begin_0 = const()[name = tensor<string, []>("input_41_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
tensor<int32, [3]> input_41_end_0 = const()[name = tensor<string, []>("input_41_end_0"), val = tensor<int32, [3]>([1, 512, 2048])];
tensor<bool, [3]> input_41_end_mask_0 = const()[name = tensor<string, []>("input_41_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
tensor<fp16, [1, 512, 2048]> input_41_cast_fp16 = slice_by_index(begin = input_41_begin_0, end = input_41_end_0, end_mask = input_41_end_mask_0, x = linear_10_cast_fp16)[name = tensor<string, []>("input_41_cast_fp16")];
tensor<int32, [3]> non_gated_3_begin_0 = const()[name = tensor<string, []>("non_gated_3_begin_0"), val = tensor<int32, [3]>([0, 0, 2048])];
tensor<int32, [3]> non_gated_3_end_0 = const()[name = tensor<string, []>("non_gated_3_end_0"), val = tensor<int32, [3]>([1, 512, 4096])];
tensor<bool, [3]> non_gated_3_end_mask_0 = const()[name = tensor<string, []>("non_gated_3_end_mask_0"), val = tensor<bool, [3]>([true, true, true])];
tensor<fp16, [1, 512, 2048]> non_gated_3_cast_fp16 = slice_by_index(begin = non_gated_3_begin_0, end = non_gated_3_end_0, end_mask = non_gated_3_end_mask_0, x = linear_10_cast_fp16)[name = tensor<string, []>("non_gated_3_cast_fp16")];
tensor<string, []> var_230_mode_0 = const()[name = tensor<string, []>("op_230_mode_0"), val = tensor<string, []>("EXACT")];
tensor<fp16, [1, 512, 2048]> var_230_cast_fp16 = gelu(mode = var_230_mode_0, x = input_41_cast_fp16)[name = tensor<string, []>("op_230_cast_fp16")];
tensor<fp16, [1, 512, 2048]> input_43_cast_fp16 = mul(x = var_230_cast_fp16, y = non_gated_3_cast_fp16)[name = tensor<string, []>("input_43_cast_fp16")];
tensor<fp16, [512, 2048]> m_encoder_layer_1_mlp_wo_weight_to_fp16 = const()[name = tensor<string, []>("m_encoder_layer_1_mlp_wo_weight_to_fp16"), val = tensor<fp16, [512, 2048]>(BLOBFILE(path = tensor<string, []>("@model_path/weights/weight.bin"), offset = tensor<uint64, []>(50687040)))];
tensor<fp16, [512]> m_encoder_layer_1_mlp_wo_bias_to_fp16 = const()[name = tensor<string, []>("m_encoder_layer_1_mlp_wo_bias_to_fp16"), val = tensor<fp16, [512]>(BLOBFILE(path = tensor<string, []>("@model_path/weights/weight.bin"), offset = tensor<uint64, []>(52784256)))];
tensor<fp16, [1, 512, 512]> linear_11_cast_fp16 = linear(bias = m_encoder_layer_1_mlp_wo_bias_to_fp16, weight = m_encoder_layer_1_mlp_wo_weight_to_fp16, x = input_43_cast_fp16)[name = tensor<string, []>("linear_11_cast_fp16")];
tensor<fp16, [1, 512, 512]> input_47_cast_fp16 = add(x = linear_11_cast_fp16, y = input_39_cast_fp16)[name = tensor<string, []>("input_47_cast_fp16")];
tensor<int32, [1]> input_49_axes_0 = const()[name = tensor<string, []>("input_49_axes_0"), val = tensor<int32, [1]>([-1])];
tensor<fp16, [512]> m_encoder_layer_1_mlp_layernorm_weight_to_fp16 = const()[name = tensor<string, []>("m_encoder_layer_1_mlp_layernorm_weight_to_fp16"), val = tensor<fp16, [512]>(BLOBFILE(path = tensor<string, []>("@model_path/weights/weight.bin"), offset = tensor<uint64, []>(52785344)))];
tensor<fp16, [512]> m_encoder_layer_1_mlp_layernorm_bias_to_fp16 = const()[name = tensor<string, []>("m_encoder_layer_1_mlp_layernorm_bias_to_fp16"), val = tensor<fp16, [512]>(BLOBFILE(path = tensor<string, []>("@model_path/weights/weight.bin"), offset = tensor<uint64, []>(52786432)))];
tensor<fp16, [1, 512, 512]> input_49_cast_fp16 = layer_norm(axes = input_49_axes_0, beta = m_encoder_layer_1_mlp_layernorm_bias_to_fp16, epsilon = var_18_to_fp16, gamma = m_encoder_layer_1_mlp_layernorm_weight_to_fp16, x = input_47_cast_fp16)[name = tensor<string, []>("input_49_cast_fp16")];
tensor<fp16, [512, 512]> m_encoder_layer_2_attention_self_query_weight_to_fp16 = const()[name = tensor<string, []>("m_encoder_layer_2_attention_self_query_weight_to_fp16"), val = tensor<fp16, [512, 512]>(BLOBFILE(path = tensor<string, []>("@model_path/weights/weight.bin"), offset = tensor<uint64, []>(52787520)))];
tensor<fp16, [512]> m_encoder_layer_2_attention_self_query_bias_to_fp16 = const()[name = tensor<string, []>("m_encoder_layer_2_attention_self_query_bias_to_fp16"), val = tensor<fp16, [512]>(BLOBFILE(path = tensor<string, []>("@model_path/weights/weight.bin"), offset = tensor<uint64, []>(53311872)))];
tensor<fp16, [1, 512, 512]> linear_12_cast_fp16 = linear(bias = m_encoder_layer_2_attention_self_query_bias_to_fp16, weight = m_encoder_layer_2_attention_self_query_weight_to_fp16, x = input_49_cast_fp16)[name = tensor<string, []>("linear_12_cast_fp16")];
tensor<fp16, [512, 512]> m_encoder_layer_2_attention_self_key_weight_to_fp16 = const()[name = tensor<string, []>("m_encoder_layer_2_attention_self_key_weight_to_fp16"), val = tensor<fp16, [512, 512]>(BLOBFILE(path = tensor<string, []>("@model_path/weights/weight.bin"), offset = tensor<uint64, []>(53312960)))];
tensor<fp16, [512]> m_encoder_layer_2_attention_self_key_bias_to_fp16 = const()[name = tensor<string, []>("m_encoder_layer_2_attention_self_key_bias_to_fp16"), val = tensor<fp16, [512]>(BLOBFILE(path = tensor<string, []>("@model_path/weights/weight.bin"), offset = tensor<uint64, []>(53837312)))];
tensor<fp16, [1, 512, 512]> linear_13_cast_fp16 = linear(bias = m_encoder_layer_2_attention_self_key_bias_to_fp16, weight = m_encoder_layer_2_attention_self_key_weight_to_fp16, x = input_49_cast_fp16)[name = tensor<string, []>("linear_13_cast_fp16")];
tensor<int32, [4]> var_256 = const()[name = tensor<string, []>("op_256"), val = tensor<int32, [4]>([1, 512, 8, 64])];
tensor<fp16, [1, 512, 8, 64]> x_27_cast_fp16 = reshape(shape = var_256, x = linear_13_cast_fp16)[name = tensor<string, []>("x_27_cast_fp16")];
tensor<fp16, [512, 512]> m_encoder_layer_2_attention_self_value_weight_to_fp16 = const()[name = tensor<string, []>("m_encoder_layer_2_attention_self_value_weight_to_fp16"), val = tensor<fp16, [512, 512]>(BLOBFILE(path = tensor<string, []>("@model_path/weights/weight.bin"), offset = tensor<uint64, []>(53838400)))];
tensor<fp16, [512]> m_encoder_layer_2_attention_self_value_bias_to_fp16 = const()[name = tensor<string, []>("m_encoder_layer_2_attention_self_value_bias_to_fp16"), val = tensor<fp16, [512]>(BLOBFILE(path = tensor<string, []>("@model_path/weights/weight.bin"), offset = tensor<uint64, []>(54362752)))];
tensor<fp16, [1, 512, 512]> linear_14_cast_fp16 = linear(bias = m_encoder_layer_2_attention_self_value_bias_to_fp16, weight = m_encoder_layer_2_attention_self_value_weight_to_fp16, x = input_49_cast_fp16)[name = tensor<string, []>("linear_14_cast_fp16")];
tensor<int32, [4]> var_265 = const()[name = tensor<string, []>("op_265"), val = tensor<int32, [4]>([1, 512, 8, 64])];
tensor<fp16, [1, 512, 8, 64]> x_31_cast_fp16 = reshape(shape = var_265, x = linear_14_cast_fp16)[name = tensor<string, []>("x_31_cast_fp16")];
tensor<int32, [4]> var_267 = const()[name = tensor<string, []>("op_267"), val = tensor<int32, [4]>([0, 2, 1, 3])];
tensor<int32, [4]> var_271 = const()[name = tensor<string, []>("op_271"), val = tensor<int32, [4]>([1, 512, 8, 64])];
tensor<fp16, [1, 512, 8, 64]> x_35_cast_fp16 = reshape(shape = var_271, x = linear_12_cast_fp16)[name = tensor<string, []>("x_35_cast_fp16")];
tensor<bool, []> attention_scores_13_transpose_x_0 = const()[name = tensor<string, []>("attention_scores_13_transpose_x_0"), val = tensor<bool, []>(false)];
tensor<bool, []> attention_scores_13_transpose_y_0 = const()[name = tensor<string, []>("attention_scores_13_transpose_y_0"), val = tensor<bool, []>(false)];
tensor<int32, [4]> transpose_16_perm_0 = const()[name = tensor<string, []>("transpose_16_perm_0"), val = tensor<int32, [4]>([0, 2, -3, -1])];
tensor<int32, [4]> transpose_17_perm_0 = const()[name = tensor<string, []>("transpose_17_perm_0"), val = tensor<int32, [4]>([0, 2, -1, -3])];
tensor<fp16, [1, 8, 64, 512]> transpose_17 = transpose(perm = transpose_17_perm_0, x = x_27_cast_fp16)[name = tensor<string, []>("transpose_25")];
tensor<fp16, [1, 8, 512, 64]> transpose_16 = transpose(perm = transpose_16_perm_0, x = x_35_cast_fp16)[name = tensor<string, []>("transpose_26")];
tensor<fp16, [1, 8, 512, 512]> attention_scores_13_cast_fp16 = matmul(transpose_x = attention_scores_13_transpose_x_0, transpose_y = attention_scores_13_transpose_y_0, x = transpose_16, y = transpose_17)[name = tensor<string, []>("attention_scores_13_cast_fp16")];
tensor<fp16, []> _inversed_attention_scores_15_y_0_to_fp16 = const()[name = tensor<string, []>("_inversed_attention_scores_15_y_0_to_fp16"), val = tensor<fp16, []>(0x1p-3)];
tensor<fp16, [1, 8, 512, 512]> _inversed_attention_scores_15_cast_fp16 = mul(x = attention_scores_13_cast_fp16, y = _inversed_attention_scores_15_y_0_to_fp16)[name = tensor<string, []>("_inversed_attention_scores_15_cast_fp16")];
tensor<fp16, [1, 8, 512, 512]> attention_scores_17_cast_fp16 = add(x = _inversed_attention_scores_15_cast_fp16, y = attention_mask_cast_fp16)[name = tensor<string, []>("attention_scores_17_cast_fp16")];
tensor<fp16, [1, 8, 512, 512]> input_51_cast_fp16 = add(x = attention_scores_17_cast_fp16, y = bias_3_to_fp16)[name = tensor<string, []>("input_51_cast_fp16")];
tensor<fp16, [1, 8, 512, 512]> input_53_cast_fp16 = softmax(axis = var_16, x = input_51_cast_fp16)[name = tensor<string, []>("input_53_cast_fp16")];
tensor<bool, []> context_layer_9_transpose_x_0 = const()[name = tensor<string, []>("context_layer_9_transpose_x_0"), val = tensor<bool, []>(false)];
tensor<bool, []> context_layer_9_transpose_y_0 = const()[name = tensor<string, []>("context_layer_9_transpose_y_0"), val = tensor<bool, []>(false)];
tensor<fp16, [1, 8, 512, 64]> value_layer_5_cast_fp16 = transpose(perm = var_267, x = x_31_cast_fp16)[name = tensor<string, []>("transpose_27")];
tensor<fp16, [1, 8, 512, 64]> context_layer_9_cast_fp16 = matmul(transpose_x = context_layer_9_transpose_x_0, transpose_y = context_layer_9_transpose_y_0, x = input_53_cast_fp16, y = value_layer_5_cast_fp16)[name = tensor<string, []>("context_layer_9_cast_fp16")];
tensor<int32, [4]> var_284 = const()[name = tensor<string, []>("op_284"), val = tensor<int32, [4]>([0, 2, 1, 3])];
tensor<int32, [3]> var_289 = const()[name = tensor<string, []>("op_289"), val = tensor<int32, [3]>([1, 512, 512])];
tensor<fp16, [1, 512, 8, 64]> var_285_cast_fp16 = transpose(perm = var_284, x = context_layer_9_cast_fp16)[name = tensor<string, []>("transpose_24")];
tensor<fp16, [1, 512, 512]> input_55_cast_fp16 = reshape(shape = var_289, x = var_285_cast_fp16)[name = tensor<string, []>("input_55_cast_fp16")];
tensor<fp16, [512, 512]> m_encoder_layer_2_attention_output_dense_weight_to_fp16 = const()[name = tensor<string, []>("m_encoder_layer_2_attention_output_dense_weight_to_fp16"), val = tensor<fp16, [512, 512]>(BLOBFILE(path = tensor<string, []>("@model_path/weights/weight.bin"), offset = tensor<uint64, []>(54363840)))];
tensor<fp16, [512]> m_encoder_layer_2_attention_output_dense_bias_to_fp16 = const()[name = tensor<string, []>("m_encoder_layer_2_attention_output_dense_bias_to_fp16"), val = tensor<fp16, [512]>(BLOBFILE(path = tensor<string, []>("@model_path/weights/weight.bin"), offset = tensor<uint64, []>(54888192)))];
tensor<fp16, [1, 512, 512]> linear_15_cast_fp16 = linear(bias = m_encoder_layer_2_attention_output_dense_bias_to_fp16, weight = m_encoder_layer_2_attention_output_dense_weight_to_fp16, x = input_55_cast_fp16)[name = tensor<string, []>("linear_15_cast_fp16")];
tensor<fp16, [1, 512, 512]> input_59_cast_fp16 = add(x = linear_15_cast_fp16, y = input_49_cast_fp16)[name = tensor<string, []>("input_59_cast_fp16")];
tensor<int32, [1]> input_61_axes_0 = const()[name = tensor<string, []>("input_61_axes_0"), val = tensor<int32, [1]>([-1])];
tensor<fp16, [512]> m_encoder_layer_2_attention_output_LayerNorm_weight_to_fp16 = const()[name = tensor<string, []>("m_encoder_layer_2_attention_output_LayerNorm_weight_to_fp16"), val = tensor<fp16, [512]>(BLOBFILE(path = tensor<string, []>("@model_path/weights/weight.bin"), offset = tensor<uint64, []>(54889280)))];
tensor<fp16, [512]> m_encoder_layer_2_attention_output_LayerNorm_bias_to_fp16 = const()[name = tensor<string, []>("m_encoder_layer_2_attention_output_LayerNorm_bias_to_fp16"), val = tensor<fp16, [512]>(BLOBFILE(path = tensor<string, []>("@model_path/weights/weight.bin"), offset = tensor<uint64, []>(54890368)))];
tensor<fp16, [1, 512, 512]> input_61_cast_fp16 = layer_norm(axes = input_61_axes_0, beta = m_encoder_layer_2_attention_output_LayerNorm_bias_to_fp16, epsilon = var_18_to_fp16, gamma = m_encoder_layer_2_attention_output_LayerNorm_weight_to_fp16, x = input_59_cast_fp16)[name = tensor<string, []>("input_61_cast_fp16")];
tensor<fp16, [4096, 512]> m_encoder_layer_2_mlp_gated_layers_weight_to_fp16 = const()[name = tensor<string, []>("m_encoder_layer_2_mlp_gated_layers_weight_to_fp16"), val = tensor<fp16, [4096, 512]>(BLOBFILE(path = tensor<string, []>("@model_path/weights/weight.bin"), offset = tensor<uint64, []>(54891456)))];
tensor<fp16, [1, 512, 4096]> linear_16_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = m_encoder_layer_2_mlp_gated_layers_weight_to_fp16, x = input_61_cast_fp16)[name = tensor<string, []>("linear_16_cast_fp16")];
tensor<int32, [3]> input_63_begin_0 = const()[name = tensor<string, []>("input_63_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
tensor<int32, [3]> input_63_end_0 = const()[name = tensor<string, []>("input_63_end_0"), val = tensor<int32, [3]>([1, 512, 2048])];
tensor<bool, [3]> input_63_end_mask_0 = const()[name = tensor<string, []>("input_63_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
tensor<fp16, [1, 512, 2048]> input_63_cast_fp16 = slice_by_index(begin = input_63_begin_0, end = input_63_end_0, end_mask = input_63_end_mask_0, x = linear_16_cast_fp16)[name = tensor<string, []>("input_63_cast_fp16")];
tensor<int32, [3]> non_gated_5_begin_0 = const()[name = tensor<string, []>("non_gated_5_begin_0"), val = tensor<int32, [3]>([0, 0, 2048])];
tensor<int32, [3]> non_gated_5_end_0 = const()[name = tensor<string, []>("non_gated_5_end_0"), val = tensor<int32, [3]>([1, 512, 4096])];
tensor<bool, [3]> non_gated_5_end_mask_0 = const()[name = tensor<string, []>("non_gated_5_end_mask_0"), val = tensor<bool, [3]>([true, true, true])];
tensor<fp16, [1, 512, 2048]> non_gated_5_cast_fp16 = slice_by_index(begin = non_gated_5_begin_0, end = non_gated_5_end_0, end_mask = non_gated_5_end_mask_0, x = linear_16_cast_fp16)[name = tensor<string, []>("non_gated_5_cast_fp16")];
tensor<string, []> var_313_mode_0 = const()[name = tensor<string, []>("op_313_mode_0"), val = tensor<string, []>("EXACT")];
tensor<fp16, [1, 512, 2048]> var_313_cast_fp16 = gelu(mode = var_313_mode_0, x = input_63_cast_fp16)[name = tensor<string, []>("op_313_cast_fp16")];
tensor<fp16, [1, 512, 2048]> input_65_cast_fp16 = mul(x = var_313_cast_fp16, y = non_gated_5_cast_fp16)[name = tensor<string, []>("input_65_cast_fp16")];
tensor<fp16, [512, 2048]> m_encoder_layer_2_mlp_wo_weight_to_fp16 = const()[name = tensor<string, []>("m_encoder_layer_2_mlp_wo_weight_to_fp16"), val = tensor<fp16, [512, 2048]>(BLOBFILE(path = tensor<string, []>("@model_path/weights/weight.bin"), offset = tensor<uint64, []>(59085824)))];
tensor<fp16, [512]> m_encoder_layer_2_mlp_wo_bias_to_fp16 = const()[name = tensor<string, []>("m_encoder_layer_2_mlp_wo_bias_to_fp16"), val = tensor<fp16, [512]>(BLOBFILE(path = tensor<string, []>("@model_path/weights/weight.bin"), offset = tensor<uint64, []>(61183040)))];
tensor<fp16, [1, 512, 512]> linear_17_cast_fp16 = linear(bias = m_encoder_layer_2_mlp_wo_bias_to_fp16, weight = m_encoder_layer_2_mlp_wo_weight_to_fp16, x = input_65_cast_fp16)[name = tensor<string, []>("linear_17_cast_fp16")];
tensor<fp16, [1, 512, 512]> input_69_cast_fp16 = add(x = linear_17_cast_fp16, y = input_61_cast_fp16)[name = tensor<string, []>("input_69_cast_fp16")];
tensor<int32, [1]> input_71_axes_0 = const()[name = tensor<string, []>("input_71_axes_0"), val = tensor<int32, [1]>([-1])];
tensor<fp16, [512]> m_encoder_layer_2_mlp_layernorm_weight_to_fp16 = const()[name = tensor<string, []>("m_encoder_layer_2_mlp_layernorm_weight_to_fp16"), val = tensor<fp16, [512]>(BLOBFILE(path = tensor<string, []>("@model_path/weights/weight.bin"), offset = tensor<uint64, []>(61184128)))];
tensor<fp16, [512]> m_encoder_layer_2_mlp_layernorm_bias_to_fp16 = const()[name = tensor<string, []>("m_encoder_layer_2_mlp_layernorm_bias_to_fp16"), val = tensor<fp16, [512]>(BLOBFILE(path = tensor<string, []>("@model_path/weights/weight.bin"), offset = tensor<uint64, []>(61185216)))];
tensor<fp16, [1, 512, 512]> input_71_cast_fp16 = layer_norm(axes = input_71_axes_0, beta = m_encoder_layer_2_mlp_layernorm_bias_to_fp16, epsilon = var_18_to_fp16, gamma = m_encoder_layer_2_mlp_layernorm_weight_to_fp16, x = input_69_cast_fp16)[name = tensor<string, []>("input_71_cast_fp16")];
tensor<fp16, [512, 512]> m_encoder_layer_3_attention_self_query_weight_to_fp16 = const()[name = tensor<string, []>("m_encoder_layer_3_attention_self_query_weight_to_fp16"), val = tensor<fp16, [512, 512]>(BLOBFILE(path = tensor<string, []>("@model_path/weights/weight.bin"), offset = tensor<uint64, []>(61186304)))];
tensor<fp16, [512]> m_encoder_layer_3_attention_self_query_bias_to_fp16 = const()[name = tensor<string, []>("m_encoder_layer_3_attention_self_query_bias_to_fp16"), val = tensor<fp16, [512]>(BLOBFILE(path = tensor<string, []>("@model_path/weights/weight.bin"), offset = tensor<uint64, []>(61710656)))];
tensor<fp16, [1, 512, 512]> linear_18_cast_fp16 = linear(bias = m_encoder_layer_3_attention_self_query_bias_to_fp16, weight = m_encoder_layer_3_attention_self_query_weight_to_fp16, x = input_71_cast_fp16)[name = tensor<string, []>("linear_18_cast_fp16")];
tensor<fp16, [512, 512]> m_encoder_layer_3_attention_self_key_weight_to_fp16 = const()[name = tensor<string, []>("m_encoder_layer_3_attention_self_key_weight_to_fp16"), val = tensor<fp16, [512, 512]>(BLOBFILE(path = tensor<string, []>("@model_path/weights/weight.bin"), offset = tensor<uint64, []>(61711744)))];
tensor<fp16, [512]> m_encoder_layer_3_attention_self_key_bias_to_fp16 = const()[name = tensor<string, []>("m_encoder_layer_3_attention_self_key_bias_to_fp16"), val = tensor<fp16, [512]>(BLOBFILE(path = tensor<string, []>("@model_path/weights/weight.bin"), offset = tensor<uint64, []>(62236096)))];
tensor<fp16, [1, 512, 512]> linear_19_cast_fp16 = linear(bias = m_encoder_layer_3_attention_self_key_bias_to_fp16, weight = m_encoder_layer_3_attention_self_key_weight_to_fp16, x = input_71_cast_fp16)[name = tensor<string, []>("linear_19_cast_fp16")];
tensor<int32, [4]> var_339 = const()[name = tensor<string, []>("op_339"), val = tensor<int32, [4]>([1, 512, 8, 64])];
tensor<fp16, [1, 512, 8, 64]> x_39_cast_fp16 = reshape(shape = var_339, x = linear_19_cast_fp16)[name = tensor<string, []>("x_39_cast_fp16")];
tensor<fp16, [512, 512]> m_encoder_layer_3_attention_self_value_weight_to_fp16 = const()[name = tensor<string, []>("m_encoder_layer_3_attention_self_value_weight_to_fp16"), val = tensor<fp16, [512, 512]>(BLOBFILE(path = tensor<string, []>("@model_path/weights/weight.bin"), offset = tensor<uint64, []>(62237184)))];
tensor<fp16, [512]> m_encoder_layer_3_attention_self_value_bias_to_fp16 = const()[name = tensor<string, []>("m_encoder_layer_3_attention_self_value_bias_to_fp16"), val = tensor<fp16, [512]>(BLOBFILE(path = tensor<string, []>("@model_path/weights/weight.bin"), offset = tensor<uint64, []>(62761536)))];
tensor<fp16, [1, 512, 512]> linear_20_cast_fp16 = linear(bias = m_encoder_layer_3_attention_self_value_bias_to_fp16, weight = m_encoder_layer_3_attention_self_value_weight_to_fp16, x = input_71_cast_fp16)[name = tensor<string, []>("linear_20_cast_fp16")];
tensor<int32, [4]> var_348 = const()[name = tensor<string, []>("op_348"), val = tensor<int32, [4]>([1, 512, 8, 64])];
tensor<fp16, [1, 512, 8, 64]> x_43_cast_fp16 = reshape(shape = var_348, x = linear_20_cast_fp16)[name = tensor<string, []>("x_43_cast_fp16")];
tensor<int32, [4]> var_350 = const()[name = tensor<string, []>("op_350"), val = tensor<int32, [4]>([0, 2, 1, 3])];
tensor<int32, [4]> var_354 = const()[name = tensor<string, []>("op_354"), val = tensor<int32, [4]>([1, 512, 8, 64])];
tensor<fp16, [1, 512, 8, 64]> x_cast_fp16 = reshape(shape = var_354, x = linear_18_cast_fp16)[name = tensor<string, []>("x_cast_fp16")];
tensor<bool, []> attention_scores_19_transpose_x_0 = const()[name = tensor<string, []>("attention_scores_19_transpose_x_0"), val = tensor<bool, []>(false)];
tensor<bool, []> attention_scores_19_transpose_y_0 = const()[name = tensor<string, []>("attention_scores_19_transpose_y_0"), val = tensor<bool, []>(false)];
tensor<int32, [4]> transpose_18_perm_0 = const()[name = tensor<string, []>("transpose_18_perm_0"), val = tensor<int32, [4]>([0, 2, -3, -1])];
tensor<int32, [4]> transpose_19_perm_0 = const()[name = tensor<string, []>("transpose_19_perm_0"), val = tensor<int32, [4]>([0, 2, -1, -3])];
tensor<fp16, [1, 8, 64, 512]> transpose_19 = transpose(perm = transpose_19_perm_0, x = x_39_cast_fp16)[name = tensor<string, []>("transpose_21")];
tensor<fp16, [1, 8, 512, 64]> transpose_18 = transpose(perm = transpose_18_perm_0, x = x_cast_fp16)[name = tensor<string, []>("transpose_22")];
tensor<fp16, [1, 8, 512, 512]> attention_scores_19_cast_fp16 = matmul(transpose_x = attention_scores_19_transpose_x_0, transpose_y = attention_scores_19_transpose_y_0, x = transpose_18, y = transpose_19)[name = tensor<string, []>("attention_scores_19_cast_fp16")];
tensor<fp16, []> _inversed_attention_scores_21_y_0_to_fp16 = const()[name = tensor<string, []>("_inversed_attention_scores_21_y_0_to_fp16"), val = tensor<fp16, []>(0x1p-3)];
tensor<fp16, [1, 8, 512, 512]> _inversed_attention_scores_21_cast_fp16 = mul(x = attention_scores_19_cast_fp16, y = _inversed_attention_scores_21_y_0_to_fp16)[name = tensor<string, []>("_inversed_attention_scores_21_cast_fp16")];
tensor<fp16, [1, 8, 512, 512]> attention_scores_cast_fp16 = add(x = _inversed_attention_scores_21_cast_fp16, y = attention_mask_cast_fp16)[name = tensor<string, []>("attention_scores_cast_fp16")];
tensor<fp16, [1, 8, 512, 512]> input_73_cast_fp16 = add(x = attention_scores_cast_fp16, y = bias_3_to_fp16)[name = tensor<string, []>("input_73_cast_fp16")];
tensor<fp16, [1, 8, 512, 512]> input_75_cast_fp16 = softmax(axis = var_16, x = input_73_cast_fp16)[name = tensor<string, []>("input_75_cast_fp16")];
tensor<bool, []> context_layer_13_transpose_x_0 = const()[name = tensor<string, []>("context_layer_13_transpose_x_0"), val = tensor<bool, []>(false)];
tensor<bool, []> context_layer_13_transpose_y_0 = const()[name = tensor<string, []>("context_layer_13_transpose_y_0"), val = tensor<bool, []>(false)];
tensor<fp16, [1, 8, 512, 64]> value_layer_cast_fp16 = transpose(perm = var_350, x = x_43_cast_fp16)[name = tensor<string, []>("transpose_23")];
tensor<fp16, [1, 8, 512, 64]> context_layer_13_cast_fp16 = matmul(transpose_x = context_layer_13_transpose_x_0, transpose_y = context_layer_13_transpose_y_0, x = input_75_cast_fp16, y = value_layer_cast_fp16)[name = tensor<string, []>("context_layer_13_cast_fp16")];
tensor<int32, [4]> var_367 = const()[name = tensor<string, []>("op_367"), val = tensor<int32, [4]>([0, 2, 1, 3])];
tensor<int32, [3]> var_372 = const()[name = tensor<string, []>("op_372"), val = tensor<int32, [3]>([1, 512, 512])];
tensor<fp16, [1, 512, 8, 64]> var_368_cast_fp16 = transpose(perm = var_367, x = context_layer_13_cast_fp16)[name = tensor<string, []>("transpose_20")];
tensor<fp16, [1, 512, 512]> input_77_cast_fp16 = reshape(shape = var_372, x = var_368_cast_fp16)[name = tensor<string, []>("input_77_cast_fp16")];
tensor<fp16, [512, 512]> m_encoder_layer_3_attention_output_dense_weight_to_fp16 = const()[name = tensor<string, []>("m_encoder_layer_3_attention_output_dense_weight_to_fp16"), val = tensor<fp16, [512, 512]>(BLOBFILE(path = tensor<string, []>("@model_path/weights/weight.bin"), offset = tensor<uint64, []>(62762624)))];
tensor<fp16, [512]> m_encoder_layer_3_attention_output_dense_bias_to_fp16 = const()[name = tensor<string, []>("m_encoder_layer_3_attention_output_dense_bias_to_fp16"), val = tensor<fp16, [512]>(BLOBFILE(path = tensor<string, []>("@model_path/weights/weight.bin"), offset = tensor<uint64, []>(63286976)))];
tensor<fp16, [1, 512, 512]> linear_21_cast_fp16 = linear(bias = m_encoder_layer_3_attention_output_dense_bias_to_fp16, weight = m_encoder_layer_3_attention_output_dense_weight_to_fp16, x = input_77_cast_fp16)[name = tensor<string, []>("linear_21_cast_fp16")];
tensor<fp16, [1, 512, 512]> input_81_cast_fp16 = add(x = linear_21_cast_fp16, y = input_71_cast_fp16)[name = tensor<string, []>("input_81_cast_fp16")];
tensor<int32, [1]> input_83_axes_0 = const()[name = tensor<string, []>("input_83_axes_0"), val = tensor<int32, [1]>([-1])];
tensor<fp16, [512]> m_encoder_layer_3_attention_output_LayerNorm_weight_to_fp16 = const()[name = tensor<string, []>("m_encoder_layer_3_attention_output_LayerNorm_weight_to_fp16"), val = tensor<fp16, [512]>(BLOBFILE(path = tensor<string, []>("@model_path/weights/weight.bin"), offset = tensor<uint64, []>(63288064)))];
tensor<fp16, [512]> m_encoder_layer_3_attention_output_LayerNorm_bias_to_fp16 = const()[name = tensor<string, []>("m_encoder_layer_3_attention_output_LayerNorm_bias_to_fp16"), val = tensor<fp16, [512]>(BLOBFILE(path = tensor<string, []>("@model_path/weights/weight.bin"), offset = tensor<uint64, []>(63289152)))];
tensor<fp16, [1, 512, 512]> input_83_cast_fp16 = layer_norm(axes = input_83_axes_0, beta = m_encoder_layer_3_attention_output_LayerNorm_bias_to_fp16, epsilon = var_18_to_fp16, gamma = m_encoder_layer_3_attention_output_LayerNorm_weight_to_fp16, x = input_81_cast_fp16)[name = tensor<string, []>("input_83_cast_fp16")];
tensor<fp16, [4096, 512]> m_encoder_layer_3_mlp_gated_layers_weight_to_fp16 = const()[name = tensor<string, []>("m_encoder_layer_3_mlp_gated_layers_weight_to_fp16"), val = tensor<fp16, [4096, 512]>(BLOBFILE(path = tensor<string, []>("@model_path/weights/weight.bin"), offset = tensor<uint64, []>(63290240)))];
tensor<fp16, [1, 512, 4096]> linear_22_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = m_encoder_layer_3_mlp_gated_layers_weight_to_fp16, x = input_83_cast_fp16)[name = tensor<string, []>("linear_22_cast_fp16")];
tensor<int32, [3]> input_85_begin_0 = const()[name = tensor<string, []>("input_85_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
tensor<int32, [3]> input_85_end_0 = const()[name = tensor<string, []>("input_85_end_0"), val = tensor<int32, [3]>([1, 512, 2048])];
tensor<bool, [3]> input_85_end_mask_0 = const()[name = tensor<string, []>("input_85_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
tensor<fp16, [1, 512, 2048]> input_85_cast_fp16 = slice_by_index(begin = input_85_begin_0, end = input_85_end_0, end_mask = input_85_end_mask_0, x = linear_22_cast_fp16)[name = tensor<string, []>("input_85_cast_fp16")];
tensor<int32, [3]> non_gated_begin_0 = const()[name = tensor<string, []>("non_gated_begin_0"), val = tensor<int32, [3]>([0, 0, 2048])];
tensor<int32, [3]> non_gated_end_0 = const()[name = tensor<string, []>("non_gated_end_0"), val = tensor<int32, [3]>([1, 512, 4096])];
tensor<bool, [3]> non_gated_end_mask_0 = const()[name = tensor<string, []>("non_gated_end_mask_0"), val = tensor<bool, [3]>([true, true, true])];
tensor<fp16, [1, 512, 2048]> non_gated_cast_fp16 = slice_by_index(begin = non_gated_begin_0, end = non_gated_end_0, end_mask = non_gated_end_mask_0, x = linear_22_cast_fp16)[name = tensor<string, []>("non_gated_cast_fp16")];
tensor<string, []> var_396_mode_0 = const()[name = tensor<string, []>("op_396_mode_0"), val = tensor<string, []>("EXACT")];
tensor<fp16, [1, 512, 2048]> var_396_cast_fp16 = gelu(mode = var_396_mode_0, x = input_85_cast_fp16)[name = tensor<string, []>("op_396_cast_fp16")];
tensor<fp16, [1, 512, 2048]> input_87_cast_fp16 = mul(x = var_396_cast_fp16, y = non_gated_cast_fp16)[name = tensor<string, []>("input_87_cast_fp16")];
tensor<fp16, [512, 2048]> m_encoder_layer_3_mlp_wo_weight_to_fp16 = const()[name = tensor<string, []>("m_encoder_layer_3_mlp_wo_weight_to_fp16"), val = tensor<fp16, [512, 2048]>(BLOBFILE(path = tensor<string, []>("@model_path/weights/weight.bin"), offset = tensor<uint64, []>(67484608)))];
tensor<fp16, [512]> m_encoder_layer_3_mlp_wo_bias_to_fp16 = const()[name = tensor<string, []>("m_encoder_layer_3_mlp_wo_bias_to_fp16"), val = tensor<fp16, [512]>(BLOBFILE(path = tensor<string, []>("@model_path/weights/weight.bin"), offset = tensor<uint64, []>(69581824)))];
tensor<fp16, [1, 512, 512]> linear_23_cast_fp16 = linear(bias = m_encoder_layer_3_mlp_wo_bias_to_fp16, weight = m_encoder_layer_3_mlp_wo_weight_to_fp16, x = input_87_cast_fp16)[name = tensor<string, []>("linear_23_cast_fp16")];
tensor<fp16, [1, 512, 512]> input_91_cast_fp16 = add(x = linear_23_cast_fp16, y = input_83_cast_fp16)[name = tensor<string, []>("input_91_cast_fp16")];
tensor<int32, [1]> hidden_states_axes_0 = const()[name = tensor<string, []>("hidden_states_axes_0"), val = tensor<int32, [1]>([-1])];
tensor<fp16, [512]> m_encoder_layer_3_mlp_layernorm_weight_to_fp16 = const()[name = tensor<string, []>("m_encoder_layer_3_mlp_layernorm_weight_to_fp16"), val = tensor<fp16, [512]>(BLOBFILE(path = tensor<string, []>("@model_path/weights/weight.bin"), offset = tensor<uint64, []>(69582912)))];
tensor<fp16, [512]> m_encoder_layer_3_mlp_layernorm_bias_to_fp16 = const()[name = tensor<string, []>("m_encoder_layer_3_mlp_layernorm_bias_to_fp16"), val = tensor<fp16, [512]>(BLOBFILE(path = tensor<string, []>("@model_path/weights/weight.bin"), offset = tensor<uint64, []>(69584000)))];
tensor<fp16, [1, 512, 512]> hidden_states_cast_fp16 = layer_norm(axes = hidden_states_axes_0, beta = m_encoder_layer_3_mlp_layernorm_bias_to_fp16, epsilon = var_18_to_fp16, gamma = m_encoder_layer_3_mlp_layernorm_weight_to_fp16, x = input_91_cast_fp16)[name = tensor<string, []>("hidden_states_cast_fp16")];
tensor<int32, [1]> var_408_axes_0 = const()[name = tensor<string, []>("op_408_axes_0"), val = tensor<int32, [1]>([-1])];
tensor<int32, [1, 512, 1]> var_408 = expand_dims(axes = var_408_axes_0, x = cast_0)[name = tensor<string, []>("op_408")];
tensor<string, []> cast_28_to_fp16_dtype_0 = const()[name = tensor<string, []>("cast_28_to_fp16_dtype_0"), val = tensor<string, []>("fp16")];
tensor<fp16, [1, 512, 1]> var_408_to_fp16 = cast(dtype = cast_28_to_fp16_dtype_0, x = var_408)[name = tensor<string, []>("cast_34")];
tensor<fp16, [1, 512, 512]> var_414_cast_fp16 = mul(x = hidden_states_cast_fp16, y = var_408_to_fp16)[name = tensor<string, []>("op_414_cast_fp16")];
tensor<int32, [1]> var_419_axes_0 = const()[name = tensor<string, []>("op_419_axes_0"), val = tensor<int32, [1]>([1])];
tensor<bool, []> var_419_keep_dims_0 = const()[name = tensor<string, []>("op_419_keep_dims_0"), val = tensor<bool, []>(false)];
tensor<fp16, [1, 512]> var_419_cast_fp16 = reduce_sum(axes = var_419_axes_0, keep_dims = var_419_keep_dims_0, x = var_414_cast_fp16)[name = tensor<string, []>("op_419_cast_fp16")];
tensor<int32, [1]> var_424_axes_0 = const()[name = tensor<string, []>("op_424_axes_0"), val = tensor<int32, [1]>([1])];
tensor<bool, []> var_424_keep_dims_0 = const()[name = tensor<string, []>("op_424_keep_dims_0"), val = tensor<bool, []>(false)];
tensor<fp16, [1, 1]> var_424_cast_fp16 = reduce_sum(axes = var_424_axes_0, keep_dims = var_424_keep_dims_0, x = var_408_to_fp16)[name = tensor<string, []>("op_424_cast_fp16")];
tensor<fp16, []> var_425_to_fp16 = const()[name = tensor<string, []>("op_425_to_fp16"), val = tensor<fp16, []>(0x1p-24)];
tensor<fp16, []> const_35_to_fp16 = const()[name = tensor<string, []>("const_35_to_fp16"), val = tensor<fp16, []>(inf)];
tensor<fp16, [1, 1]> clip_0_cast_fp16 = clip(alpha = var_425_to_fp16, beta = const_35_to_fp16, x = var_424_cast_fp16)[name = tensor<string, []>("clip_0_cast_fp16")];
tensor<fp16, [1, 512]> input_cast_fp16 = real_div(x = var_419_cast_fp16, y = clip_0_cast_fp16)[name = tensor<string, []>("input_cast_fp16")];
tensor<int32, [1]> var_431 = const()[name = tensor<string, []>("op_431"), val = tensor<int32, [1]>([-1])];
tensor<bool, []> var_432 = const()[name = tensor<string, []>("op_432"), val = tensor<bool, []>(true)];
tensor<fp16, [1, 1]> var_434_cast_fp16 = reduce_l2_norm(axes = var_431, keep_dims = var_432, x = input_cast_fp16)[name = tensor<string, []>("op_434_cast_fp16")];
tensor<fp16, []> var_435_to_fp16 = const()[name = tensor<string, []>("op_435_to_fp16"), val = tensor<fp16, []>(0x1p-24)];
tensor<fp16, [1, 1]> var_436_cast_fp16 = maximum(x = var_434_cast_fp16, y = var_435_to_fp16)[name = tensor<string, []>("op_436_cast_fp16")];
tensor<int32, [2]> denom_reps_0 = const()[name = tensor<string, []>("denom_reps_0"), val = tensor<int32, [2]>([1, 512])];
tensor<fp16, [1, 512]> denom_cast_fp16 = tile(reps = denom_reps_0, x = var_436_cast_fp16)[name = tensor<string, []>("denom_cast_fp16")];
tensor<fp16, [1, 512]> embedding = real_div(x = input_cast_fp16, y = denom_cast_fp16)[name = tensor<string, []>("op_438_cast_fp16")];
} -> (embedding);
} |