program(1.3) [buildInfo = dict({{"coremlc-component-MIL", "3520.4.1"}, {"coremlc-version", "3520.5.1"}, {"coremltools-component-torch", "2.11.0"}, {"coremltools-source-dialect", "TorchScript"}, {"coremltools-version", "9.0"}})] { func main(tensor attention_mask, tensor tokens) { int32 inputs_embeds_batch_dims_0 = const()[name = string("inputs_embeds_batch_dims_0"), val = int32(0)]; bool inputs_embeds_validate_indices_0 = const()[name = string("inputs_embeds_validate_indices_0"), val = bool(false)]; tensor bert_embeddings_word_embeddings_weight_to_fp16 = const()[name = string("bert_embeddings_word_embeddings_weight_to_fp16"), val = tensor(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(64)))]; string tokens_to_int16_dtype_0 = const()[name = string("tokens_to_int16_dtype_0"), val = string("int16")]; string cast_53_dtype_0 = const()[name = string("cast_53_dtype_0"), val = string("int32")]; int32 greater_equal_0_y_0 = const()[name = string("greater_equal_0_y_0"), val = int32(0)]; tensor tokens_to_int16 = cast(dtype = tokens_to_int16_dtype_0, x = tokens)[name = string("cast_58")]; tensor cast_53 = cast(dtype = cast_53_dtype_0, x = tokens_to_int16)[name = string("cast_57")]; tensor greater_equal_0 = greater_equal(x = cast_53, y = greater_equal_0_y_0)[name = string("greater_equal_0")]; int32 slice_by_index_0 = const()[name = string("slice_by_index_0"), val = int32(178)]; tensor add_0 = add(x = cast_53, y = slice_by_index_0)[name = string("add_0")]; tensor select_0 = select(a = cast_53, b = add_0, cond = greater_equal_0)[name = string("select_0")]; int32 inputs_embeds_cast_fp16_cast_uint16_axis_0 = const()[name = string("inputs_embeds_cast_fp16_cast_uint16_axis_0"), val = int32(0)]; string select_0_to_int16_dtype_0 = const()[name = string("select_0_to_int16_dtype_0"), val = string("int16")]; tensor select_0_to_int16 = cast(dtype = select_0_to_int16_dtype_0, x = select_0)[name = string("cast_56")]; tensor inputs_embeds_cast_fp16_cast_uint16_cast_uint16 = gather(axis = inputs_embeds_cast_fp16_cast_uint16_axis_0, batch_dims = inputs_embeds_batch_dims_0, indices = select_0_to_int16, validate_indices = inputs_embeds_validate_indices_0, x = bert_embeddings_word_embeddings_weight_to_fp16)[name = string("inputs_embeds_cast_fp16_cast_uint16_cast_uint16")]; tensor token_type_embeddings_1_to_fp16 = const()[name = string("token_type_embeddings_1_to_fp16"), val = tensor(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(45696)))]; tensor embeddings_1_cast_fp16 = add(x = inputs_embeds_cast_fp16_cast_uint16_cast_uint16, y = token_type_embeddings_1_to_fp16)[name = string("embeddings_1_cast_fp16")]; tensor position_embeddings_1_to_fp16 = const()[name = string("position_embeddings_1_to_fp16"), val = tensor(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(60352)))]; tensor input_5_cast_fp16 = add(x = embeddings_1_cast_fp16, y = position_embeddings_1_to_fp16)[name = string("input_5_cast_fp16")]; tensor input_7_axes_0 = const()[name = string("input_7_axes_0"), val = tensor([-1])]; tensor bert_embeddings_LayerNorm_weight_to_fp16 = const()[name = string("bert_embeddings_LayerNorm_weight_to_fp16"), val = tensor(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(75008)))]; tensor bert_embeddings_LayerNorm_bias_to_fp16 = const()[name = string("bert_embeddings_LayerNorm_bias_to_fp16"), val = tensor(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(75328)))]; fp16 var_34_to_fp16 = const()[name = string("op_34_to_fp16"), val = fp16(0x1p-24)]; tensor input_7_cast_fp16 = layer_norm(axes = input_7_axes_0, beta = bert_embeddings_LayerNorm_bias_to_fp16, epsilon = var_34_to_fp16, gamma = bert_embeddings_LayerNorm_weight_to_fp16, x = input_5_cast_fp16)[name = string("input_7_cast_fp16")]; tensor var_79_axes_0 = const()[name = string("op_79_axes_0"), val = tensor([1])]; tensor var_79 = expand_dims(axes = var_79_axes_0, x = attention_mask)[name = string("op_79")]; tensor var_81_axes_0 = const()[name = string("op_81_axes_0"), val = tensor([2])]; tensor var_81 = expand_dims(axes = var_81_axes_0, x = var_79)[name = string("op_81")]; tensor var_90_reps_0 = const()[name = string("op_90_reps_0"), val = tensor([1, 1, 57, 1])]; tensor var_90 = tile(reps = var_90_reps_0, x = var_81)[name = string("op_90")]; fp16 var_96_to_fp16 = const()[name = string("op_96_to_fp16"), val = fp16(0x1p+0)]; string var_95_to_fp16_dtype_0 = const()[name = string("op_95_to_fp16_dtype_0"), val = string("fp16")]; tensor var_90_to_fp16 = cast(dtype = var_95_to_fp16_dtype_0, x = var_90)[name = string("cast_55")]; tensor inverted_mask_cast_fp16 = sub(x = var_96_to_fp16, y = var_90_to_fp16)[name = string("inverted_mask_cast_fp16")]; string var_103_dtype_0 = const()[name = string("op_103_dtype_0"), val = string("bool")]; fp16 var_104_to_fp16 = const()[name = string("op_104_to_fp16"), val = fp16(-inf)]; tensor inverted_mask_cast_fp16_to_bool = cast(dtype = var_103_dtype_0, x = inverted_mask_cast_fp16)[name = string("cast_54")]; tensor attention_mask_cast_fp16 = select(a = var_104_to_fp16, b = inverted_mask_cast_fp16, cond = inverted_mask_cast_fp16_to_bool)[name = string("attention_mask_cast_fp16")]; tensor bert_encoder_embedding_hidden_mapping_in_weight_to_fp16 = const()[name = string("bert_encoder_embedding_hidden_mapping_in_weight_to_fp16"), val = tensor(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(75648)))]; tensor bert_encoder_embedding_hidden_mapping_in_bias_to_fp16 = const()[name = string("bert_encoder_embedding_hidden_mapping_in_bias_to_fp16"), val = tensor(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(272320)))]; tensor linear_0_cast_fp16 = linear(bias = bert_encoder_embedding_hidden_mapping_in_bias_to_fp16, weight = bert_encoder_embedding_hidden_mapping_in_weight_to_fp16, x = input_7_cast_fp16)[name = string("linear_0_cast_fp16")]; tensor bert_encoder_albert_layer_groups_0_albert_layers_0_attention_query_weight_to_fp16 = const()[name = string("bert_encoder_albert_layer_groups_0_albert_layers_0_attention_query_weight_to_fp16"), val = tensor(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(273920)))]; tensor bert_encoder_albert_layer_groups_0_albert_layers_0_attention_query_bias_to_fp16 = const()[name = string("bert_encoder_albert_layer_groups_0_albert_layers_0_attention_query_bias_to_fp16"), val = tensor(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(1453632)))]; tensor linear_1_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_query_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_query_weight_to_fp16, x = linear_0_cast_fp16)[name = string("linear_1_cast_fp16")]; tensor var_143 = const()[name = string("op_143"), val = tensor([1, 57, 12, 64])]; tensor x_3_cast_fp16 = reshape(shape = var_143, x = linear_1_cast_fp16)[name = string("x_3_cast_fp16")]; tensor bert_encoder_albert_layer_groups_0_albert_layers_0_attention_key_weight_to_fp16 = const()[name = string("bert_encoder_albert_layer_groups_0_albert_layers_0_attention_key_weight_to_fp16"), val = tensor(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(1455232)))]; tensor bert_encoder_albert_layer_groups_0_albert_layers_0_attention_key_bias_to_fp16 = const()[name = string("bert_encoder_albert_layer_groups_0_albert_layers_0_attention_key_bias_to_fp16"), val = tensor(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(2634944)))]; tensor linear_2_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_key_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_key_weight_to_fp16, x = linear_0_cast_fp16)[name = string("linear_2_cast_fp16")]; tensor var_152 = const()[name = string("op_152"), val = tensor([1, 57, 12, 64])]; tensor x_7_cast_fp16 = reshape(shape = var_152, x = linear_2_cast_fp16)[name = string("x_7_cast_fp16")]; tensor bert_encoder_albert_layer_groups_0_albert_layers_0_attention_value_weight_to_fp16 = const()[name = string("bert_encoder_albert_layer_groups_0_albert_layers_0_attention_value_weight_to_fp16"), val = tensor(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(2636544)))]; tensor bert_encoder_albert_layer_groups_0_albert_layers_0_attention_value_bias_to_fp16 = const()[name = string("bert_encoder_albert_layer_groups_0_albert_layers_0_attention_value_bias_to_fp16"), val = tensor(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(3816256)))]; tensor linear_3_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_value_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_value_weight_to_fp16, x = linear_0_cast_fp16)[name = string("linear_3_cast_fp16")]; tensor var_161 = const()[name = string("op_161"), val = tensor([1, 57, 12, 64])]; tensor x_11_cast_fp16 = reshape(shape = var_161, x = linear_3_cast_fp16)[name = string("x_11_cast_fp16")]; tensor transpose_72_perm_0 = const()[name = string("transpose_72_perm_0"), val = tensor([0, 2, 1, 3])]; tensor transpose_73_perm_0 = const()[name = string("transpose_73_perm_0"), val = tensor([0, 2, 1, 3])]; tensor transpose_74_perm_0 = const()[name = string("transpose_74_perm_0"), val = tensor([0, 2, 1, 3])]; tensor transpose_74 = transpose(perm = transpose_74_perm_0, x = x_11_cast_fp16)[name = string("transpose_154")]; tensor transpose_73 = transpose(perm = transpose_73_perm_0, x = x_7_cast_fp16)[name = string("transpose_155")]; tensor transpose_72 = transpose(perm = transpose_72_perm_0, x = x_3_cast_fp16)[name = string("transpose_156")]; tensor attention_output_1_cast_fp16 = scaled_dot_product_attention(attn_mask = attention_mask_cast_fp16, key = transpose_73, query = transpose_72, value = transpose_74)[name = string("attention_output_1_cast_fp16")]; tensor attention_output_3_perm_0 = const()[name = string("attention_output_3_perm_0"), val = tensor([0, 2, 1, 3])]; tensor var_167 = const()[name = string("op_167"), val = tensor([1, 57, 768])]; tensor attention_output_3_cast_fp16 = transpose(perm = attention_output_3_perm_0, x = attention_output_1_cast_fp16)[name = string("transpose_153")]; tensor input_9_cast_fp16 = reshape(shape = var_167, x = attention_output_3_cast_fp16)[name = string("input_9_cast_fp16")]; tensor bert_encoder_albert_layer_groups_0_albert_layers_0_attention_dense_weight_to_fp16 = const()[name = string("bert_encoder_albert_layer_groups_0_albert_layers_0_attention_dense_weight_to_fp16"), val = tensor(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(3817856)))]; tensor bert_encoder_albert_layer_groups_0_albert_layers_0_attention_dense_bias_to_fp16 = const()[name = string("bert_encoder_albert_layer_groups_0_albert_layers_0_attention_dense_bias_to_fp16"), val = tensor(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(4997568)))]; tensor linear_4_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_dense_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_dense_weight_to_fp16, x = input_9_cast_fp16)[name = string("linear_4_cast_fp16")]; tensor input_11_cast_fp16 = add(x = linear_0_cast_fp16, y = linear_4_cast_fp16)[name = string("input_11_cast_fp16")]; tensor input_13_axes_0 = const()[name = string("input_13_axes_0"), val = tensor([-1])]; tensor bert_encoder_albert_layer_groups_0_albert_layers_0_attention_LayerNorm_weight_to_fp16 = const()[name = string("bert_encoder_albert_layer_groups_0_albert_layers_0_attention_LayerNorm_weight_to_fp16"), val = tensor(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(4999168)))]; tensor bert_encoder_albert_layer_groups_0_albert_layers_0_attention_LayerNorm_bias_to_fp16 = const()[name = string("bert_encoder_albert_layer_groups_0_albert_layers_0_attention_LayerNorm_bias_to_fp16"), val = tensor(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(5000768)))]; fp16 var_118_to_fp16 = const()[name = string("op_118_to_fp16"), val = fp16(0x1p-24)]; tensor input_13_cast_fp16 = layer_norm(axes = input_13_axes_0, beta = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_LayerNorm_bias_to_fp16, epsilon = var_118_to_fp16, gamma = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_LayerNorm_weight_to_fp16, x = input_11_cast_fp16)[name = string("input_13_cast_fp16")]; tensor bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_weight_to_fp16 = const()[name = string("bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_weight_to_fp16"), val = tensor(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(5002368)))]; tensor bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_bias_to_fp16 = const()[name = string("bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_bias_to_fp16"), val = tensor(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(8148160)))]; tensor linear_5_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_weight_to_fp16, x = input_13_cast_fp16)[name = string("linear_5_cast_fp16")]; string input_17_mode_0 = const()[name = string("input_17_mode_0"), val = string("TANH_APPROXIMATION")]; tensor input_17_cast_fp16 = gelu(mode = input_17_mode_0, x = linear_5_cast_fp16)[name = string("input_17_cast_fp16")]; tensor bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_output_weight_to_fp16 = const()[name = string("bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_output_weight_to_fp16"), val = tensor(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(8152320)))]; tensor bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_output_bias_to_fp16 = const()[name = string("bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_output_bias_to_fp16"), val = tensor(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(11298112)))]; tensor linear_6_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_output_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_output_weight_to_fp16, x = input_17_cast_fp16)[name = string("linear_6_cast_fp16")]; tensor input_19_cast_fp16 = add(x = linear_6_cast_fp16, y = input_13_cast_fp16)[name = string("input_19_cast_fp16")]; tensor hidden_states_3_axes_0 = const()[name = string("hidden_states_3_axes_0"), val = tensor([-1])]; tensor bert_encoder_albert_layer_groups_0_albert_layers_0_full_layer_layer_norm_weight_to_fp16 = const()[name = string("bert_encoder_albert_layer_groups_0_albert_layers_0_full_layer_layer_norm_weight_to_fp16"), val = tensor(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(11299712)))]; tensor bert_encoder_albert_layer_groups_0_albert_layers_0_full_layer_layer_norm_bias_to_fp16 = const()[name = string("bert_encoder_albert_layer_groups_0_albert_layers_0_full_layer_layer_norm_bias_to_fp16"), val = tensor(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(11301312)))]; tensor hidden_states_3_cast_fp16 = layer_norm(axes = hidden_states_3_axes_0, beta = bert_encoder_albert_layer_groups_0_albert_layers_0_full_layer_layer_norm_bias_to_fp16, epsilon = var_118_to_fp16, gamma = bert_encoder_albert_layer_groups_0_albert_layers_0_full_layer_layer_norm_weight_to_fp16, x = input_19_cast_fp16)[name = string("hidden_states_3_cast_fp16")]; tensor linear_7_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_query_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_query_weight_to_fp16, x = hidden_states_3_cast_fp16)[name = string("linear_7_cast_fp16")]; tensor var_218 = const()[name = string("op_218"), val = tensor([1, 57, 12, 64])]; tensor x_15_cast_fp16 = reshape(shape = var_218, x = linear_7_cast_fp16)[name = string("x_15_cast_fp16")]; tensor linear_8_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_key_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_key_weight_to_fp16, x = hidden_states_3_cast_fp16)[name = string("linear_8_cast_fp16")]; tensor var_227 = const()[name = string("op_227"), val = tensor([1, 57, 12, 64])]; tensor x_19_cast_fp16 = reshape(shape = var_227, x = linear_8_cast_fp16)[name = string("x_19_cast_fp16")]; tensor linear_9_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_value_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_value_weight_to_fp16, x = hidden_states_3_cast_fp16)[name = string("linear_9_cast_fp16")]; tensor var_236 = const()[name = string("op_236"), val = tensor([1, 57, 12, 64])]; tensor x_23_cast_fp16 = reshape(shape = var_236, x = linear_9_cast_fp16)[name = string("x_23_cast_fp16")]; tensor transpose_75_perm_0 = const()[name = string("transpose_75_perm_0"), val = tensor([0, 2, 1, 3])]; tensor transpose_76_perm_0 = const()[name = string("transpose_76_perm_0"), val = tensor([0, 2, 1, 3])]; tensor transpose_77_perm_0 = const()[name = string("transpose_77_perm_0"), val = tensor([0, 2, 1, 3])]; tensor transpose_77 = transpose(perm = transpose_77_perm_0, x = x_23_cast_fp16)[name = string("transpose_150")]; tensor transpose_76 = transpose(perm = transpose_76_perm_0, x = x_19_cast_fp16)[name = string("transpose_151")]; tensor transpose_75 = transpose(perm = transpose_75_perm_0, x = x_15_cast_fp16)[name = string("transpose_152")]; tensor attention_output_5_cast_fp16 = scaled_dot_product_attention(attn_mask = attention_mask_cast_fp16, key = transpose_76, query = transpose_75, value = transpose_77)[name = string("attention_output_5_cast_fp16")]; tensor attention_output_7_perm_0 = const()[name = string("attention_output_7_perm_0"), val = tensor([0, 2, 1, 3])]; tensor var_242 = const()[name = string("op_242"), val = tensor([1, 57, 768])]; tensor attention_output_7_cast_fp16 = transpose(perm = attention_output_7_perm_0, x = attention_output_5_cast_fp16)[name = string("transpose_149")]; tensor input_21_cast_fp16 = reshape(shape = var_242, x = attention_output_7_cast_fp16)[name = string("input_21_cast_fp16")]; tensor linear_10_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_dense_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_dense_weight_to_fp16, x = input_21_cast_fp16)[name = string("linear_10_cast_fp16")]; tensor input_23_cast_fp16 = add(x = hidden_states_3_cast_fp16, y = linear_10_cast_fp16)[name = string("input_23_cast_fp16")]; tensor input_25_axes_0 = const()[name = string("input_25_axes_0"), val = tensor([-1])]; tensor input_25_cast_fp16 = layer_norm(axes = input_25_axes_0, beta = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_LayerNorm_bias_to_fp16, epsilon = var_118_to_fp16, gamma = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_LayerNorm_weight_to_fp16, x = input_23_cast_fp16)[name = string("input_25_cast_fp16")]; tensor linear_11_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_weight_to_fp16, x = input_25_cast_fp16)[name = string("linear_11_cast_fp16")]; string input_29_mode_0 = const()[name = string("input_29_mode_0"), val = string("TANH_APPROXIMATION")]; tensor input_29_cast_fp16 = gelu(mode = input_29_mode_0, x = linear_11_cast_fp16)[name = string("input_29_cast_fp16")]; tensor linear_12_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_output_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_output_weight_to_fp16, x = input_29_cast_fp16)[name = string("linear_12_cast_fp16")]; tensor input_31_cast_fp16 = add(x = linear_12_cast_fp16, y = input_25_cast_fp16)[name = string("input_31_cast_fp16")]; tensor hidden_states_5_axes_0 = const()[name = string("hidden_states_5_axes_0"), val = tensor([-1])]; tensor hidden_states_5_cast_fp16 = layer_norm(axes = hidden_states_5_axes_0, beta = bert_encoder_albert_layer_groups_0_albert_layers_0_full_layer_layer_norm_bias_to_fp16, epsilon = var_118_to_fp16, gamma = bert_encoder_albert_layer_groups_0_albert_layers_0_full_layer_layer_norm_weight_to_fp16, x = input_31_cast_fp16)[name = string("hidden_states_5_cast_fp16")]; tensor linear_13_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_query_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_query_weight_to_fp16, x = hidden_states_5_cast_fp16)[name = string("linear_13_cast_fp16")]; tensor var_293 = const()[name = string("op_293"), val = tensor([1, 57, 12, 64])]; tensor x_27_cast_fp16 = reshape(shape = var_293, x = linear_13_cast_fp16)[name = string("x_27_cast_fp16")]; tensor linear_14_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_key_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_key_weight_to_fp16, x = hidden_states_5_cast_fp16)[name = string("linear_14_cast_fp16")]; tensor var_302 = const()[name = string("op_302"), val = tensor([1, 57, 12, 64])]; tensor x_31_cast_fp16 = reshape(shape = var_302, x = linear_14_cast_fp16)[name = string("x_31_cast_fp16")]; tensor linear_15_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_value_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_value_weight_to_fp16, x = hidden_states_5_cast_fp16)[name = string("linear_15_cast_fp16")]; tensor var_311 = const()[name = string("op_311"), val = tensor([1, 57, 12, 64])]; tensor x_35_cast_fp16 = reshape(shape = var_311, x = linear_15_cast_fp16)[name = string("x_35_cast_fp16")]; tensor transpose_78_perm_0 = const()[name = string("transpose_78_perm_0"), val = tensor([0, 2, 1, 3])]; tensor transpose_79_perm_0 = const()[name = string("transpose_79_perm_0"), val = tensor([0, 2, 1, 3])]; tensor transpose_80_perm_0 = const()[name = string("transpose_80_perm_0"), val = tensor([0, 2, 1, 3])]; tensor transpose_80 = transpose(perm = transpose_80_perm_0, x = x_35_cast_fp16)[name = string("transpose_146")]; tensor transpose_79 = transpose(perm = transpose_79_perm_0, x = x_31_cast_fp16)[name = string("transpose_147")]; tensor transpose_78 = transpose(perm = transpose_78_perm_0, x = x_27_cast_fp16)[name = string("transpose_148")]; tensor attention_output_9_cast_fp16 = scaled_dot_product_attention(attn_mask = attention_mask_cast_fp16, key = transpose_79, query = transpose_78, value = transpose_80)[name = string("attention_output_9_cast_fp16")]; tensor attention_output_11_perm_0 = const()[name = string("attention_output_11_perm_0"), val = tensor([0, 2, 1, 3])]; tensor var_317 = const()[name = string("op_317"), val = tensor([1, 57, 768])]; tensor attention_output_11_cast_fp16 = transpose(perm = attention_output_11_perm_0, x = attention_output_9_cast_fp16)[name = string("transpose_145")]; tensor input_33_cast_fp16 = reshape(shape = var_317, x = attention_output_11_cast_fp16)[name = string("input_33_cast_fp16")]; tensor linear_16_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_dense_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_dense_weight_to_fp16, x = input_33_cast_fp16)[name = string("linear_16_cast_fp16")]; tensor input_35_cast_fp16 = add(x = hidden_states_5_cast_fp16, y = linear_16_cast_fp16)[name = string("input_35_cast_fp16")]; tensor input_37_axes_0 = const()[name = string("input_37_axes_0"), val = tensor([-1])]; tensor input_37_cast_fp16 = layer_norm(axes = input_37_axes_0, beta = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_LayerNorm_bias_to_fp16, epsilon = var_118_to_fp16, gamma = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_LayerNorm_weight_to_fp16, x = input_35_cast_fp16)[name = string("input_37_cast_fp16")]; tensor linear_17_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_weight_to_fp16, x = input_37_cast_fp16)[name = string("linear_17_cast_fp16")]; string input_41_mode_0 = const()[name = string("input_41_mode_0"), val = string("TANH_APPROXIMATION")]; tensor input_41_cast_fp16 = gelu(mode = input_41_mode_0, x = linear_17_cast_fp16)[name = string("input_41_cast_fp16")]; tensor linear_18_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_output_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_output_weight_to_fp16, x = input_41_cast_fp16)[name = string("linear_18_cast_fp16")]; tensor input_43_cast_fp16 = add(x = linear_18_cast_fp16, y = input_37_cast_fp16)[name = string("input_43_cast_fp16")]; tensor hidden_states_7_axes_0 = const()[name = string("hidden_states_7_axes_0"), val = tensor([-1])]; tensor hidden_states_7_cast_fp16 = layer_norm(axes = hidden_states_7_axes_0, beta = bert_encoder_albert_layer_groups_0_albert_layers_0_full_layer_layer_norm_bias_to_fp16, epsilon = var_118_to_fp16, gamma = bert_encoder_albert_layer_groups_0_albert_layers_0_full_layer_layer_norm_weight_to_fp16, x = input_43_cast_fp16)[name = string("hidden_states_7_cast_fp16")]; tensor linear_19_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_query_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_query_weight_to_fp16, x = hidden_states_7_cast_fp16)[name = string("linear_19_cast_fp16")]; tensor var_368 = const()[name = string("op_368"), val = tensor([1, 57, 12, 64])]; tensor x_39_cast_fp16 = reshape(shape = var_368, x = linear_19_cast_fp16)[name = string("x_39_cast_fp16")]; tensor linear_20_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_key_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_key_weight_to_fp16, x = hidden_states_7_cast_fp16)[name = string("linear_20_cast_fp16")]; tensor var_377 = const()[name = string("op_377"), val = tensor([1, 57, 12, 64])]; tensor x_43_cast_fp16 = reshape(shape = var_377, x = linear_20_cast_fp16)[name = string("x_43_cast_fp16")]; tensor linear_21_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_value_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_value_weight_to_fp16, x = hidden_states_7_cast_fp16)[name = string("linear_21_cast_fp16")]; tensor var_386 = const()[name = string("op_386"), val = tensor([1, 57, 12, 64])]; tensor x_47_cast_fp16 = reshape(shape = var_386, x = linear_21_cast_fp16)[name = string("x_47_cast_fp16")]; tensor transpose_81_perm_0 = const()[name = string("transpose_81_perm_0"), val = tensor([0, 2, 1, 3])]; tensor transpose_82_perm_0 = const()[name = string("transpose_82_perm_0"), val = tensor([0, 2, 1, 3])]; tensor transpose_83_perm_0 = const()[name = string("transpose_83_perm_0"), val = tensor([0, 2, 1, 3])]; tensor transpose_83 = transpose(perm = transpose_83_perm_0, x = x_47_cast_fp16)[name = string("transpose_142")]; tensor transpose_82 = transpose(perm = transpose_82_perm_0, x = x_43_cast_fp16)[name = string("transpose_143")]; tensor transpose_81 = transpose(perm = transpose_81_perm_0, x = x_39_cast_fp16)[name = string("transpose_144")]; tensor attention_output_13_cast_fp16 = scaled_dot_product_attention(attn_mask = attention_mask_cast_fp16, key = transpose_82, query = transpose_81, value = transpose_83)[name = string("attention_output_13_cast_fp16")]; tensor attention_output_15_perm_0 = const()[name = string("attention_output_15_perm_0"), val = tensor([0, 2, 1, 3])]; tensor var_392 = const()[name = string("op_392"), val = tensor([1, 57, 768])]; tensor attention_output_15_cast_fp16 = transpose(perm = attention_output_15_perm_0, x = attention_output_13_cast_fp16)[name = string("transpose_141")]; tensor input_45_cast_fp16 = reshape(shape = var_392, x = attention_output_15_cast_fp16)[name = string("input_45_cast_fp16")]; tensor linear_22_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_dense_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_dense_weight_to_fp16, x = input_45_cast_fp16)[name = string("linear_22_cast_fp16")]; tensor input_47_cast_fp16 = add(x = hidden_states_7_cast_fp16, y = linear_22_cast_fp16)[name = string("input_47_cast_fp16")]; tensor input_49_axes_0 = const()[name = string("input_49_axes_0"), val = tensor([-1])]; tensor input_49_cast_fp16 = layer_norm(axes = input_49_axes_0, beta = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_LayerNorm_bias_to_fp16, epsilon = var_118_to_fp16, gamma = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_LayerNorm_weight_to_fp16, x = input_47_cast_fp16)[name = string("input_49_cast_fp16")]; tensor linear_23_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_weight_to_fp16, x = input_49_cast_fp16)[name = string("linear_23_cast_fp16")]; string input_53_mode_0 = const()[name = string("input_53_mode_0"), val = string("TANH_APPROXIMATION")]; tensor input_53_cast_fp16 = gelu(mode = input_53_mode_0, x = linear_23_cast_fp16)[name = string("input_53_cast_fp16")]; tensor linear_24_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_output_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_output_weight_to_fp16, x = input_53_cast_fp16)[name = string("linear_24_cast_fp16")]; tensor input_55_cast_fp16 = add(x = linear_24_cast_fp16, y = input_49_cast_fp16)[name = string("input_55_cast_fp16")]; tensor hidden_states_9_axes_0 = const()[name = string("hidden_states_9_axes_0"), val = tensor([-1])]; tensor hidden_states_9_cast_fp16 = layer_norm(axes = hidden_states_9_axes_0, beta = bert_encoder_albert_layer_groups_0_albert_layers_0_full_layer_layer_norm_bias_to_fp16, epsilon = var_118_to_fp16, gamma = bert_encoder_albert_layer_groups_0_albert_layers_0_full_layer_layer_norm_weight_to_fp16, x = input_55_cast_fp16)[name = string("hidden_states_9_cast_fp16")]; tensor linear_25_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_query_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_query_weight_to_fp16, x = hidden_states_9_cast_fp16)[name = string("linear_25_cast_fp16")]; tensor var_443 = const()[name = string("op_443"), val = tensor([1, 57, 12, 64])]; tensor x_51_cast_fp16 = reshape(shape = var_443, x = linear_25_cast_fp16)[name = string("x_51_cast_fp16")]; tensor linear_26_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_key_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_key_weight_to_fp16, x = hidden_states_9_cast_fp16)[name = string("linear_26_cast_fp16")]; tensor var_452 = const()[name = string("op_452"), val = tensor([1, 57, 12, 64])]; tensor x_55_cast_fp16 = reshape(shape = var_452, x = linear_26_cast_fp16)[name = string("x_55_cast_fp16")]; tensor linear_27_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_value_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_value_weight_to_fp16, x = hidden_states_9_cast_fp16)[name = string("linear_27_cast_fp16")]; tensor var_461 = const()[name = string("op_461"), val = tensor([1, 57, 12, 64])]; tensor x_59_cast_fp16 = reshape(shape = var_461, x = linear_27_cast_fp16)[name = string("x_59_cast_fp16")]; tensor transpose_84_perm_0 = const()[name = string("transpose_84_perm_0"), val = tensor([0, 2, 1, 3])]; tensor transpose_85_perm_0 = const()[name = string("transpose_85_perm_0"), val = tensor([0, 2, 1, 3])]; tensor transpose_86_perm_0 = const()[name = string("transpose_86_perm_0"), val = tensor([0, 2, 1, 3])]; tensor transpose_86 = transpose(perm = transpose_86_perm_0, x = x_59_cast_fp16)[name = string("transpose_138")]; tensor transpose_85 = transpose(perm = transpose_85_perm_0, x = x_55_cast_fp16)[name = string("transpose_139")]; tensor transpose_84 = transpose(perm = transpose_84_perm_0, x = x_51_cast_fp16)[name = string("transpose_140")]; tensor attention_output_17_cast_fp16 = scaled_dot_product_attention(attn_mask = attention_mask_cast_fp16, key = transpose_85, query = transpose_84, value = transpose_86)[name = string("attention_output_17_cast_fp16")]; tensor attention_output_19_perm_0 = const()[name = string("attention_output_19_perm_0"), val = tensor([0, 2, 1, 3])]; tensor var_467 = const()[name = string("op_467"), val = tensor([1, 57, 768])]; tensor attention_output_19_cast_fp16 = transpose(perm = attention_output_19_perm_0, x = attention_output_17_cast_fp16)[name = string("transpose_137")]; tensor input_57_cast_fp16 = reshape(shape = var_467, x = attention_output_19_cast_fp16)[name = string("input_57_cast_fp16")]; tensor linear_28_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_dense_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_dense_weight_to_fp16, x = input_57_cast_fp16)[name = string("linear_28_cast_fp16")]; tensor input_59_cast_fp16 = add(x = hidden_states_9_cast_fp16, y = linear_28_cast_fp16)[name = string("input_59_cast_fp16")]; tensor input_61_axes_0 = const()[name = string("input_61_axes_0"), val = tensor([-1])]; tensor input_61_cast_fp16 = layer_norm(axes = input_61_axes_0, beta = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_LayerNorm_bias_to_fp16, epsilon = var_118_to_fp16, gamma = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_LayerNorm_weight_to_fp16, x = input_59_cast_fp16)[name = string("input_61_cast_fp16")]; tensor linear_29_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_weight_to_fp16, x = input_61_cast_fp16)[name = string("linear_29_cast_fp16")]; string input_65_mode_0 = const()[name = string("input_65_mode_0"), val = string("TANH_APPROXIMATION")]; tensor input_65_cast_fp16 = gelu(mode = input_65_mode_0, x = linear_29_cast_fp16)[name = string("input_65_cast_fp16")]; tensor linear_30_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_output_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_output_weight_to_fp16, x = input_65_cast_fp16)[name = string("linear_30_cast_fp16")]; tensor input_67_cast_fp16 = add(x = linear_30_cast_fp16, y = input_61_cast_fp16)[name = string("input_67_cast_fp16")]; tensor hidden_states_11_axes_0 = const()[name = string("hidden_states_11_axes_0"), val = tensor([-1])]; tensor hidden_states_11_cast_fp16 = layer_norm(axes = hidden_states_11_axes_0, beta = bert_encoder_albert_layer_groups_0_albert_layers_0_full_layer_layer_norm_bias_to_fp16, epsilon = var_118_to_fp16, gamma = bert_encoder_albert_layer_groups_0_albert_layers_0_full_layer_layer_norm_weight_to_fp16, x = input_67_cast_fp16)[name = string("hidden_states_11_cast_fp16")]; tensor linear_31_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_query_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_query_weight_to_fp16, x = hidden_states_11_cast_fp16)[name = string("linear_31_cast_fp16")]; tensor var_518 = const()[name = string("op_518"), val = tensor([1, 57, 12, 64])]; tensor x_63_cast_fp16 = reshape(shape = var_518, x = linear_31_cast_fp16)[name = string("x_63_cast_fp16")]; tensor linear_32_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_key_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_key_weight_to_fp16, x = hidden_states_11_cast_fp16)[name = string("linear_32_cast_fp16")]; tensor var_527 = const()[name = string("op_527"), val = tensor([1, 57, 12, 64])]; tensor x_67_cast_fp16 = reshape(shape = var_527, x = linear_32_cast_fp16)[name = string("x_67_cast_fp16")]; tensor linear_33_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_value_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_value_weight_to_fp16, x = hidden_states_11_cast_fp16)[name = string("linear_33_cast_fp16")]; tensor var_536 = const()[name = string("op_536"), val = tensor([1, 57, 12, 64])]; tensor x_71_cast_fp16 = reshape(shape = var_536, x = linear_33_cast_fp16)[name = string("x_71_cast_fp16")]; tensor transpose_87_perm_0 = const()[name = string("transpose_87_perm_0"), val = tensor([0, 2, 1, 3])]; tensor transpose_88_perm_0 = const()[name = string("transpose_88_perm_0"), val = tensor([0, 2, 1, 3])]; tensor transpose_89_perm_0 = const()[name = string("transpose_89_perm_0"), val = tensor([0, 2, 1, 3])]; tensor transpose_89 = transpose(perm = transpose_89_perm_0, x = x_71_cast_fp16)[name = string("transpose_134")]; tensor transpose_88 = transpose(perm = transpose_88_perm_0, x = x_67_cast_fp16)[name = string("transpose_135")]; tensor transpose_87 = transpose(perm = transpose_87_perm_0, x = x_63_cast_fp16)[name = string("transpose_136")]; tensor attention_output_21_cast_fp16 = scaled_dot_product_attention(attn_mask = attention_mask_cast_fp16, key = transpose_88, query = transpose_87, value = transpose_89)[name = string("attention_output_21_cast_fp16")]; tensor attention_output_23_perm_0 = const()[name = string("attention_output_23_perm_0"), val = tensor([0, 2, 1, 3])]; tensor var_542 = const()[name = string("op_542"), val = tensor([1, 57, 768])]; tensor attention_output_23_cast_fp16 = transpose(perm = attention_output_23_perm_0, x = attention_output_21_cast_fp16)[name = string("transpose_133")]; tensor input_69_cast_fp16 = reshape(shape = var_542, x = attention_output_23_cast_fp16)[name = string("input_69_cast_fp16")]; tensor linear_34_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_dense_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_dense_weight_to_fp16, x = input_69_cast_fp16)[name = string("linear_34_cast_fp16")]; tensor input_71_cast_fp16 = add(x = hidden_states_11_cast_fp16, y = linear_34_cast_fp16)[name = string("input_71_cast_fp16")]; tensor input_73_axes_0 = const()[name = string("input_73_axes_0"), val = tensor([-1])]; tensor input_73_cast_fp16 = layer_norm(axes = input_73_axes_0, beta = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_LayerNorm_bias_to_fp16, epsilon = var_118_to_fp16, gamma = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_LayerNorm_weight_to_fp16, x = input_71_cast_fp16)[name = string("input_73_cast_fp16")]; tensor linear_35_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_weight_to_fp16, x = input_73_cast_fp16)[name = string("linear_35_cast_fp16")]; string input_77_mode_0 = const()[name = string("input_77_mode_0"), val = string("TANH_APPROXIMATION")]; tensor input_77_cast_fp16 = gelu(mode = input_77_mode_0, x = linear_35_cast_fp16)[name = string("input_77_cast_fp16")]; tensor linear_36_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_output_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_output_weight_to_fp16, x = input_77_cast_fp16)[name = string("linear_36_cast_fp16")]; tensor input_79_cast_fp16 = add(x = linear_36_cast_fp16, y = input_73_cast_fp16)[name = string("input_79_cast_fp16")]; tensor hidden_states_13_axes_0 = const()[name = string("hidden_states_13_axes_0"), val = tensor([-1])]; tensor hidden_states_13_cast_fp16 = layer_norm(axes = hidden_states_13_axes_0, beta = bert_encoder_albert_layer_groups_0_albert_layers_0_full_layer_layer_norm_bias_to_fp16, epsilon = var_118_to_fp16, gamma = bert_encoder_albert_layer_groups_0_albert_layers_0_full_layer_layer_norm_weight_to_fp16, x = input_79_cast_fp16)[name = string("hidden_states_13_cast_fp16")]; tensor linear_37_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_query_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_query_weight_to_fp16, x = hidden_states_13_cast_fp16)[name = string("linear_37_cast_fp16")]; tensor var_593 = const()[name = string("op_593"), val = tensor([1, 57, 12, 64])]; tensor x_75_cast_fp16 = reshape(shape = var_593, x = linear_37_cast_fp16)[name = string("x_75_cast_fp16")]; tensor linear_38_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_key_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_key_weight_to_fp16, x = hidden_states_13_cast_fp16)[name = string("linear_38_cast_fp16")]; tensor var_602 = const()[name = string("op_602"), val = tensor([1, 57, 12, 64])]; tensor x_79_cast_fp16 = reshape(shape = var_602, x = linear_38_cast_fp16)[name = string("x_79_cast_fp16")]; tensor linear_39_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_value_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_value_weight_to_fp16, x = hidden_states_13_cast_fp16)[name = string("linear_39_cast_fp16")]; tensor var_611 = const()[name = string("op_611"), val = tensor([1, 57, 12, 64])]; tensor x_83_cast_fp16 = reshape(shape = var_611, x = linear_39_cast_fp16)[name = string("x_83_cast_fp16")]; tensor transpose_90_perm_0 = const()[name = string("transpose_90_perm_0"), val = tensor([0, 2, 1, 3])]; tensor transpose_91_perm_0 = const()[name = string("transpose_91_perm_0"), val = tensor([0, 2, 1, 3])]; tensor transpose_92_perm_0 = const()[name = string("transpose_92_perm_0"), val = tensor([0, 2, 1, 3])]; tensor transpose_92 = transpose(perm = transpose_92_perm_0, x = x_83_cast_fp16)[name = string("transpose_130")]; tensor transpose_91 = transpose(perm = transpose_91_perm_0, x = x_79_cast_fp16)[name = string("transpose_131")]; tensor transpose_90 = transpose(perm = transpose_90_perm_0, x = x_75_cast_fp16)[name = string("transpose_132")]; tensor attention_output_25_cast_fp16 = scaled_dot_product_attention(attn_mask = attention_mask_cast_fp16, key = transpose_91, query = transpose_90, value = transpose_92)[name = string("attention_output_25_cast_fp16")]; tensor attention_output_27_perm_0 = const()[name = string("attention_output_27_perm_0"), val = tensor([0, 2, 1, 3])]; tensor var_617 = const()[name = string("op_617"), val = tensor([1, 57, 768])]; tensor attention_output_27_cast_fp16 = transpose(perm = attention_output_27_perm_0, x = attention_output_25_cast_fp16)[name = string("transpose_129")]; tensor input_81_cast_fp16 = reshape(shape = var_617, x = attention_output_27_cast_fp16)[name = string("input_81_cast_fp16")]; tensor linear_40_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_dense_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_dense_weight_to_fp16, x = input_81_cast_fp16)[name = string("linear_40_cast_fp16")]; tensor input_83_cast_fp16 = add(x = hidden_states_13_cast_fp16, y = linear_40_cast_fp16)[name = string("input_83_cast_fp16")]; tensor input_85_axes_0 = const()[name = string("input_85_axes_0"), val = tensor([-1])]; tensor input_85_cast_fp16 = layer_norm(axes = input_85_axes_0, beta = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_LayerNorm_bias_to_fp16, epsilon = var_118_to_fp16, gamma = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_LayerNorm_weight_to_fp16, x = input_83_cast_fp16)[name = string("input_85_cast_fp16")]; tensor linear_41_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_weight_to_fp16, x = input_85_cast_fp16)[name = string("linear_41_cast_fp16")]; string input_89_mode_0 = const()[name = string("input_89_mode_0"), val = string("TANH_APPROXIMATION")]; tensor input_89_cast_fp16 = gelu(mode = input_89_mode_0, x = linear_41_cast_fp16)[name = string("input_89_cast_fp16")]; tensor linear_42_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_output_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_output_weight_to_fp16, x = input_89_cast_fp16)[name = string("linear_42_cast_fp16")]; tensor input_91_cast_fp16 = add(x = linear_42_cast_fp16, y = input_85_cast_fp16)[name = string("input_91_cast_fp16")]; tensor hidden_states_15_axes_0 = const()[name = string("hidden_states_15_axes_0"), val = tensor([-1])]; tensor hidden_states_15_cast_fp16 = layer_norm(axes = hidden_states_15_axes_0, beta = bert_encoder_albert_layer_groups_0_albert_layers_0_full_layer_layer_norm_bias_to_fp16, epsilon = var_118_to_fp16, gamma = bert_encoder_albert_layer_groups_0_albert_layers_0_full_layer_layer_norm_weight_to_fp16, x = input_91_cast_fp16)[name = string("hidden_states_15_cast_fp16")]; tensor linear_43_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_query_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_query_weight_to_fp16, x = hidden_states_15_cast_fp16)[name = string("linear_43_cast_fp16")]; tensor var_668 = const()[name = string("op_668"), val = tensor([1, 57, 12, 64])]; tensor x_87_cast_fp16 = reshape(shape = var_668, x = linear_43_cast_fp16)[name = string("x_87_cast_fp16")]; tensor linear_44_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_key_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_key_weight_to_fp16, x = hidden_states_15_cast_fp16)[name = string("linear_44_cast_fp16")]; tensor var_677 = const()[name = string("op_677"), val = tensor([1, 57, 12, 64])]; tensor x_91_cast_fp16 = reshape(shape = var_677, x = linear_44_cast_fp16)[name = string("x_91_cast_fp16")]; tensor linear_45_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_value_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_value_weight_to_fp16, x = hidden_states_15_cast_fp16)[name = string("linear_45_cast_fp16")]; tensor var_686 = const()[name = string("op_686"), val = tensor([1, 57, 12, 64])]; tensor x_95_cast_fp16 = reshape(shape = var_686, x = linear_45_cast_fp16)[name = string("x_95_cast_fp16")]; tensor transpose_93_perm_0 = const()[name = string("transpose_93_perm_0"), val = tensor([0, 2, 1, 3])]; tensor transpose_94_perm_0 = const()[name = string("transpose_94_perm_0"), val = tensor([0, 2, 1, 3])]; tensor transpose_95_perm_0 = const()[name = string("transpose_95_perm_0"), val = tensor([0, 2, 1, 3])]; tensor transpose_95 = transpose(perm = transpose_95_perm_0, x = x_95_cast_fp16)[name = string("transpose_126")]; tensor transpose_94 = transpose(perm = transpose_94_perm_0, x = x_91_cast_fp16)[name = string("transpose_127")]; tensor transpose_93 = transpose(perm = transpose_93_perm_0, x = x_87_cast_fp16)[name = string("transpose_128")]; tensor attention_output_29_cast_fp16 = scaled_dot_product_attention(attn_mask = attention_mask_cast_fp16, key = transpose_94, query = transpose_93, value = transpose_95)[name = string("attention_output_29_cast_fp16")]; tensor attention_output_31_perm_0 = const()[name = string("attention_output_31_perm_0"), val = tensor([0, 2, 1, 3])]; tensor var_692 = const()[name = string("op_692"), val = tensor([1, 57, 768])]; tensor attention_output_31_cast_fp16 = transpose(perm = attention_output_31_perm_0, x = attention_output_29_cast_fp16)[name = string("transpose_125")]; tensor input_93_cast_fp16 = reshape(shape = var_692, x = attention_output_31_cast_fp16)[name = string("input_93_cast_fp16")]; tensor linear_46_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_dense_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_dense_weight_to_fp16, x = input_93_cast_fp16)[name = string("linear_46_cast_fp16")]; tensor input_95_cast_fp16 = add(x = hidden_states_15_cast_fp16, y = linear_46_cast_fp16)[name = string("input_95_cast_fp16")]; tensor input_97_axes_0 = const()[name = string("input_97_axes_0"), val = tensor([-1])]; tensor input_97_cast_fp16 = layer_norm(axes = input_97_axes_0, beta = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_LayerNorm_bias_to_fp16, epsilon = var_118_to_fp16, gamma = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_LayerNorm_weight_to_fp16, x = input_95_cast_fp16)[name = string("input_97_cast_fp16")]; tensor linear_47_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_weight_to_fp16, x = input_97_cast_fp16)[name = string("linear_47_cast_fp16")]; string input_101_mode_0 = const()[name = string("input_101_mode_0"), val = string("TANH_APPROXIMATION")]; tensor input_101_cast_fp16 = gelu(mode = input_101_mode_0, x = linear_47_cast_fp16)[name = string("input_101_cast_fp16")]; tensor linear_48_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_output_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_ffn_output_weight_to_fp16, x = input_101_cast_fp16)[name = string("linear_48_cast_fp16")]; tensor input_103_cast_fp16 = add(x = linear_48_cast_fp16, y = input_97_cast_fp16)[name = string("input_103_cast_fp16")]; tensor hidden_states_17_axes_0 = const()[name = string("hidden_states_17_axes_0"), val = tensor([-1])]; tensor hidden_states_17_cast_fp16 = layer_norm(axes = hidden_states_17_axes_0, beta = bert_encoder_albert_layer_groups_0_albert_layers_0_full_layer_layer_norm_bias_to_fp16, epsilon = var_118_to_fp16, gamma = bert_encoder_albert_layer_groups_0_albert_layers_0_full_layer_layer_norm_weight_to_fp16, x = input_103_cast_fp16)[name = string("hidden_states_17_cast_fp16")]; tensor linear_49_cast_fp16 = linear(bias = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_query_bias_to_fp16, weight = bert_encoder_albert_layer_groups_0_albert_layers_0_attention_query_weight_to_fp16, x = hidden_states_17_cast_fp16)[name = string("linear_49_cast_fp16")]; tensor var_743 = const()[name = string("op_743"), val = tensor([1, 57, 12, 64])]; tensor x_99_cast_fp16 = reshape(shape = var_743, x = linear_49_cast_fp16)[name = string("x_99_cast_fp16")]; tensor