diff --git "a/CoreMLModels/clip_text_s2.mlmodelc/model.mil" "b/CoreMLModels/clip_text_s2.mlmodelc/model.mil" new file mode 100644--- /dev/null +++ "b/CoreMLModels/clip_text_s2.mlmodelc/model.mil" @@ -0,0 +1,719 @@ +program(1.0) +[buildInfo = dict, tensor>({{"coremlc-component-MIL", "3405.2.1"}, {"coremlc-version", "3404.23.1"}, {"coremltools-component-torch", "2.6.0+cu124"}, {"coremltools-source-dialect", "TorchScript"}, {"coremltools-version", "8.3.0"}})] +{ + func main(tensor input_text) { + tensor var_18 = const()[name = tensor("op_18"), val = tensor(-1)]; + tensor token_emb_1_axis_0 = const()[name = tensor("token_emb_1_axis_0"), val = tensor(0)]; + tensor token_emb_1_batch_dims_0 = const()[name = tensor("token_emb_1_batch_dims_0"), val = tensor(0)]; + tensor token_emb_1_validate_indices_0 = const()[name = tensor("token_emb_1_validate_indices_0"), val = tensor(false)]; + tensor text_encoder_embedding_layer_weight_to_fp16 = const()[name = tensor("text_encoder_embedding_layer_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(64)))]; + tensor token_emb_1_cast_fp16 = gather(axis = token_emb_1_axis_0, batch_dims = token_emb_1_batch_dims_0, indices = input_text, validate_indices = token_emb_1_validate_indices_0, x = text_encoder_embedding_layer_weight_to_fp16)[name = tensor("token_emb_1_cast_fp16")]; + tensor const_1_to_fp16 = const()[name = tensor("const_1_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(50593920)))]; + tensor input_1_cast_fp16 = add(x = token_emb_1_cast_fp16, y = const_1_to_fp16)[name = tensor("input_1_cast_fp16")]; + tensor var_67_axes_0 = const()[name = tensor("op_67_axes_0"), val = tensor([-1])]; + tensor text_encoder_transformer_0_pre_norm_mha_0_weight_to_fp16 = const()[name = tensor("text_encoder_transformer_0_pre_norm_mha_0_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(50672832)))]; + tensor text_encoder_transformer_0_pre_norm_mha_0_bias_to_fp16 = const()[name = tensor("text_encoder_transformer_0_pre_norm_mha_0_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(50673920)))]; + tensor var_5_to_fp16 = const()[name = tensor("op_5_to_fp16"), val = tensor(0x1.5p-17)]; + tensor var_67_cast_fp16 = layer_norm(axes = var_67_axes_0, beta = text_encoder_transformer_0_pre_norm_mha_0_bias_to_fp16, epsilon = var_5_to_fp16, gamma = text_encoder_transformer_0_pre_norm_mha_0_weight_to_fp16, x = input_1_cast_fp16)[name = tensor("op_67_cast_fp16")]; + tensor text_encoder_transformer_0_pre_norm_mha_1_qkv_proj_weight_to_fp16 = const()[name = tensor("text_encoder_transformer_0_pre_norm_mha_1_qkv_proj_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(50675008)))]; + tensor text_encoder_transformer_0_pre_norm_mha_1_qkv_proj_bias_to_fp16 = const()[name = tensor("text_encoder_transformer_0_pre_norm_mha_1_qkv_proj_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(52247936)))]; + tensor linear_0_cast_fp16 = linear(bias = text_encoder_transformer_0_pre_norm_mha_1_qkv_proj_bias_to_fp16, weight = text_encoder_transformer_0_pre_norm_mha_1_qkv_proj_weight_to_fp16, x = var_67_cast_fp16)[name = tensor("linear_0_cast_fp16")]; + tensor var_79 = const()[name = tensor("op_79"), val = tensor([1, 77, 3, 8, -1])]; + tensor qkv_1_cast_fp16 = reshape(shape = var_79, x = linear_0_cast_fp16)[name = tensor("qkv_1_cast_fp16")]; + tensor var_81_perm_0 = const()[name = tensor("op_81_perm_0"), val = tensor([0, 3, 2, 1, 4])]; + tensor query_1_begin_0 = const()[name = tensor("query_1_begin_0"), val = tensor([0, 0, 0, 0, 0])]; + tensor query_1_end_0 = const()[name = tensor("query_1_end_0"), val = tensor([1, 8, 1, 77, 64])]; + tensor query_1_end_mask_0 = const()[name = tensor("query_1_end_mask_0"), val = tensor([true, true, false, true, true])]; + tensor query_1_squeeze_mask_0 = const()[name = tensor("query_1_squeeze_mask_0"), val = tensor([false, false, true, false, false])]; + tensor var_81_cast_fp16 = transpose(perm = var_81_perm_0, x = qkv_1_cast_fp16)[name = tensor("transpose_25")]; + tensor query_1_cast_fp16 = slice_by_index(begin = query_1_begin_0, end = query_1_end_0, end_mask = query_1_end_mask_0, squeeze_mask = query_1_squeeze_mask_0, x = var_81_cast_fp16)[name = tensor("query_1_cast_fp16")]; + tensor key_1_begin_0 = const()[name = tensor("key_1_begin_0"), val = tensor([0, 0, 1, 0, 0])]; + tensor key_1_end_0 = const()[name = tensor("key_1_end_0"), val = tensor([1, 8, 2, 77, 64])]; + tensor key_1_end_mask_0 = const()[name = tensor("key_1_end_mask_0"), val = tensor([true, true, false, true, true])]; + tensor key_1_squeeze_mask_0 = const()[name = tensor("key_1_squeeze_mask_0"), val = tensor([false, false, true, false, false])]; + tensor key_1_cast_fp16 = slice_by_index(begin = key_1_begin_0, end = key_1_end_0, end_mask = key_1_end_mask_0, squeeze_mask = key_1_squeeze_mask_0, x = var_81_cast_fp16)[name = tensor("key_1_cast_fp16")]; + tensor value_1_begin_0 = const()[name = tensor("value_1_begin_0"), val = tensor([0, 0, 2, 0, 0])]; + tensor value_1_end_0 = const()[name = tensor("value_1_end_0"), val = tensor([1, 8, 3, 77, 64])]; + tensor value_1_end_mask_0 = const()[name = tensor("value_1_end_mask_0"), val = tensor([true, true, false, true, true])]; + tensor value_1_squeeze_mask_0 = const()[name = tensor("value_1_squeeze_mask_0"), val = tensor([false, false, true, false, false])]; + tensor value_1_cast_fp16 = slice_by_index(begin = value_1_begin_0, end = value_1_end_0, end_mask = value_1_end_mask_0, squeeze_mask = value_1_squeeze_mask_0, x = var_81_cast_fp16)[name = tensor("value_1_cast_fp16")]; + tensor var_92_to_fp16 = const()[name = tensor("op_92_to_fp16"), val = tensor(0x1p-3)]; + tensor query_3_cast_fp16 = mul(x = query_1_cast_fp16, y = var_92_to_fp16)[name = tensor("query_3_cast_fp16")]; + tensor attn_1_transpose_x_1 = const()[name = tensor("attn_1_transpose_x_1"), val = tensor(false)]; + tensor attn_1_transpose_y_1 = const()[name = tensor("attn_1_transpose_y_1"), val = tensor(true)]; + tensor attn_1_cast_fp16 = matmul(transpose_x = attn_1_transpose_x_1, transpose_y = attn_1_transpose_y_1, x = query_3_cast_fp16, y = key_1_cast_fp16)[name = tensor("attn_1_cast_fp16")]; + tensor attn_as_float_1_cast_fp16 = softmax(axis = var_18, x = attn_1_cast_fp16)[name = tensor("attn_as_float_1_cast_fp16")]; + tensor out_1_transpose_x_0 = const()[name = tensor("out_1_transpose_x_0"), val = tensor(false)]; + tensor out_1_transpose_y_0 = const()[name = tensor("out_1_transpose_y_0"), val = tensor(false)]; + tensor out_1_cast_fp16 = matmul(transpose_x = out_1_transpose_x_0, transpose_y = out_1_transpose_y_0, x = attn_as_float_1_cast_fp16, y = value_1_cast_fp16)[name = tensor("out_1_cast_fp16")]; + tensor var_101_perm_0 = const()[name = tensor("op_101_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_102 = const()[name = tensor("op_102"), val = tensor([1, 77, -1])]; + tensor var_101_cast_fp16 = transpose(perm = var_101_perm_0, x = out_1_cast_fp16)[name = tensor("transpose_24")]; + tensor input_9_cast_fp16 = reshape(shape = var_102, x = var_101_cast_fp16)[name = tensor("input_9_cast_fp16")]; + tensor text_encoder_transformer_0_pre_norm_mha_1_out_proj_weight_to_fp16 = const()[name = tensor("text_encoder_transformer_0_pre_norm_mha_1_out_proj_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(52251072)))]; + tensor text_encoder_transformer_0_pre_norm_mha_1_out_proj_bias_to_fp16 = const()[name = tensor("text_encoder_transformer_0_pre_norm_mha_1_out_proj_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(52775424)))]; + tensor linear_1_cast_fp16 = linear(bias = text_encoder_transformer_0_pre_norm_mha_1_out_proj_bias_to_fp16, weight = text_encoder_transformer_0_pre_norm_mha_1_out_proj_weight_to_fp16, x = input_9_cast_fp16)[name = tensor("linear_1_cast_fp16")]; + tensor x_5_cast_fp16 = add(x = linear_1_cast_fp16, y = input_1_cast_fp16)[name = tensor("x_5_cast_fp16")]; + tensor var_116_axes_0 = const()[name = tensor("op_116_axes_0"), val = tensor([-1])]; + tensor text_encoder_transformer_0_pre_norm_ffn_0_weight_to_fp16 = const()[name = tensor("text_encoder_transformer_0_pre_norm_ffn_0_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(52776512)))]; + tensor text_encoder_transformer_0_pre_norm_ffn_0_bias_to_fp16 = const()[name = tensor("text_encoder_transformer_0_pre_norm_ffn_0_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(52777600)))]; + tensor var_116_cast_fp16 = layer_norm(axes = var_116_axes_0, beta = text_encoder_transformer_0_pre_norm_ffn_0_bias_to_fp16, epsilon = var_5_to_fp16, gamma = text_encoder_transformer_0_pre_norm_ffn_0_weight_to_fp16, x = x_5_cast_fp16)[name = tensor("op_116_cast_fp16")]; + tensor text_encoder_transformer_0_pre_norm_ffn_1_weight_to_fp16 = const()[name = tensor("text_encoder_transformer_0_pre_norm_ffn_1_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(52778688)))]; + tensor text_encoder_transformer_0_pre_norm_ffn_1_bias_to_fp16 = const()[name = tensor("text_encoder_transformer_0_pre_norm_ffn_1_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(54875904)))]; + tensor linear_2_cast_fp16 = linear(bias = text_encoder_transformer_0_pre_norm_ffn_1_bias_to_fp16, weight = text_encoder_transformer_0_pre_norm_ffn_1_weight_to_fp16, x = var_116_cast_fp16)[name = tensor("linear_2_cast_fp16")]; + tensor input_19_mode_0 = const()[name = tensor("input_19_mode_0"), val = tensor("EXACT")]; + tensor input_19_cast_fp16 = gelu(mode = input_19_mode_0, x = linear_2_cast_fp16)[name = tensor("input_19_cast_fp16")]; + tensor text_encoder_transformer_0_pre_norm_ffn_4_weight_to_fp16 = const()[name = tensor("text_encoder_transformer_0_pre_norm_ffn_4_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(54880064)))]; + tensor text_encoder_transformer_0_pre_norm_ffn_4_bias_to_fp16 = const()[name = tensor("text_encoder_transformer_0_pre_norm_ffn_4_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(56977280)))]; + tensor linear_3_cast_fp16 = linear(bias = text_encoder_transformer_0_pre_norm_ffn_4_bias_to_fp16, weight = text_encoder_transformer_0_pre_norm_ffn_4_weight_to_fp16, x = input_19_cast_fp16)[name = tensor("linear_3_cast_fp16")]; + tensor x_7_cast_fp16 = add(x = x_5_cast_fp16, y = linear_3_cast_fp16)[name = tensor("x_7_cast_fp16")]; + tensor var_143_axes_0 = const()[name = tensor("op_143_axes_0"), val = tensor([-1])]; + tensor text_encoder_transformer_1_pre_norm_mha_0_weight_to_fp16 = const()[name = tensor("text_encoder_transformer_1_pre_norm_mha_0_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(56978368)))]; + tensor text_encoder_transformer_1_pre_norm_mha_0_bias_to_fp16 = const()[name = tensor("text_encoder_transformer_1_pre_norm_mha_0_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(56979456)))]; + tensor var_143_cast_fp16 = layer_norm(axes = var_143_axes_0, beta = text_encoder_transformer_1_pre_norm_mha_0_bias_to_fp16, epsilon = var_5_to_fp16, gamma = text_encoder_transformer_1_pre_norm_mha_0_weight_to_fp16, x = x_7_cast_fp16)[name = tensor("op_143_cast_fp16")]; + tensor text_encoder_transformer_1_pre_norm_mha_1_qkv_proj_weight_to_fp16 = const()[name = tensor("text_encoder_transformer_1_pre_norm_mha_1_qkv_proj_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(56980544)))]; + tensor text_encoder_transformer_1_pre_norm_mha_1_qkv_proj_bias_to_fp16 = const()[name = tensor("text_encoder_transformer_1_pre_norm_mha_1_qkv_proj_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(58553472)))]; + tensor linear_4_cast_fp16 = linear(bias = text_encoder_transformer_1_pre_norm_mha_1_qkv_proj_bias_to_fp16, weight = text_encoder_transformer_1_pre_norm_mha_1_qkv_proj_weight_to_fp16, x = var_143_cast_fp16)[name = tensor("linear_4_cast_fp16")]; + tensor var_155 = const()[name = tensor("op_155"), val = tensor([1, 77, 3, 8, -1])]; + tensor qkv_5_cast_fp16 = reshape(shape = var_155, x = linear_4_cast_fp16)[name = tensor("qkv_5_cast_fp16")]; + tensor var_157_perm_0 = const()[name = tensor("op_157_perm_0"), val = tensor([0, 3, 2, 1, 4])]; + tensor query_5_begin_0 = const()[name = tensor("query_5_begin_0"), val = tensor([0, 0, 0, 0, 0])]; + tensor query_5_end_0 = const()[name = tensor("query_5_end_0"), val = tensor([1, 8, 1, 77, 64])]; + tensor query_5_end_mask_0 = const()[name = tensor("query_5_end_mask_0"), val = tensor([true, true, false, true, true])]; + tensor query_5_squeeze_mask_0 = const()[name = tensor("query_5_squeeze_mask_0"), val = tensor([false, false, true, false, false])]; + tensor var_157_cast_fp16 = transpose(perm = var_157_perm_0, x = qkv_5_cast_fp16)[name = tensor("transpose_23")]; + tensor query_5_cast_fp16 = slice_by_index(begin = query_5_begin_0, end = query_5_end_0, end_mask = query_5_end_mask_0, squeeze_mask = query_5_squeeze_mask_0, x = var_157_cast_fp16)[name = tensor("query_5_cast_fp16")]; + tensor key_5_begin_0 = const()[name = tensor("key_5_begin_0"), val = tensor([0, 0, 1, 0, 0])]; + tensor key_5_end_0 = const()[name = tensor("key_5_end_0"), val = tensor([1, 8, 2, 77, 64])]; + tensor key_5_end_mask_0 = const()[name = tensor("key_5_end_mask_0"), val = tensor([true, true, false, true, true])]; + tensor key_5_squeeze_mask_0 = const()[name = tensor("key_5_squeeze_mask_0"), val = tensor([false, false, true, false, false])]; + tensor key_5_cast_fp16 = slice_by_index(begin = key_5_begin_0, end = key_5_end_0, end_mask = key_5_end_mask_0, squeeze_mask = key_5_squeeze_mask_0, x = var_157_cast_fp16)[name = tensor("key_5_cast_fp16")]; + tensor value_3_begin_0 = const()[name = tensor("value_3_begin_0"), val = tensor([0, 0, 2, 0, 0])]; + tensor value_3_end_0 = const()[name = tensor("value_3_end_0"), val = tensor([1, 8, 3, 77, 64])]; + tensor value_3_end_mask_0 = const()[name = tensor("value_3_end_mask_0"), val = tensor([true, true, false, true, true])]; + tensor value_3_squeeze_mask_0 = const()[name = tensor("value_3_squeeze_mask_0"), val = tensor([false, false, true, false, false])]; + tensor value_3_cast_fp16 = slice_by_index(begin = value_3_begin_0, end = value_3_end_0, end_mask = value_3_end_mask_0, squeeze_mask = value_3_squeeze_mask_0, x = var_157_cast_fp16)[name = tensor("value_3_cast_fp16")]; + tensor var_168_to_fp16 = const()[name = tensor("op_168_to_fp16"), val = tensor(0x1p-3)]; + tensor query_7_cast_fp16 = mul(x = query_5_cast_fp16, y = var_168_to_fp16)[name = tensor("query_7_cast_fp16")]; + tensor attn_5_transpose_x_1 = const()[name = tensor("attn_5_transpose_x_1"), val = tensor(false)]; + tensor attn_5_transpose_y_1 = const()[name = tensor("attn_5_transpose_y_1"), val = tensor(true)]; + tensor attn_5_cast_fp16 = matmul(transpose_x = attn_5_transpose_x_1, transpose_y = attn_5_transpose_y_1, x = query_7_cast_fp16, y = key_5_cast_fp16)[name = tensor("attn_5_cast_fp16")]; + tensor attn_as_float_3_cast_fp16 = softmax(axis = var_18, x = attn_5_cast_fp16)[name = tensor("attn_as_float_3_cast_fp16")]; + tensor out_3_transpose_x_0 = const()[name = tensor("out_3_transpose_x_0"), val = tensor(false)]; + tensor out_3_transpose_y_0 = const()[name = tensor("out_3_transpose_y_0"), val = tensor(false)]; + tensor out_3_cast_fp16 = matmul(transpose_x = out_3_transpose_x_0, transpose_y = out_3_transpose_y_0, x = attn_as_float_3_cast_fp16, y = value_3_cast_fp16)[name = tensor("out_3_cast_fp16")]; + tensor var_177_perm_0 = const()[name = tensor("op_177_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_178 = const()[name = tensor("op_178"), val = tensor([1, 77, -1])]; + tensor var_177_cast_fp16 = transpose(perm = var_177_perm_0, x = out_3_cast_fp16)[name = tensor("transpose_22")]; + tensor input_31_cast_fp16 = reshape(shape = var_178, x = var_177_cast_fp16)[name = tensor("input_31_cast_fp16")]; + tensor text_encoder_transformer_1_pre_norm_mha_1_out_proj_weight_to_fp16 = const()[name = tensor("text_encoder_transformer_1_pre_norm_mha_1_out_proj_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(58556608)))]; + tensor text_encoder_transformer_1_pre_norm_mha_1_out_proj_bias_to_fp16 = const()[name = tensor("text_encoder_transformer_1_pre_norm_mha_1_out_proj_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(59080960)))]; + tensor linear_5_cast_fp16 = linear(bias = text_encoder_transformer_1_pre_norm_mha_1_out_proj_bias_to_fp16, weight = text_encoder_transformer_1_pre_norm_mha_1_out_proj_weight_to_fp16, x = input_31_cast_fp16)[name = tensor("linear_5_cast_fp16")]; + tensor x_11_cast_fp16 = add(x = linear_5_cast_fp16, y = x_7_cast_fp16)[name = tensor("x_11_cast_fp16")]; + tensor var_192_axes_0 = const()[name = tensor("op_192_axes_0"), val = tensor([-1])]; + tensor text_encoder_transformer_1_pre_norm_ffn_0_weight_to_fp16 = const()[name = tensor("text_encoder_transformer_1_pre_norm_ffn_0_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(59082048)))]; + tensor text_encoder_transformer_1_pre_norm_ffn_0_bias_to_fp16 = const()[name = tensor("text_encoder_transformer_1_pre_norm_ffn_0_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(59083136)))]; + tensor var_192_cast_fp16 = layer_norm(axes = var_192_axes_0, beta = text_encoder_transformer_1_pre_norm_ffn_0_bias_to_fp16, epsilon = var_5_to_fp16, gamma = text_encoder_transformer_1_pre_norm_ffn_0_weight_to_fp16, x = x_11_cast_fp16)[name = tensor("op_192_cast_fp16")]; + tensor text_encoder_transformer_1_pre_norm_ffn_1_weight_to_fp16 = const()[name = tensor("text_encoder_transformer_1_pre_norm_ffn_1_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(59084224)))]; + tensor text_encoder_transformer_1_pre_norm_ffn_1_bias_to_fp16 = const()[name = tensor("text_encoder_transformer_1_pre_norm_ffn_1_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(61181440)))]; + tensor linear_6_cast_fp16 = linear(bias = text_encoder_transformer_1_pre_norm_ffn_1_bias_to_fp16, weight = text_encoder_transformer_1_pre_norm_ffn_1_weight_to_fp16, x = var_192_cast_fp16)[name = tensor("linear_6_cast_fp16")]; + tensor input_41_mode_0 = const()[name = tensor("input_41_mode_0"), val = tensor("EXACT")]; + tensor input_41_cast_fp16 = gelu(mode = input_41_mode_0, x = linear_6_cast_fp16)[name = tensor("input_41_cast_fp16")]; + tensor text_encoder_transformer_1_pre_norm_ffn_4_weight_to_fp16 = const()[name = tensor("text_encoder_transformer_1_pre_norm_ffn_4_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(61185600)))]; + tensor text_encoder_transformer_1_pre_norm_ffn_4_bias_to_fp16 = const()[name = tensor("text_encoder_transformer_1_pre_norm_ffn_4_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(63282816)))]; + tensor linear_7_cast_fp16 = linear(bias = text_encoder_transformer_1_pre_norm_ffn_4_bias_to_fp16, weight = text_encoder_transformer_1_pre_norm_ffn_4_weight_to_fp16, x = input_41_cast_fp16)[name = tensor("linear_7_cast_fp16")]; + tensor x_13_cast_fp16 = add(x = x_11_cast_fp16, y = linear_7_cast_fp16)[name = tensor("x_13_cast_fp16")]; + tensor var_219_axes_0 = const()[name = tensor("op_219_axes_0"), val = tensor([-1])]; + tensor text_encoder_transformer_2_pre_norm_mha_0_weight_to_fp16 = const()[name = tensor("text_encoder_transformer_2_pre_norm_mha_0_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(63283904)))]; + tensor text_encoder_transformer_2_pre_norm_mha_0_bias_to_fp16 = const()[name = tensor("text_encoder_transformer_2_pre_norm_mha_0_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(63284992)))]; + tensor var_219_cast_fp16 = layer_norm(axes = var_219_axes_0, beta = text_encoder_transformer_2_pre_norm_mha_0_bias_to_fp16, epsilon = var_5_to_fp16, gamma = text_encoder_transformer_2_pre_norm_mha_0_weight_to_fp16, x = x_13_cast_fp16)[name = tensor("op_219_cast_fp16")]; + tensor text_encoder_transformer_2_pre_norm_mha_1_qkv_proj_weight_to_fp16 = const()[name = tensor("text_encoder_transformer_2_pre_norm_mha_1_qkv_proj_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(63286080)))]; + tensor text_encoder_transformer_2_pre_norm_mha_1_qkv_proj_bias_to_fp16 = const()[name = tensor("text_encoder_transformer_2_pre_norm_mha_1_qkv_proj_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(64859008)))]; + tensor linear_8_cast_fp16 = linear(bias = text_encoder_transformer_2_pre_norm_mha_1_qkv_proj_bias_to_fp16, weight = text_encoder_transformer_2_pre_norm_mha_1_qkv_proj_weight_to_fp16, x = var_219_cast_fp16)[name = tensor("linear_8_cast_fp16")]; + tensor var_231 = const()[name = tensor("op_231"), val = tensor([1, 77, 3, 8, -1])]; + tensor qkv_9_cast_fp16 = reshape(shape = var_231, x = linear_8_cast_fp16)[name = tensor("qkv_9_cast_fp16")]; + tensor var_233_perm_0 = const()[name = tensor("op_233_perm_0"), val = tensor([0, 3, 2, 1, 4])]; + tensor query_9_begin_0 = const()[name = tensor("query_9_begin_0"), val = tensor([0, 0, 0, 0, 0])]; + tensor query_9_end_0 = const()[name = tensor("query_9_end_0"), val = tensor([1, 8, 1, 77, 64])]; + tensor query_9_end_mask_0 = const()[name = tensor("query_9_end_mask_0"), val = tensor([true, true, false, true, true])]; + tensor query_9_squeeze_mask_0 = const()[name = tensor("query_9_squeeze_mask_0"), val = tensor([false, false, true, false, false])]; + tensor var_233_cast_fp16 = transpose(perm = var_233_perm_0, x = qkv_9_cast_fp16)[name = tensor("transpose_21")]; + tensor query_9_cast_fp16 = slice_by_index(begin = query_9_begin_0, end = query_9_end_0, end_mask = query_9_end_mask_0, squeeze_mask = query_9_squeeze_mask_0, x = var_233_cast_fp16)[name = tensor("query_9_cast_fp16")]; + tensor key_9_begin_0 = const()[name = tensor("key_9_begin_0"), val = tensor([0, 0, 1, 0, 0])]; + tensor key_9_end_0 = const()[name = tensor("key_9_end_0"), val = tensor([1, 8, 2, 77, 64])]; + tensor key_9_end_mask_0 = const()[name = tensor("key_9_end_mask_0"), val = tensor([true, true, false, true, true])]; + tensor key_9_squeeze_mask_0 = const()[name = tensor("key_9_squeeze_mask_0"), val = tensor([false, false, true, false, false])]; + tensor key_9_cast_fp16 = slice_by_index(begin = key_9_begin_0, end = key_9_end_0, end_mask = key_9_end_mask_0, squeeze_mask = key_9_squeeze_mask_0, x = var_233_cast_fp16)[name = tensor("key_9_cast_fp16")]; + tensor value_5_begin_0 = const()[name = tensor("value_5_begin_0"), val = tensor([0, 0, 2, 0, 0])]; + tensor value_5_end_0 = const()[name = tensor("value_5_end_0"), val = tensor([1, 8, 3, 77, 64])]; + tensor value_5_end_mask_0 = const()[name = tensor("value_5_end_mask_0"), val = tensor([true, true, false, true, true])]; + tensor value_5_squeeze_mask_0 = const()[name = tensor("value_5_squeeze_mask_0"), val = tensor([false, false, true, false, false])]; + tensor value_5_cast_fp16 = slice_by_index(begin = value_5_begin_0, end = value_5_end_0, end_mask = value_5_end_mask_0, squeeze_mask = value_5_squeeze_mask_0, x = var_233_cast_fp16)[name = tensor("value_5_cast_fp16")]; + tensor var_244_to_fp16 = const()[name = tensor("op_244_to_fp16"), val = tensor(0x1p-3)]; + tensor query_11_cast_fp16 = mul(x = query_9_cast_fp16, y = var_244_to_fp16)[name = tensor("query_11_cast_fp16")]; + tensor attn_9_transpose_x_1 = const()[name = tensor("attn_9_transpose_x_1"), val = tensor(false)]; + tensor attn_9_transpose_y_1 = const()[name = tensor("attn_9_transpose_y_1"), val = tensor(true)]; + tensor attn_9_cast_fp16 = matmul(transpose_x = attn_9_transpose_x_1, transpose_y = attn_9_transpose_y_1, x = query_11_cast_fp16, y = key_9_cast_fp16)[name = tensor("attn_9_cast_fp16")]; + tensor attn_as_float_5_cast_fp16 = softmax(axis = var_18, x = attn_9_cast_fp16)[name = tensor("attn_as_float_5_cast_fp16")]; + tensor out_5_transpose_x_0 = const()[name = tensor("out_5_transpose_x_0"), val = tensor(false)]; + tensor out_5_transpose_y_0 = const()[name = tensor("out_5_transpose_y_0"), val = tensor(false)]; + tensor out_5_cast_fp16 = matmul(transpose_x = out_5_transpose_x_0, transpose_y = out_5_transpose_y_0, x = attn_as_float_5_cast_fp16, y = value_5_cast_fp16)[name = tensor("out_5_cast_fp16")]; + tensor var_253_perm_0 = const()[name = tensor("op_253_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_254 = const()[name = tensor("op_254"), val = tensor([1, 77, -1])]; + tensor var_253_cast_fp16 = transpose(perm = var_253_perm_0, x = out_5_cast_fp16)[name = tensor("transpose_20")]; + tensor input_53_cast_fp16 = reshape(shape = var_254, x = var_253_cast_fp16)[name = tensor("input_53_cast_fp16")]; + tensor text_encoder_transformer_2_pre_norm_mha_1_out_proj_weight_to_fp16 = const()[name = tensor("text_encoder_transformer_2_pre_norm_mha_1_out_proj_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(64862144)))]; + tensor text_encoder_transformer_2_pre_norm_mha_1_out_proj_bias_to_fp16 = const()[name = tensor("text_encoder_transformer_2_pre_norm_mha_1_out_proj_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(65386496)))]; + tensor linear_9_cast_fp16 = linear(bias = text_encoder_transformer_2_pre_norm_mha_1_out_proj_bias_to_fp16, weight = text_encoder_transformer_2_pre_norm_mha_1_out_proj_weight_to_fp16, x = input_53_cast_fp16)[name = tensor("linear_9_cast_fp16")]; + tensor x_17_cast_fp16 = add(x = linear_9_cast_fp16, y = x_13_cast_fp16)[name = tensor("x_17_cast_fp16")]; + tensor var_268_axes_0 = const()[name = tensor("op_268_axes_0"), val = tensor([-1])]; + tensor text_encoder_transformer_2_pre_norm_ffn_0_weight_to_fp16 = const()[name = tensor("text_encoder_transformer_2_pre_norm_ffn_0_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(65387584)))]; + tensor text_encoder_transformer_2_pre_norm_ffn_0_bias_to_fp16 = const()[name = tensor("text_encoder_transformer_2_pre_norm_ffn_0_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(65388672)))]; + tensor var_268_cast_fp16 = layer_norm(axes = var_268_axes_0, beta = text_encoder_transformer_2_pre_norm_ffn_0_bias_to_fp16, epsilon = var_5_to_fp16, gamma = text_encoder_transformer_2_pre_norm_ffn_0_weight_to_fp16, x = x_17_cast_fp16)[name = tensor("op_268_cast_fp16")]; + tensor text_encoder_transformer_2_pre_norm_ffn_1_weight_to_fp16 = const()[name = tensor("text_encoder_transformer_2_pre_norm_ffn_1_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(65389760)))]; + tensor text_encoder_transformer_2_pre_norm_ffn_1_bias_to_fp16 = const()[name = tensor("text_encoder_transformer_2_pre_norm_ffn_1_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(67486976)))]; + tensor linear_10_cast_fp16 = linear(bias = text_encoder_transformer_2_pre_norm_ffn_1_bias_to_fp16, weight = text_encoder_transformer_2_pre_norm_ffn_1_weight_to_fp16, x = var_268_cast_fp16)[name = tensor("linear_10_cast_fp16")]; + tensor input_63_mode_0 = const()[name = tensor("input_63_mode_0"), val = tensor("EXACT")]; + tensor input_63_cast_fp16 = gelu(mode = input_63_mode_0, x = linear_10_cast_fp16)[name = tensor("input_63_cast_fp16")]; + tensor text_encoder_transformer_2_pre_norm_ffn_4_weight_to_fp16 = const()[name = tensor("text_encoder_transformer_2_pre_norm_ffn_4_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(67491136)))]; + tensor text_encoder_transformer_2_pre_norm_ffn_4_bias_to_fp16 = const()[name = tensor("text_encoder_transformer_2_pre_norm_ffn_4_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(69588352)))]; + tensor linear_11_cast_fp16 = linear(bias = text_encoder_transformer_2_pre_norm_ffn_4_bias_to_fp16, weight = text_encoder_transformer_2_pre_norm_ffn_4_weight_to_fp16, x = input_63_cast_fp16)[name = tensor("linear_11_cast_fp16")]; + tensor x_19_cast_fp16 = add(x = x_17_cast_fp16, y = linear_11_cast_fp16)[name = tensor("x_19_cast_fp16")]; + tensor var_295_axes_0 = const()[name = tensor("op_295_axes_0"), val = tensor([-1])]; + tensor text_encoder_transformer_3_pre_norm_mha_0_weight_to_fp16 = const()[name = tensor("text_encoder_transformer_3_pre_norm_mha_0_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(69589440)))]; + tensor text_encoder_transformer_3_pre_norm_mha_0_bias_to_fp16 = const()[name = tensor("text_encoder_transformer_3_pre_norm_mha_0_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(69590528)))]; + tensor var_295_cast_fp16 = layer_norm(axes = var_295_axes_0, beta = text_encoder_transformer_3_pre_norm_mha_0_bias_to_fp16, epsilon = var_5_to_fp16, gamma = text_encoder_transformer_3_pre_norm_mha_0_weight_to_fp16, x = x_19_cast_fp16)[name = tensor("op_295_cast_fp16")]; + tensor text_encoder_transformer_3_pre_norm_mha_1_qkv_proj_weight_to_fp16 = const()[name = tensor("text_encoder_transformer_3_pre_norm_mha_1_qkv_proj_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(69591616)))]; + tensor text_encoder_transformer_3_pre_norm_mha_1_qkv_proj_bias_to_fp16 = const()[name = tensor("text_encoder_transformer_3_pre_norm_mha_1_qkv_proj_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(71164544)))]; + tensor linear_12_cast_fp16 = linear(bias = text_encoder_transformer_3_pre_norm_mha_1_qkv_proj_bias_to_fp16, weight = text_encoder_transformer_3_pre_norm_mha_1_qkv_proj_weight_to_fp16, x = var_295_cast_fp16)[name = tensor("linear_12_cast_fp16")]; + tensor var_307 = const()[name = tensor("op_307"), val = tensor([1, 77, 3, 8, -1])]; + tensor qkv_13_cast_fp16 = reshape(shape = var_307, x = linear_12_cast_fp16)[name = tensor("qkv_13_cast_fp16")]; + tensor var_309_perm_0 = const()[name = tensor("op_309_perm_0"), val = tensor([0, 3, 2, 1, 4])]; + tensor query_13_begin_0 = const()[name = tensor("query_13_begin_0"), val = tensor([0, 0, 0, 0, 0])]; + tensor query_13_end_0 = const()[name = tensor("query_13_end_0"), val = tensor([1, 8, 1, 77, 64])]; + tensor query_13_end_mask_0 = const()[name = tensor("query_13_end_mask_0"), val = tensor([true, true, false, true, true])]; + tensor query_13_squeeze_mask_0 = const()[name = tensor("query_13_squeeze_mask_0"), val = tensor([false, false, true, false, false])]; + tensor var_309_cast_fp16 = transpose(perm = var_309_perm_0, x = qkv_13_cast_fp16)[name = tensor("transpose_19")]; + tensor query_13_cast_fp16 = slice_by_index(begin = query_13_begin_0, end = query_13_end_0, end_mask = query_13_end_mask_0, squeeze_mask = query_13_squeeze_mask_0, x = var_309_cast_fp16)[name = tensor("query_13_cast_fp16")]; + tensor key_13_begin_0 = const()[name = tensor("key_13_begin_0"), val = tensor([0, 0, 1, 0, 0])]; + tensor key_13_end_0 = const()[name = tensor("key_13_end_0"), val = tensor([1, 8, 2, 77, 64])]; + tensor key_13_end_mask_0 = const()[name = tensor("key_13_end_mask_0"), val = tensor([true, true, false, true, true])]; + tensor key_13_squeeze_mask_0 = const()[name = tensor("key_13_squeeze_mask_0"), val = tensor([false, false, true, false, false])]; + tensor key_13_cast_fp16 = slice_by_index(begin = key_13_begin_0, end = key_13_end_0, end_mask = key_13_end_mask_0, squeeze_mask = key_13_squeeze_mask_0, x = var_309_cast_fp16)[name = tensor("key_13_cast_fp16")]; + tensor value_7_begin_0 = const()[name = tensor("value_7_begin_0"), val = tensor([0, 0, 2, 0, 0])]; + tensor value_7_end_0 = const()[name = tensor("value_7_end_0"), val = tensor([1, 8, 3, 77, 64])]; + tensor value_7_end_mask_0 = const()[name = tensor("value_7_end_mask_0"), val = tensor([true, true, false, true, true])]; + tensor value_7_squeeze_mask_0 = const()[name = tensor("value_7_squeeze_mask_0"), val = tensor([false, false, true, false, false])]; + tensor value_7_cast_fp16 = slice_by_index(begin = value_7_begin_0, end = value_7_end_0, end_mask = value_7_end_mask_0, squeeze_mask = value_7_squeeze_mask_0, x = var_309_cast_fp16)[name = tensor("value_7_cast_fp16")]; + tensor var_320_to_fp16 = const()[name = tensor("op_320_to_fp16"), val = tensor(0x1p-3)]; + tensor query_15_cast_fp16 = mul(x = query_13_cast_fp16, y = var_320_to_fp16)[name = tensor("query_15_cast_fp16")]; + tensor attn_13_transpose_x_1 = const()[name = tensor("attn_13_transpose_x_1"), val = tensor(false)]; + tensor attn_13_transpose_y_1 = const()[name = tensor("attn_13_transpose_y_1"), val = tensor(true)]; + tensor attn_13_cast_fp16 = matmul(transpose_x = attn_13_transpose_x_1, transpose_y = attn_13_transpose_y_1, x = query_15_cast_fp16, y = key_13_cast_fp16)[name = tensor("attn_13_cast_fp16")]; + tensor attn_as_float_7_cast_fp16 = softmax(axis = var_18, x = attn_13_cast_fp16)[name = tensor("attn_as_float_7_cast_fp16")]; + tensor out_7_transpose_x_0 = const()[name = tensor("out_7_transpose_x_0"), val = tensor(false)]; + tensor out_7_transpose_y_0 = const()[name = tensor("out_7_transpose_y_0"), val = tensor(false)]; + tensor out_7_cast_fp16 = matmul(transpose_x = out_7_transpose_x_0, transpose_y = out_7_transpose_y_0, x = attn_as_float_7_cast_fp16, y = value_7_cast_fp16)[name = tensor("out_7_cast_fp16")]; + tensor var_329_perm_0 = const()[name = tensor("op_329_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_330 = const()[name = tensor("op_330"), val = tensor([1, 77, -1])]; + tensor var_329_cast_fp16 = transpose(perm = var_329_perm_0, x = out_7_cast_fp16)[name = tensor("transpose_18")]; + tensor input_75_cast_fp16 = reshape(shape = var_330, x = var_329_cast_fp16)[name = tensor("input_75_cast_fp16")]; + tensor text_encoder_transformer_3_pre_norm_mha_1_out_proj_weight_to_fp16 = const()[name = tensor("text_encoder_transformer_3_pre_norm_mha_1_out_proj_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(71167680)))]; + tensor text_encoder_transformer_3_pre_norm_mha_1_out_proj_bias_to_fp16 = const()[name = tensor("text_encoder_transformer_3_pre_norm_mha_1_out_proj_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(71692032)))]; + tensor linear_13_cast_fp16 = linear(bias = text_encoder_transformer_3_pre_norm_mha_1_out_proj_bias_to_fp16, weight = text_encoder_transformer_3_pre_norm_mha_1_out_proj_weight_to_fp16, x = input_75_cast_fp16)[name = tensor("linear_13_cast_fp16")]; + tensor x_23_cast_fp16 = add(x = linear_13_cast_fp16, y = x_19_cast_fp16)[name = tensor("x_23_cast_fp16")]; + tensor var_344_axes_0 = const()[name = tensor("op_344_axes_0"), val = tensor([-1])]; + tensor text_encoder_transformer_3_pre_norm_ffn_0_weight_to_fp16 = const()[name = tensor("text_encoder_transformer_3_pre_norm_ffn_0_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(71693120)))]; + tensor text_encoder_transformer_3_pre_norm_ffn_0_bias_to_fp16 = const()[name = tensor("text_encoder_transformer_3_pre_norm_ffn_0_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(71694208)))]; + tensor var_344_cast_fp16 = layer_norm(axes = var_344_axes_0, beta = text_encoder_transformer_3_pre_norm_ffn_0_bias_to_fp16, epsilon = var_5_to_fp16, gamma = text_encoder_transformer_3_pre_norm_ffn_0_weight_to_fp16, x = x_23_cast_fp16)[name = tensor("op_344_cast_fp16")]; + tensor text_encoder_transformer_3_pre_norm_ffn_1_weight_to_fp16 = const()[name = tensor("text_encoder_transformer_3_pre_norm_ffn_1_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(71695296)))]; + tensor text_encoder_transformer_3_pre_norm_ffn_1_bias_to_fp16 = const()[name = tensor("text_encoder_transformer_3_pre_norm_ffn_1_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(73792512)))]; + tensor linear_14_cast_fp16 = linear(bias = text_encoder_transformer_3_pre_norm_ffn_1_bias_to_fp16, weight = text_encoder_transformer_3_pre_norm_ffn_1_weight_to_fp16, x = var_344_cast_fp16)[name = tensor("linear_14_cast_fp16")]; + tensor input_85_mode_0 = const()[name = tensor("input_85_mode_0"), val = tensor("EXACT")]; + tensor input_85_cast_fp16 = gelu(mode = input_85_mode_0, x = linear_14_cast_fp16)[name = tensor("input_85_cast_fp16")]; + tensor text_encoder_transformer_3_pre_norm_ffn_4_weight_to_fp16 = const()[name = tensor("text_encoder_transformer_3_pre_norm_ffn_4_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(73796672)))]; + tensor text_encoder_transformer_3_pre_norm_ffn_4_bias_to_fp16 = const()[name = tensor("text_encoder_transformer_3_pre_norm_ffn_4_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(75893888)))]; + tensor linear_15_cast_fp16 = linear(bias = text_encoder_transformer_3_pre_norm_ffn_4_bias_to_fp16, weight = text_encoder_transformer_3_pre_norm_ffn_4_weight_to_fp16, x = input_85_cast_fp16)[name = tensor("linear_15_cast_fp16")]; + tensor x_25_cast_fp16 = add(x = x_23_cast_fp16, y = linear_15_cast_fp16)[name = tensor("x_25_cast_fp16")]; + tensor var_371_axes_0 = const()[name = tensor("op_371_axes_0"), val = tensor([-1])]; + tensor text_encoder_transformer_4_pre_norm_mha_0_weight_to_fp16 = const()[name = tensor("text_encoder_transformer_4_pre_norm_mha_0_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(75894976)))]; + tensor text_encoder_transformer_4_pre_norm_mha_0_bias_to_fp16 = const()[name = tensor("text_encoder_transformer_4_pre_norm_mha_0_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(75896064)))]; + tensor var_371_cast_fp16 = layer_norm(axes = var_371_axes_0, beta = text_encoder_transformer_4_pre_norm_mha_0_bias_to_fp16, epsilon = var_5_to_fp16, gamma = text_encoder_transformer_4_pre_norm_mha_0_weight_to_fp16, x = x_25_cast_fp16)[name = tensor("op_371_cast_fp16")]; + tensor text_encoder_transformer_4_pre_norm_mha_1_qkv_proj_weight_to_fp16 = const()[name = tensor("text_encoder_transformer_4_pre_norm_mha_1_qkv_proj_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(75897152)))]; + tensor text_encoder_transformer_4_pre_norm_mha_1_qkv_proj_bias_to_fp16 = const()[name = tensor("text_encoder_transformer_4_pre_norm_mha_1_qkv_proj_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(77470080)))]; + tensor linear_16_cast_fp16 = linear(bias = text_encoder_transformer_4_pre_norm_mha_1_qkv_proj_bias_to_fp16, weight = text_encoder_transformer_4_pre_norm_mha_1_qkv_proj_weight_to_fp16, x = var_371_cast_fp16)[name = tensor("linear_16_cast_fp16")]; + tensor var_383 = const()[name = tensor("op_383"), val = tensor([1, 77, 3, 8, -1])]; + tensor qkv_17_cast_fp16 = reshape(shape = var_383, x = linear_16_cast_fp16)[name = tensor("qkv_17_cast_fp16")]; + tensor var_385_perm_0 = const()[name = tensor("op_385_perm_0"), val = tensor([0, 3, 2, 1, 4])]; + tensor query_17_begin_0 = const()[name = tensor("query_17_begin_0"), val = tensor([0, 0, 0, 0, 0])]; + tensor query_17_end_0 = const()[name = tensor("query_17_end_0"), val = tensor([1, 8, 1, 77, 64])]; + tensor query_17_end_mask_0 = const()[name = tensor("query_17_end_mask_0"), val = tensor([true, true, false, true, true])]; + tensor query_17_squeeze_mask_0 = const()[name = tensor("query_17_squeeze_mask_0"), val = tensor([false, false, true, false, false])]; + tensor var_385_cast_fp16 = transpose(perm = var_385_perm_0, x = qkv_17_cast_fp16)[name = tensor("transpose_17")]; + tensor query_17_cast_fp16 = slice_by_index(begin = query_17_begin_0, end = query_17_end_0, end_mask = query_17_end_mask_0, squeeze_mask = query_17_squeeze_mask_0, x = var_385_cast_fp16)[name = tensor("query_17_cast_fp16")]; + tensor key_17_begin_0 = const()[name = tensor("key_17_begin_0"), val = tensor([0, 0, 1, 0, 0])]; + tensor key_17_end_0 = const()[name = tensor("key_17_end_0"), val = tensor([1, 8, 2, 77, 64])]; + tensor key_17_end_mask_0 = const()[name = tensor("key_17_end_mask_0"), val = tensor([true, true, false, true, true])]; + tensor key_17_squeeze_mask_0 = const()[name = tensor("key_17_squeeze_mask_0"), val = tensor([false, false, true, false, false])]; + tensor key_17_cast_fp16 = slice_by_index(begin = key_17_begin_0, end = key_17_end_0, end_mask = key_17_end_mask_0, squeeze_mask = key_17_squeeze_mask_0, x = var_385_cast_fp16)[name = tensor("key_17_cast_fp16")]; + tensor value_9_begin_0 = const()[name = tensor("value_9_begin_0"), val = tensor([0, 0, 2, 0, 0])]; + tensor value_9_end_0 = const()[name = tensor("value_9_end_0"), val = tensor([1, 8, 3, 77, 64])]; + tensor value_9_end_mask_0 = const()[name = tensor("value_9_end_mask_0"), val = tensor([true, true, false, true, true])]; + tensor value_9_squeeze_mask_0 = const()[name = tensor("value_9_squeeze_mask_0"), val = tensor([false, false, true, false, false])]; + tensor value_9_cast_fp16 = slice_by_index(begin = value_9_begin_0, end = value_9_end_0, end_mask = value_9_end_mask_0, squeeze_mask = value_9_squeeze_mask_0, x = var_385_cast_fp16)[name = tensor("value_9_cast_fp16")]; + tensor var_396_to_fp16 = const()[name = tensor("op_396_to_fp16"), val = tensor(0x1p-3)]; + tensor query_19_cast_fp16 = mul(x = query_17_cast_fp16, y = var_396_to_fp16)[name = tensor("query_19_cast_fp16")]; + tensor attn_17_transpose_x_1 = const()[name = tensor("attn_17_transpose_x_1"), val = tensor(false)]; + tensor attn_17_transpose_y_1 = const()[name = tensor("attn_17_transpose_y_1"), val = tensor(true)]; + tensor attn_17_cast_fp16 = matmul(transpose_x = attn_17_transpose_x_1, transpose_y = attn_17_transpose_y_1, x = query_19_cast_fp16, y = key_17_cast_fp16)[name = tensor("attn_17_cast_fp16")]; + tensor attn_as_float_9_cast_fp16 = softmax(axis = var_18, x = attn_17_cast_fp16)[name = tensor("attn_as_float_9_cast_fp16")]; + tensor out_9_transpose_x_0 = const()[name = tensor("out_9_transpose_x_0"), val = tensor(false)]; + tensor out_9_transpose_y_0 = const()[name = tensor("out_9_transpose_y_0"), val = tensor(false)]; + tensor out_9_cast_fp16 = matmul(transpose_x = out_9_transpose_x_0, transpose_y = out_9_transpose_y_0, x = attn_as_float_9_cast_fp16, y = value_9_cast_fp16)[name = tensor("out_9_cast_fp16")]; + tensor var_405_perm_0 = const()[name = tensor("op_405_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_406 = const()[name = tensor("op_406"), val = tensor([1, 77, -1])]; + tensor var_405_cast_fp16 = transpose(perm = var_405_perm_0, x = out_9_cast_fp16)[name = tensor("transpose_16")]; + tensor input_97_cast_fp16 = reshape(shape = var_406, x = var_405_cast_fp16)[name = tensor("input_97_cast_fp16")]; + tensor text_encoder_transformer_4_pre_norm_mha_1_out_proj_weight_to_fp16 = const()[name = tensor("text_encoder_transformer_4_pre_norm_mha_1_out_proj_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(77473216)))]; + tensor text_encoder_transformer_4_pre_norm_mha_1_out_proj_bias_to_fp16 = const()[name = tensor("text_encoder_transformer_4_pre_norm_mha_1_out_proj_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(77997568)))]; + tensor linear_17_cast_fp16 = linear(bias = text_encoder_transformer_4_pre_norm_mha_1_out_proj_bias_to_fp16, weight = text_encoder_transformer_4_pre_norm_mha_1_out_proj_weight_to_fp16, x = input_97_cast_fp16)[name = tensor("linear_17_cast_fp16")]; + tensor x_29_cast_fp16 = add(x = linear_17_cast_fp16, y = x_25_cast_fp16)[name = tensor("x_29_cast_fp16")]; + tensor var_420_axes_0 = const()[name = tensor("op_420_axes_0"), val = tensor([-1])]; + tensor text_encoder_transformer_4_pre_norm_ffn_0_weight_to_fp16 = const()[name = tensor("text_encoder_transformer_4_pre_norm_ffn_0_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(77998656)))]; + tensor text_encoder_transformer_4_pre_norm_ffn_0_bias_to_fp16 = const()[name = tensor("text_encoder_transformer_4_pre_norm_ffn_0_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(77999744)))]; + tensor var_420_cast_fp16 = layer_norm(axes = var_420_axes_0, beta = text_encoder_transformer_4_pre_norm_ffn_0_bias_to_fp16, epsilon = var_5_to_fp16, gamma = text_encoder_transformer_4_pre_norm_ffn_0_weight_to_fp16, x = x_29_cast_fp16)[name = tensor("op_420_cast_fp16")]; + tensor text_encoder_transformer_4_pre_norm_ffn_1_weight_to_fp16 = const()[name = tensor("text_encoder_transformer_4_pre_norm_ffn_1_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(78000832)))]; + tensor text_encoder_transformer_4_pre_norm_ffn_1_bias_to_fp16 = const()[name = tensor("text_encoder_transformer_4_pre_norm_ffn_1_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(80098048)))]; + tensor linear_18_cast_fp16 = linear(bias = text_encoder_transformer_4_pre_norm_ffn_1_bias_to_fp16, weight = text_encoder_transformer_4_pre_norm_ffn_1_weight_to_fp16, x = var_420_cast_fp16)[name = tensor("linear_18_cast_fp16")]; + tensor input_107_mode_0 = const()[name = tensor("input_107_mode_0"), val = tensor("EXACT")]; + tensor input_107_cast_fp16 = gelu(mode = input_107_mode_0, x = linear_18_cast_fp16)[name = tensor("input_107_cast_fp16")]; + tensor text_encoder_transformer_4_pre_norm_ffn_4_weight_to_fp16 = const()[name = tensor("text_encoder_transformer_4_pre_norm_ffn_4_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(80102208)))]; + tensor text_encoder_transformer_4_pre_norm_ffn_4_bias_to_fp16 = const()[name = tensor("text_encoder_transformer_4_pre_norm_ffn_4_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(82199424)))]; + tensor linear_19_cast_fp16 = linear(bias = text_encoder_transformer_4_pre_norm_ffn_4_bias_to_fp16, weight = text_encoder_transformer_4_pre_norm_ffn_4_weight_to_fp16, x = input_107_cast_fp16)[name = tensor("linear_19_cast_fp16")]; + tensor x_31_cast_fp16 = add(x = x_29_cast_fp16, y = linear_19_cast_fp16)[name = tensor("x_31_cast_fp16")]; + tensor var_447_axes_0 = const()[name = tensor("op_447_axes_0"), val = tensor([-1])]; + tensor text_encoder_transformer_5_pre_norm_mha_0_weight_to_fp16 = const()[name = tensor("text_encoder_transformer_5_pre_norm_mha_0_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(82200512)))]; + tensor text_encoder_transformer_5_pre_norm_mha_0_bias_to_fp16 = const()[name = tensor("text_encoder_transformer_5_pre_norm_mha_0_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(82201600)))]; + tensor var_447_cast_fp16 = layer_norm(axes = var_447_axes_0, beta = text_encoder_transformer_5_pre_norm_mha_0_bias_to_fp16, epsilon = var_5_to_fp16, gamma = text_encoder_transformer_5_pre_norm_mha_0_weight_to_fp16, x = x_31_cast_fp16)[name = tensor("op_447_cast_fp16")]; + tensor text_encoder_transformer_5_pre_norm_mha_1_qkv_proj_weight_to_fp16 = const()[name = tensor("text_encoder_transformer_5_pre_norm_mha_1_qkv_proj_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(82202688)))]; + tensor text_encoder_transformer_5_pre_norm_mha_1_qkv_proj_bias_to_fp16 = const()[name = tensor("text_encoder_transformer_5_pre_norm_mha_1_qkv_proj_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(83775616)))]; + tensor linear_20_cast_fp16 = linear(bias = text_encoder_transformer_5_pre_norm_mha_1_qkv_proj_bias_to_fp16, weight = text_encoder_transformer_5_pre_norm_mha_1_qkv_proj_weight_to_fp16, x = var_447_cast_fp16)[name = tensor("linear_20_cast_fp16")]; + tensor var_459 = const()[name = tensor("op_459"), val = tensor([1, 77, 3, 8, -1])]; + tensor qkv_21_cast_fp16 = reshape(shape = var_459, x = linear_20_cast_fp16)[name = tensor("qkv_21_cast_fp16")]; + tensor var_461_perm_0 = const()[name = tensor("op_461_perm_0"), val = tensor([0, 3, 2, 1, 4])]; + tensor query_21_begin_0 = const()[name = tensor("query_21_begin_0"), val = tensor([0, 0, 0, 0, 0])]; + tensor query_21_end_0 = const()[name = tensor("query_21_end_0"), val = tensor([1, 8, 1, 77, 64])]; + tensor query_21_end_mask_0 = const()[name = tensor("query_21_end_mask_0"), val = tensor([true, true, false, true, true])]; + tensor query_21_squeeze_mask_0 = const()[name = tensor("query_21_squeeze_mask_0"), val = tensor([false, false, true, false, false])]; + tensor var_461_cast_fp16 = transpose(perm = var_461_perm_0, x = qkv_21_cast_fp16)[name = tensor("transpose_15")]; + tensor query_21_cast_fp16 = slice_by_index(begin = query_21_begin_0, end = query_21_end_0, end_mask = query_21_end_mask_0, squeeze_mask = query_21_squeeze_mask_0, x = var_461_cast_fp16)[name = tensor("query_21_cast_fp16")]; + tensor key_21_begin_0 = const()[name = tensor("key_21_begin_0"), val = tensor([0, 0, 1, 0, 0])]; + tensor key_21_end_0 = const()[name = tensor("key_21_end_0"), val = tensor([1, 8, 2, 77, 64])]; + tensor key_21_end_mask_0 = const()[name = tensor("key_21_end_mask_0"), val = tensor([true, true, false, true, true])]; + tensor key_21_squeeze_mask_0 = const()[name = tensor("key_21_squeeze_mask_0"), val = tensor([false, false, true, false, false])]; + tensor key_21_cast_fp16 = slice_by_index(begin = key_21_begin_0, end = key_21_end_0, end_mask = key_21_end_mask_0, squeeze_mask = key_21_squeeze_mask_0, x = var_461_cast_fp16)[name = tensor("key_21_cast_fp16")]; + tensor value_11_begin_0 = const()[name = tensor("value_11_begin_0"), val = tensor([0, 0, 2, 0, 0])]; + tensor value_11_end_0 = const()[name = tensor("value_11_end_0"), val = tensor([1, 8, 3, 77, 64])]; + tensor value_11_end_mask_0 = const()[name = tensor("value_11_end_mask_0"), val = tensor([true, true, false, true, true])]; + tensor value_11_squeeze_mask_0 = const()[name = tensor("value_11_squeeze_mask_0"), val = tensor([false, false, true, false, false])]; + tensor value_11_cast_fp16 = slice_by_index(begin = value_11_begin_0, end = value_11_end_0, end_mask = value_11_end_mask_0, squeeze_mask = value_11_squeeze_mask_0, x = var_461_cast_fp16)[name = tensor("value_11_cast_fp16")]; + tensor var_472_to_fp16 = const()[name = tensor("op_472_to_fp16"), val = tensor(0x1p-3)]; + tensor query_23_cast_fp16 = mul(x = query_21_cast_fp16, y = var_472_to_fp16)[name = tensor("query_23_cast_fp16")]; + tensor attn_21_transpose_x_1 = const()[name = tensor("attn_21_transpose_x_1"), val = tensor(false)]; + tensor attn_21_transpose_y_1 = const()[name = tensor("attn_21_transpose_y_1"), val = tensor(true)]; + tensor attn_21_cast_fp16 = matmul(transpose_x = attn_21_transpose_x_1, transpose_y = attn_21_transpose_y_1, x = query_23_cast_fp16, y = key_21_cast_fp16)[name = tensor("attn_21_cast_fp16")]; + tensor attn_as_float_11_cast_fp16 = softmax(axis = var_18, x = attn_21_cast_fp16)[name = tensor("attn_as_float_11_cast_fp16")]; + tensor out_11_transpose_x_0 = const()[name = tensor("out_11_transpose_x_0"), val = tensor(false)]; + tensor out_11_transpose_y_0 = const()[name = tensor("out_11_transpose_y_0"), val = tensor(false)]; + tensor out_11_cast_fp16 = matmul(transpose_x = out_11_transpose_x_0, transpose_y = out_11_transpose_y_0, x = attn_as_float_11_cast_fp16, y = value_11_cast_fp16)[name = tensor("out_11_cast_fp16")]; + tensor var_481_perm_0 = const()[name = tensor("op_481_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_482 = const()[name = tensor("op_482"), val = tensor([1, 77, -1])]; + tensor var_481_cast_fp16 = transpose(perm = var_481_perm_0, x = out_11_cast_fp16)[name = tensor("transpose_14")]; + tensor input_119_cast_fp16 = reshape(shape = var_482, x = var_481_cast_fp16)[name = tensor("input_119_cast_fp16")]; + tensor text_encoder_transformer_5_pre_norm_mha_1_out_proj_weight_to_fp16 = const()[name = tensor("text_encoder_transformer_5_pre_norm_mha_1_out_proj_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(83778752)))]; + tensor text_encoder_transformer_5_pre_norm_mha_1_out_proj_bias_to_fp16 = const()[name = tensor("text_encoder_transformer_5_pre_norm_mha_1_out_proj_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(84303104)))]; + tensor linear_21_cast_fp16 = linear(bias = text_encoder_transformer_5_pre_norm_mha_1_out_proj_bias_to_fp16, weight = text_encoder_transformer_5_pre_norm_mha_1_out_proj_weight_to_fp16, x = input_119_cast_fp16)[name = tensor("linear_21_cast_fp16")]; + tensor x_35_cast_fp16 = add(x = linear_21_cast_fp16, y = x_31_cast_fp16)[name = tensor("x_35_cast_fp16")]; + tensor var_496_axes_0 = const()[name = tensor("op_496_axes_0"), val = tensor([-1])]; + tensor text_encoder_transformer_5_pre_norm_ffn_0_weight_to_fp16 = const()[name = tensor("text_encoder_transformer_5_pre_norm_ffn_0_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(84304192)))]; + tensor text_encoder_transformer_5_pre_norm_ffn_0_bias_to_fp16 = const()[name = tensor("text_encoder_transformer_5_pre_norm_ffn_0_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(84305280)))]; + tensor var_496_cast_fp16 = layer_norm(axes = var_496_axes_0, beta = text_encoder_transformer_5_pre_norm_ffn_0_bias_to_fp16, epsilon = var_5_to_fp16, gamma = text_encoder_transformer_5_pre_norm_ffn_0_weight_to_fp16, x = x_35_cast_fp16)[name = tensor("op_496_cast_fp16")]; + tensor text_encoder_transformer_5_pre_norm_ffn_1_weight_to_fp16 = const()[name = tensor("text_encoder_transformer_5_pre_norm_ffn_1_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(84306368)))]; + tensor text_encoder_transformer_5_pre_norm_ffn_1_bias_to_fp16 = const()[name = tensor("text_encoder_transformer_5_pre_norm_ffn_1_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(86403584)))]; + tensor linear_22_cast_fp16 = linear(bias = text_encoder_transformer_5_pre_norm_ffn_1_bias_to_fp16, weight = text_encoder_transformer_5_pre_norm_ffn_1_weight_to_fp16, x = var_496_cast_fp16)[name = tensor("linear_22_cast_fp16")]; + tensor input_129_mode_0 = const()[name = tensor("input_129_mode_0"), val = tensor("EXACT")]; + tensor input_129_cast_fp16 = gelu(mode = input_129_mode_0, x = linear_22_cast_fp16)[name = tensor("input_129_cast_fp16")]; + tensor text_encoder_transformer_5_pre_norm_ffn_4_weight_to_fp16 = const()[name = tensor("text_encoder_transformer_5_pre_norm_ffn_4_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(86407744)))]; + tensor text_encoder_transformer_5_pre_norm_ffn_4_bias_to_fp16 = const()[name = tensor("text_encoder_transformer_5_pre_norm_ffn_4_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(88504960)))]; + tensor linear_23_cast_fp16 = linear(bias = text_encoder_transformer_5_pre_norm_ffn_4_bias_to_fp16, weight = text_encoder_transformer_5_pre_norm_ffn_4_weight_to_fp16, x = input_129_cast_fp16)[name = tensor("linear_23_cast_fp16")]; + tensor x_37_cast_fp16 = add(x = x_35_cast_fp16, y = linear_23_cast_fp16)[name = tensor("x_37_cast_fp16")]; + tensor var_523_axes_0 = const()[name = tensor("op_523_axes_0"), val = tensor([-1])]; + tensor text_encoder_transformer_6_pre_norm_mha_0_weight_to_fp16 = const()[name = tensor("text_encoder_transformer_6_pre_norm_mha_0_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(88506048)))]; + tensor text_encoder_transformer_6_pre_norm_mha_0_bias_to_fp16 = const()[name = tensor("text_encoder_transformer_6_pre_norm_mha_0_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(88507136)))]; + tensor var_523_cast_fp16 = layer_norm(axes = var_523_axes_0, beta = text_encoder_transformer_6_pre_norm_mha_0_bias_to_fp16, epsilon = var_5_to_fp16, gamma = text_encoder_transformer_6_pre_norm_mha_0_weight_to_fp16, x = x_37_cast_fp16)[name = tensor("op_523_cast_fp16")]; + tensor text_encoder_transformer_6_pre_norm_mha_1_qkv_proj_weight_to_fp16 = const()[name = tensor("text_encoder_transformer_6_pre_norm_mha_1_qkv_proj_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(88508224)))]; + tensor text_encoder_transformer_6_pre_norm_mha_1_qkv_proj_bias_to_fp16 = const()[name = tensor("text_encoder_transformer_6_pre_norm_mha_1_qkv_proj_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(90081152)))]; + tensor linear_24_cast_fp16 = linear(bias = text_encoder_transformer_6_pre_norm_mha_1_qkv_proj_bias_to_fp16, weight = text_encoder_transformer_6_pre_norm_mha_1_qkv_proj_weight_to_fp16, x = var_523_cast_fp16)[name = tensor("linear_24_cast_fp16")]; + tensor var_535 = const()[name = tensor("op_535"), val = tensor([1, 77, 3, 8, -1])]; + tensor qkv_25_cast_fp16 = reshape(shape = var_535, x = linear_24_cast_fp16)[name = tensor("qkv_25_cast_fp16")]; + tensor var_537_perm_0 = const()[name = tensor("op_537_perm_0"), val = tensor([0, 3, 2, 1, 4])]; + tensor query_25_begin_0 = const()[name = tensor("query_25_begin_0"), val = tensor([0, 0, 0, 0, 0])]; + tensor query_25_end_0 = const()[name = tensor("query_25_end_0"), val = tensor([1, 8, 1, 77, 64])]; + tensor query_25_end_mask_0 = const()[name = tensor("query_25_end_mask_0"), val = tensor([true, true, false, true, true])]; + tensor query_25_squeeze_mask_0 = const()[name = tensor("query_25_squeeze_mask_0"), val = tensor([false, false, true, false, false])]; + tensor var_537_cast_fp16 = transpose(perm = var_537_perm_0, x = qkv_25_cast_fp16)[name = tensor("transpose_13")]; + tensor query_25_cast_fp16 = slice_by_index(begin = query_25_begin_0, end = query_25_end_0, end_mask = query_25_end_mask_0, squeeze_mask = query_25_squeeze_mask_0, x = var_537_cast_fp16)[name = tensor("query_25_cast_fp16")]; + tensor key_25_begin_0 = const()[name = tensor("key_25_begin_0"), val = tensor([0, 0, 1, 0, 0])]; + tensor key_25_end_0 = const()[name = tensor("key_25_end_0"), val = tensor([1, 8, 2, 77, 64])]; + tensor key_25_end_mask_0 = const()[name = tensor("key_25_end_mask_0"), val = tensor([true, true, false, true, true])]; + tensor key_25_squeeze_mask_0 = const()[name = tensor("key_25_squeeze_mask_0"), val = tensor([false, false, true, false, false])]; + tensor key_25_cast_fp16 = slice_by_index(begin = key_25_begin_0, end = key_25_end_0, end_mask = key_25_end_mask_0, squeeze_mask = key_25_squeeze_mask_0, x = var_537_cast_fp16)[name = tensor("key_25_cast_fp16")]; + tensor value_13_begin_0 = const()[name = tensor("value_13_begin_0"), val = tensor([0, 0, 2, 0, 0])]; + tensor value_13_end_0 = const()[name = tensor("value_13_end_0"), val = tensor([1, 8, 3, 77, 64])]; + tensor value_13_end_mask_0 = const()[name = tensor("value_13_end_mask_0"), val = tensor([true, true, false, true, true])]; + tensor value_13_squeeze_mask_0 = const()[name = tensor("value_13_squeeze_mask_0"), val = tensor([false, false, true, false, false])]; + tensor value_13_cast_fp16 = slice_by_index(begin = value_13_begin_0, end = value_13_end_0, end_mask = value_13_end_mask_0, squeeze_mask = value_13_squeeze_mask_0, x = var_537_cast_fp16)[name = tensor("value_13_cast_fp16")]; + tensor var_548_to_fp16 = const()[name = tensor("op_548_to_fp16"), val = tensor(0x1p-3)]; + tensor query_27_cast_fp16 = mul(x = query_25_cast_fp16, y = var_548_to_fp16)[name = tensor("query_27_cast_fp16")]; + tensor attn_25_transpose_x_1 = const()[name = tensor("attn_25_transpose_x_1"), val = tensor(false)]; + tensor attn_25_transpose_y_1 = const()[name = tensor("attn_25_transpose_y_1"), val = tensor(true)]; + tensor attn_25_cast_fp16 = matmul(transpose_x = attn_25_transpose_x_1, transpose_y = attn_25_transpose_y_1, x = query_27_cast_fp16, y = key_25_cast_fp16)[name = tensor("attn_25_cast_fp16")]; + tensor attn_as_float_13_cast_fp16 = softmax(axis = var_18, x = attn_25_cast_fp16)[name = tensor("attn_as_float_13_cast_fp16")]; + tensor out_13_transpose_x_0 = const()[name = tensor("out_13_transpose_x_0"), val = tensor(false)]; + tensor out_13_transpose_y_0 = const()[name = tensor("out_13_transpose_y_0"), val = tensor(false)]; + tensor out_13_cast_fp16 = matmul(transpose_x = out_13_transpose_x_0, transpose_y = out_13_transpose_y_0, x = attn_as_float_13_cast_fp16, y = value_13_cast_fp16)[name = tensor("out_13_cast_fp16")]; + tensor var_557_perm_0 = const()[name = tensor("op_557_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_558 = const()[name = tensor("op_558"), val = tensor([1, 77, -1])]; + tensor var_557_cast_fp16 = transpose(perm = var_557_perm_0, x = out_13_cast_fp16)[name = tensor("transpose_12")]; + tensor input_141_cast_fp16 = reshape(shape = var_558, x = var_557_cast_fp16)[name = tensor("input_141_cast_fp16")]; + tensor text_encoder_transformer_6_pre_norm_mha_1_out_proj_weight_to_fp16 = const()[name = tensor("text_encoder_transformer_6_pre_norm_mha_1_out_proj_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(90084288)))]; + tensor text_encoder_transformer_6_pre_norm_mha_1_out_proj_bias_to_fp16 = const()[name = tensor("text_encoder_transformer_6_pre_norm_mha_1_out_proj_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(90608640)))]; + tensor linear_25_cast_fp16 = linear(bias = text_encoder_transformer_6_pre_norm_mha_1_out_proj_bias_to_fp16, weight = text_encoder_transformer_6_pre_norm_mha_1_out_proj_weight_to_fp16, x = input_141_cast_fp16)[name = tensor("linear_25_cast_fp16")]; + tensor x_41_cast_fp16 = add(x = linear_25_cast_fp16, y = x_37_cast_fp16)[name = tensor("x_41_cast_fp16")]; + tensor var_572_axes_0 = const()[name = tensor("op_572_axes_0"), val = tensor([-1])]; + tensor text_encoder_transformer_6_pre_norm_ffn_0_weight_to_fp16 = const()[name = tensor("text_encoder_transformer_6_pre_norm_ffn_0_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(90609728)))]; + tensor text_encoder_transformer_6_pre_norm_ffn_0_bias_to_fp16 = const()[name = tensor("text_encoder_transformer_6_pre_norm_ffn_0_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(90610816)))]; + tensor var_572_cast_fp16 = layer_norm(axes = var_572_axes_0, beta = text_encoder_transformer_6_pre_norm_ffn_0_bias_to_fp16, epsilon = var_5_to_fp16, gamma = text_encoder_transformer_6_pre_norm_ffn_0_weight_to_fp16, x = x_41_cast_fp16)[name = tensor("op_572_cast_fp16")]; + tensor text_encoder_transformer_6_pre_norm_ffn_1_weight_to_fp16 = const()[name = tensor("text_encoder_transformer_6_pre_norm_ffn_1_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(90611904)))]; + tensor text_encoder_transformer_6_pre_norm_ffn_1_bias_to_fp16 = const()[name = tensor("text_encoder_transformer_6_pre_norm_ffn_1_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(92709120)))]; + tensor linear_26_cast_fp16 = linear(bias = text_encoder_transformer_6_pre_norm_ffn_1_bias_to_fp16, weight = text_encoder_transformer_6_pre_norm_ffn_1_weight_to_fp16, x = var_572_cast_fp16)[name = tensor("linear_26_cast_fp16")]; + tensor input_151_mode_0 = const()[name = tensor("input_151_mode_0"), val = tensor("EXACT")]; + tensor input_151_cast_fp16 = gelu(mode = input_151_mode_0, x = linear_26_cast_fp16)[name = tensor("input_151_cast_fp16")]; + tensor text_encoder_transformer_6_pre_norm_ffn_4_weight_to_fp16 = const()[name = tensor("text_encoder_transformer_6_pre_norm_ffn_4_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(92713280)))]; + tensor text_encoder_transformer_6_pre_norm_ffn_4_bias_to_fp16 = const()[name = tensor("text_encoder_transformer_6_pre_norm_ffn_4_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(94810496)))]; + tensor linear_27_cast_fp16 = linear(bias = text_encoder_transformer_6_pre_norm_ffn_4_bias_to_fp16, weight = text_encoder_transformer_6_pre_norm_ffn_4_weight_to_fp16, x = input_151_cast_fp16)[name = tensor("linear_27_cast_fp16")]; + tensor x_43_cast_fp16 = add(x = x_41_cast_fp16, y = linear_27_cast_fp16)[name = tensor("x_43_cast_fp16")]; + tensor var_599_axes_0 = const()[name = tensor("op_599_axes_0"), val = tensor([-1])]; + tensor text_encoder_transformer_7_pre_norm_mha_0_weight_to_fp16 = const()[name = tensor("text_encoder_transformer_7_pre_norm_mha_0_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(94811584)))]; + tensor text_encoder_transformer_7_pre_norm_mha_0_bias_to_fp16 = const()[name = tensor("text_encoder_transformer_7_pre_norm_mha_0_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(94812672)))]; + tensor var_599_cast_fp16 = layer_norm(axes = var_599_axes_0, beta = text_encoder_transformer_7_pre_norm_mha_0_bias_to_fp16, epsilon = var_5_to_fp16, gamma = text_encoder_transformer_7_pre_norm_mha_0_weight_to_fp16, x = x_43_cast_fp16)[name = tensor("op_599_cast_fp16")]; + tensor text_encoder_transformer_7_pre_norm_mha_1_qkv_proj_weight_to_fp16 = const()[name = tensor("text_encoder_transformer_7_pre_norm_mha_1_qkv_proj_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(94813760)))]; + tensor text_encoder_transformer_7_pre_norm_mha_1_qkv_proj_bias_to_fp16 = const()[name = tensor("text_encoder_transformer_7_pre_norm_mha_1_qkv_proj_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(96386688)))]; + tensor linear_28_cast_fp16 = linear(bias = text_encoder_transformer_7_pre_norm_mha_1_qkv_proj_bias_to_fp16, weight = text_encoder_transformer_7_pre_norm_mha_1_qkv_proj_weight_to_fp16, x = var_599_cast_fp16)[name = tensor("linear_28_cast_fp16")]; + tensor var_611 = const()[name = tensor("op_611"), val = tensor([1, 77, 3, 8, -1])]; + tensor qkv_29_cast_fp16 = reshape(shape = var_611, x = linear_28_cast_fp16)[name = tensor("qkv_29_cast_fp16")]; + tensor var_613_perm_0 = const()[name = tensor("op_613_perm_0"), val = tensor([0, 3, 2, 1, 4])]; + tensor query_29_begin_0 = const()[name = tensor("query_29_begin_0"), val = tensor([0, 0, 0, 0, 0])]; + tensor query_29_end_0 = const()[name = tensor("query_29_end_0"), val = tensor([1, 8, 1, 77, 64])]; + tensor query_29_end_mask_0 = const()[name = tensor("query_29_end_mask_0"), val = tensor([true, true, false, true, true])]; + tensor query_29_squeeze_mask_0 = const()[name = tensor("query_29_squeeze_mask_0"), val = tensor([false, false, true, false, false])]; + tensor var_613_cast_fp16 = transpose(perm = var_613_perm_0, x = qkv_29_cast_fp16)[name = tensor("transpose_11")]; + tensor query_29_cast_fp16 = slice_by_index(begin = query_29_begin_0, end = query_29_end_0, end_mask = query_29_end_mask_0, squeeze_mask = query_29_squeeze_mask_0, x = var_613_cast_fp16)[name = tensor("query_29_cast_fp16")]; + tensor key_29_begin_0 = const()[name = tensor("key_29_begin_0"), val = tensor([0, 0, 1, 0, 0])]; + tensor key_29_end_0 = const()[name = tensor("key_29_end_0"), val = tensor([1, 8, 2, 77, 64])]; + tensor key_29_end_mask_0 = const()[name = tensor("key_29_end_mask_0"), val = tensor([true, true, false, true, true])]; + tensor key_29_squeeze_mask_0 = const()[name = tensor("key_29_squeeze_mask_0"), val = tensor([false, false, true, false, false])]; + tensor key_29_cast_fp16 = slice_by_index(begin = key_29_begin_0, end = key_29_end_0, end_mask = key_29_end_mask_0, squeeze_mask = key_29_squeeze_mask_0, x = var_613_cast_fp16)[name = tensor("key_29_cast_fp16")]; + tensor value_15_begin_0 = const()[name = tensor("value_15_begin_0"), val = tensor([0, 0, 2, 0, 0])]; + tensor value_15_end_0 = const()[name = tensor("value_15_end_0"), val = tensor([1, 8, 3, 77, 64])]; + tensor value_15_end_mask_0 = const()[name = tensor("value_15_end_mask_0"), val = tensor([true, true, false, true, true])]; + tensor value_15_squeeze_mask_0 = const()[name = tensor("value_15_squeeze_mask_0"), val = tensor([false, false, true, false, false])]; + tensor value_15_cast_fp16 = slice_by_index(begin = value_15_begin_0, end = value_15_end_0, end_mask = value_15_end_mask_0, squeeze_mask = value_15_squeeze_mask_0, x = var_613_cast_fp16)[name = tensor("value_15_cast_fp16")]; + tensor var_624_to_fp16 = const()[name = tensor("op_624_to_fp16"), val = tensor(0x1p-3)]; + tensor query_31_cast_fp16 = mul(x = query_29_cast_fp16, y = var_624_to_fp16)[name = tensor("query_31_cast_fp16")]; + tensor attn_29_transpose_x_1 = const()[name = tensor("attn_29_transpose_x_1"), val = tensor(false)]; + tensor attn_29_transpose_y_1 = const()[name = tensor("attn_29_transpose_y_1"), val = tensor(true)]; + tensor attn_29_cast_fp16 = matmul(transpose_x = attn_29_transpose_x_1, transpose_y = attn_29_transpose_y_1, x = query_31_cast_fp16, y = key_29_cast_fp16)[name = tensor("attn_29_cast_fp16")]; + tensor attn_as_float_15_cast_fp16 = softmax(axis = var_18, x = attn_29_cast_fp16)[name = tensor("attn_as_float_15_cast_fp16")]; + tensor out_15_transpose_x_0 = const()[name = tensor("out_15_transpose_x_0"), val = tensor(false)]; + tensor out_15_transpose_y_0 = const()[name = tensor("out_15_transpose_y_0"), val = tensor(false)]; + tensor out_15_cast_fp16 = matmul(transpose_x = out_15_transpose_x_0, transpose_y = out_15_transpose_y_0, x = attn_as_float_15_cast_fp16, y = value_15_cast_fp16)[name = tensor("out_15_cast_fp16")]; + tensor var_633_perm_0 = const()[name = tensor("op_633_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_634 = const()[name = tensor("op_634"), val = tensor([1, 77, -1])]; + tensor var_633_cast_fp16 = transpose(perm = var_633_perm_0, x = out_15_cast_fp16)[name = tensor("transpose_10")]; + tensor input_163_cast_fp16 = reshape(shape = var_634, x = var_633_cast_fp16)[name = tensor("input_163_cast_fp16")]; + tensor text_encoder_transformer_7_pre_norm_mha_1_out_proj_weight_to_fp16 = const()[name = tensor("text_encoder_transformer_7_pre_norm_mha_1_out_proj_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(96389824)))]; + tensor text_encoder_transformer_7_pre_norm_mha_1_out_proj_bias_to_fp16 = const()[name = tensor("text_encoder_transformer_7_pre_norm_mha_1_out_proj_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(96914176)))]; + tensor linear_29_cast_fp16 = linear(bias = text_encoder_transformer_7_pre_norm_mha_1_out_proj_bias_to_fp16, weight = text_encoder_transformer_7_pre_norm_mha_1_out_proj_weight_to_fp16, x = input_163_cast_fp16)[name = tensor("linear_29_cast_fp16")]; + tensor x_47_cast_fp16 = add(x = linear_29_cast_fp16, y = x_43_cast_fp16)[name = tensor("x_47_cast_fp16")]; + tensor var_648_axes_0 = const()[name = tensor("op_648_axes_0"), val = tensor([-1])]; + tensor text_encoder_transformer_7_pre_norm_ffn_0_weight_to_fp16 = const()[name = tensor("text_encoder_transformer_7_pre_norm_ffn_0_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(96915264)))]; + tensor text_encoder_transformer_7_pre_norm_ffn_0_bias_to_fp16 = const()[name = tensor("text_encoder_transformer_7_pre_norm_ffn_0_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(96916352)))]; + tensor var_648_cast_fp16 = layer_norm(axes = var_648_axes_0, beta = text_encoder_transformer_7_pre_norm_ffn_0_bias_to_fp16, epsilon = var_5_to_fp16, gamma = text_encoder_transformer_7_pre_norm_ffn_0_weight_to_fp16, x = x_47_cast_fp16)[name = tensor("op_648_cast_fp16")]; + tensor text_encoder_transformer_7_pre_norm_ffn_1_weight_to_fp16 = const()[name = tensor("text_encoder_transformer_7_pre_norm_ffn_1_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(96917440)))]; + tensor text_encoder_transformer_7_pre_norm_ffn_1_bias_to_fp16 = const()[name = tensor("text_encoder_transformer_7_pre_norm_ffn_1_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(99014656)))]; + tensor linear_30_cast_fp16 = linear(bias = text_encoder_transformer_7_pre_norm_ffn_1_bias_to_fp16, weight = text_encoder_transformer_7_pre_norm_ffn_1_weight_to_fp16, x = var_648_cast_fp16)[name = tensor("linear_30_cast_fp16")]; + tensor input_173_mode_0 = const()[name = tensor("input_173_mode_0"), val = tensor("EXACT")]; + tensor input_173_cast_fp16 = gelu(mode = input_173_mode_0, x = linear_30_cast_fp16)[name = tensor("input_173_cast_fp16")]; + tensor text_encoder_transformer_7_pre_norm_ffn_4_weight_to_fp16 = const()[name = tensor("text_encoder_transformer_7_pre_norm_ffn_4_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(99018816)))]; + tensor text_encoder_transformer_7_pre_norm_ffn_4_bias_to_fp16 = const()[name = tensor("text_encoder_transformer_7_pre_norm_ffn_4_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(101116032)))]; + tensor linear_31_cast_fp16 = linear(bias = text_encoder_transformer_7_pre_norm_ffn_4_bias_to_fp16, weight = text_encoder_transformer_7_pre_norm_ffn_4_weight_to_fp16, x = input_173_cast_fp16)[name = tensor("linear_31_cast_fp16")]; + tensor x_49_cast_fp16 = add(x = x_47_cast_fp16, y = linear_31_cast_fp16)[name = tensor("x_49_cast_fp16")]; + tensor var_675_axes_0 = const()[name = tensor("op_675_axes_0"), val = tensor([-1])]; + tensor text_encoder_transformer_8_pre_norm_mha_0_weight_to_fp16 = const()[name = tensor("text_encoder_transformer_8_pre_norm_mha_0_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(101117120)))]; + tensor text_encoder_transformer_8_pre_norm_mha_0_bias_to_fp16 = const()[name = tensor("text_encoder_transformer_8_pre_norm_mha_0_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(101118208)))]; + tensor var_675_cast_fp16 = layer_norm(axes = var_675_axes_0, beta = text_encoder_transformer_8_pre_norm_mha_0_bias_to_fp16, epsilon = var_5_to_fp16, gamma = text_encoder_transformer_8_pre_norm_mha_0_weight_to_fp16, x = x_49_cast_fp16)[name = tensor("op_675_cast_fp16")]; + tensor text_encoder_transformer_8_pre_norm_mha_1_qkv_proj_weight_to_fp16 = const()[name = tensor("text_encoder_transformer_8_pre_norm_mha_1_qkv_proj_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(101119296)))]; + tensor text_encoder_transformer_8_pre_norm_mha_1_qkv_proj_bias_to_fp16 = const()[name = tensor("text_encoder_transformer_8_pre_norm_mha_1_qkv_proj_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(102692224)))]; + tensor linear_32_cast_fp16 = linear(bias = text_encoder_transformer_8_pre_norm_mha_1_qkv_proj_bias_to_fp16, weight = text_encoder_transformer_8_pre_norm_mha_1_qkv_proj_weight_to_fp16, x = var_675_cast_fp16)[name = tensor("linear_32_cast_fp16")]; + tensor var_687 = const()[name = tensor("op_687"), val = tensor([1, 77, 3, 8, -1])]; + tensor qkv_33_cast_fp16 = reshape(shape = var_687, x = linear_32_cast_fp16)[name = tensor("qkv_33_cast_fp16")]; + tensor var_689_perm_0 = const()[name = tensor("op_689_perm_0"), val = tensor([0, 3, 2, 1, 4])]; + tensor query_33_begin_0 = const()[name = tensor("query_33_begin_0"), val = tensor([0, 0, 0, 0, 0])]; + tensor query_33_end_0 = const()[name = tensor("query_33_end_0"), val = tensor([1, 8, 1, 77, 64])]; + tensor query_33_end_mask_0 = const()[name = tensor("query_33_end_mask_0"), val = tensor([true, true, false, true, true])]; + tensor query_33_squeeze_mask_0 = const()[name = tensor("query_33_squeeze_mask_0"), val = tensor([false, false, true, false, false])]; + tensor var_689_cast_fp16 = transpose(perm = var_689_perm_0, x = qkv_33_cast_fp16)[name = tensor("transpose_9")]; + tensor query_33_cast_fp16 = slice_by_index(begin = query_33_begin_0, end = query_33_end_0, end_mask = query_33_end_mask_0, squeeze_mask = query_33_squeeze_mask_0, x = var_689_cast_fp16)[name = tensor("query_33_cast_fp16")]; + tensor key_33_begin_0 = const()[name = tensor("key_33_begin_0"), val = tensor([0, 0, 1, 0, 0])]; + tensor key_33_end_0 = const()[name = tensor("key_33_end_0"), val = tensor([1, 8, 2, 77, 64])]; + tensor key_33_end_mask_0 = const()[name = tensor("key_33_end_mask_0"), val = tensor([true, true, false, true, true])]; + tensor key_33_squeeze_mask_0 = const()[name = tensor("key_33_squeeze_mask_0"), val = tensor([false, false, true, false, false])]; + tensor key_33_cast_fp16 = slice_by_index(begin = key_33_begin_0, end = key_33_end_0, end_mask = key_33_end_mask_0, squeeze_mask = key_33_squeeze_mask_0, x = var_689_cast_fp16)[name = tensor("key_33_cast_fp16")]; + tensor value_17_begin_0 = const()[name = tensor("value_17_begin_0"), val = tensor([0, 0, 2, 0, 0])]; + tensor value_17_end_0 = const()[name = tensor("value_17_end_0"), val = tensor([1, 8, 3, 77, 64])]; + tensor value_17_end_mask_0 = const()[name = tensor("value_17_end_mask_0"), val = tensor([true, true, false, true, true])]; + tensor value_17_squeeze_mask_0 = const()[name = tensor("value_17_squeeze_mask_0"), val = tensor([false, false, true, false, false])]; + tensor value_17_cast_fp16 = slice_by_index(begin = value_17_begin_0, end = value_17_end_0, end_mask = value_17_end_mask_0, squeeze_mask = value_17_squeeze_mask_0, x = var_689_cast_fp16)[name = tensor("value_17_cast_fp16")]; + tensor var_700_to_fp16 = const()[name = tensor("op_700_to_fp16"), val = tensor(0x1p-3)]; + tensor query_35_cast_fp16 = mul(x = query_33_cast_fp16, y = var_700_to_fp16)[name = tensor("query_35_cast_fp16")]; + tensor attn_33_transpose_x_1 = const()[name = tensor("attn_33_transpose_x_1"), val = tensor(false)]; + tensor attn_33_transpose_y_1 = const()[name = tensor("attn_33_transpose_y_1"), val = tensor(true)]; + tensor attn_33_cast_fp16 = matmul(transpose_x = attn_33_transpose_x_1, transpose_y = attn_33_transpose_y_1, x = query_35_cast_fp16, y = key_33_cast_fp16)[name = tensor("attn_33_cast_fp16")]; + tensor attn_as_float_17_cast_fp16 = softmax(axis = var_18, x = attn_33_cast_fp16)[name = tensor("attn_as_float_17_cast_fp16")]; + tensor out_17_transpose_x_0 = const()[name = tensor("out_17_transpose_x_0"), val = tensor(false)]; + tensor out_17_transpose_y_0 = const()[name = tensor("out_17_transpose_y_0"), val = tensor(false)]; + tensor out_17_cast_fp16 = matmul(transpose_x = out_17_transpose_x_0, transpose_y = out_17_transpose_y_0, x = attn_as_float_17_cast_fp16, y = value_17_cast_fp16)[name = tensor("out_17_cast_fp16")]; + tensor var_709_perm_0 = const()[name = tensor("op_709_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_710 = const()[name = tensor("op_710"), val = tensor([1, 77, -1])]; + tensor var_709_cast_fp16 = transpose(perm = var_709_perm_0, x = out_17_cast_fp16)[name = tensor("transpose_8")]; + tensor input_185_cast_fp16 = reshape(shape = var_710, x = var_709_cast_fp16)[name = tensor("input_185_cast_fp16")]; + tensor text_encoder_transformer_8_pre_norm_mha_1_out_proj_weight_to_fp16 = const()[name = tensor("text_encoder_transformer_8_pre_norm_mha_1_out_proj_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(102695360)))]; + tensor text_encoder_transformer_8_pre_norm_mha_1_out_proj_bias_to_fp16 = const()[name = tensor("text_encoder_transformer_8_pre_norm_mha_1_out_proj_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(103219712)))]; + tensor linear_33_cast_fp16 = linear(bias = text_encoder_transformer_8_pre_norm_mha_1_out_proj_bias_to_fp16, weight = text_encoder_transformer_8_pre_norm_mha_1_out_proj_weight_to_fp16, x = input_185_cast_fp16)[name = tensor("linear_33_cast_fp16")]; + tensor x_53_cast_fp16 = add(x = linear_33_cast_fp16, y = x_49_cast_fp16)[name = tensor("x_53_cast_fp16")]; + tensor var_724_axes_0 = const()[name = tensor("op_724_axes_0"), val = tensor([-1])]; + tensor text_encoder_transformer_8_pre_norm_ffn_0_weight_to_fp16 = const()[name = tensor("text_encoder_transformer_8_pre_norm_ffn_0_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(103220800)))]; + tensor text_encoder_transformer_8_pre_norm_ffn_0_bias_to_fp16 = const()[name = tensor("text_encoder_transformer_8_pre_norm_ffn_0_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(103221888)))]; + tensor var_724_cast_fp16 = layer_norm(axes = var_724_axes_0, beta = text_encoder_transformer_8_pre_norm_ffn_0_bias_to_fp16, epsilon = var_5_to_fp16, gamma = text_encoder_transformer_8_pre_norm_ffn_0_weight_to_fp16, x = x_53_cast_fp16)[name = tensor("op_724_cast_fp16")]; + tensor text_encoder_transformer_8_pre_norm_ffn_1_weight_to_fp16 = const()[name = tensor("text_encoder_transformer_8_pre_norm_ffn_1_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(103222976)))]; + tensor text_encoder_transformer_8_pre_norm_ffn_1_bias_to_fp16 = const()[name = tensor("text_encoder_transformer_8_pre_norm_ffn_1_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(105320192)))]; + tensor linear_34_cast_fp16 = linear(bias = text_encoder_transformer_8_pre_norm_ffn_1_bias_to_fp16, weight = text_encoder_transformer_8_pre_norm_ffn_1_weight_to_fp16, x = var_724_cast_fp16)[name = tensor("linear_34_cast_fp16")]; + tensor input_195_mode_0 = const()[name = tensor("input_195_mode_0"), val = tensor("EXACT")]; + tensor input_195_cast_fp16 = gelu(mode = input_195_mode_0, x = linear_34_cast_fp16)[name = tensor("input_195_cast_fp16")]; + tensor text_encoder_transformer_8_pre_norm_ffn_4_weight_to_fp16 = const()[name = tensor("text_encoder_transformer_8_pre_norm_ffn_4_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(105324352)))]; + tensor text_encoder_transformer_8_pre_norm_ffn_4_bias_to_fp16 = const()[name = tensor("text_encoder_transformer_8_pre_norm_ffn_4_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(107421568)))]; + tensor linear_35_cast_fp16 = linear(bias = text_encoder_transformer_8_pre_norm_ffn_4_bias_to_fp16, weight = text_encoder_transformer_8_pre_norm_ffn_4_weight_to_fp16, x = input_195_cast_fp16)[name = tensor("linear_35_cast_fp16")]; + tensor x_55_cast_fp16 = add(x = x_53_cast_fp16, y = linear_35_cast_fp16)[name = tensor("x_55_cast_fp16")]; + tensor var_751_axes_0 = const()[name = tensor("op_751_axes_0"), val = tensor([-1])]; + tensor text_encoder_transformer_9_pre_norm_mha_0_weight_to_fp16 = const()[name = tensor("text_encoder_transformer_9_pre_norm_mha_0_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(107422656)))]; + tensor text_encoder_transformer_9_pre_norm_mha_0_bias_to_fp16 = const()[name = tensor("text_encoder_transformer_9_pre_norm_mha_0_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(107423744)))]; + tensor var_751_cast_fp16 = layer_norm(axes = var_751_axes_0, beta = text_encoder_transformer_9_pre_norm_mha_0_bias_to_fp16, epsilon = var_5_to_fp16, gamma = text_encoder_transformer_9_pre_norm_mha_0_weight_to_fp16, x = x_55_cast_fp16)[name = tensor("op_751_cast_fp16")]; + tensor text_encoder_transformer_9_pre_norm_mha_1_qkv_proj_weight_to_fp16 = const()[name = tensor("text_encoder_transformer_9_pre_norm_mha_1_qkv_proj_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(107424832)))]; + tensor text_encoder_transformer_9_pre_norm_mha_1_qkv_proj_bias_to_fp16 = const()[name = tensor("text_encoder_transformer_9_pre_norm_mha_1_qkv_proj_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(108997760)))]; + tensor linear_36_cast_fp16 = linear(bias = text_encoder_transformer_9_pre_norm_mha_1_qkv_proj_bias_to_fp16, weight = text_encoder_transformer_9_pre_norm_mha_1_qkv_proj_weight_to_fp16, x = var_751_cast_fp16)[name = tensor("linear_36_cast_fp16")]; + tensor var_763 = const()[name = tensor("op_763"), val = tensor([1, 77, 3, 8, -1])]; + tensor qkv_37_cast_fp16 = reshape(shape = var_763, x = linear_36_cast_fp16)[name = tensor("qkv_37_cast_fp16")]; + tensor var_765_perm_0 = const()[name = tensor("op_765_perm_0"), val = tensor([0, 3, 2, 1, 4])]; + tensor query_37_begin_0 = const()[name = tensor("query_37_begin_0"), val = tensor([0, 0, 0, 0, 0])]; + tensor query_37_end_0 = const()[name = tensor("query_37_end_0"), val = tensor([1, 8, 1, 77, 64])]; + tensor query_37_end_mask_0 = const()[name = tensor("query_37_end_mask_0"), val = tensor([true, true, false, true, true])]; + tensor query_37_squeeze_mask_0 = const()[name = tensor("query_37_squeeze_mask_0"), val = tensor([false, false, true, false, false])]; + tensor var_765_cast_fp16 = transpose(perm = var_765_perm_0, x = qkv_37_cast_fp16)[name = tensor("transpose_7")]; + tensor query_37_cast_fp16 = slice_by_index(begin = query_37_begin_0, end = query_37_end_0, end_mask = query_37_end_mask_0, squeeze_mask = query_37_squeeze_mask_0, x = var_765_cast_fp16)[name = tensor("query_37_cast_fp16")]; + tensor key_37_begin_0 = const()[name = tensor("key_37_begin_0"), val = tensor([0, 0, 1, 0, 0])]; + tensor key_37_end_0 = const()[name = tensor("key_37_end_0"), val = tensor([1, 8, 2, 77, 64])]; + tensor key_37_end_mask_0 = const()[name = tensor("key_37_end_mask_0"), val = tensor([true, true, false, true, true])]; + tensor key_37_squeeze_mask_0 = const()[name = tensor("key_37_squeeze_mask_0"), val = tensor([false, false, true, false, false])]; + tensor key_37_cast_fp16 = slice_by_index(begin = key_37_begin_0, end = key_37_end_0, end_mask = key_37_end_mask_0, squeeze_mask = key_37_squeeze_mask_0, x = var_765_cast_fp16)[name = tensor("key_37_cast_fp16")]; + tensor value_19_begin_0 = const()[name = tensor("value_19_begin_0"), val = tensor([0, 0, 2, 0, 0])]; + tensor value_19_end_0 = const()[name = tensor("value_19_end_0"), val = tensor([1, 8, 3, 77, 64])]; + tensor value_19_end_mask_0 = const()[name = tensor("value_19_end_mask_0"), val = tensor([true, true, false, true, true])]; + tensor value_19_squeeze_mask_0 = const()[name = tensor("value_19_squeeze_mask_0"), val = tensor([false, false, true, false, false])]; + tensor value_19_cast_fp16 = slice_by_index(begin = value_19_begin_0, end = value_19_end_0, end_mask = value_19_end_mask_0, squeeze_mask = value_19_squeeze_mask_0, x = var_765_cast_fp16)[name = tensor("value_19_cast_fp16")]; + tensor var_776_to_fp16 = const()[name = tensor("op_776_to_fp16"), val = tensor(0x1p-3)]; + tensor query_39_cast_fp16 = mul(x = query_37_cast_fp16, y = var_776_to_fp16)[name = tensor("query_39_cast_fp16")]; + tensor attn_37_transpose_x_1 = const()[name = tensor("attn_37_transpose_x_1"), val = tensor(false)]; + tensor attn_37_transpose_y_1 = const()[name = tensor("attn_37_transpose_y_1"), val = tensor(true)]; + tensor attn_37_cast_fp16 = matmul(transpose_x = attn_37_transpose_x_1, transpose_y = attn_37_transpose_y_1, x = query_39_cast_fp16, y = key_37_cast_fp16)[name = tensor("attn_37_cast_fp16")]; + tensor attn_as_float_19_cast_fp16 = softmax(axis = var_18, x = attn_37_cast_fp16)[name = tensor("attn_as_float_19_cast_fp16")]; + tensor out_19_transpose_x_0 = const()[name = tensor("out_19_transpose_x_0"), val = tensor(false)]; + tensor out_19_transpose_y_0 = const()[name = tensor("out_19_transpose_y_0"), val = tensor(false)]; + tensor out_19_cast_fp16 = matmul(transpose_x = out_19_transpose_x_0, transpose_y = out_19_transpose_y_0, x = attn_as_float_19_cast_fp16, y = value_19_cast_fp16)[name = tensor("out_19_cast_fp16")]; + tensor var_785_perm_0 = const()[name = tensor("op_785_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_786 = const()[name = tensor("op_786"), val = tensor([1, 77, -1])]; + tensor var_785_cast_fp16 = transpose(perm = var_785_perm_0, x = out_19_cast_fp16)[name = tensor("transpose_6")]; + tensor input_207_cast_fp16 = reshape(shape = var_786, x = var_785_cast_fp16)[name = tensor("input_207_cast_fp16")]; + tensor text_encoder_transformer_9_pre_norm_mha_1_out_proj_weight_to_fp16 = const()[name = tensor("text_encoder_transformer_9_pre_norm_mha_1_out_proj_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(109000896)))]; + tensor text_encoder_transformer_9_pre_norm_mha_1_out_proj_bias_to_fp16 = const()[name = tensor("text_encoder_transformer_9_pre_norm_mha_1_out_proj_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(109525248)))]; + tensor linear_37_cast_fp16 = linear(bias = text_encoder_transformer_9_pre_norm_mha_1_out_proj_bias_to_fp16, weight = text_encoder_transformer_9_pre_norm_mha_1_out_proj_weight_to_fp16, x = input_207_cast_fp16)[name = tensor("linear_37_cast_fp16")]; + tensor x_59_cast_fp16 = add(x = linear_37_cast_fp16, y = x_55_cast_fp16)[name = tensor("x_59_cast_fp16")]; + tensor var_800_axes_0 = const()[name = tensor("op_800_axes_0"), val = tensor([-1])]; + tensor text_encoder_transformer_9_pre_norm_ffn_0_weight_to_fp16 = const()[name = tensor("text_encoder_transformer_9_pre_norm_ffn_0_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(109526336)))]; + tensor text_encoder_transformer_9_pre_norm_ffn_0_bias_to_fp16 = const()[name = tensor("text_encoder_transformer_9_pre_norm_ffn_0_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(109527424)))]; + tensor var_800_cast_fp16 = layer_norm(axes = var_800_axes_0, beta = text_encoder_transformer_9_pre_norm_ffn_0_bias_to_fp16, epsilon = var_5_to_fp16, gamma = text_encoder_transformer_9_pre_norm_ffn_0_weight_to_fp16, x = x_59_cast_fp16)[name = tensor("op_800_cast_fp16")]; + tensor text_encoder_transformer_9_pre_norm_ffn_1_weight_to_fp16 = const()[name = tensor("text_encoder_transformer_9_pre_norm_ffn_1_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(109528512)))]; + tensor text_encoder_transformer_9_pre_norm_ffn_1_bias_to_fp16 = const()[name = tensor("text_encoder_transformer_9_pre_norm_ffn_1_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(111625728)))]; + tensor linear_38_cast_fp16 = linear(bias = text_encoder_transformer_9_pre_norm_ffn_1_bias_to_fp16, weight = text_encoder_transformer_9_pre_norm_ffn_1_weight_to_fp16, x = var_800_cast_fp16)[name = tensor("linear_38_cast_fp16")]; + tensor input_217_mode_0 = const()[name = tensor("input_217_mode_0"), val = tensor("EXACT")]; + tensor input_217_cast_fp16 = gelu(mode = input_217_mode_0, x = linear_38_cast_fp16)[name = tensor("input_217_cast_fp16")]; + tensor text_encoder_transformer_9_pre_norm_ffn_4_weight_to_fp16 = const()[name = tensor("text_encoder_transformer_9_pre_norm_ffn_4_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(111629888)))]; + tensor text_encoder_transformer_9_pre_norm_ffn_4_bias_to_fp16 = const()[name = tensor("text_encoder_transformer_9_pre_norm_ffn_4_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(113727104)))]; + tensor linear_39_cast_fp16 = linear(bias = text_encoder_transformer_9_pre_norm_ffn_4_bias_to_fp16, weight = text_encoder_transformer_9_pre_norm_ffn_4_weight_to_fp16, x = input_217_cast_fp16)[name = tensor("linear_39_cast_fp16")]; + tensor x_61_cast_fp16 = add(x = x_59_cast_fp16, y = linear_39_cast_fp16)[name = tensor("x_61_cast_fp16")]; + tensor var_827_axes_0 = const()[name = tensor("op_827_axes_0"), val = tensor([-1])]; + tensor text_encoder_transformer_10_pre_norm_mha_0_weight_to_fp16 = const()[name = tensor("text_encoder_transformer_10_pre_norm_mha_0_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(113728192)))]; + tensor text_encoder_transformer_10_pre_norm_mha_0_bias_to_fp16 = const()[name = tensor("text_encoder_transformer_10_pre_norm_mha_0_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(113729280)))]; + tensor var_827_cast_fp16 = layer_norm(axes = var_827_axes_0, beta = text_encoder_transformer_10_pre_norm_mha_0_bias_to_fp16, epsilon = var_5_to_fp16, gamma = text_encoder_transformer_10_pre_norm_mha_0_weight_to_fp16, x = x_61_cast_fp16)[name = tensor("op_827_cast_fp16")]; + tensor text_encoder_transformer_10_pre_norm_mha_1_qkv_proj_weight_to_fp16 = const()[name = tensor("text_encoder_transformer_10_pre_norm_mha_1_qkv_proj_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(113730368)))]; + tensor text_encoder_transformer_10_pre_norm_mha_1_qkv_proj_bias_to_fp16 = const()[name = tensor("text_encoder_transformer_10_pre_norm_mha_1_qkv_proj_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(115303296)))]; + tensor linear_40_cast_fp16 = linear(bias = text_encoder_transformer_10_pre_norm_mha_1_qkv_proj_bias_to_fp16, weight = text_encoder_transformer_10_pre_norm_mha_1_qkv_proj_weight_to_fp16, x = var_827_cast_fp16)[name = tensor("linear_40_cast_fp16")]; + tensor var_839 = const()[name = tensor("op_839"), val = tensor([1, 77, 3, 8, -1])]; + tensor qkv_41_cast_fp16 = reshape(shape = var_839, x = linear_40_cast_fp16)[name = tensor("qkv_41_cast_fp16")]; + tensor var_841_perm_0 = const()[name = tensor("op_841_perm_0"), val = tensor([0, 3, 2, 1, 4])]; + tensor query_41_begin_0 = const()[name = tensor("query_41_begin_0"), val = tensor([0, 0, 0, 0, 0])]; + tensor query_41_end_0 = const()[name = tensor("query_41_end_0"), val = tensor([1, 8, 1, 77, 64])]; + tensor query_41_end_mask_0 = const()[name = tensor("query_41_end_mask_0"), val = tensor([true, true, false, true, true])]; + tensor query_41_squeeze_mask_0 = const()[name = tensor("query_41_squeeze_mask_0"), val = tensor([false, false, true, false, false])]; + tensor var_841_cast_fp16 = transpose(perm = var_841_perm_0, x = qkv_41_cast_fp16)[name = tensor("transpose_5")]; + tensor query_41_cast_fp16 = slice_by_index(begin = query_41_begin_0, end = query_41_end_0, end_mask = query_41_end_mask_0, squeeze_mask = query_41_squeeze_mask_0, x = var_841_cast_fp16)[name = tensor("query_41_cast_fp16")]; + tensor key_41_begin_0 = const()[name = tensor("key_41_begin_0"), val = tensor([0, 0, 1, 0, 0])]; + tensor key_41_end_0 = const()[name = tensor("key_41_end_0"), val = tensor([1, 8, 2, 77, 64])]; + tensor key_41_end_mask_0 = const()[name = tensor("key_41_end_mask_0"), val = tensor([true, true, false, true, true])]; + tensor key_41_squeeze_mask_0 = const()[name = tensor("key_41_squeeze_mask_0"), val = tensor([false, false, true, false, false])]; + tensor key_41_cast_fp16 = slice_by_index(begin = key_41_begin_0, end = key_41_end_0, end_mask = key_41_end_mask_0, squeeze_mask = key_41_squeeze_mask_0, x = var_841_cast_fp16)[name = tensor("key_41_cast_fp16")]; + tensor value_21_begin_0 = const()[name = tensor("value_21_begin_0"), val = tensor([0, 0, 2, 0, 0])]; + tensor value_21_end_0 = const()[name = tensor("value_21_end_0"), val = tensor([1, 8, 3, 77, 64])]; + tensor value_21_end_mask_0 = const()[name = tensor("value_21_end_mask_0"), val = tensor([true, true, false, true, true])]; + tensor value_21_squeeze_mask_0 = const()[name = tensor("value_21_squeeze_mask_0"), val = tensor([false, false, true, false, false])]; + tensor value_21_cast_fp16 = slice_by_index(begin = value_21_begin_0, end = value_21_end_0, end_mask = value_21_end_mask_0, squeeze_mask = value_21_squeeze_mask_0, x = var_841_cast_fp16)[name = tensor("value_21_cast_fp16")]; + tensor var_852_to_fp16 = const()[name = tensor("op_852_to_fp16"), val = tensor(0x1p-3)]; + tensor query_43_cast_fp16 = mul(x = query_41_cast_fp16, y = var_852_to_fp16)[name = tensor("query_43_cast_fp16")]; + tensor attn_41_transpose_x_1 = const()[name = tensor("attn_41_transpose_x_1"), val = tensor(false)]; + tensor attn_41_transpose_y_1 = const()[name = tensor("attn_41_transpose_y_1"), val = tensor(true)]; + tensor attn_41_cast_fp16 = matmul(transpose_x = attn_41_transpose_x_1, transpose_y = attn_41_transpose_y_1, x = query_43_cast_fp16, y = key_41_cast_fp16)[name = tensor("attn_41_cast_fp16")]; + tensor attn_as_float_21_cast_fp16 = softmax(axis = var_18, x = attn_41_cast_fp16)[name = tensor("attn_as_float_21_cast_fp16")]; + tensor out_21_transpose_x_0 = const()[name = tensor("out_21_transpose_x_0"), val = tensor(false)]; + tensor out_21_transpose_y_0 = const()[name = tensor("out_21_transpose_y_0"), val = tensor(false)]; + tensor out_21_cast_fp16 = matmul(transpose_x = out_21_transpose_x_0, transpose_y = out_21_transpose_y_0, x = attn_as_float_21_cast_fp16, y = value_21_cast_fp16)[name = tensor("out_21_cast_fp16")]; + tensor var_861_perm_0 = const()[name = tensor("op_861_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_862 = const()[name = tensor("op_862"), val = tensor([1, 77, -1])]; + tensor var_861_cast_fp16 = transpose(perm = var_861_perm_0, x = out_21_cast_fp16)[name = tensor("transpose_4")]; + tensor input_229_cast_fp16 = reshape(shape = var_862, x = var_861_cast_fp16)[name = tensor("input_229_cast_fp16")]; + tensor text_encoder_transformer_10_pre_norm_mha_1_out_proj_weight_to_fp16 = const()[name = tensor("text_encoder_transformer_10_pre_norm_mha_1_out_proj_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(115306432)))]; + tensor text_encoder_transformer_10_pre_norm_mha_1_out_proj_bias_to_fp16 = const()[name = tensor("text_encoder_transformer_10_pre_norm_mha_1_out_proj_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(115830784)))]; + tensor linear_41_cast_fp16 = linear(bias = text_encoder_transformer_10_pre_norm_mha_1_out_proj_bias_to_fp16, weight = text_encoder_transformer_10_pre_norm_mha_1_out_proj_weight_to_fp16, x = input_229_cast_fp16)[name = tensor("linear_41_cast_fp16")]; + tensor x_65_cast_fp16 = add(x = linear_41_cast_fp16, y = x_61_cast_fp16)[name = tensor("x_65_cast_fp16")]; + tensor var_876_axes_0 = const()[name = tensor("op_876_axes_0"), val = tensor([-1])]; + tensor text_encoder_transformer_10_pre_norm_ffn_0_weight_to_fp16 = const()[name = tensor("text_encoder_transformer_10_pre_norm_ffn_0_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(115831872)))]; + tensor text_encoder_transformer_10_pre_norm_ffn_0_bias_to_fp16 = const()[name = tensor("text_encoder_transformer_10_pre_norm_ffn_0_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(115832960)))]; + tensor var_876_cast_fp16 = layer_norm(axes = var_876_axes_0, beta = text_encoder_transformer_10_pre_norm_ffn_0_bias_to_fp16, epsilon = var_5_to_fp16, gamma = text_encoder_transformer_10_pre_norm_ffn_0_weight_to_fp16, x = x_65_cast_fp16)[name = tensor("op_876_cast_fp16")]; + tensor text_encoder_transformer_10_pre_norm_ffn_1_weight_to_fp16 = const()[name = tensor("text_encoder_transformer_10_pre_norm_ffn_1_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(115834048)))]; + tensor text_encoder_transformer_10_pre_norm_ffn_1_bias_to_fp16 = const()[name = tensor("text_encoder_transformer_10_pre_norm_ffn_1_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(117931264)))]; + tensor linear_42_cast_fp16 = linear(bias = text_encoder_transformer_10_pre_norm_ffn_1_bias_to_fp16, weight = text_encoder_transformer_10_pre_norm_ffn_1_weight_to_fp16, x = var_876_cast_fp16)[name = tensor("linear_42_cast_fp16")]; + tensor input_239_mode_0 = const()[name = tensor("input_239_mode_0"), val = tensor("EXACT")]; + tensor input_239_cast_fp16 = gelu(mode = input_239_mode_0, x = linear_42_cast_fp16)[name = tensor("input_239_cast_fp16")]; + tensor text_encoder_transformer_10_pre_norm_ffn_4_weight_to_fp16 = const()[name = tensor("text_encoder_transformer_10_pre_norm_ffn_4_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(117935424)))]; + tensor text_encoder_transformer_10_pre_norm_ffn_4_bias_to_fp16 = const()[name = tensor("text_encoder_transformer_10_pre_norm_ffn_4_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(120032640)))]; + tensor linear_43_cast_fp16 = linear(bias = text_encoder_transformer_10_pre_norm_ffn_4_bias_to_fp16, weight = text_encoder_transformer_10_pre_norm_ffn_4_weight_to_fp16, x = input_239_cast_fp16)[name = tensor("linear_43_cast_fp16")]; + tensor x_67_cast_fp16 = add(x = x_65_cast_fp16, y = linear_43_cast_fp16)[name = tensor("x_67_cast_fp16")]; + tensor var_903_axes_0 = const()[name = tensor("op_903_axes_0"), val = tensor([-1])]; + tensor text_encoder_transformer_11_pre_norm_mha_0_weight_to_fp16 = const()[name = tensor("text_encoder_transformer_11_pre_norm_mha_0_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(120033728)))]; + tensor text_encoder_transformer_11_pre_norm_mha_0_bias_to_fp16 = const()[name = tensor("text_encoder_transformer_11_pre_norm_mha_0_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(120034816)))]; + tensor var_903_cast_fp16 = layer_norm(axes = var_903_axes_0, beta = text_encoder_transformer_11_pre_norm_mha_0_bias_to_fp16, epsilon = var_5_to_fp16, gamma = text_encoder_transformer_11_pre_norm_mha_0_weight_to_fp16, x = x_67_cast_fp16)[name = tensor("op_903_cast_fp16")]; + tensor text_encoder_transformer_11_pre_norm_mha_1_qkv_proj_weight_to_fp16 = const()[name = tensor("text_encoder_transformer_11_pre_norm_mha_1_qkv_proj_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(120035904)))]; + tensor text_encoder_transformer_11_pre_norm_mha_1_qkv_proj_bias_to_fp16 = const()[name = tensor("text_encoder_transformer_11_pre_norm_mha_1_qkv_proj_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(121608832)))]; + tensor linear_44_cast_fp16 = linear(bias = text_encoder_transformer_11_pre_norm_mha_1_qkv_proj_bias_to_fp16, weight = text_encoder_transformer_11_pre_norm_mha_1_qkv_proj_weight_to_fp16, x = var_903_cast_fp16)[name = tensor("linear_44_cast_fp16")]; + tensor var_915 = const()[name = tensor("op_915"), val = tensor([1, 77, 3, 8, -1])]; + tensor qkv_45_cast_fp16 = reshape(shape = var_915, x = linear_44_cast_fp16)[name = tensor("qkv_45_cast_fp16")]; + tensor var_917_perm_0 = const()[name = tensor("op_917_perm_0"), val = tensor([0, 3, 2, 1, 4])]; + tensor query_45_begin_0 = const()[name = tensor("query_45_begin_0"), val = tensor([0, 0, 0, 0, 0])]; + tensor query_45_end_0 = const()[name = tensor("query_45_end_0"), val = tensor([1, 8, 1, 77, 64])]; + tensor query_45_end_mask_0 = const()[name = tensor("query_45_end_mask_0"), val = tensor([true, true, false, true, true])]; + tensor query_45_squeeze_mask_0 = const()[name = tensor("query_45_squeeze_mask_0"), val = tensor([false, false, true, false, false])]; + tensor var_917_cast_fp16 = transpose(perm = var_917_perm_0, x = qkv_45_cast_fp16)[name = tensor("transpose_3")]; + tensor query_45_cast_fp16 = slice_by_index(begin = query_45_begin_0, end = query_45_end_0, end_mask = query_45_end_mask_0, squeeze_mask = query_45_squeeze_mask_0, x = var_917_cast_fp16)[name = tensor("query_45_cast_fp16")]; + tensor key_45_begin_0 = const()[name = tensor("key_45_begin_0"), val = tensor([0, 0, 1, 0, 0])]; + tensor key_45_end_0 = const()[name = tensor("key_45_end_0"), val = tensor([1, 8, 2, 77, 64])]; + tensor key_45_end_mask_0 = const()[name = tensor("key_45_end_mask_0"), val = tensor([true, true, false, true, true])]; + tensor key_45_squeeze_mask_0 = const()[name = tensor("key_45_squeeze_mask_0"), val = tensor([false, false, true, false, false])]; + tensor key_45_cast_fp16 = slice_by_index(begin = key_45_begin_0, end = key_45_end_0, end_mask = key_45_end_mask_0, squeeze_mask = key_45_squeeze_mask_0, x = var_917_cast_fp16)[name = tensor("key_45_cast_fp16")]; + tensor value_begin_0 = const()[name = tensor("value_begin_0"), val = tensor([0, 0, 2, 0, 0])]; + tensor value_end_0 = const()[name = tensor("value_end_0"), val = tensor([1, 8, 3, 77, 64])]; + tensor value_end_mask_0 = const()[name = tensor("value_end_mask_0"), val = tensor([true, true, false, true, true])]; + tensor value_squeeze_mask_0 = const()[name = tensor("value_squeeze_mask_0"), val = tensor([false, false, true, false, false])]; + tensor value_cast_fp16 = slice_by_index(begin = value_begin_0, end = value_end_0, end_mask = value_end_mask_0, squeeze_mask = value_squeeze_mask_0, x = var_917_cast_fp16)[name = tensor("value_cast_fp16")]; + tensor var_928_to_fp16 = const()[name = tensor("op_928_to_fp16"), val = tensor(0x1p-3)]; + tensor query_cast_fp16 = mul(x = query_45_cast_fp16, y = var_928_to_fp16)[name = tensor("query_cast_fp16")]; + tensor attn_45_transpose_x_1 = const()[name = tensor("attn_45_transpose_x_1"), val = tensor(false)]; + tensor attn_45_transpose_y_1 = const()[name = tensor("attn_45_transpose_y_1"), val = tensor(true)]; + tensor attn_45_cast_fp16 = matmul(transpose_x = attn_45_transpose_x_1, transpose_y = attn_45_transpose_y_1, x = query_cast_fp16, y = key_45_cast_fp16)[name = tensor("attn_45_cast_fp16")]; + tensor attn_as_float_cast_fp16 = softmax(axis = var_18, x = attn_45_cast_fp16)[name = tensor("attn_as_float_cast_fp16")]; + tensor out_transpose_x_0 = const()[name = tensor("out_transpose_x_0"), val = tensor(false)]; + tensor out_transpose_y_0 = const()[name = tensor("out_transpose_y_0"), val = tensor(false)]; + tensor out_cast_fp16 = matmul(transpose_x = out_transpose_x_0, transpose_y = out_transpose_y_0, x = attn_as_float_cast_fp16, y = value_cast_fp16)[name = tensor("out_cast_fp16")]; + tensor var_937_perm_0 = const()[name = tensor("op_937_perm_0"), val = tensor([0, 2, 1, 3])]; + tensor var_938 = const()[name = tensor("op_938"), val = tensor([1, 77, -1])]; + tensor var_937_cast_fp16 = transpose(perm = var_937_perm_0, x = out_cast_fp16)[name = tensor("transpose_2")]; + tensor input_251_cast_fp16 = reshape(shape = var_938, x = var_937_cast_fp16)[name = tensor("input_251_cast_fp16")]; + tensor text_encoder_transformer_11_pre_norm_mha_1_out_proj_weight_to_fp16 = const()[name = tensor("text_encoder_transformer_11_pre_norm_mha_1_out_proj_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(121611968)))]; + tensor text_encoder_transformer_11_pre_norm_mha_1_out_proj_bias_to_fp16 = const()[name = tensor("text_encoder_transformer_11_pre_norm_mha_1_out_proj_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(122136320)))]; + tensor linear_45_cast_fp16 = linear(bias = text_encoder_transformer_11_pre_norm_mha_1_out_proj_bias_to_fp16, weight = text_encoder_transformer_11_pre_norm_mha_1_out_proj_weight_to_fp16, x = input_251_cast_fp16)[name = tensor("linear_45_cast_fp16")]; + tensor x_71_cast_fp16 = add(x = linear_45_cast_fp16, y = x_67_cast_fp16)[name = tensor("x_71_cast_fp16")]; + tensor var_952_axes_0 = const()[name = tensor("op_952_axes_0"), val = tensor([-1])]; + tensor text_encoder_transformer_11_pre_norm_ffn_0_weight_to_fp16 = const()[name = tensor("text_encoder_transformer_11_pre_norm_ffn_0_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(122137408)))]; + tensor text_encoder_transformer_11_pre_norm_ffn_0_bias_to_fp16 = const()[name = tensor("text_encoder_transformer_11_pre_norm_ffn_0_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(122138496)))]; + tensor var_952_cast_fp16 = layer_norm(axes = var_952_axes_0, beta = text_encoder_transformer_11_pre_norm_ffn_0_bias_to_fp16, epsilon = var_5_to_fp16, gamma = text_encoder_transformer_11_pre_norm_ffn_0_weight_to_fp16, x = x_71_cast_fp16)[name = tensor("op_952_cast_fp16")]; + tensor text_encoder_transformer_11_pre_norm_ffn_1_weight_to_fp16 = const()[name = tensor("text_encoder_transformer_11_pre_norm_ffn_1_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(122139584)))]; + tensor text_encoder_transformer_11_pre_norm_ffn_1_bias_to_fp16 = const()[name = tensor("text_encoder_transformer_11_pre_norm_ffn_1_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(124236800)))]; + tensor linear_46_cast_fp16 = linear(bias = text_encoder_transformer_11_pre_norm_ffn_1_bias_to_fp16, weight = text_encoder_transformer_11_pre_norm_ffn_1_weight_to_fp16, x = var_952_cast_fp16)[name = tensor("linear_46_cast_fp16")]; + tensor input_261_mode_0 = const()[name = tensor("input_261_mode_0"), val = tensor("EXACT")]; + tensor input_261_cast_fp16 = gelu(mode = input_261_mode_0, x = linear_46_cast_fp16)[name = tensor("input_261_cast_fp16")]; + tensor text_encoder_transformer_11_pre_norm_ffn_4_weight_to_fp16 = const()[name = tensor("text_encoder_transformer_11_pre_norm_ffn_4_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(124240960)))]; + tensor text_encoder_transformer_11_pre_norm_ffn_4_bias_to_fp16 = const()[name = tensor("text_encoder_transformer_11_pre_norm_ffn_4_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(126338176)))]; + tensor linear_47_cast_fp16 = linear(bias = text_encoder_transformer_11_pre_norm_ffn_4_bias_to_fp16, weight = text_encoder_transformer_11_pre_norm_ffn_4_weight_to_fp16, x = input_261_cast_fp16)[name = tensor("linear_47_cast_fp16")]; + tensor x_cast_fp16 = add(x = x_71_cast_fp16, y = linear_47_cast_fp16)[name = tensor("x_cast_fp16")]; + tensor var_974_axes_0 = const()[name = tensor("op_974_axes_0"), val = tensor([-1])]; + tensor text_encoder_final_layer_norm_weight_to_fp16 = const()[name = tensor("text_encoder_final_layer_norm_weight_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(126339264)))]; + tensor text_encoder_final_layer_norm_bias_to_fp16 = const()[name = tensor("text_encoder_final_layer_norm_bias_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(126340352)))]; + tensor var_974_cast_fp16 = layer_norm(axes = var_974_axes_0, beta = text_encoder_final_layer_norm_bias_to_fp16, epsilon = var_5_to_fp16, gamma = text_encoder_final_layer_norm_weight_to_fp16, x = x_cast_fp16)[name = tensor("op_974_cast_fp16")]; + tensor var_977 = const()[name = tensor("op_977"), val = tensor([0])]; + tensor var_978_axis_0 = const()[name = tensor("op_978_axis_0"), val = tensor(-1)]; + tensor var_978_keep_dims_0 = const()[name = tensor("op_978_keep_dims_0"), val = tensor(false)]; + tensor var_978_output_dtype_0 = const()[name = tensor("op_978_output_dtype_0"), val = tensor("int32")]; + tensor var_978 = reduce_argmax(axis = var_978_axis_0, keep_dims = var_978_keep_dims_0, output_dtype = var_978_output_dtype_0, x = input_text)[name = tensor("op_978")]; + tensor stack_0_axis_0 = const()[name = tensor("stack_0_axis_0"), val = tensor(1)]; + tensor stack_0 = stack(axis = stack_0_axis_0, values = (var_977, var_978))[name = tensor("stack_0")]; + tensor greater_equal_0_y_0 = const()[name = tensor("greater_equal_0_y_0"), val = tensor(0)]; + tensor greater_equal_0 = greater_equal(x = stack_0, y = greater_equal_0_y_0)[name = tensor("greater_equal_0")]; + tensor slice_by_size_0 = const()[name = tensor("slice_by_size_0"), val = tensor([1, 77])]; + tensor add_0 = add(x = stack_0, y = slice_by_size_0)[name = tensor("add_0")]; + tensor select_0 = select(a = stack_0, b = add_0, cond = greater_equal_0)[name = tensor("select_0")]; + tensor token_emb_transpose_batch_dims_0 = const()[name = tensor("token_emb_transpose_batch_dims_0"), val = tensor(0)]; + tensor token_emb_transpose_validate_indices_0 = const()[name = tensor("token_emb_transpose_validate_indices_0"), val = tensor(false)]; + tensor select_0_to_uint16_dtype_0 = const()[name = tensor("select_0_to_uint16_dtype_0"), val = tensor("uint16")]; + tensor select_0_to_uint16 = cast(dtype = select_0_to_uint16_dtype_0, x = select_0)[name = tensor("cast_103")]; + tensor token_emb_transpose_cast_fp16_cast_uint16 = gather_nd(batch_dims = token_emb_transpose_batch_dims_0, indices = select_0_to_uint16, validate_indices = token_emb_transpose_validate_indices_0, x = var_974_cast_fp16)[name = tensor("token_emb_transpose_cast_fp16_cast_uint16")]; + tensor transpose_1_to_fp16 = const()[name = tensor("transpose_1_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(126341440)))]; + tensor input_bias_0_to_fp16 = const()[name = tensor("input_bias_0_to_fp16"), val = tensor(BLOBFILE(path = tensor("@model_path/weights/weight.bin"), offset = tensor(126865792)))]; + tensor input_cast_fp16 = linear(bias = input_bias_0_to_fp16, weight = transpose_1_to_fp16, x = token_emb_transpose_cast_fp16_cast_uint16)[name = tensor("input_cast_fp16")]; + tensor var_984 = const()[name = tensor("op_984"), val = tensor([-1])]; + tensor var_985 = const()[name = tensor("op_985"), val = tensor(true)]; + tensor var_987_cast_fp16 = reduce_l2_norm(axes = var_984, keep_dims = var_985, x = input_cast_fp16)[name = tensor("op_987_cast_fp16")]; + tensor var_988_to_fp16 = const()[name = tensor("op_988_to_fp16"), val = tensor(0x1p-24)]; + tensor var_989_cast_fp16 = maximum(x = var_987_cast_fp16, y = var_988_to_fp16)[name = tensor("op_989_cast_fp16")]; + tensor denom_reps_0 = const()[name = tensor("denom_reps_0"), val = tensor([1, 512])]; + tensor denom_cast_fp16 = tile(reps = denom_reps_0, x = var_989_cast_fp16)[name = tensor("denom_cast_fp16")]; + tensor output_embeddings = real_div(x = input_cast_fp16, y = denom_cast_fp16)[name = tensor("op_991_cast_fp16")]; + } -> (output_embeddings); +} \ No newline at end of file