convert_model.py

import uform
import torch
import coremltools as ct
from os.path import join
from argparse import ArgumentParser

class TextEncoder(torch.nn.Module):
    def __init__(self, model):
        super().__init__()
        self.model = model.eval()
        
    def forward(self, input_ids, attention_mask):
        features = self.model.forward_features(
            input_ids, attention_mask
        )

        embeddings = self.model.forward_embedding(
            features, attention_mask
        )

        return features, embeddings
    
    
class ImageEncoder(torch.nn.Module):
    def __init__(self, model):
        super().__init__()
        self.model = model.eval()
        
    def forward(self, image):
        features = self.model.forward_features(
            image
        )

        embeddings = self.model.forward_embedding(
            features
        )

        return features, embeddings

def convert_model(opts):
    src_model = uform.get_model(opts.model_name)
    input_ids = torch.ones(1, src_model.text_encoder.max_position_embeddings, dtype=torch.int32)
    attention_mask = torch.ones(1, src_model.text_encoder.max_position_embeddings, dtype=torch.int32)
    image = torch.ones(1, 3, src_model.image_encoder.image_size, src_model.image_encoder.image_size, dtype=torch.float32)

    print('Tracing models…')
    image_encoder = ImageEncoder(src_model.image_encoder).eval()
    image_encoder = torch.jit.trace(image_encoder, image)
    text_encoder = TextEncoder(src_model.text_encoder).eval()
    text_encoder = torch.jit.trace(text_encoder, (input_ids, attention_mask))

    print('Converting models…')

    if opts.image_batchsize_lb == opts.image_batchsize_ub:
        image_batch_dim_shape = opts.image_batchsize_lb 
    else:
        image_batch_dim_shape = ct.RangeDim(lower_bound=opts.image_batchsize_lb, upper_bound=opts.image_batchsize_ub, default=1)

    image_encoder = ct.convert(
        image_encoder,
        convert_to='mlprogram',
        inputs=[
            ct.TensorType(
                name='image',
                shape=(image_batch_dim_shape,) + image.shape[1:],
                dtype=image.numpy().dtype
            )],
        outputs=[
            ct.TensorType(
                name='features'
            ),
            ct.TensorType(
                name='embeddings'
            )
        ],
        compute_precision=ct.precision.FLOAT16 if opts.use_fp16 else ct.precision.FLOAT32
    )

    if opts.text_batchsize_lb == opts.text_batchsize_ub:
        text_batch_dim_shape = opts.text_batchsize_lb 
    else:
        text_batch_dim_shape = ct.RangeDim(lower_bound=opts.text_batchsize_lb, upper_bound=opts.text_batchsize_ub, default=1)

    text_encoder = ct.convert(
        text_encoder,
        convert_to='mlprogram',
        inputs=[
            ct.TensorType(
                name='input_ids',
                shape=(text_batch_dim_shape,) + input_ids.shape[1:],
                dtype=input_ids.numpy().dtype
            ),
            ct.TensorType(
                name='attention_mask',
                shape=(text_batch_dim_shape,) + attention_mask.shape[1:],
                dtype=attention_mask.numpy().dtype
            )],
        outputs=[
            ct.TensorType(
                name="features"
            ),
            ct.TensorType(
                name="embeddings"
            )
        ],
        compute_precision=ct.precision.FLOAT16 if opts.use_fp16 else ct.precision.FLOAT32
    )

    print('Image encoder:', image_encoder, sep='\n')
    print('Text encoder:', text_encoder, sep='\n')

    image_encoder.save(join(opts.output_dir, f"{opts.model_name.replace('/', '.')}.image-encoder.mlpackage"))
    text_encoder.save(join(opts.output_dir, f"{opts.model_name.replace('/', '.')}.text-encoder.mlpackage"))


if __name__ == '__main__':
    opts = ArgumentParser()
    opts.add_argument('--model_name',
                      action='store',
                      type=str,
                      help='UForm model name')

    opts.add_argument('--text_batchsize_lb',
                      action='store',
                      type=int,
                      help='lower bound of batch size for text encoder')

    opts.add_argument('--text_batchsize_ub',
                      action='store',
                      type=int,
                      help='upper bound of batch size for text encoder')
    
    opts.add_argument('--image_batchsize_lb',
                      action='store',
                      type=int,
                      help='lower bound of batch size for image encoder')

    opts.add_argument('--image_batchsize_ub',
                      action='store',
                      type=int,
                      help='upper bound of batch size for image encoder')
    
    opts.add_argument('-use_fp16',
                      action='store_true',
                      help='whether to use fp16 for inference or not')
    
    opts.add_argument('--output_dir',
                      action='store',
                      type=str,
                      help='ouput directory')

    convert_model(opts.parse_args())