| | """
|
| | Download the weights in ./checkpoints beforehand for fast inference
|
| | wget https://storage.googleapis.com/sfr-vision-language-research/BLIP/models/model*_base_caption.pth
|
| | wget https://storage.googleapis.com/sfr-vision-language-research/BLIP/models/model*_vqa.pth
|
| | wget https://storage.googleapis.com/sfr-vision-language-research/BLIP/models/model_base_retrieval_coco.pth
|
| | """
|
| |
|
| | from pathlib import Path
|
| |
|
| | from PIL import Image
|
| | import torch
|
| | from torchvision import transforms
|
| | from torchvision.transforms.functional import InterpolationMode
|
| | import cog
|
| |
|
| | from models.blip import blip_decoder
|
| | from models.blip_vqa import blip_vqa
|
| | from models.blip_itm import blip_itm
|
| |
|
| |
|
| | class Predictor(cog.Predictor):
|
| | def setup(self):
|
| | self.device = "cuda:0"
|
| |
|
| | self.models = {
|
| | 'image_captioning': blip_decoder(pretrained='checkpoints/model*_base_caption.pth',
|
| | image_size=384, vit='base'),
|
| | 'visual_question_answering': blip_vqa(pretrained='checkpoints/model*_vqa.pth',
|
| | image_size=480, vit='base'),
|
| | 'image_text_matching': blip_itm(pretrained='checkpoints/model_base_retrieval_coco.pth',
|
| | image_size=384, vit='base')
|
| | }
|
| |
|
| | @cog.input(
|
| | "image",
|
| | type=Path,
|
| | help="input image",
|
| | )
|
| | @cog.input(
|
| | "task",
|
| | type=str,
|
| | default='image_captioning',
|
| | options=['image_captioning', 'visual_question_answering', 'image_text_matching'],
|
| | help="Choose a task.",
|
| | )
|
| | @cog.input(
|
| | "question",
|
| | type=str,
|
| | default=None,
|
| | help="Type question for the input image for visual question answering task.",
|
| | )
|
| | @cog.input(
|
| | "caption",
|
| | type=str,
|
| | default=None,
|
| | help="Type caption for the input image for image text matching task.",
|
| | )
|
| | def predict(self, image, task, question, caption):
|
| | if task == 'visual_question_answering':
|
| | assert question is not None, 'Please type a question for visual question answering task.'
|
| | if task == 'image_text_matching':
|
| | assert caption is not None, 'Please type a caption for mage text matching task.'
|
| |
|
| | im = load_image(image, image_size=480 if task == 'visual_question_answering' else 384, device=self.device)
|
| | model = self.models[task]
|
| | model.eval()
|
| | model = model.to(self.device)
|
| |
|
| | if task == 'image_captioning':
|
| | with torch.no_grad():
|
| | caption = model.generate(im, sample=False, num_beams=3, max_length=20, min_length=5)
|
| | return 'Caption: ' + caption[0]
|
| |
|
| | if task == 'visual_question_answering':
|
| | with torch.no_grad():
|
| | answer = model(im, question, train=False, inference='generate')
|
| | return 'Answer: ' + answer[0]
|
| |
|
| |
|
| | itm_output = model(im, caption, match_head='itm')
|
| | itm_score = torch.nn.functional.softmax(itm_output, dim=1)[:, 1]
|
| | itc_score = model(im, caption, match_head='itc')
|
| | return f'The image and text is matched with a probability of {itm_score.item():.4f}.\n' \
|
| | f'The image feature and text feature has a cosine similarity of {itc_score.item():.4f}.'
|
| |
|
| |
|
| | def load_image(image, image_size, device):
|
| | raw_image = Image.open(str(image)).convert('RGB')
|
| |
|
| | w, h = raw_image.size
|
| |
|
| | transform = transforms.Compose([
|
| | transforms.Resize((image_size, image_size), interpolation=InterpolationMode.BICUBIC),
|
| | transforms.ToTensor(),
|
| | transforms.Normalize((0.48145466, 0.4578275, 0.40821073), (0.26862954, 0.26130258, 0.27577711))
|
| | ])
|
| | image = transform(raw_image).unsqueeze(0).to(device)
|
| | return image
|
| |
|
