import torch import torch.nn as nn import torch.nn.functional as F import PIL.Image from typing import List from friday.util import expand2square, pad_and_stack from transformers import AutoModel, AutoImageProcessor from friday.util.s2wrapper import forward as multiscale_forward class FastVitVisionTower(nn.Module): def __init__(self, pretrained_model_name_or_path, model_params={}, pad_to_square=True, **kwargs): super().__init__() self.is_loaded = False self.pretrained_model_name_or_path = pretrained_model_name_or_path self.model_params = model_params self.pad_to_square = pad_to_square self.load_model() @property def output_dim(self): return self.vision_tower.config.embed_dim if self.vision_tower else None def load_model(self): if self.is_loaded: return self.image_processor = AutoImageProcessor.from_pretrained(self.pretrained_model_name_or_path) self.image_processor.crop_size = self.image_processor.size self.vision_tower = AutoModel.from_pretrained( self.pretrained_model_name_or_path, **self.model_params, ) self.vision_tower.requires_grad_(False) self.is_loaded = True def preprocess_images(self, imgs: List[PIL.Image.Image], pad_and_stack_tensors=True) -> torch.Tensor: img_mean = tuple(int(x * 255) for x in self.image_processor.image_mean) if self.pad_to_square: imgs = [expand2square(img, img_mean) for img in imgs] imgs = [self.image_processor(img, do_resize=True, do_center_crop=False, return_tensors="pt")['pixel_values'][0] for img in imgs] if pad_and_stack_tensors: imgs = pad_and_stack(imgs, pad_value=0.0) imgs = imgs.to(dtype=torch.float32, device=self.device) return imgs def forward(self, images): if type(images) is list: image_features = [] for image in images: image_feature = self.vision_tower( image.to(device=self.device, dtype=self.dtype).unsqueeze(0) ) image_features.append(image_feature) else: image_features = self.vision_tower( images.to(device=self.device, dtype=self.dtype), ) return image_features @property def dummy_feature(self): return torch.zeros(1, self.embed_dim, device=self.device, dtype=self.dtype) @property def dtype(self): return self.vision_tower.dtype @property def device(self): return self.vision_tower.device @property def config(self): if self.is_loaded: return self.vision_tower.config else: return self.cfg_only @property def hidden_size(self): return self.config.embed_dim @property def num_patches(self): return (self.config.image_size // self.config.patch_size) ** 2 class FastVitVisionTowerS2(FastVitVisionTower): def __init__(self, pretrained_model_name_or_path, s2_scales, model_params={}, **kwargs): self.s2_scales = list(map(int, s2_scales.split(','))) self.s2_scales.sort() self.s2_split_size = self.s2_scales[0] self.s2_image_size = self.s2_scales[-1] super().__init__(pretrained_model_name_or_path, model_params) self.multiscale_forward = multiscale_forward @property def output_dim(self): return (2*self.vision_tower.config.embed_dim) if self.vision_tower else None def load_model(self): if self.is_loaded: return super().load_model() self.image_processor.size = self.image_processor.crop_size = { "height": self.s2_image_size, "width": self.s2_image_size } def forward_feature(self, images): image_size = self.vision_tower.config.image_size if images.shape[2] != image_size or images.shape[3] != image_size: images = F.interpolate( images, size=(image_size, image_size), mode="bilinear", align_corners=False, antialias=True ) return self.vision_tower( images.to(device=self.device, dtype=self.dtype), ) def forward(self, images): if type(images) is list: image_features = [] for image in images: image_feature = self.multiscale_forward( self.forward_feature, image.unsqueeze(0), img_sizes=self.s2_scales, max_split_size=self.s2_split_size ) image_features.append(image_feature) else: image_features = self.multiscale_forward( self.forward_feature, images, img_sizes=self.s2_scales, max_split_size=self.s2_split_size ) return image_features @property def hidden_size(self): return self.config.embed_dim * len(self.s2_scales)