import math import torch from transformers.image_processing_backends import TorchvisionBackend from transformers.image_processing_utils import BatchFeature, SizeDict from transformers.image_utils import PILImageResampling from transformers.processing_utils import ImagesKwargs class LocateAnythingImageProcessorKwargs(ImagesKwargs, total=False): patch_size: int merge_kernel_size: tuple[int, int] in_token_limit: int class LocateAnythingImageProcessor(TorchvisionBackend): resample = PILImageResampling.BICUBIC image_mean = [0.5, 0.5, 0.5] image_std = [0.5, 0.5, 0.5] patch_size = 14 merge_kernel_size = [2, 2] in_token_limit = 25600 do_rescale = True do_normalize = True do_convert_rgb = True model_input_names = ["pixel_values", "image_grid_thw"] valid_kwargs = LocateAnythingImageProcessorKwargs def _target_size(self, height, width, patch_size, merge_kernel_size, in_token_limit): if (width // patch_size) * (height // patch_size) > in_token_limit: scale = math.sqrt(in_token_limit / ((width // patch_size) * (height // patch_size))) height, width = int(height * scale), int(width * scale) pad_h = merge_kernel_size[0] * patch_size pad_w = merge_kernel_size[1] * patch_size return math.ceil(height / pad_h) * pad_h, math.ceil(width / pad_w) * pad_w def _preprocess( self, images, resample, do_rescale, rescale_factor, do_normalize, image_mean, image_std, patch_size, merge_kernel_size, in_token_limit, return_tensors, **kwargs, ): patches_batch = [] grids = [] for image in images: _, height, width = image.shape target_h, target_w = self._target_size( height, width, patch_size, merge_kernel_size, in_token_limit ) if (target_h, target_w) != (height, width): image = self.resize(image, SizeDict(height=target_h, width=target_w), resample=resample) image = self.rescale_and_normalize( image, do_rescale, rescale_factor, do_normalize, image_mean, image_std ) channels, height, width = image.shape grid_h, grid_w = height // patch_size, width // patch_size patches = image.reshape(channels, grid_h, patch_size, grid_w, patch_size) patches = patches.permute(1, 3, 0, 2, 4).reshape(-1, channels, patch_size, patch_size) patches_batch.append(patches) grids.append([1, grid_h, grid_w]) return BatchFeature( data={ "pixel_values": torch.cat(patches_batch, dim=0), "image_grid_thw": torch.tensor(grids, dtype=torch.long), }, tensor_type=return_tensors, ) __all__ = ["LocateAnythingImageProcessor"]