File size: 2,933 Bytes
1983b12
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
import math

import torch
from transformers.image_processing_backends import TorchvisionBackend
from transformers.image_processing_utils import BatchFeature, SizeDict
from transformers.image_utils import PILImageResampling
from transformers.processing_utils import ImagesKwargs


class LocateAnythingImageProcessorKwargs(ImagesKwargs, total=False):
    patch_size: int
    merge_kernel_size: tuple[int, int]
    in_token_limit: int


class LocateAnythingImageProcessor(TorchvisionBackend):
    resample = PILImageResampling.BICUBIC
    image_mean = [0.5, 0.5, 0.5]
    image_std = [0.5, 0.5, 0.5]
    patch_size = 14
    merge_kernel_size = [2, 2]
    in_token_limit = 25600
    do_rescale = True
    do_normalize = True
    do_convert_rgb = True
    model_input_names = ["pixel_values", "image_grid_thw"]
    valid_kwargs = LocateAnythingImageProcessorKwargs

    def _target_size(self, height, width, patch_size, merge_kernel_size, in_token_limit):
        if (width // patch_size) * (height // patch_size) > in_token_limit:
            scale = math.sqrt(in_token_limit / ((width // patch_size) * (height // patch_size)))
            height, width = int(height * scale), int(width * scale)
        pad_h = merge_kernel_size[0] * patch_size
        pad_w = merge_kernel_size[1] * patch_size
        return math.ceil(height / pad_h) * pad_h, math.ceil(width / pad_w) * pad_w

    def _preprocess(
        self,
        images,
        resample,
        do_rescale,
        rescale_factor,
        do_normalize,
        image_mean,
        image_std,
        patch_size,
        merge_kernel_size,
        in_token_limit,
        return_tensors,
        **kwargs,
    ):
        patches_batch = []
        grids = []
        for image in images:
            _, height, width = image.shape
            target_h, target_w = self._target_size(
                height, width, patch_size, merge_kernel_size, in_token_limit
            )
            if (target_h, target_w) != (height, width):
                image = self.resize(image, SizeDict(height=target_h, width=target_w), resample=resample)
            image = self.rescale_and_normalize(
                image, do_rescale, rescale_factor, do_normalize, image_mean, image_std
            )
            channels, height, width = image.shape
            grid_h, grid_w = height // patch_size, width // patch_size
            patches = image.reshape(channels, grid_h, patch_size, grid_w, patch_size)
            patches = patches.permute(1, 3, 0, 2, 4).reshape(-1, channels, patch_size, patch_size)
            patches_batch.append(patches)
            grids.append([1, grid_h, grid_w])

        return BatchFeature(
            data={
                "pixel_values": torch.cat(patches_batch, dim=0),
                "image_grid_thw": torch.tensor(grids, dtype=torch.long),
            },
            tensor_type=return_tensors,
        )


__all__ = ["LocateAnythingImageProcessor"]