File size: 12,824 Bytes
48e1b03 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 | # π¨π¨π¨π¨π¨π¨π¨π¨π¨π¨π¨π¨π¨π¨π¨π¨π¨π¨π¨π¨π¨π¨π¨π¨π¨π¨π¨π¨π¨π¨π¨π¨π¨π¨π¨π¨π¨π¨π¨π¨π¨π¨π¨π¨π¨π¨π¨π¨
# This file was automatically generated from src/transformers/models/hy_v4_flash/modular_hy_v4_flash.py.
# Do NOT edit this file manually as any edits will be overwritten by the generation of
# the file from the modular. If any change should be done, please apply the change to the
# modular_hy_v4_flash.py file directly. One of our CI enforces this.
# π¨π¨π¨π¨π¨π¨π¨π¨π¨π¨π¨π¨π¨π¨π¨π¨π¨π¨π¨π¨π¨π¨π¨π¨π¨π¨π¨π¨π¨π¨π¨π¨π¨π¨π¨π¨π¨π¨π¨π¨π¨π¨π¨π¨π¨π¨π¨π¨
# Copyright 2026 Tencent HunYuan Team and The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import math
import warnings
from collections.abc import Iterable
import torch
from torchvision.transforms.v2 import functional as tvF
from ...image_processing_backends import TorchvisionBackend
from ...image_processing_utils import BatchFeature
from ...image_transforms import group_images_by_shape, reorder_images
from ...image_utils import OPENAI_CLIP_MEAN, OPENAI_CLIP_STD, ImageInput, PILImageResampling, SizeDict
from ...processing_utils import ImagesKwargs, Unpack
from ...utils import TensorType, auto_docstring
# Adapted from transformers.models.hy_v4_flash.image_processing_hy_v4_flash.HYV4FlashImageProcessorKwargs
class HYV4FlashImageProcessorKwargs(ImagesKwargs, total=False):
r"""
min_pixels (`int`, *optional*, defaults to `512 * 512`):
The min pixels of the image to resize the image.
max_pixels (`int`, *optional*, defaults to `2048 * 2048`):
The max pixels of the image to resize the image.
patch_size (`int`, *optional*, defaults to 16):
The spatial patch size of the vision encoder.
temporal_patch_size (`int`, *optional*, defaults to 1):
The temporal patch size of the vision encoder.
merge_size (`int`, *optional*, defaults to 2):
The merge size of the vision encoder to llm encoder.
"""
min_pixels: int
max_pixels: int
patch_size: int
temporal_patch_size: int
merge_size: int
def smart_resize(
height: int, width: int, factor: int = 28, min_pixels: int = 56 * 56, max_pixels: int = 14 * 14 * 4 * 1280
):
"""Rescales the image so that the following conditions are met:
1. Both dimensions (height and width) are divisible by 'factor'.
2. The total number of pixels is within the range ['min_pixels', 'max_pixels'].
3. The aspect ratio of the image is maintained as closely as possible.
"""
if max(height, width) / min(height, width) > 200:
raise ValueError(
f"absolute aspect ratio must be smaller than 200, got {max(height, width) / min(height, width)}"
)
h_bar = round(height / factor) * factor
w_bar = round(width / factor) * factor
if h_bar * w_bar > max_pixels:
beta = math.sqrt((height * width) / max_pixels)
h_bar = max(factor, math.floor(height / beta / factor) * factor)
w_bar = max(factor, math.floor(width / beta / factor) * factor)
elif h_bar * w_bar < min_pixels:
beta = math.sqrt(min_pixels / (height * width))
h_bar = math.ceil(height * beta / factor) * factor
w_bar = math.ceil(width * beta / factor) * factor
return h_bar, w_bar
@auto_docstring
class HYV4FlashImageProcessor(TorchvisionBackend):
do_resize = True
resample = PILImageResampling.BICUBIC
size = {"shortest_edge": 512 * 512, "longest_edge": 2048 * 2048}
default_to_square = False
do_rescale = True
do_normalize = True
image_mean = OPENAI_CLIP_MEAN
image_std = OPENAI_CLIP_STD
do_convert_rgb = True
patch_size = 16
temporal_patch_size = 1
merge_size = 2
valid_kwargs = HYV4FlashImageProcessorKwargs
model_input_names = ["pixel_values", "image_grid_thw"]
spatial_patch_size = 1
def __init__(self, **kwargs: Unpack[HYV4FlashImageProcessorKwargs]):
# backward compatibility: override size with min_pixels and max_pixels if they are provided
size = kwargs.pop("size", None)
size = dict(self.size) if size is None else size
if (min_pixels := kwargs.pop("min_pixels", None)) is not None:
size["shortest_edge"] = min_pixels
size.pop("min_pixels", None)
if (max_pixels := kwargs.pop("max_pixels", None)) is not None:
size["longest_edge"] = max_pixels
size.pop("max_pixels", None)
super().__init__(size=size, **kwargs)
def _standardize_kwargs(
self,
size: int | Iterable[int] | dict[str, int] | SizeDict | None = None,
min_pixels: int | None = None,
max_pixels: int | None = None,
**kwargs,
) -> dict:
if min_pixels is not None or max_pixels is not None:
warnings.warn(
"Passing `min_pixels` and `max_pixels` to a processor call is deprecated and will be removed in v5.23. "
"Pass in `size={'longest_edge': xxx, 'shortest_edge': xxx} to override the target size.`",
FutureWarning,
)
size_dict = dict(size) if isinstance(size, (dict, SizeDict)) else {}
if min_pixels is not None:
size_dict["shortest_edge"] = min_pixels
if max_pixels is not None:
size_dict["longest_edge"] = max_pixels
size = SizeDict(**size_dict)
return super()._standardize_kwargs(size=size, **kwargs)
@auto_docstring
def preprocess(
self,
images: ImageInput,
**kwargs: Unpack[HYV4FlashImageProcessorKwargs],
) -> BatchFeature:
return super().preprocess(images, **kwargs)
def resize(
self,
images: "torch.Tensor",
size: SizeDict,
resample: "PILImageResampling | tvF.InterpolationMode | int | None",
factor: int,
**kwargs,
) -> "torch.Tensor":
"""Resize dynamically based on input image aspect ratio."""
if not size.shortest_edge or not size.longest_edge:
raise ValueError(f"`size` dict must contain 'shortest_edge' and 'longest_edge' keys but got {size}.")
height, width = images.shape[-2:]
resized_height, resized_width = smart_resize(
height,
width,
factor=factor,
min_pixels=size.shortest_edge,
max_pixels=size.longest_edge,
)
return super().resize(
image=images,
size=SizeDict(height=resized_height, width=resized_width),
# The reference HunyuanOCR processor calls `PIL.Image.resize` without a
# resampling argument, which uses BICUBIC for RGB images. Its config has
# `resample=1` (LANCZOS), but the original implementation never uses it.
resample=PILImageResampling.BICUBIC,
)
def patchify(
self,
images: "torch.Tensor",
patch_size: int,
merge_size: int,
temporal_patch_size: int,
) -> tuple["torch.Tensor", int, int]:
"""Patchifies each image into flat layout of shape (`seq_len`, `patch_dim`) so we can concat dynamically shaped pixels."""
batch_size, channel, resized_height, resized_width = images.shape
grid_h, grid_w = resized_height // patch_size, resized_width // patch_size
patches = images.reshape(
batch_size,
channel,
grid_h // merge_size,
merge_size,
patch_size,
grid_w // merge_size,
merge_size,
patch_size,
)
# Flatten patches in row-major order in comparison to Qwen
patches = patches.permute(0, 2, 3, 5, 6, 1, 4, 7)
flatten_patches = (
patches.unsqueeze(6)
.expand(-1, -1, -1, -1, -1, -1, temporal_patch_size, -1, -1)
.reshape(
batch_size,
grid_h * grid_w,
channel * temporal_patch_size * patch_size * patch_size,
)
)
return flatten_patches, grid_h, grid_w
def _preprocess(
self,
images: list["torch.Tensor"],
do_resize: bool,
size: SizeDict,
resample: "PILImageResampling | tvF.InterpolationMode | int | None",
do_rescale: bool,
rescale_factor: float,
do_normalize: bool,
image_mean: float | list[float] | None,
image_std: float | list[float] | None,
patch_size: int,
temporal_patch_size: int,
merge_size: int,
disable_grouping: bool | None,
return_tensors: str | TensorType | None,
**kwargs,
) -> BatchFeature:
grouped_images, grouped_images_index = group_images_by_shape(images, disable_grouping=disable_grouping)
resized_images_grouped = {}
for shape, stacked_images in grouped_images.items():
if do_resize:
stacked_images = self.resize(
images=stacked_images,
size=size,
resample=resample,
factor=patch_size * merge_size,
)
resized_images_grouped[shape] = stacked_images
resized_images = reorder_images(resized_images_grouped, grouped_images_index)
grouped_images, grouped_images_index = group_images_by_shape(resized_images, disable_grouping=disable_grouping)
processed_images_grouped = {}
processed_grids = {}
for shape, stacked_images in grouped_images.items():
stacked_images = self.rescale_and_normalize(
stacked_images, do_rescale, rescale_factor, do_normalize, image_mean, image_std
)
patches, grid_h, grid_w = self.patchify(
stacked_images,
patch_size=patch_size,
merge_size=merge_size,
temporal_patch_size=temporal_patch_size,
)
processed_images_grouped[shape] = patches
processed_grids[shape] = [[1, grid_h, grid_w]] * len(stacked_images)
processed_images = reorder_images(processed_images_grouped, grouped_images_index)
processed_grids_ordered = reorder_images(processed_grids, grouped_images_index)
pixel_values = processed_images[0] if len(processed_images) == 1 else torch.cat(processed_images, dim=0)
image_grid_thw = torch.tensor(processed_grids_ordered, dtype=torch.long)
return BatchFeature(
data={"pixel_values": pixel_values, "image_grid_thw": image_grid_thw}, tensor_type=return_tensors
)
def get_number_of_image_patches(
self, height: int, width: int, images_kwargs: dict | None = None
) -> tuple[int, int]:
"""
A utility that returns number of image patches for a given image size.
Note: Do not remove this method! It is used by vLLM to infer the number of patches and placeholders
without an image input.
Args:
height (`int`):
Height of the input image.
width (`int`):
Width of the input image.
images_kwargs (`dict`, *optional*)
Any kwargs to override defaults of the image processor.
Returns:
`tuple[int, int]`: Number of image patches per image, as a `(height, width)` grid.
"""
images_kwargs = images_kwargs or {}
min_pixels = images_kwargs["min_pixels"] if "min_pixels" in images_kwargs else self.size["shortest_edge"]
max_pixels = images_kwargs["max_pixels"] if "max_pixels" in images_kwargs else self.size["longest_edge"]
patch_size = images_kwargs.get("patch_size", self.patch_size)
merge_size = images_kwargs.get("merge_size", self.merge_size)
factor = patch_size * merge_size
resized_height, resized_width = smart_resize(
height, width, factor, min_pixels=min_pixels, max_pixels=max_pixels
)
return resized_height // patch_size, resized_width // patch_size
__all__ = ["HYV4FlashImageProcessor"]
|