hy4-flash-vision-encoder / native_source /processing_hy_v4_flash.py
gatilin's picture
Add files using upload-large-folder tool
48e1b03 verified
Raw History Blame Contribute Delete
6.37 kB
# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
# This file was automatically generated from src/transformers/models/hy_v4_flash/modular_hy_v4_flash.py.
# Do NOT edit this file manually as any edits will be overwritten by the generation of
# the file from the modular. If any change should be done, please apply the change to the
# modular_hy_v4_flash.py file directly. One of our CI enforces this.
# 🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
# Copyright 2026 Tencent HunYuan Team and The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from ...processing_utils import MultiModalData, ProcessorMixin
from ...utils import auto_docstring
@auto_docstring
class HYV4FlashProcessor(ProcessorMixin):
return_mm_token_type_ids = True
def __init__(self, image_processor=None, tokenizer=None, video_processor=None, chat_template=None, **kwargs):
self.image_token = "<|image_pad|>" if not hasattr(tokenizer, "image_token") else tokenizer.image_token
self.video_token = "<|video_pad|>" if not hasattr(tokenizer, "video_token") else tokenizer.video_token
self.image_token_id = (
tokenizer.image_token_id
if getattr(tokenizer, "image_token_id", None)
else tokenizer.convert_tokens_to_ids(self.image_token)
)
self.video_token_id = (
tokenizer.video_token_id
if getattr(tokenizer, "video_token_id", None)
else tokenizer.convert_tokens_to_ids(self.video_token)
)
super().__init__(image_processor, tokenizer, video_processor, chat_template=chat_template)
def replace_image_token(self, image_inputs: dict, image_idx: int, **kwargs) -> str:
_, grid_h, grid_w = image_inputs["image_grid_thw"][image_idx]
merge_size = kwargs.get("merge_size") or self.image_processor.merge_size
return self.image_token * self._num_frame_tokens(grid_h, grid_w, merge_size)
def replace_video_token(self, video_inputs: dict, video_idx: int, **kwargs) -> str:
grid_t, grid_h, grid_w = video_inputs["video_grid_thw"][video_idx]
merge_size = kwargs.get("merge_size") or self.video_processor.merge_size
return self.video_token * (grid_t * self._num_frame_tokens(grid_h, grid_w, merge_size))
def _get_num_multimodal_tokens(self, image_sizes=None, video_sizes=None, **kwargs):
"""
Computes the number of placeholder tokens needed for multimodal inputs with the given sizes.
"""
vision_data = {}
if image_sizes is not None:
images_kwargs = {**kwargs, **kwargs.get("images_kwargs", {})}
merge_size = images_kwargs.get("merge_size") or self.image_processor.merge_size
grids = [
self.image_processor.get_number_of_image_patches(height, width, images_kwargs)
for height, width in image_sizes
]
vision_data["num_image_patches"] = [grid_h * grid_w for grid_h, grid_w in grids]
vision_data["num_image_tokens"] = [self._num_frame_tokens(*grid, merge_size) for grid in grids]
if video_sizes is not None:
videos_kwargs = {**kwargs, **kwargs.get("videos_kwargs", {})}
merge_size = videos_kwargs.get("merge_size") or self.video_processor.merge_size
num_video_tokens = []
for num_frames, height, width in video_sizes:
_, grid_t, grid_h, grid_w = self.video_processor.get_num_of_video_patches(
num_frames, height, width, videos_kwargs, return_grid=True
)
num_video_tokens.append(grid_t * self._num_frame_tokens(grid_h, grid_w, merge_size))
vision_data["num_video_tokens"] = num_video_tokens
return MultiModalData(**vision_data)
def post_process_image_text_to_text(
self, generated_outputs, skip_special_tokens=True, clean_up_tokenization_spaces=False, **kwargs
):
"""
Post-process the output of the model to decode the text.
Args:
generated_outputs (`torch.Tensor` or `np.ndarray`):
The output of the model `generate` function. The output is expected to be a tensor of shape `(batch_size, sequence_length)`
or `(sequence_length,)`.
skip_special_tokens (`bool`, *optional*, defaults to `True`):
Whether or not to remove special tokens in the output. Argument passed to the tokenizer's `batch_decode` method.
clean_up_tokenization_spaces (`bool`, *optional*, defaults to `False`):
Whether or not to clean up the tokenization spaces. Argument passed to the tokenizer's `batch_decode` method.
**kwargs:
Additional arguments to be passed to the tokenizer's `batch_decode method`.
Returns:
`list[str]`: The decoded text.
"""
return self.tokenizer.batch_decode(
generated_outputs,
skip_special_tokens=skip_special_tokens,
clean_up_tokenization_spaces=clean_up_tokenization_spaces,
**kwargs,
)
@property
def model_input_names(self):
return super().model_input_names + ["mm_token_type_ids"]
@staticmethod
def _num_frame_tokens(grid_h: int, grid_w: int, merge_size: int) -> int:
# One extra newline token per merged row
return (grid_h // merge_size) * (grid_w // merge_size + 1)
__all__ = ["HYV4FlashProcessor"]