Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text
+trainer_state.json filter=lfs diff=lfs merge=lfs -text

added_tokens.json

@@ -0,0 +1,34 @@
+{
+  "</think>": 151668,
+  "</tool_call>": 151658,
+  "</tool_response>": 151666,
+  "<image>": 151669,
+  "<think>": 151667,
+  "<tool_call>": 151657,
+  "<tool_response>": 151665,
+  "<|audio_end|>": 151674,
+  "<|audio_start|>": 151673,
+  "<|audio|>": 151672,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|stream_end|>": 151671,
+  "<|stream_start|>": 151670,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

chat_template.json

@@ -0,0 +1,3 @@
+{
+  "chat_template": "\n{%- set identifier = 'im' %}\n{% for message in messages %}\n    {% if message['role'] == 'stream' %}\n        {% set identifier = 'stream' %}\n    {% else %}\n        {% set identifier = 'im' %}\n    {% endif %}\n    {% if message['role'] is not none %}\n        {{- '<|' + identifier + '_start|>' + message['role'] + '\n' -}}\n    {% endif %}\n    {% if message['content'] is string %}\n        {{- message['content'] + '<|' + identifier + '_end|>\n' -}}\n    {% else %}\n        {% for content in message['content'] %}\n            {% if content['type'] == 'image' or 'image' in content or 'image_url' in content %}\n                {% if 'time' in content %}\n                    {{- 'Time ' + content['time'] | round(1) | string + 's: ' -}}\n                {% endif %}\n                {{- image_token + '\n' -}}\n            {% elif content['type'] == 'video' or 'video' in content or 'video_url' in content %}\n                {% for i in range(content['num_frames']) %}\n                    {% if 'timestamps' in content %}\n                        {{- 'Time ' + content['timestamps'][i] | round(1) | string + 's:' -}}\n                    {% endif %}\n                    {% if i < content['num_frames'] - 1 %}\n                        {{- image_token + ',' -}}\n                        {% if 'audio_split' in content and content['audio_split'][i] > 0 %}\n                            {{- '<|audio_start|>' + audio_token * content['audio_split'][i] + '<|audio_end|>,' -}}\n                        {% endif %}\n                    {% else %}\n                        {{- image_token -}}\n                        {% if 'audio_split' in content and content['audio_split'][i] > 0 %}\n                            {{- ',<|audio_start|>' + audio_token * content['audio_split'][i] + '<|audio_end|>\n' -}}\n                        {% else %}\n                            {{- '\n' -}}\n                        {% endif %}\n                    {% endif %}\n                {% endfor %}\n            {% elif content['type'] == 'audio' or 'audio' in content or 'audio_url' in content %}\n                {% for i in range(content['num_frames']) %}\n                    {% if 'timestamps' in content %}\n                        {{- 'Time ' + content['timestamps'][i] | round(1) | string + 's:' -}}\n                    {% endif %}\n                    {% if i < content['num_frames'] - 1 %}\n                        {{- '<|audio_start|>' + audio_token + '<|audio_end|>,' -}}\n                    {% else %}\n                        {{- '<|audio_start|>' + audio_token + '<|audio_end|>\n' -}}\n                    {% endif %}\n                {% endfor %}\n            {% elif content['type'] == 'text' or 'text' in content %}\n                {{- content['text'] -}}\n            {% endif %}\n        {% endfor %}\n        {% if message['role'] is not none %}\n            {{- '<|' + identifier + '_end|>\n' -}}\n        {% endif %}\n    {% endif %}\n{% endfor %}\n{% if add_generation_prompt %}\n    {{- '<|im_start|>assistant\n' -}}\n    {% if not add_think_prompt %}\n        {{- '<think>\n\n</think>\n\n' -}}\n    {% endif %}\n{% endif %}\n"
+}

config.json

@@ -0,0 +1,87 @@
+{
+  "architectures": [
+    "Videollama3Qwen3ForCausalLM"
+  ],
+  "auto_map": {
+    "AutoConfig": "configuration_videollama3.Videollama3Qwen3Config",
+    "AutoModelForCausalLM": "modeling_videollama3_qwen3.Videollama3Qwen3ForCausalLM"
+  },
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "audio_encoder": "Cyril666/whisper-large-v3-encoder",
+  "audio_encoder_lr": 5e-06,
+  "audio_hidden_size": 1280,
+  "audio_projector_lr": 0.001,
+  "audio_projector_type": "dmlp2x_gelu",
+  "audio_token_index": 151672,
+  "bos_token_id": 151643,
+  "embedding_lr": null,
+  "eos_token_id": 151645,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 2048,
+  "image_aspect_ratio": "square",
+  "image_size": -1,
+  "image_token_index": 151669,
+  "image_token_length": 1,
+  "initializer_range": 0.02,
+  "intermediate_size": 6144,
+  "is_alignment": false,
+  "llm_lr": null,
+  "loss_reduction_scope": "batch",
+  "max_frames": 180,
+  "max_position_embeddings": 40960,
+  "max_window_layers": 28,
+  "mm_hidden_size": 1024,
+  "mm_projector_lr": 1e-05,
+  "mm_projector_type": "mlp2x_gelu",
+  "mm_vision_select_feature": "patch",
+  "mm_vision_select_layer": -1,
+  "model_type": "videollama3_qwen3",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 28,
+  "num_key_value_heads": 8,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 1000000,
+  "sliding_window": null,
+  "tie_word_embeddings": true,
+  "tokenizer_model_max_length": 16384,
+  "tokenizer_padding_side": "right",
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.51.3",
+  "use_cache": true,
+  "use_mm_proj": true,
+  "use_reconstruct": false,
+  "use_sliding_window": false,
+  "use_token_compression": false,
+  "use_vision_teacher": false,
+  "use_visual_expert": false,
+  "vision_encoder": null,
+  "vision_encoder_lr": null,
+  "vision_encoder_teacher": null,
+  "vision_hidden_size": 1024,
+  "vision_projector_lr": null,
+  "vision_projector_type": "mlp2x_gelu",
+  "visual_expert_lr": null,
+  "vocab_size": 151936,
+  "vision_encoder_config": {
+    "head_dim": 128,
+    "hidden_act": "silu",
+    "hidden_size": 1024,
+    "initializer_range": 0.02,
+    "intermediate_size": 3072,
+    "layer_norm_eps": 1e-06,
+    "max_window_layers": 28,
+    "num_attention_heads": 16,
+    "num_channels": 3,
+    "num_hidden_layers": 28,
+    "num_key_value_heads": 8,
+    "patch_size": 14,
+    "rms_norm_eps": 1e-06,
+    "rope_scaling": null,
+    "rope_theta": 1000000,
+    "sliding_window": null,
+    "torch_dtype": "bfloat16"
+  }
+}

configuration_sfl_encoder.py

@@ -0,0 +1,79 @@
+# Adopted from https://github.com/huggingface/transformers/blob/main/src/transformers/models/siglip/configuration_siglip.py.
+# Below is the original copyright:
+# coding=utf-8
+# Copyright 2024 The HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""VideoLLaMA3 vision encoder model configuration."""
+
+from transformers import Qwen2Config, Qwen3Config
+
+
+class SFLVisionEncoderConfigFromQwen2(Qwen2Config):
+
+    model_type = "sfl_vision_encoder_qwen2"
+
+    def __init__(
+        self,
+        hidden_size=1536,
+        intermediate_size=8960,
+        num_hidden_layers=12,
+        num_attention_heads=12,
+        num_channels=3,
+        patch_size=14,
+        layer_norm_eps=1e-6,
+        attention_dropout=0.0,
+        num_key_value_heads=2,
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+
+        self.hidden_size = hidden_size
+        self.intermediate_size = intermediate_size
+        self.num_hidden_layers = num_hidden_layers
+        self.num_attention_heads = num_attention_heads
+        self.num_channels = num_channels
+        self.patch_size = patch_size
+        self.attention_dropout = attention_dropout
+        self.num_key_value_heads = num_key_value_heads
+        self.layer_norm_eps = layer_norm_eps
+
+
+class SFLVisionEncoderConfigFromQwen3(Qwen3Config):
+
+    model_type = "sfl_vision_encoder_qwen3"
+
+    def __init__(
+        self,
+        hidden_size=1536,
+        intermediate_size=8960,
+        num_hidden_layers=12,
+        num_attention_heads=12,
+        num_channels=3,
+        patch_size=14,
+        layer_norm_eps=1e-6,
+        attention_dropout=0.0,
+        num_key_value_heads=2,
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+
+        self.hidden_size = hidden_size
+        self.intermediate_size = intermediate_size
+        self.num_hidden_layers = num_hidden_layers
+        self.num_attention_heads = num_attention_heads
+        self.num_channels = num_channels
+        self.patch_size = patch_size
+        self.attention_dropout = attention_dropout
+        self.num_key_value_heads = num_key_value_heads
+        self.layer_norm_eps = layer_norm_eps

configuration_videollama3.py

@@ -0,0 +1,76 @@
+"""VideoLLaMA3 model configuration."""
+
+import importlib.util
+import os.path as osp
+from typing import Optional, Dict, Any
+
+from transformers import AutoConfig, AutoModel, PretrainedConfig, Qwen2Config, Qwen3Config, Qwen2AudioEncoderConfig
+
+try:
+    from .configuration_sfl_encoder import SFLVisionEncoderConfigFromQwen3
+except ModuleNotFoundError:
+    spec = importlib.util.spec_from_file_location(
+        "configuration_sfl_encoder",
+        osp.join(osp.dirname(__file__), "configuration_sfl_encoder.py"),
+    )
+    configuration_sfl_encoder = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(configuration_sfl_encoder)
+    SFLVisionEncoderConfigFromQwen3 = getattr(
+        configuration_sfl_encoder,
+        "SFLVisionEncoderConfigFromQwen3",
+    )
+
+try:
+    from .modeling_sfl_encoder_qwen3 import SFLVisionEncoderModelFromQwen3
+except ModuleNotFoundError:
+    spec = importlib.util.spec_from_file_location(
+        "modeling_sfl_encoder_qwen3",
+        osp.join(osp.dirname(__file__), "modeling_sfl_encoder_qwen3.py"),
+    )
+    modeling_sfl_encoder_qwen3 = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(modeling_sfl_encoder_qwen3)
+    SFLVisionEncoderModelFromQwen3 = getattr(
+        modeling_sfl_encoder_qwen3,
+        "SFLVisionEncoderModelFromQwen3",
+    )
+
+AutoConfig.register("sfl_vision_encoder_qwen3", SFLVisionEncoderConfigFromQwen3)
+AutoModel.register(SFLVisionEncoderConfigFromQwen3, SFLVisionEncoderModelFromQwen3)
+
+
+class Videollama3Qwen3Config(Qwen3Config):
+
+    model_type = "videollama3_qwen3"
+    sub_configs = {"vision_encoder_config": SFLVisionEncoderConfigFromQwen3, "audio_encoder_config": Qwen2AudioEncoderConfig}
+
+    def __init__(
+        self,
+        vision_encoder: Optional[str] = None,
+        audio_encoder: Optional[str] = None,
+        vision_encoder_config: Dict[str, Any] = {},
+        audio_encoder_config: Dict[str, Any] = {},
+        vision_projector_type: str = "mlp2x_gelu",
+        audio_projector_type: str = "mlp2x_gelu",
+        use_token_compression: bool = True,
+        image_token_index: int = -1,
+        audio_token_index: int = -1,
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+        self.model_type = "videollama3_qwen3"
+
+        self.vision_encoder = vision_encoder
+        if vision_encoder_config is not None and not isinstance(vision_encoder_config, PretrainedConfig):
+            vision_encoder_config = SFLVisionEncoderConfigFromQwen3(**vision_encoder_config)
+        self.vision_encoder_config = vision_encoder_config
+
+        self.audio_encoder = audio_encoder
+        if audio_encoder_config is not None and not isinstance(audio_encoder_config, PretrainedConfig):
+            audio_encoder_config = Qwen2AudioEncoderConfig(**audio_encoder_config)
+        self.audio_encoder_config = audio_encoder_config
+
+        self.vision_projector_type = vision_projector_type
+        self.audio_projector_type = audio_projector_type
+        self.use_token_compression = use_token_compression
+        self.image_token_index = image_token_index
+        self.audio_token_index = audio_token_index

generation_config.json

@@ -0,0 +1,13 @@
+{
+  "bos_token_id": 151643,
+  "do_sample": true,
+  "eos_token_id": [
+    151645,
+    151643
+  ],
+  "pad_token_id": 151643,
+  "temperature": 0.6,
+  "top_k": 20,
+  "top_p": 0.95,
+  "transformers_version": "4.51.3"
+}

image_processing_sfl.py

@@ -0,0 +1,476 @@
+# Adopted from https://github.com/huggingface/transformers/blob/main/src/transformers/models/qwen2_vl/image_processing_qwen2_vl.py.
+# Below is the original copyright:
+# Copyright 2024 The Qwen team, Alibaba Group and the HuggingFace Inc. team. All rights reserved.
+#
+# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
+# and OPT implementations in this library. It has been modified from its
+# original forms to accommodate minor architectural differences compared
+# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Image processor class for VideoLLaMA3."""
+
+import math
+from typing import Dict, List, Optional, Union
+
+import numpy as np
+
+import torch
+from transformers.image_processing_utils import BaseImageProcessor, BatchFeature
+from transformers.image_utils import ImageInput
+from transformers.image_transforms import (
+    convert_to_rgb,
+    resize,
+    to_channel_dimension_format,
+)
+from transformers.image_utils import (
+    OPENAI_CLIP_MEAN,
+    OPENAI_CLIP_STD,
+    ChannelDimension,
+    ImageInput,
+    PILImageResampling,
+    get_image_size,
+    infer_channel_dimension_format,
+    is_scaled_image,
+    is_valid_image,
+    make_list_of_images,
+    to_numpy_array,
+)
+try:
+    from transformers.image_utils import VideoInput
+except:
+    from transformers.video_utils import VideoInput
+from transformers.utils import TensorType, is_vision_available, logging
+
+
+logger = logging.get_logger(__name__)
+
+
+if is_vision_available():
+    from PIL import Image
+
+
+def is_valid_video(video) -> bool:
+    if isinstance(video, (list, tuple)):
+        return all(is_valid_image(frame) for frame in video)
+    elif isinstance(video, np.ndarray):
+        return video.ndim == 4
+    elif isinstance(video, torch.Tensor):
+        return video.ndim == 4
+    return False
+
+
+def make_batched_images(images) -> List[List[ImageInput]]:
+    """
+    Accepts images in list or nested list format, and makes a list of images for preprocessing.
+
+    Args:
+        images (`Union[List[List[ImageInput]], List[ImageInput], ImageInput]`):
+            The input image.
+
+    Returns:
+        list: A list of images.
+    """
+    if isinstance(images, (list, tuple)):
+        # list of images/videos
+        if not all(is_valid_video(image) or is_valid_image(image) for image in images):
+            raise ValueError(f"Could not make batched images from {images}")
+        return images
+    elif is_valid_video(images) or is_valid_image(images):
+        # single image/video
+        return [images]
+
+    raise ValueError(f"Could not make batched images from {images}")
+
+
+def simple_batched_resize(
+    images, factor: int = 28, min_tokens: int = 4 * 4, max_tokens: int = 16384, input_data_format: str = None
+):
+    min_pixels = min_tokens * factor * factor
+    max_pixels = max_tokens * factor * factor
+
+    num_images = 0
+    for image in images:
+        if is_valid_video(image):
+            num_images += len(image)
+        else:
+            num_images += 1
+
+    image_sizes = []
+    for image in images:
+        if is_valid_video(image):
+            image = image[0]
+        if isinstance(image, Image.Image):
+            width, height = image.size
+        else:
+            height, width = get_image_size(image, channel_dim=input_data_format)
+        image_sizes.append([height, width])
+
+    tmp_image_sizes = []
+    for height, width in image_sizes:
+        h_bar = round(height / factor) * factor
+        w_bar = round(width / factor) * factor
+        if h_bar * w_bar > (max_pixels // num_images):
+            beta = math.sqrt((height * width) / (max_pixels // num_images))
+            h_bar = math.floor(height / beta / factor) * factor
+            w_bar = math.floor(width / beta / factor) * factor
+        # per image min_pixels
+        if h_bar * w_bar < min_pixels:
+            beta = math.sqrt(min_pixels / (height * width))
+            h_bar = math.ceil(height * beta / factor) * factor
+            w_bar = math.ceil(width * beta / factor) * factor
+        tmp_image_sizes.append((h_bar, w_bar))
+    image_sizes = tmp_image_sizes
+    return image_sizes
+
+
+def batched_resize(
+    images, factors: List[int], min_tokens: int = 4 * 4, max_tokens: int = 16384, input_data_format: str = None
+):
+    image_sizes = []
+    for image in images:
+        if is_valid_video(image):
+            num_frame = len(image)
+            image = image[0]
+        else:
+            num_frame = 1
+        if isinstance(image, Image.Image):
+            width, height = image.size
+        else:
+            height, width = get_image_size(image, channel_dim=input_data_format)
+        image_sizes.append([num_frame, height, width])
+
+    # global max_pixels
+    smart_scale_factors = 1.0
+    total_tokens = 0
+    for (num_frame, height, width), factor in zip(image_sizes, factors):
+        total_tokens += num_frame * math.ceil(height / factor) * math.ceil(width / factor)
+
+    # TODO: add min_pixels
+    if total_tokens > max_tokens:
+        beta = math.sqrt(total_tokens / max_tokens)
+        tmp_image_sizes = []
+        for (_, height, width), factor in zip(image_sizes, factors):
+            h_bar = math.floor(height / beta / factor) * factor
+            w_bar = math.floor(width / beta / factor) * factor
+            tmp_image_sizes.append((h_bar, w_bar))
+        image_sizes = tmp_image_sizes
+    else:
+        tmp_image_sizes = []
+        for (_, height, width), factor in zip(image_sizes, factors):
+            height = round(height / factor) * factor
+            width = round(width / factor) * factor
+            tmp_image_sizes.append((height, width))
+        image_sizes = tmp_image_sizes
+
+    return image_sizes
+
+
+class SFLImageProcessor(BaseImageProcessor):
+    r"""
+    Constructs a DAMOVL image processor that dynamically resizes images based on the original images.
+
+    Args:
+        do_resize (`bool`, *optional*, defaults to `True`):
+            Whether to resize the image's (height, width) dimensions.
+        resample (`PILImageResampling`, *optional*, defaults to `Resampling.BICUBIC`):
+            Resampling filter to use when resizing the image.
+        do_rescale (`bool`, *optional*, defaults to `True`):
+            Whether to rescale the image by the specified scale `rescale_factor`.
+        rescale_factor (`int` or `float`, *optional*, defaults to `1/255`):
+            Scale factor to use if rescaling the image.
+        do_normalize (`bool`, *optional*, defaults to `True`):
+            Whether to normalize the image.
+        image_mean (`float` or `List[float]`, *optional*, defaults to `[0.48145466, 0.4578275, 0.40821073]`):
+            Mean to use if normalizing the image. This is a float or list of floats for each channel in the image.
+        image_std (`float` or `List[float]`, *optional*, defaults to `[0.26862954, 0.26130258, 0.27577711]`):
+            Standard deviation to use if normalizing the image. This is a float or list of floats for each channel in the image.
+        do_convert_rgb (`bool`, *optional*, defaults to `True`):
+            Whether to convert the image to RGB.
+        min_pixels (`int`, *optional*, defaults to `56 * 56`):
+            The min pixels of the image to resize the image.
+        max_pixels (`int`, *optional*, defaults to `28 * 28 * 1280`):
+            The max pixels of the image to resize the image.
+        patch_size (`int`, *optional*, defaults to 14):
+            The spacial patch size of the vision encoder.
+    """
+
+    model_input_names = ["pixel_values", "grid_sizes", "merge_sizes"]
+
+    def __init__(
+        self,
+        do_resize: bool = True,
+        resample: PILImageResampling = PILImageResampling.BICUBIC,
+        do_rescale: bool = True,
+        rescale_factor: Union[int, float] = 1 / 255,
+        do_normalize: bool = True,
+        image_mean: Optional[Union[float, List[float]]] = None,
+        image_std: Optional[Union[float, List[float]]] = None,
+        do_convert_rgb: bool = True,
+        min_tokens: int = 4 * 4,
+        max_tokens: int = 16384,
+        patch_size: int = 14,
+        **kwargs,
+    ) -> None:
+        super().__init__(**kwargs)
+        self.do_resize = do_resize
+        self.resample = resample
+        self.do_rescale = do_rescale
+        self.rescale_factor = rescale_factor
+        self.do_normalize = do_normalize
+        self.image_mean = image_mean if image_mean is not None else OPENAI_CLIP_MEAN
+        self.image_std = image_std if image_std is not None else OPENAI_CLIP_STD
+        self.min_tokens = min_tokens
+        self.max_tokens = max_tokens
+        self.patch_size = patch_size
+        self.do_convert_rgb = do_convert_rgb
+
+    def _preprocess(
+        self,
+        images: Union[ImageInput, VideoInput],
+        target_size: List[int],
+        merge_size: int = 1,
+        do_resize: bool = None,
+        resample: PILImageResampling = None,
+        do_rescale: bool = None,
+        rescale_factor: float = None,
+        do_normalize: bool = None,
+        image_mean: Optional[Union[float, List[float]]] = None,
+        image_std: Optional[Union[float, List[float]]] = None,
+        do_convert_rgb: bool = None,
+        data_format: Optional[ChannelDimension] = ChannelDimension.FIRST,
+        input_data_format: Optional[Union[str, ChannelDimension]] = None,
+    ):
+        """
+        Preprocess an image or batch of images. Copy of the `preprocess` method from `CLIPImageProcessor`.
+
+        Args:
+            images (`ImageInput`):
+                Image or batch of images to preprocess. Expects pixel values ranging from 0 to 255. If pixel values range from 0 to 1, set `do_rescale=False`.
+            target_size (`List[int]`):
+                The target size to resize the image to. Should be a list of two integers: [target_height, target_width].
+            merge_size (`int`, *optional*, defaults to `1`):
+                The merge size after the vision encoder.
+            do_resize (`bool`, *optional*, defaults to `self.do_resize`):
+                Whether to resize the image.
+            resample (`PILImageResampling`, *optional*, defaults to `self.resample`):
+                Resampling filter to use if resizing the image. This can be one of the `PILImageResampling` enums.
+            do_rescale (`bool`, *optional*, defaults to `self.do_rescale`):
+                Whether to rescale the image.
+            rescale_factor (`float`, *optional*, defaults to `self.rescale_factor`):
+                Scale factor to use if rescaling the image.
+            do_normalize (`bool`, *optional*, defaults to `self.do_normalize`):
+                Whether to normalize the image.
+            image_mean (`float` or `List[float]`, *optional*, defaults to `self.image_mean`):
+                Mean to use if normalizing the image. Can be a float or a list of floats corresponding to the number of channels in the image.
+            image_std (`float` or `List[float]`, *optional*, defaults to `self.image_std`):
+                Standard deviation to use if normalizing the image. Can be a float or a list of floats corresponding to the number of channels in the image.
+            do_convert_rgb (`bool`, *optional*, defaults to `self.do_convert_rgb`):
+                Whether to convert the image to RGB.
+            data_format (`ChannelDimension`, *optional*, defaults to `ChannelDimension.FIRST`):
+                The channel dimension format for the output image. Can be one of:
+                - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format.
+                - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format.
+                - Unset: Use the channel dimension format of the input image.
+            input_data_format (`ChannelDimension` or `str`, *optional*):
+                The channel dimension format for the input image. Can be one of:
+                - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format.
+                - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format.
+                - `"none"` or `ChannelDimension.NONE`: image in (height, width) format.   - `"none"` or `ChannelDimension.NONE`: image in (height, width) format.
+        """
+        images = make_list_of_images(images)
+
+        if do_convert_rgb:
+            images = [convert_to_rgb(image) for image in images]
+
+        # All transformations expect numpy arrays.
+        images = [to_numpy_array(image) for image in images]
+
+        if is_scaled_image(images[0]) and do_rescale:
+            logger.warning_once(
+                "It looks like you are trying to rescale already rescaled images. If the input"
+                " images have pixel values between 0 and 1, set `do_rescale=False` to avoid rescaling them again."
+            )
+        if input_data_format is None:
+            # We assume that all images have the same channel dimension format.
+            input_data_format = infer_channel_dimension_format(images[0])
+
+        height, width = get_image_size(images[0], channel_dim=input_data_format)
+        resized_height, resized_width = height, width
+        processed_images = []
+        for image in images:
+            if do_resize:
+                resized_height, resized_width = target_size
+                image = resize(
+                    image, size=(resized_height, resized_width), resample=resample, input_data_format=input_data_format
+                )
+
+            if do_rescale:
+                image = self.rescale(image, scale=rescale_factor, input_data_format=input_data_format)
+
+            if do_normalize:
+                image = self.normalize(
+                    image=image, mean=image_mean, std=image_std, input_data_format=input_data_format
+                )
+
+            image = to_channel_dimension_format(image, data_format, input_channel_dim=input_data_format)
+            processed_images.append(image)
+
+        patches = np.array(processed_images)
+        if data_format == ChannelDimension.LAST:
+            patches = patches.transpose(0, 3, 1, 2)
+        t = patches.shape[0]
+        channel = patches.shape[1]
+        grid_h, grid_w = resized_height // self.patch_size, resized_width // self.patch_size
+        patches = patches.reshape(
+            t,
+            channel,
+            grid_h // merge_size,
+            merge_size,
+            self.patch_size,
+            grid_w // merge_size,
+            merge_size,
+            self.patch_size,
+        )
+        patches = patches.transpose(0, 2, 5, 3, 6, 1, 4, 7)
+        flatten_patches = patches.reshape(
+            t * grid_h * grid_w, channel * self.patch_size * self.patch_size
+        )
+
+        return flatten_patches, (t, grid_h, grid_w)
+
+    def preprocess(
+        self,
+        images: ImageInput,
+        do_resize: bool = None,
+        resample: PILImageResampling = None,
+        do_rescale: bool = None,
+        rescale_factor: float = None,
+        do_normalize: bool = None,
+        image_mean: Optional[Union[float, List[float]]] = None,
+        image_std: Optional[Union[float, List[float]]] = None,
+        do_convert_rgb: bool = None,
+        merge_size: Optional[Union[int, List[int]]] = None,
+        return_tensors: Optional[Union[str, TensorType]] = None,
+        data_format: Optional[ChannelDimension] = ChannelDimension.FIRST,
+        input_data_format: Optional[Union[str, ChannelDimension]] = None,
+    ):
+        """
+        Args:
+            images (`ImageInput`):
+                Image to preprocess. Expects a single or batch of images with pixel values ranging from 0 to 255. If
+                passing in images with pixel values between 0 and 1, set `do_rescale=False`.
+            do_resize (`bool`, *optional*, defaults to `self.do_resize`):
+                Whether to resize the image.
+            resample (`int`, *optional*, defaults to `self.resample`):
+                Resampling filter to use if resizing the image. This can be one of the enum `PILImageResampling`. Only
+                has an effect if `do_resize` is set to `True`.
+            do_rescale (`bool`, *optional*, defaults to `self.do_rescale`):
+                Whether to rescale the image.
+            rescale_factor (`float`, *optional*, defaults to `self.rescale_factor`):
+                Rescale factor to rescale the image by if `do_rescale` is set to `True`.
+            do_normalize (`bool`, *optional*, defaults to `self.do_normalize`):
+                Whether to normalize the image.
+            image_mean (`float` or `List[float]`, *optional*, defaults to `self.image_mean`):
+                Image mean to use for normalization. Only has an effect if `do_normalize` is set to `True`.
+            image_std (`float` or `List[float]`, *optional*, defaults to `self.image_std`):
+                Image standard deviation to use for normalization. Only has an effect if `do_normalize` is set to
+                `True`.
+            do_convert_rgb (`bool`, *optional*, defaults to `self.do_convert_rgb`):
+                Whether to convert the image to RGB.
+            return_tensors (`str` or `TensorType`, *optional*):
+                The type of tensors to return. Can be one of:
+                - Unset: Return a list of `np.ndarray`.
+                - `TensorType.TENSORFLOW` or `'tf'`: Return a batch of type `tf.Tensor`.
+                - `TensorType.PYTORCH` or `'pt'`: Return a batch of type `torch.Tensor`.
+                - `TensorType.NUMPY` or `'np'`: Return a batch of type `np.ndarray`.
+                - `TensorType.JAX` or `'jax'`: Return a batch of type `jax.numpy.ndarray`.
+            data_format (`ChannelDimension` or `str`, *optional*, defaults to `ChannelDimension.FIRST`):
+                The channel dimension format for the output image. Can be one of:
+                - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format.
+                - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format.
+                - Unset: Use the channel dimension format of the input image.
+            input_data_format (`ChannelDimension` or `str`, *optional*):
+                The channel dimension format for the input image. If unset, the channel dimension format is inferred
+                from the input image. Can be one of:
+                - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format.
+                - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format.
+                - `"none"` or `ChannelDimension.NONE`: image in (height, width) format.
+
+        """
+        do_resize = do_resize if do_resize is not None else self.do_resize
+        resample = resample if resample is not None else self.resample
+        do_rescale = do_rescale if do_rescale is not None else self.do_rescale
+        rescale_factor = rescale_factor if rescale_factor is not None else self.rescale_factor
+        do_normalize = do_normalize if do_normalize is not None else self.do_normalize
+        image_mean = image_mean if image_mean is not None else self.image_mean
+        image_std = image_std if image_std is not None else self.image_std
+        merge_size = merge_size if merge_size is not None else self.merge_size
+        do_convert_rgb = do_convert_rgb if do_convert_rgb is not None else self.do_convert_rgb
+
+        images = make_batched_images(images)
+
+        if isinstance(merge_size, (list, tuple)):
+            assert len(merge_size) == len(images), "Merge size must be the same length as images."
+            merge_sizes = merge_size
+        else:
+            merge_sizes = [merge_size for _ in images]
+
+        if all(merge_size == merge_sizes[0] for merge_size in merge_sizes):
+            target_sizes = simple_batched_resize(
+                images,
+                factor=self.patch_size * merge_sizes[0],
+                min_tokens=self.min_tokens,
+                max_tokens=self.max_tokens,
+                input_data_format=input_data_format,
+            )
+        else:
+            target_sizes = batched_resize(
+                images,
+                factors=[self.patch_size * merge_size for merge_size in merge_sizes],
+                min_tokens=self.min_tokens,
+                max_tokens=self.max_tokens,
+                input_data_format=input_data_format,
+            )
+
+        pixel_values, grid_sizes = [], []
+        for image, merge_size, target_size in zip(images, merge_sizes, target_sizes):
+            patches, grid_size = self._preprocess(
+                image,
+                target_size=target_size,
+                merge_size=merge_size,
+                do_resize=do_resize,
+                resample=resample,
+                do_rescale=do_rescale,
+                rescale_factor=rescale_factor,
+                do_normalize=do_normalize,
+                image_mean=image_mean,
+                image_std=image_std,
+                data_format=data_format,
+                do_convert_rgb=do_convert_rgb,
+                input_data_format=input_data_format,
+            )
+            pixel_values.append(patches)
+            grid_sizes.append(grid_size)
+
+        pixel_values = np.concatenate(pixel_values, axis=0)
+        grid_sizes = np.array(grid_sizes)
+        merge_sizes = np.array(merge_sizes)
+
+        data = {
+            "pixel_values": pixel_values,
+            "grid_sizes": grid_sizes,
+            "merge_sizes": merge_sizes,
+        }
+
+        return BatchFeature(data=data, tensor_type=return_tensors)

model-00001-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cb961d8369bc4582a7e4ee1ed830e157ade0d181696a7343208d82dd3e95c832
+size 4993416376

model-00002-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b665473c46a773ede1c33e86fbe0146dbb87a7034ac422a3420a8ee030e08a5
+size 1278741680

modeling_qwen2_audio_encoder.py

@@ -0,0 +1,106 @@
+from transformers import Qwen2AudioEncoder
+import torch
+from torch import nn
+from transformers.modeling_outputs import BaseModelOutput
+import torch.nn.functional as F
+
+class Qwen2AudioEncoderModel(Qwen2AudioEncoder):
+    def forward(
+        self,
+        input_features,
+        attention_mask=None,
+        head_mask=None,
+        output_attentions=None,
+        output_hidden_states=None,
+        return_dict=None,
+    ):
+        r"""
+        Args:
+            attention_mask (`torch.Tensor`)`, *optional*):
+                Qwen2Audio does not support masking of the `input_features`, this argument is preserved for compatibility,
+                but it is not used. By default the silence in the input log mel spectrogram are ignored.
+            head_mask (`torch.Tensor` of shape `(encoder_layers, encoder_attention_heads)`, *optional*):
+                Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`:
+
+                - 1 indicates the head is **not masked**,
+                - 0 indicates the head is **masked**.
+            output_attentions (`bool`, *optional*):
+                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
+                returned tensors for more detail.
+            output_hidden_states (`bool`, *optional*):
+                Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors
+                for more detail.
+            return_dict (`bool`, *optional*):
+                Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
+        """
+
+        expected_seq_length = self.config.max_source_positions * self.conv1.stride[0] * self.conv2.stride[0]
+
+        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+        output_hidden_states = (
+            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+        )
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+        # Ignore copy
+        input_features = input_features.to(dtype=self.conv1.weight.dtype, device=self.conv1.weight.device)
+
+        inputs_embeds = nn.functional.gelu(self.conv1(input_features))
+        inputs_embeds = nn.functional.gelu(self.conv2(inputs_embeds))
+
+        inputs_embeds = inputs_embeds.permute(0, 2, 1)
+        embed_pos = self.embed_positions.weight
+
+        hidden_states = inputs_embeds + embed_pos[: inputs_embeds.shape[1], :]
+        hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training)
+
+        encoder_states = () if output_hidden_states else None
+        all_attentions = () if output_attentions else None
+
+        # check if head_mask has a correct number of layers specified if desired
+        if head_mask is not None:
+            assert head_mask.size()[0] == (len(self.layers)), (
+                f"The head_mask should be specified for {len(self.layers)} layers, but it is for {head_mask.size()[0]}."
+            )
+
+        for idx, encoder_layer in enumerate(self.layers):
+            if output_hidden_states:
+                encoder_states = encoder_states + (hidden_states,)
+            # add LayerDrop (see https://huggingface.co/papers/1909.11556 for description)
+            to_drop = False
+            if self.training:
+                dropout_probability = torch.rand([])
+                if dropout_probability < self.layerdrop:  # skip the layer
+                    to_drop = True
+
+            # Ignore copy
+            if to_drop:
+                layer_outputs = (None, None)
+            else:
+                layer_outputs = encoder_layer(
+                    hidden_states,
+                    attention_mask,
+                    layer_head_mask=(head_mask[idx] if head_mask is not None else None),
+                    output_attentions=output_attentions,
+                )
+
+                hidden_states = layer_outputs[0]
+
+            if output_attentions:
+                all_attentions = all_attentions + (layer_outputs[1],)
+
+        # Ignore copy
+        # hidden_states = hidden_states.permute(0, 2, 1)
+        # hidden_states = self.avg_pooler(hidden_states)
+        # hidden_states = F.max_pool1d(hidden_states, kernel_size=2)
+        # hidden_states = hidden_states.permute(0, 2, 1)
+
+        hidden_states = self.layer_norm(hidden_states)
+        if output_hidden_states:
+            encoder_states = encoder_states + (hidden_states,)
+
+        if not return_dict:
+            return tuple(v for v in [hidden_states, encoder_states, all_attentions] if v is not None)
+        return BaseModelOutput(
+            last_hidden_state=hidden_states, hidden_states=encoder_states, attentions=all_attentions
+        )

modeling_sfl_encoder_qwen3.py

@@ -0,0 +1,720 @@
+from torch import nn
+import torch
+import math
+import warnings
+from functools import partial
+from .configuration_sfl_encoder import SFLVisionEncoderConfigFromQwen3
+from transformers.modeling_utils import PreTrainedModel
+from transformers.models.qwen3.modeling_qwen3 import Qwen3Model, Qwen3Attention, rotate_half, Qwen3DecoderLayer
+from typing import List, Optional, Tuple, Union
+from transformers.modeling_outputs import BaseModelOutputWithPast
+from transformers.processing_utils import Unpack
+from transformers.modeling_flash_attention_utils import FlashAttentionKwargs
+from transformers.cache_utils import Cache, DynamicCache
+from transformers.utils import logging, is_flash_attn_greater_or_equal_2_10, is_flash_attn_2_available
+from transformers.modeling_rope_utils import ROPE_INIT_FUNCTIONS
+from torch.nn.init import _calculate_fan_in_and_fan_out
+import torch.nn.functional as F
+if is_flash_attn_2_available():
+    from transformers.modeling_flash_attention_utils import _flash_attention_forward
+    from flash_attn import flash_attn_varlen_func
+
+logger = logging.get_logger(__name__)
+
+def _trunc_normal_(tensor, mean, std, a, b):
+    # Cut & paste from PyTorch official master until it's in a few official releases - RW
+    # Method based on https://people.sc.fsu.edu/~jburkardt/presentations/truncated_normal.pdf
+    def norm_cdf(x):
+        # Computes standard normal cumulative distribution function
+        return (1.0 + math.erf(x / math.sqrt(2.0))) / 2.0
+
+    if (mean < a - 2 * std) or (mean > b + 2 * std):
+        warnings.warn(
+            "mean is more than 2 std from [a, b] in nn.init.trunc_normal_. "
+            "The distribution of values may be incorrect.",
+            stacklevel=2,
+        )
+
+    # Values are generated by using a truncated uniform distribution and
+    # then using the inverse CDF for the normal distribution.
+    # Get upper and lower cdf values
+    l = norm_cdf((a - mean) / std)
+    u = norm_cdf((b - mean) / std)
+
+    # Uniformly fill tensor with values from [l, u], then translate to
+    # [2l-1, 2u-1].
+    tensor.uniform_(2 * l - 1, 2 * u - 1)
+
+    # Use inverse cdf transform for normal distribution to get truncated
+    # standard normal
+    tensor.erfinv_()
+
+    # Transform to proper mean, std
+    tensor.mul_(std * math.sqrt(2.0))
+    tensor.add_(mean)
+
+    # Clamp to ensure it's in the proper range
+    tensor.clamp_(min=a, max=b)
+
+
+def trunc_normal_tf_(
+    tensor: torch.Tensor, mean: float = 0.0, std: float = 1.0, a: float = -2.0, b: float = 2.0
+) -> torch.Tensor:
+    """Fills the input Tensor with values drawn from a truncated
+    normal distribution. The values are effectively drawn from the
+    normal distribution :math:`\\mathcal{N}(\text{mean}, \text{std}^2)`
+    with values outside :math:`[a, b]` redrawn until they are within
+    the bounds. The method used for generating the random values works
+    best when :math:`a \\leq \text{mean} \\leq b`.
+
+    NOTE: this 'tf' variant behaves closer to Tensorflow / JAX impl where the
+    bounds [a, b] are applied when sampling the normal distribution with mean=0, std=1.0
+    and the result is subsequently scaled and shifted by the mean and std args.
+
+    Args:
+        tensor: an n-dimensional `torch.Tensor`
+        mean: the mean of the normal distribution
+        std: the standard deviation of the normal distribution
+        a: the minimum cutoff value
+        b: the maximum cutoff value
+    """
+    with torch.no_grad():
+        _trunc_normal_(tensor, 0, 1.0, a, b)
+        tensor.mul_(std).add_(mean)
+
+
+def variance_scaling_(tensor, scale=1.0, mode="fan_in", distribution="normal"):
+    fan_in, fan_out = _calculate_fan_in_and_fan_out(tensor)
+    if mode == "fan_in":
+        denom = fan_in
+    elif mode == "fan_out":
+        denom = fan_out
+    elif mode == "fan_avg":
+        denom = (fan_in + fan_out) / 2
+
+    variance = scale / denom
+
+    if distribution == "truncated_normal":
+        # constant is stddev of standard normal truncated to (-2, 2)
+        trunc_normal_tf_(tensor, std=math.sqrt(variance) / 0.87962566103423978)
+    elif distribution == "normal":
+        with torch.no_grad():
+            tensor.normal_(std=math.sqrt(variance))
+    elif distribution == "uniform":
+        bound = math.sqrt(3 * variance)
+        with torch.no_grad():
+            tensor.uniform_(-bound, bound)
+    else:
+        raise ValueError(f"invalid distribution {distribution}")
+    
+
+def lecun_normal_(tensor):
+    variance_scaling_(tensor, mode="fan_in", distribution="truncated_normal")
+
+
+class SFLVisionEncoderEmbeddings(nn.Module):
+
+    def __init__(self, config: SFLVisionEncoderConfigFromQwen3):
+        super().__init__()
+        self.config = config
+        self.embed_dim = config.hidden_size
+        self.patch_size = config.patch_size
+
+        self.patch_embedding = nn.Conv2d(
+            in_channels=config.num_channels,
+            out_channels=self.embed_dim,
+            kernel_size=self.patch_size,
+            stride=self.patch_size,
+            padding="valid",
+        )
+
+    def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+        hidden_states = hidden_states.view(
+            -1, self.config.num_channels, self.patch_size, self.patch_size
+        )
+        patch_embeds = self.patch_embedding(hidden_states)  # shape = [*, width, grid, grid]
+        # embeddings = patch_embeds.flatten(2).transpose(1, 2)
+        embeddings = patch_embeds.view(-1, self.embed_dim)
+
+        return embeddings
+    
+
+class VisualRotaryEmbedding(nn.Module):
+    def __init__(
+        self,
+        dim=None,
+        max_position_embeddings=2048,
+        base=10000,
+        device=None,
+        scaling_factor=1.0,
+        rope_type="default",
+        config = None,
+    ):
+        super().__init__()
+        # TODO (joao): remove the `if` below, only used for BC
+        self.rope_kwargs = {}
+        if config is None:
+            logger.warning_once(
+                "`Qwen2VLRotaryEmbedding` can now be fully parameterized by passing the model config through the "
+                "`config` argument. All other arguments will be removed in v4.46"
+            )
+            self.rope_kwargs = {
+                "rope_type": rope_type,
+                "factor": scaling_factor,
+                "dim": dim,
+                "base": base,
+                "max_position_embeddings": max_position_embeddings,
+            }
+            self.rope_type = rope_type
+            self.max_seq_len_cached = max_position_embeddings
+            self.original_max_seq_len = max_position_embeddings
+        else:
+            # BC: "rope_type" was originally "type"
+            if config.rope_scaling is not None:
+                self.rope_type = config.rope_scaling.get("rope_type", config.rope_scaling.get("type"))
+            else:
+                self.rope_type = "default"
+            self.max_seq_len_cached = config.max_position_embeddings
+            self.original_max_seq_len = config.max_position_embeddings
+
+        self.config = config
+        self.rope_init_fn = ROPE_INIT_FUNCTIONS[self.rope_type]
+
+        inv_freq, self.attention_scaling = self.rope_init_fn(self.config, device, **self.rope_kwargs)
+        self.register_buffer("inv_freq", inv_freq, persistent=False)
+        self.original_inv_freq = self.inv_freq
+
+    def _dynamic_frequency_update(self, position_ids, device):
+        """
+        dynamic RoPE layers should recompute `inv_freq` in the following situations:
+        1 - growing beyond the cached sequence length (allow scaling)
+        2 - the current sequence length is in the original scale (avoid losing precision with small sequences)
+        """
+        seq_len = torch.max(position_ids) + 1
+        if seq_len > self.max_seq_len_cached:  # growth
+            inv_freq, self.attention_scaling = self.rope_init_fn(
+                self.config, device, seq_len=seq_len, **self.rope_kwargs
+            )
+            self.register_buffer("inv_freq", inv_freq, persistent=False)  # TODO joao: may break with compilation
+            self.max_seq_len_cached = seq_len
+
+        if seq_len < self.original_max_seq_len and self.max_seq_len_cached > self.original_max_seq_len:  # reset
+            self.register_buffer("inv_freq", self.original_inv_freq, persistent=False)
+            self.max_seq_len_cached = self.original_max_seq_len
+
+    @torch.no_grad()
+    def forward(self, x, position_ids):
+        if "dynamic" in self.rope_type:
+            self._dynamic_frequency_update(position_ids, device=x.device)
+
+        inv_freq_expanded = self.inv_freq[None, None, :, None].float().expand(2, position_ids.shape[1], -1, 1)
+        position_ids_expanded = position_ids[:, :, None, :].float()  # shape (2, bs, 1, positions)
+        # Force float32 (see https://github.com/huggingface/transformers/pull/29285)
+        device_type = x.device.type
+        device_type = device_type if isinstance(device_type, str) and device_type != "mps" else "cpu"
+        with torch.autocast(device_type=device_type, enabled=False):
+            freqs = (inv_freq_expanded.float() @ position_ids_expanded.float()).transpose(2, 3)
+            emb = torch.cat((freqs, freqs), dim=-1)
+            cos = emb.cos()
+            sin = emb.sin()
+
+        # Advanced RoPE types (e.g. yarn) apply a post-processing scaling factor, equivalent to scaling attention
+        cos = cos * self.attention_scaling
+        sin = sin * self.attention_scaling
+
+        return cos.to(dtype=x.dtype), sin.to(dtype=x.dtype)
+
+
+def apply_multimodal_rotary_pos_emb(q, k, cos, sin, unsqueeze_dim=1):
+    rope_section = [cos.shape[-1] // 2, cos.shape[-1] // 2]
+    cos = torch.cat([m[i % 2] for i, m in enumerate(cos.split(rope_section, dim=-1))], dim=-1).unsqueeze(unsqueeze_dim)
+    sin = torch.cat([m[i % 2] for i, m in enumerate(sin.split(rope_section, dim=-1))], dim=-1).unsqueeze(unsqueeze_dim)
+
+    q_embed = (q * cos) + (rotate_half(q) * sin)
+    k_embed = (k * cos) + (rotate_half(k) * sin)
+    return q_embed, k_embed
+
+
+class SFLQwen3Attention(Qwen3Attention):
+    """Multi-headed attention from 'Attention Is All You Need' paper"""
+
+    # Copied from transformers.models.llama.modeling_llama.LlamaFlashAttention2.__init__
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.is_causal = False
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        position_embeddings: Tuple[torch.Tensor, torch.Tensor],
+        attention_mask: Optional[torch.Tensor],
+        past_key_value: Optional[Cache] = None,
+        cache_position: Optional[torch.LongTensor] = None,
+        cu_seqlens: Optional[torch.Tensor] = None,
+        **kwargs: Unpack[FlashAttentionKwargs],
+    ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
+        input_shape = hidden_states.shape[:-1]
+        hidden_shape = (*input_shape, -1, self.head_dim)
+
+        query_states = self.q_norm(self.q_proj(hidden_states).view(hidden_shape)).transpose(1, 2)
+        key_states = self.k_norm(self.k_proj(hidden_states).view(hidden_shape)).transpose(1, 2)
+        value_states = self.v_proj(hidden_states).view(hidden_shape).transpose(1, 2)
+
+        cos, sin = position_embeddings
+        query_states, key_states = apply_multimodal_rotary_pos_emb(query_states, key_states, cos, sin)
+
+        if past_key_value is not None:
+            # sin and cos are specific to RoPE models; cache_position needed for the static cache
+            cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position}
+            key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs)
+
+        # This is before the transpose
+        seq_len = query_states.shape[2]
+
+        # In PEFT, usually we cast the layer norms in float32 for training stability reasons
+        # therefore the input hidden states gets silently casted in float32. Hence, we need
+        # cast them back in the correct dtype just to be sure everything works as expected.
+        # This might slowdown training & inference so it is recommended to not cast the LayerNorms
+        # in fp32. (usually our RMSNorm modules handle it correctly)
+        target_dtype = None
+        if query_states.dtype == torch.float32:
+            if torch.is_autocast_enabled():
+                target_dtype = torch.get_autocast_gpu_dtype()
+            # Handle the case where the model is quantized
+            elif hasattr(self.config, "_pre_quantization_dtype"):
+                target_dtype = self.config._pre_quantization_dtype
+            else:
+                target_dtype = next(layer for layer in self.modules() if isinstance(layer, torch.nn.Linear)).weight.dtype
+
+        # FA2 always relies on the value set in the module, so remove it if present in kwargs to avoid passing it twice
+        kwargs.pop("is_causal", None)
+
+        # Reashape to the expected shape for Flash Attention
+        query_states = query_states.transpose(1, 2).squeeze(0)
+        key_states = key_states.transpose(1, 2).squeeze(0)
+        value_states = value_states.transpose(1, 2).squeeze(0)
+
+        max_seqlen = (cu_seqlens[1:] - cu_seqlens[:-1]).max().item()
+        attn_output = flash_attn_varlen_func(
+            query_states,
+            key_states,
+            value_states,
+            cu_seqlens_q=cu_seqlens,
+            cu_seqlens_k=cu_seqlens,
+            max_seqlen_q=max_seqlen,
+            max_seqlen_k=max_seqlen,
+            dropout_p=0.0 if not self.training else self.attention_dropout,
+            causal=self.is_causal
+        )
+
+        # attn_output = _flash_attention_forward(
+        #     query_states,
+        #     key_states,
+        #     value_states,
+        #     attention_mask,
+        #     q_len,
+        #     position_ids=position_ids,
+        #     dropout=dropout_rate,
+        #     sliding_window=sliding_window,
+        #     is_causal=self.is_causal,
+        #     use_top_left_mask=self._flash_attn_uses_top_left_mask,
+        # )
+
+        attn_output = attn_output.reshape(*input_shape, -1).contiguous()
+        attn_output = self.o_proj(attn_output)
+        return attn_output, None
+    
+
+class SFLQwen3DecoderLayer(Qwen3DecoderLayer):
+    def __init__(self, config: SFLVisionEncoderConfigFromQwen3, layer_idx: int):
+        super(SFLQwen3DecoderLayer, self).__init__(config, layer_idx)
+        self.self_attn = SFLQwen3Attention(config, layer_idx)
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_value: Optional[Cache] = None,
+        output_attentions: Optional[bool] = False,
+        use_cache: Optional[bool] = False,
+        cache_position: Optional[torch.LongTensor] = None,
+        position_embeddings: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,  # necessary, but kept here for BC
+        cu_seqlens: Optional[torch.Tensor] = None,
+        **kwargs: Unpack[FlashAttentionKwargs],
+    ) -> Tuple[torch.FloatTensor, Optional[Tuple[torch.FloatTensor, torch.FloatTensor]]]:
+        residual = hidden_states
+
+        hidden_states = self.input_layernorm(hidden_states)
+
+        # Self Attention
+        hidden_states, self_attn_weights = self.self_attn(
+            hidden_states=hidden_states,
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            past_key_value=past_key_value,
+            output_attentions=output_attentions,
+            use_cache=use_cache,
+            cache_position=cache_position,
+            position_embeddings=position_embeddings,
+            cu_seqlens=cu_seqlens,
+            **kwargs,
+        )
+        hidden_states = residual + hidden_states
+
+        # Fully Connected
+        residual = hidden_states
+        hidden_states = self.post_attention_layernorm(hidden_states)
+        hidden_states = self.mlp(hidden_states)
+        hidden_states = residual + hidden_states
+
+        outputs = (hidden_states,)
+        if output_attentions:
+            outputs += (self_attn_weights,)
+
+        return outputs
+    
+    
+class SFLVisionEncoderFromQwen3Model(Qwen3Model):
+    config_class = SFLVisionEncoderConfigFromQwen3
+    def __init__(self, config: SFLVisionEncoderConfigFromQwen3):
+        super().__init__(config)
+        self.layers = nn.ModuleList(
+            [SFLQwen3DecoderLayer(config, layer_idx) for layer_idx in range(config.num_hidden_layers)]
+        )
+        self.rotary_emb = VisualRotaryEmbedding(config=config)
+        del self.embed_tokens
+
+    @staticmethod
+    def _prepare_4d_causal_attention_mask_with_cache_position(
+        attention_mask: torch.Tensor,
+        sequence_length: int,
+        target_length: int,
+        dtype: torch.dtype,
+        device: torch.device,
+        cache_position: torch.Tensor,
+        batch_size: int,
+        config: SFLVisionEncoderConfigFromQwen3,
+        past_key_values: Cache,
+    ):
+        """
+        Override the original causal mask method to create full attention mask instead.
+        Creates a full attention 4D mask of shape `(batch_size, 1, query_length, key_value_length)` 
+        from a 2D mask of shape `(batch_size, key_value_length)`.
+        
+        For vision encoding, we want full attention between all patches, not causal attention.
+        """
+        if attention_mask is not None and attention_mask.dim() == 4:
+            # In this case we assume that the mask comes already in inverted form and requires no inversion or slicing.
+            full_attention_mask = attention_mask
+        else:
+            # Create full attention mask (all zeros, meaning attend to all positions)
+            # We only mask based on the provided attention_mask for padding
+            if attention_mask is not None:
+                # Use the provided attention_mask to handle padding
+                min_dtype = torch.finfo(dtype).min
+                full_attention_mask = torch.zeros(
+                    (sequence_length, target_length), dtype=dtype, device=device
+                )
+                # Expand to 4D
+                full_attention_mask = full_attention_mask[None, None, :, :].expand(batch_size, 1, -1, -1)
+                
+                # Apply padding mask if provided
+                full_attention_mask = full_attention_mask.clone()  # copy to contiguous memory for in-place edit
+                if attention_mask.shape[-1] > target_length:
+                    attention_mask = attention_mask[:, :target_length]
+                mask_length = attention_mask.shape[-1]
+                padding_mask = attention_mask[:, None, None, :] == 0
+                full_attention_mask[:, :, :, :mask_length] = full_attention_mask[:, :, :, :mask_length].masked_fill(
+                    padding_mask, min_dtype
+                )
+            else:
+                # No attention mask provided, create all-zeros mask (full attention)
+                full_attention_mask = torch.zeros(
+                    (batch_size, 1, sequence_length, target_length), dtype=dtype, device=device
+                )
+        return full_attention_mask
+
+    def get_rope_index(self, grid_sizes, merge_sizes, position_ids):
+        position_ids = position_ids.contiguous()
+        """
+        Generate position indices for RoPE:
+        - Vision tokens (vision_mask=True): use 2D position encoding like (0,0), (0,1), (0,2), (1,0), (1,1), (1,2)
+        - Text tokens (vision_mask=False): use 1D position encoding like (3,3), (4,4), (5,5)
+        """
+        batch_size = grid_sizes.shape[0]
+
+        # Vision Part: Generate 2D position indices for vision tokens
+        vision_pos_ids = []
+        for (t, h, w), merge_size in zip(grid_sizes, merge_sizes):
+            # Generate height position indices
+            hpos_ids = torch.arange(h).unsqueeze(1).expand(-1, w).to(position_ids.device)
+            hpos_ids = hpos_ids.reshape(
+                h // merge_size,
+                merge_size,
+                w // merge_size,
+                merge_size,
+            )
+            hpos_ids = hpos_ids.permute(0, 2, 1, 3)
+            hpos_ids = hpos_ids.flatten()
+
+            # Generate width position indices  
+            wpos_ids = torch.arange(w).unsqueeze(0).expand(h, -1).to(position_ids.device)
+            wpos_ids = wpos_ids.reshape(
+                h // merge_size,
+                merge_size,
+                w // merge_size,
+                merge_size,
+            )
+            wpos_ids = wpos_ids.permute(0, 2, 1, 3)
+            wpos_ids = wpos_ids.flatten()
+            
+            # Stack height and width to create 2D positions
+            vision_pos_ids.append(torch.stack([hpos_ids, wpos_ids], dim=-1).repeat(t, 1))
+
+        num_start_idx = 0
+        for batch_idx in range(batch_size):
+            pos_len = vision_pos_ids[batch_idx].shape[0]
+            position_ids[:, 0, num_start_idx: num_start_idx+pos_len] = vision_pos_ids[batch_idx].permute(1, 0)
+            num_start_idx += pos_len
+        
+        return position_ids # shape: (2, batch_size, seq_len)
+    
+    # def get_rope_index(self, grid_sizes, merge_sizes, position_ids):
+    #     position_ids = position_ids.contiguous()
+    #     """
+    #     Generate polar (r, φ) position indices for RoPE:
+    #     - Vision tokens (vision_mask=True): use 2D polar coordinates
+    #     r = sqrt((h - c_h)^2 + (w - c_w)^2)
+    #     φ = atan2(h - c_h, w - c_w)
+    #     - Text tokens (vision_mask=False): keep 1D indices unchanged
+    #     """
+    #     batch_size = grid_sizes.shape[0]
+
+    #     vision_pos_ids = []
+    #     for (t, h, w), merge_size in zip(grid_sizes, merge_sizes):
+    #         device = position_ids.device
+
+    #         h_idx = torch.arange(h, device=device)
+    #         w_idx = torch.arange(w, device=device)
+    #         hh, ww = torch.meshgrid(h_idx, w_idx, indexing='ij')
+
+    #         hh = hh.reshape(h // merge_size, merge_size, w // merge_size, merge_size)
+    #         ww = ww.reshape(h // merge_size, merge_size, w // merge_size, merge_size)
+    #         hh = hh.permute(0, 2, 1, 3).flatten()
+    #         ww = ww.permute(0, 2, 1, 3).flatten()
+
+    #         center_h = (h - 1) / 2
+    #         center_w = (w - 1) / 2
+
+    #         rh = hh.float() - center_h
+    #         rw = ww.float() - center_w
+    #         r = torch.sqrt(rh ** 2 + rw ** 2)
+    #         phi = torch.atan2(rh, rw)  # [-pi, pi]
+
+    #         # r_norm = r / r.max()
+    #         # phi_norm = (phi + math.pi) / (2 * math.pi)
+
+    #         vision_pos_ids.append(torch.stack([r, phi], dim=-1).repeat(t, 1))
+
+    #     num_start_idx = 0
+    #     for batch_idx in range(batch_size):
+    #         pos_len = vision_pos_ids[batch_idx].shape[0]
+    #         position_ids[:, 0, num_start_idx:num_start_idx + pos_len] = vision_pos_ids[batch_idx].permute(1, 0)
+    #         num_start_idx += pos_len
+
+    #     return position_ids  # shape: (2, batch_size, seq_len)
+
+
+    def forward(
+        self,
+        input_ids: Optional[torch.LongTensor] = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_values: Optional[Cache] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        use_cache: Optional[bool] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        cache_position: Optional[torch.LongTensor] = None,
+        grid_sizes: Optional[torch.Tensor] = None,
+        merge_sizes: Optional[torch.Tensor] = None,
+        **flash_attn_kwargs: Unpack[FlashAttentionKwargs],
+    ) -> BaseModelOutputWithPast:
+        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+        output_hidden_states = (
+            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+        )
+        use_cache = use_cache if use_cache is not None else self.config.use_cache
+
+        if (input_ids is None) ^ (inputs_embeds is not None):
+            raise ValueError("You must specify exactly one of input_ids or inputs_embeds")
+
+        if self.gradient_checkpointing and self.training and use_cache:
+            logger.warning_once(
+                "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`."
+            )
+            use_cache = False
+
+        # TODO (joao): remove this exception in v4.56 -- it exists for users that try to pass a legacy cache
+        if not isinstance(past_key_values, (type(None), Cache)):
+            raise ValueError("The `past_key_values` should be either a `Cache` object or `None`.")
+
+        if inputs_embeds is None:
+            inputs_embeds = self.embed_tokens(input_ids)
+
+        if use_cache and past_key_values is None:
+            past_key_values = DynamicCache()
+
+        if cache_position is None:
+            past_seen_tokens = past_key_values.get_seq_length() if past_key_values is not None else 0
+            cache_position = torch.arange(
+                past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device
+            )
+
+        # the hard coded `2` is for temporal, height and width.
+        if position_ids is None:
+            position_ids = cache_position.view(1, 1, -1).expand(2, inputs_embeds.shape[0], -1)
+        elif position_ids.dim() == 2:
+            position_ids = position_ids[None, ...].expand(2, position_ids.shape[0], -1)
+        position_ids = self.get_rope_index(grid_sizes, merge_sizes, position_ids)
+
+        causal_mask = None
+
+        hidden_states = inputs_embeds
+
+        # create position embeddings to be shared across the decoder layers
+        position_embeddings = self.rotary_emb(hidden_states, position_ids)
+
+        # decoder layers
+        all_hidden_states = () if output_hidden_states else None
+        all_self_attns = () if output_attentions else None
+
+        # Calculate cumulative sequence lengths for the grid sizes
+        cu_seqlens = torch.repeat_interleave(grid_sizes[:, 1] * grid_sizes[:, 2], grid_sizes[:, 0]).cumsum(dim=0, dtype=torch.int32)
+        cu_seqlens = F.pad(cu_seqlens, (1, 0), value=0)
+
+        for decoder_layer in self.layers[: self.config.num_hidden_layers]:
+            if output_hidden_states:
+                all_hidden_states += (hidden_states,)
+
+            if self.gradient_checkpointing and self.training:
+                layer_outputs = self._gradient_checkpointing_func(
+                    partial(decoder_layer.__call__, **flash_attn_kwargs),
+                    hidden_states,
+                    causal_mask,
+                    position_ids,
+                    past_key_values,
+                    output_attentions,
+                    use_cache,
+                    cache_position,
+                    position_embeddings,
+                    cu_seqlens,
+                )
+            else:
+                layer_outputs = decoder_layer(
+                    hidden_states,
+                    attention_mask=causal_mask,
+                    position_ids=position_ids,
+                    past_key_value=past_key_values,
+                    output_attentions=output_attentions,
+                    use_cache=use_cache,
+                    cache_position=cache_position,
+                    position_embeddings=position_embeddings,
+                    cu_seqlens=cu_seqlens,
+                    **flash_attn_kwargs,
+                )
+
+            hidden_states = layer_outputs[0]
+
+            if output_attentions:
+                all_self_attns += (layer_outputs[1],)
+
+        hidden_states = self.norm(hidden_states)
+
+        # add hidden states from the last decoder layer
+        if output_hidden_states:
+            all_hidden_states += (hidden_states,)
+
+        return BaseModelOutputWithPast(
+            last_hidden_state=hidden_states,
+            past_key_values=past_key_values if use_cache else None,
+            hidden_states=all_hidden_states,
+            attentions=all_self_attns,
+        )
+
+
+class SFLVisionEncoderModelFromQwen3(PreTrainedModel):
+
+    config_class = SFLVisionEncoderConfigFromQwen3
+    base_model_prefix = "sfl_vision_encoder_qwen3"
+    main_input_name = "pixel_values"
+    supports_gradient_checkpointing = True
+    _no_split_modules = [
+        "SFLVisionEncoderEmbeddings",
+    ]
+    _supports_flash_attn_2 = True
+    _supports_sdpa = True
+
+    def __init__(self, config: SFLVisionEncoderConfigFromQwen3):
+        super().__init__(config=config)
+        self.embeddings = SFLVisionEncoderEmbeddings(config)
+        self.encoder = SFLVisionEncoderFromQwen3Model(config)
+
+        self.post_init()
+
+
+    def forward(self, pixel_values, grid_sizes, merge_sizes=None) -> torch.Tensor:
+        hidden_states = self.embeddings(pixel_values)
+        encoder_output = self.encoder(
+            inputs_embeds=hidden_states[None, ...],
+            grid_sizes=grid_sizes,
+            merge_sizes=merge_sizes,
+            output_hidden_states=True,
+        )
+        hidden_states = encoder_output.hidden_states
+        # hidden_states = torch.cat([
+        #     hidden_states[7],
+        #     hidden_states[14],
+        #     hidden_states[21],
+        #     hidden_states[28],
+        # ], dim=-1).squeeze(0)
+        hidden_states = hidden_states[-1].squeeze(0)
+
+        hidden_states_chunks = hidden_states.split(grid_sizes.prod(dim=1).tolist(), dim=0)
+        outputs = []
+
+        for hidden_states, grid_size, merge_size in zip(hidden_states_chunks, grid_sizes, merge_sizes):
+            # NOTE: previous implementation, which supports downsampling with any factor
+            c = hidden_states.shape[-1]
+            hidden_states = hidden_states.view(
+                grid_size[0], grid_size[1] // merge_size, grid_size[2] // merge_size, merge_size, merge_size,  c
+            ).permute(0, 1, 3, 2, 4, 5)
+            hidden_states = hidden_states.reshape(
+                grid_size[0], grid_size[1], grid_size[2], c
+            ).permute(0, 3, 1, 2)
+            hidden_states = torch.nn.functional.interpolate(
+                hidden_states,
+                size=(grid_size[1] // merge_size, grid_size[2] // merge_size),
+                mode='bilinear'
+            )
+            hidden_states = hidden_states.permute(0, 2, 3, 1).view(-1, c)
+
+            # NOTE: simplified implementation, which only supports downsampling with integer factor
+            # NOTE: this implementation is mathematically equivalent to the previous one when merge_size is 1 or 2 but may cause slightly different results
+            # hidden_states = hidden_states.view(-1, merge_size * merge_size, hidden_states.size(-1))
+            # hidden_states = hidden_states.mean(dim=1)
+
+            outputs.append(hidden_states)
+        return torch.cat(outputs, dim=0)
+    
+
+    def _init_weights(self, module):
+        """Initialize the weights"""
+        if isinstance(module, (nn.Linear, nn.Conv2d)):
+            lecun_normal_(module.weight)
+            if module.bias is not None:
+                nn.init.zeros_(module.bias)
+        elif isinstance(module, nn.LayerNorm):
+            module.bias.data.zero_()
+            module.weight.data.fill_(1.0)

modeling_videollama3_qwen3.py

@@ -0,0 +1,647 @@
+# Adopted from https://github.com/haotian-liu/LLaVA.
+# Below is the original copyright:
+# Copyright 2023 Haotian Liu
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""PyTorch VideoLLaMA3 model."""
+
+import importlib.util
+import os.path as osp
+import re
+from abc import ABC, abstractmethod
+from typing import List, Optional, Tuple, Union
+
+import torch
+import torch.nn as nn
+import torch.utils.checkpoint
+import math
+
+from transformers import AutoModel, Qwen3ForCausalLM, Qwen3Model
+from transformers.generation.utils import GenerateOutput
+from transformers.modeling_outputs import CausalLMOutputWithPast
+
+from .modeling_qwen2_audio_encoder import Qwen2AudioEncoderModel
+
+try:
+    from .configuration_videollama3 import Videollama3Qwen3Config
+except ModuleNotFoundError:
+    spec = importlib.util.spec_from_file_location(
+        "configuration_videollama3",
+        osp.join(osp.dirname(__file__), "configuration_videollama3.py"),
+    )
+    configuration_videollama3 = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(configuration_videollama3)
+    Videollama3Qwen3Config = getattr(
+        configuration_videollama3,
+        "Videollama3Qwen3Config",
+    )
+
+
+def build_mlp(depth, hidden_size, output_hidden_size):
+    modules = [nn.Linear(hidden_size, output_hidden_size)]
+    for _ in range(1, depth):
+        modules.append(nn.GELU())
+        modules.append(nn.Linear(output_hidden_size, output_hidden_size))
+    return nn.Sequential(*modules)
+
+
+def build_vision_projector(config, delay_load=False, **kwargs):
+    # videollama3 projector only support image-wise operation now, i.e., prohibit the temporal aggregation
+    projector_type = getattr(config, 'vision_projector_type', 'linear')
+    if projector_type == "linear":
+        # NOTE: for both linear and mlp2x_gelu projector type, mean pooling is adopted to aggreate video features
+        return nn.Linear(config.mm_hidden_size, config.hidden_size)
+    elif projector_type.startswith("mlp"):
+        return MlpGeluProjector(config.vision_encoder_config.hidden_size, config.hidden_size, projector_type)
+    else:
+        raise ValueError(f'Unknown projector type: {projector_type}')
+    
+def build_audio_projector(config, delay_load=False, **kwargs):
+    projector_type = getattr(config, 'audio_projector_type', 'linear')
+    if projector_type == "linear":
+        # NOTE: for both linear and mlp2x_gelu projector type, mean pooling is adopted to aggreate video features
+        return nn.Linear(config.mm_hidden_size, config.hidden_size)
+    elif projector_type.startswith("mlp"):
+        return MlpGeluProjector(config.audio_encoder_config.d_model, config.hidden_size, projector_type)
+    elif projector_type.startswith("dmlp"):
+        return MlpGeluDownsampleProjector(config.audio_encoder_config.d_model, config.hidden_size, projector_type)
+    else:
+        raise ValueError(f'Unknown projector type: {projector_type}')
+
+
+class MlpGeluProjector(nn.Module):
+
+    def __init__(self, mm_hidden_size, hidden_size, projector_type):
+        super().__init__()
+
+        mlp_gelu_match = re.match(r"^mlp(\d+)x_gelu$", projector_type)
+        mlp_depth = int(mlp_gelu_match.group(1))
+
+        self.readout = build_mlp(mlp_depth, mm_hidden_size, hidden_size)
+
+    def forward(self, x):
+        x = self.readout(x)
+        return x
+    
+
+class MlpGeluDownsampleProjector(nn.Module):
+    def __init__(self, mm_hidden_size, hidden_size, projector_type):
+        super().__init__()
+        self.downsample = nn.Linear(mm_hidden_size*8, mm_hidden_size)
+
+        mlp_gelu_match = re.match(r"^dmlp(\d+)x_gelu$", projector_type)
+        mlp_depth = int(mlp_gelu_match.group(1))
+
+        self.readout = build_mlp(mlp_depth, mm_hidden_size, hidden_size)
+
+    def forward(self, x):
+        B, S, D = x.shape
+
+        group = 8
+        S8 = (S // group) * group
+        x = x[:, :S8, :]
+        x = x.reshape(B, S8 // group, group * D)
+        x = self.downsample(x)
+        x = self.readout(x)
+        return x
+
+
+class Videollama3MetaModel:
+
+    def __init__(self, config):
+        super(Videollama3MetaModel, self).__init__(config)
+        if config.vision_encoder is not None:
+            self.vision_encoder = AutoModel.from_pretrained(
+                config.vision_encoder,
+                attn_implementation=self.config._attn_implementation,
+                torch_dtype=self.dtype,
+            )
+            self.config.vision_encoder_config = self.vision_encoder.config
+            self.config.vision_encoder = None
+        elif config.vision_encoder_config is not None:
+            self.vision_encoder = AutoModel.from_config(
+                self.config.vision_encoder_config,
+                attn_implementation=self.config._attn_implementation,
+                torch_dtype=self.dtype,
+            )
+        else:
+            raise ValueError("Vision encoder is not provided in config")
+        
+        if config.audio_encoder is not None:
+            self.audio_encoder = Qwen2AudioEncoderModel.from_pretrained(
+                config.audio_encoder,
+                attn_implementation=self.config._attn_implementation,
+                torch_dtype=self.dtype,
+            )
+            self.config.audio_encoder_config = self.audio_encoder.config
+            self.config.audio_encoder = None
+        elif config.audio_encoder_config is not None:
+            self.audio_encoder = Qwen2AudioEncoderModel.from_config(
+                self.config.audio_encoder_config,
+                attn_implementation=self.config._attn_implementation,
+                torch_dtype=self.dtype,
+            )
+        else:
+            raise ValueError("Audio encoder is not provided in config")
+
+        self.vision_projector = build_vision_projector(config)
+        self.audio_projector = build_audio_projector(config)
+
+    def get_vision_encoder(self):
+        return self.vision_encoder
+
+    def get_audio_encoder(self):
+        return self.audio_encoder
+
+    def get_vision_projector(self):
+        return self.vision_projector
+    
+    def get_audio_projector(self):
+        return self.audio_projector
+
+
+class Videollama3Qwen3Model(Videollama3MetaModel, Qwen3Model):
+
+    config_class = Videollama3Qwen3Config
+
+    def __init__(self, config: Videollama3Qwen3Config):
+        super(Videollama3Qwen3Model, self).__init__(config)
+
+
+class Videollama3MetaForCausalLM(ABC):
+
+    @abstractmethod
+    def get_model(self):
+        pass
+
+    def get_vision_encoder(self):
+        return self.get_model().get_vision_encoder()
+
+    def get_audio_encoder(self):
+        return self.get_model().get_audio_encoder()
+
+    def get_vision_projector(self):
+        return self.get_model().get_vision_projector()
+    
+    def get_audio_projector(self):
+        return self.get_model().get_audio_projector()
+
+    def encode_images(
+        self,
+        pixel_values: torch.FloatTensor,
+        grid_sizes: torch.LongTensor,
+        merge_sizes: torch.LongTensor,
+    ) -> torch.FloatTensor:
+        mm_features = self.get_model().get_vision_encoder()(
+            pixel_values=pixel_values,
+            grid_sizes=grid_sizes,
+            merge_sizes=merge_sizes,
+        )
+        mm_features = self.get_model().vision_projector(mm_features)
+        return mm_features
+    
+    def encode_audios(
+        self,
+        input_features: torch.FloatTensor,
+        audio_attention_mask: Optional[torch.Tensor] = None,
+    ) -> torch.FloatTensor:
+        mm_features = self.get_model().get_audio_encoder()(input_features).last_hidden_state
+        mm_features_projector = self.get_model().audio_projector(mm_features)
+        features = []
+        for f, m in zip(mm_features_projector, audio_attention_mask):
+            valid_length = math.ceil(m.sum() / 200) * 12
+            features.append(f[:valid_length])
+        mm_features_projector = torch.cat(features, dim=0)
+        return mm_features_projector
+
+    def _get_valid_visual_tokens(
+        self,
+        mm_features: torch.FloatTensor,
+        batched_num_patches: torch.LongTensor,
+        modals: List[str],
+    ):
+        valid_masks = []
+        for num_patches, modal in zip(batched_num_patches, modals):
+            valid_mask = torch.full((num_patches, ), modal != "text", dtype=torch.bool, device=mm_features.device)
+            valid_masks.append(valid_mask)
+        mm_features = mm_features[torch.cat(valid_masks)]
+        return mm_features
+
+    def _maybe_truncate_visual_tokens(
+        self,
+        mm_features: torch.FloatTensor,
+        compression_mask: torch.BoolTensor,
+        batched_num_patches: torch.LongTensor,
+        modals: List[str],
+        input_ids: torch.LongTensor,
+        position_ids: Optional[torch.LongTensor] = None,
+    ):
+        if position_ids is None or mm_features.shape[0] == input_ids.eq(self.config.image_token_index).sum():
+            return mm_features, compression_mask
+
+        truncation_mask = []
+        for num_patches, modal in zip(batched_num_patches, modals):
+            if modal == "text":
+                truncation_mask.append(torch.ones((0,), dtype=torch.bool, device=input_ids.device))
+            else:
+                truncation_mask.append(torch.ones((num_patches,), dtype=torch.bool, device=input_ids.device))
+
+        seq_end_indices = torch.nonzero(position_ids == 0)[:, 0]
+        seq_end_indices = seq_end_indices[seq_end_indices > 0].tolist()+ [len(input_ids)]
+        seq_start_indices = [0] + seq_end_indices[:-1]
+        num_visual_tokens = [
+            input_ids[start:end].eq(self.config.image_token_index).sum()
+            for start, end in zip(seq_start_indices, seq_end_indices)
+        ]
+
+        for n, mask in zip(num_visual_tokens, truncation_mask):
+            if len(mask) > 0:
+                mask[n:] = False
+        truncation_mask = torch.cat(truncation_mask)
+
+        return mm_features[truncation_mask], compression_mask[truncation_mask]
+
+    def _get_compression_mask(
+        self,
+        pixel_values: torch.FloatTensor,
+        batched_num_patches: torch.LongTensor,
+        grid_sizes: torch.LongTensor,
+        merge_sizes: torch.LongTensor,
+        modals: List[str],
+        threshold: float = 0.1,
+        min_tokens: int = 1,
+    ) -> torch.BoolTensor:
+        batched_images = pixel_values.split(grid_sizes.prod(dim=1).tolist(), dim=0)
+        compression_masks = []
+
+        for images, num_patches, grid_size, merge_size, modal in zip(
+            batched_images, batched_num_patches, grid_sizes, merge_sizes, modals
+        ):
+            t, h, w = grid_size
+            if modal == "image" or (modal == "video" and t == 1):
+                compression_masks.append(torch.ones((num_patches,), dtype=torch.bool, device=images.device))
+
+            elif modal == "video":
+                # NOTE: video token compressor
+                images = images.view(t, (h // merge_size) * (w // merge_size), -1)
+
+                pixel_diff = images[1:] - images[:-1]
+                pixel_diff = torch.abs(pixel_diff).mean(dim=-1) * 255
+                pixel_diff = torch.cat([torch.full_like(pixel_diff[0:1], threshold + 1), pixel_diff], dim=0)
+                mask = pixel_diff > threshold
+                padding_ids = torch.nonzero(mask.sum(dim=1) < min_tokens)[:, 0]
+                # mask[padding_ids, torch.randperm(min_tokens)] = 1
+                mask[padding_ids, :min_tokens] = 1
+                compression_masks.append(mask.flatten())
+
+            else:
+                # in case of psuedo image
+                compression_masks.append(torch.ones((0,), dtype=torch.bool, device=images.device))
+
+        return torch.cat(compression_masks)
+
+    def _compress_visual_tokens(
+        self,
+        compression_mask: torch.BoolTensor,
+        mm_features: torch.FloatTensor,
+        input_ids: torch.LongTensor,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        labels: Optional[torch.LongTensor] = None,
+    ):
+        mm_features = mm_features[compression_mask]
+        image_selected = (input_ids == self.config.image_token_index)
+
+        text_masks = torch.logical_not(image_selected)
+        text_masks[image_selected] = compression_mask
+        input_ids = input_ids[text_masks]
+
+        if attention_mask is not None:
+            attention_mask = attention_mask[text_masks]
+        if labels is not None:
+            labels = labels[text_masks]
+        if position_ids is not None:
+            # FIXME: assume the first position_id is always 0
+            position_ids = position_ids[text_masks]
+            pos_start = [0] + torch.nonzero(position_ids == 0)[:, 0].tolist()
+            pos_end = pos_start[1:] + [len(input_ids)]
+            position_ids = torch.cat([torch.arange(end - start, device=input_ids.device) for start, end in zip(pos_start, pos_end)])
+
+        return mm_features, input_ids, attention_mask, position_ids, labels
+
+    def prepare_inputs_labels_for_multimodal(
+        self,
+        input_ids: torch.LongTensor = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_values: Optional[List[torch.FloatTensor]] = None,
+        labels: Optional[torch.LongTensor] = None,
+        pixel_values: Optional[torch.FloatTensor] = None,
+        grid_sizes: Optional[torch.LongTensor] = None,
+        merge_sizes: Optional[torch.LongTensor] = None,
+        audio_input_features: Optional[torch.FloatTensor] = None,
+        audio_attention_mask: Optional[torch.Tensor] = None,
+        modals: Optional[List[str]] = None,
+    ):
+        vision_encoder = self.get_vision_encoder()
+        # NOTE: text-only situation
+        if vision_encoder is None or (pixel_values is None and audio_input_features is None) or input_ids.shape[1] == 1:
+            return input_ids, attention_mask, position_ids, past_key_values, None, labels
+
+        # 1. flatten text inputs
+        B, N = input_ids.shape
+        input_ids = input_ids.view(B * N)
+        if attention_mask is not None:
+            attention_mask = attention_mask.view(B * N)
+        if position_ids is not None:
+            position_ids = position_ids.view(B * N)
+        if labels is not None:
+            labels = labels.view(B * N)
+
+        # 2. embed visual tokens
+        image_selected, audio_selected, mm_features_teacher = None, None, None
+        if pixel_values is not None:
+            batched_num_patches = grid_sizes.prod(dim=1).div(merge_sizes ** 2).long()
+            mm_features = self.encode_images(pixel_values, grid_sizes, merge_sizes)
+            mm_features = self._get_valid_visual_tokens(mm_features, batched_num_patches, modals)
+
+            compression_mask = self._get_compression_mask(
+                pixel_values, batched_num_patches, grid_sizes, merge_sizes, modals
+            )
+            mm_features, compression_mask = self._maybe_truncate_visual_tokens(
+                mm_features, compression_mask, batched_num_patches, modals, input_ids, position_ids
+            )
+
+            # 2.1 compress visual tokens
+            if self.config.use_token_compression:
+                assert B == 1, "Token compression is only supported for batch_size=1"
+                mm_features, input_ids, attention_mask, labels, position_ids = self._compress_visual_tokens(
+                    compression_mask, mm_features, input_ids, attention_mask, labels, position_ids
+                )
+            # 2.2 replace multimodal tokens with features
+            image_selected = (input_ids == self.config.image_token_index)
+            input_ids[image_selected] = 0
+
+            num_vision_tokens = image_selected.sum()
+            if mm_features.size(0) > num_vision_tokens:
+                print(f"Number of vision_features ({mm_features.size(0)}) exceeds the number of image tokens ({num_vision_tokens}). Automative truncated.")
+                mm_features = mm_features[:num_vision_tokens]
+        # 3. embed audio tokens
+        if audio_input_features is not None:
+            audio_features = self.encode_audios(audio_input_features, audio_attention_mask)
+            # audio_features = audio_features.to(input_ids.device).flatten(0, 1)
+            audio_selected = (input_ids == self.config.audio_token_index)
+            input_ids[audio_selected] = 0
+
+            num_audio_tokens = audio_selected.sum()
+            if audio_features.size(0) > num_audio_tokens:
+                print(f"Number of audio_features ({audio_features.size(0)}) exceeds the number of audio tokens ({num_audio_tokens}). Automative truncated.")
+                audio_features = audio_features[:num_audio_tokens]
+
+        # 4. embed text tokens
+        inputs_embeds = self.get_model().embed_tokens(input_ids).clone()
+        if image_selected is not None:
+            inputs_embeds[image_selected] = inputs_embeds[image_selected] * 0.0 + mm_features
+        if audio_selected is not None:
+            inputs_embeds[audio_selected] = inputs_embeds[audio_selected] * 0.0 + audio_features
+
+        # 5. reshape back to batched format
+        C = inputs_embeds.shape[-1]
+        inputs_embeds = inputs_embeds.reshape(B, -1, C)
+        if attention_mask is not None:
+            attention_mask = attention_mask.view(B, -1)
+        if labels is not None:
+            labels = labels.view(B, -1)
+        if position_ids is not None:
+            position_ids = position_ids.view(B, -1)
+
+        return None, attention_mask, position_ids, past_key_values, inputs_embeds, labels
+
+
+class Videollama3Qwen3ForCausalLM(Qwen3ForCausalLM, Videollama3MetaForCausalLM):
+
+    config_class = Videollama3Qwen3Config
+
+    def __init__(self, config, **kwargs):
+        super(Qwen3ForCausalLM, self).__init__(config)
+        self.model = Videollama3Qwen3Model(config)
+        self.vocab_size = config.vocab_size
+        self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    def get_model(self):
+        return self.model
+    
+    @classmethod
+    def _load_pretrained_model(
+        cls,
+        model,
+        state_dict,
+        checkpoint_files,
+        pretrained_model_name_or_path,
+        ignore_mismatched_sizes=False,
+        sharded_metadata=None,
+        device_map=None,
+        disk_offload_folder=None,
+        offload_state_dict=None,
+        dtype=None,
+        hf_quantizer=None,
+        keep_in_fp32_regex=None,
+        device_mesh=None,
+        key_mapping=None,
+        weights_only=True,
+    ):
+        """
+        Override to handle nested vision_encoder and audio_encoder keys before calling parent's load method.
+        Remaps keys from 'model.vision_encoder.vision_encoder.*' to 'model.vision_encoder.*'
+        and 'model.audio_encoder.audio_encoder.*' to 'model.audio_encoder.*'
+        """
+        # If state_dict is provided and needs remapping, do it here
+        if state_dict is not None:
+            needs_remapping = any(k.startswith('model.vision_encoder.vision_encoder.') or k.startswith("model.audio_encoder.audio_encoder.") for k in state_dict.keys())
+            if needs_remapping:
+                print("Detected nested encoder keys, remapping 'model.vision_encoder.vision_encoder.*' -> 'model.vision_encoder.*' and 'model.audio_encoder.audio_encoder.*' -> 'model.audio_encoder.*'")
+                new_state_dict = {}
+                for k, v in state_dict.items():
+                    if k.startswith('model.vision_encoder.vision_encoder.'):
+                        # Remap: model.vision_encoder.vision_encoder.xxx -> model.vision_encoder.xxx
+                        new_key = k.replace('model.vision_encoder.vision_encoder.', 'model.vision_encoder.')
+                        new_state_dict[new_key] = v
+                    elif k.startswith('model.audio_encoder.audio_encoder.'):
+                        # Remap: model.audio_encoder.audio_encoder.xxx -> model.audio_encoder.xxx
+                        new_key = k.replace('model.audio_encoder.audio_encoder.', 'model.audio_encoder.')
+                        new_state_dict[new_key] = v
+                    else:
+                        new_state_dict[k] = v
+                state_dict = new_state_dict
+        
+        # For checkpoint files, we need to add key_mapping to remap the keys during loading
+        if checkpoint_files is not None and key_mapping is None:
+            # Check if we need remapping by loading the first checkpoint
+            from transformers.modeling_utils import load_state_dict
+            checkpoint = {}
+            checkpoint_files_list = checkpoint_files if isinstance(checkpoint_files, list) else [checkpoint_files]
+            for ckpt_file in checkpoint_files_list:
+                ckpt = load_state_dict(ckpt_file, map_location="cpu", weights_only=weights_only)
+                checkpoint.update(ckpt)
+            needs_remapping = any(k.startswith('model.vision_encoder.vision_encoder.') or k.startswith("model.audio_encoder.audio_encoder.") for k in checkpoint.keys())
+
+            if needs_remapping:
+                print("Detected nested encoder keys in checkpoint, adding key mapping for vision_encoder and audio_encoder")
+                key_mapping = {}
+                for k in checkpoint.keys():
+                    if k.startswith('model.vision_encoder.vision_encoder.'):
+                        new_key = k.replace('model.vision_encoder.vision_encoder.', 'model.vision_encoder.')
+                        key_mapping[k] = new_key
+                    elif k.startswith('model.audio_encoder.audio_encoder.'):
+                        new_key = k.replace('model.audio_encoder.audio_encoder.', 'model.audio_encoder.')
+                        key_mapping[k] = new_key
+            del checkpoint
+        
+        return super()._load_pretrained_model(
+            model=model,
+            state_dict=state_dict,
+            checkpoint_files=checkpoint_files,
+            pretrained_model_name_or_path=pretrained_model_name_or_path,
+            ignore_mismatched_sizes=ignore_mismatched_sizes,
+            sharded_metadata=sharded_metadata,
+            device_map=device_map,
+            disk_offload_folder=disk_offload_folder,
+            offload_state_dict=offload_state_dict,
+            dtype=dtype,
+            hf_quantizer=hf_quantizer,
+            keep_in_fp32_regex=keep_in_fp32_regex,
+            device_mesh=device_mesh,
+            key_mapping=key_mapping,
+            weights_only=weights_only,
+        )
+
+    # NOTE: arguments are copied from transformers==4.46.3
+    def forward(
+        self,
+        input_ids: torch.LongTensor = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_values: Optional[List[torch.FloatTensor]] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        labels: Optional[torch.LongTensor] = None,
+        use_cache: Optional[bool] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        return_dict: Optional[bool] = None,
+        cache_position: Optional[torch.LongTensor] = None,
+        num_logits_to_keep: int = 0,
+        # multimodal inputs
+        pixel_values: Optional[torch.FloatTensor] = None,
+        grid_sizes: Optional[torch.LongTensor] = None,
+        merge_sizes: Optional[torch.LongTensor] = None,
+        modals: Optional[List[str]] = None,
+        **loss_kwargs,
+    ) -> Union[Tuple, CausalLMOutputWithPast]:
+        if inputs_embeds is None:
+            (
+                input_ids,
+                attention_mask,
+                position_ids,
+                past_key_values,
+                inputs_embeds,
+                labels,
+            ) = self.prepare_inputs_labels_for_multimodal(
+                input_ids=input_ids,
+                attention_mask=attention_mask,
+                position_ids=position_ids,
+                past_key_values=past_key_values,
+                labels=labels,
+                pixel_values=pixel_values,
+                grid_sizes=grid_sizes,
+                merge_sizes=merge_sizes,
+                modals=modals,
+            )
+
+        return super().forward(
+            input_ids=input_ids,
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            past_key_values=past_key_values,
+            inputs_embeds=inputs_embeds,
+            labels=labels,
+            use_cache=use_cache,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+            cache_position=cache_position,
+            num_logits_to_keep=num_logits_to_keep,
+            **loss_kwargs,
+        )
+
+    @torch.no_grad()
+    def generate(
+        self,
+        # multimodal inputs
+        pixel_values: Optional[torch.FloatTensor] = None,
+        grid_sizes: Optional[torch.LongTensor] = None,
+        merge_sizes: Optional[torch.LongTensor] = None,
+        audio_input_features: Optional[torch.FloatTensor] = None,
+        audio_attention_mask: Optional[torch.Tensor] = None,
+        modals: Optional[List[str]] = None,
+        **kwargs,
+    ) -> Union[GenerateOutput, torch.LongTensor]:
+        input_ids = kwargs.pop("input_ids", None)
+        attention_mask = kwargs.pop("attention_mask", None)
+        position_ids = kwargs.pop("position_ids", None)
+        past_key_values = kwargs.pop("past_key_values", None)
+
+        if "inputs_embeds" in kwargs:
+            raise NotImplementedError("`inputs_embeds` is not supported")
+
+        if pixel_values is not None or audio_input_features is not None:
+            (
+                input_ids,
+                attention_mask,
+                position_ids,
+                past_key_values,
+                inputs_embeds,
+                labels,
+            ) = self.prepare_inputs_labels_for_multimodal(
+                input_ids=input_ids,
+                attention_mask=attention_mask,
+                position_ids=position_ids,
+                past_key_values=past_key_values,
+                labels=None,
+                pixel_values=pixel_values,
+                grid_sizes=grid_sizes,
+                merge_sizes=merge_sizes,
+                audio_input_features=audio_input_features,
+                audio_attention_mask=audio_attention_mask,
+                modals=modals,
+            )
+        else:
+            inputs_embeds = self.get_model().embed_tokens(input_ids)
+
+        return super().generate(
+            position_ids=position_ids,
+            attention_mask=attention_mask,
+            inputs_embeds=inputs_embeds,
+            **kwargs
+        )
+
+    def prepare_inputs_for_generation(self, input_ids, past_key_values=None, inputs_embeds=None, **kwargs):
+        images = kwargs.pop("images", None)
+        _inputs = super().prepare_inputs_for_generation(
+            input_ids, past_key_values=past_key_values, inputs_embeds=inputs_embeds, **kwargs
+        )
+        if images is not None:
+            _inputs['images'] = images
+        return _inputs

preprocessor_config.json

@@ -0,0 +1,39 @@
+{
+  "auto_map": {
+    "AutoImageProcessor": "image_processing_sfl.SFLImageProcessor",
+    "AutoProcessor": "processing_videollama3_qwen3.Videollama3Qwen3Processor"
+  },
+  "chunk_length": 30,
+  "dither": 0.0,
+  "feature_extractor_type": "Qwen2AudioEncoderProcessor",
+  "feature_size": 128,
+  "hop_length": 160,
+  "n_fft": 400,
+  "n_samples": 480000,
+  "nb_max_frames": 3000,
+  "padding_side": "right",
+  "padding_value": 0.0,
+  "processor_class": "Videollama3Qwen3Processor",
+  "return_attention_mask": true,
+  "sampling_rate": 16000,
+  "do_convert_rgb": true,
+  "do_normalize": true,
+  "do_rescale": true,
+  "do_resize": true,
+  "image_mean": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "image_processor_type": "SFLImageProcessor",
+  "image_std": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "max_tokens": 10240,
+  "min_tokens": 16,
+  "patch_size": 14,
+  "resample": 3,
+  "rescale_factor": 0.00392156862745098
+}

processing_videollama3_qwen3.py

@@ -0,0 +1,1004 @@
+"""Processor class for VideoLLaMA3."""
+
+import copy
+import importlib.util
+import os
+import os.path as osp
+import warnings
+from collections import defaultdict
+from typing import Any, List, Union, Dict, Optional, Tuple, TypedDict
+
+import cv2
+import ffmpeg
+import imageio
+import json
+import math
+import numpy as np
+import torch
+import transformers
+from decord import VideoReader, cpu
+from PIL import Image
+from transformers.feature_extraction_utils import BatchFeature
+from transformers.image_utils import ImageInput
+from transformers.processing_utils import ProcessingKwargs, ProcessorMixin, Unpack
+from transformers.tokenization_utils_base import PreTokenizedInput, TextInput
+
+try:
+    from . import image_processing_sfl
+    from .image_processing_sfl import (
+        is_valid_image, is_valid_video,
+    )
+    # from . import audio_processing_qwen2_audio
+    from transformers import WhisperFeatureExtractor
+except ModuleNotFoundError:
+    spec = importlib.util.spec_from_file_location(
+        "image_processing_sfl",
+        osp.join(osp.dirname(__file__), "image_processing_sfl.py"),
+    )
+    image_processing_sfl = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(image_processing_sfl)
+    is_valid_image = getattr(image_processing_sfl, "is_valid_image")
+    is_valid_video = getattr(image_processing_sfl, "is_valid_video")
+
+# constants
+DEFAULT_IMAGE_TOKEN = "<image>"
+DEFAULT_AUDIO_TOKEN = "<|audio|>"
+IGNORE_INDEX = -100
+
+# Type aliases
+Conversation = List[Dict[str, Any]]
+SingleImage = Union[Image.Image, np.ndarray, torch.Tensor]
+SingleVideo = Union[List[SingleImage], np.ndarray, torch.Tensor]
+BatchedImage = List[Union[SingleImage, SingleVideo]]
+BatchedNamedImage = List[Tuple[str, Union[SingleImage, SingleVideo]]]
+
+
+def _custom_import(class_name: str):
+    try:
+        attribute_class = getattr(transformers, class_name)
+    except AttributeError:
+        if "image" in class_name.lower():
+            attribute_class = getattr(image_processing_sfl, class_name)
+    return attribute_class
+
+
+def is_named_image(image) -> bool:
+    return isinstance(image, (list, tuple)) and \
+        len(image) == 2 and \
+        isinstance(image[0], str) and \
+        image[0] in ["image", "video"] and \
+        (is_valid_image(image[1]) or is_valid_video(image[1]))
+
+
+def make_batched_images(images) -> List[List[ImageInput]]:
+    if isinstance(images, (list, tuple)) and all(is_named_image(image) for image in images):
+        # list of named images
+        return [image[0] for image in images], [image[1] for image in images]
+    elif isinstance(images, (list, tuple)) and all(is_valid_image(image) or is_valid_video(image) for image in images):
+        # list of images/videos
+        batch = []
+        for image in images:
+            if is_valid_video(image):
+                batch.append(("video", image))
+            elif is_valid_image(image):
+                batch.append(("image", image))
+            else:
+                raise ValueError(f"Could not make batched images from {images}")
+        return [x[0] for x in batch], [x[1] for x in batch]
+    elif is_named_image(images):
+        # named images
+        return [images[0]], [image[1]]
+    elif is_valid_video(images):
+        # single video
+        return ["video"], [images]
+    elif is_valid_image(images):
+        # single image
+        return ["image"], [images]
+
+    raise ValueError(f"Could not make batched images from {images}")
+
+
+def frame_sample(duration, mode='uniform', num_frames=None, vid_fps=None, fps=None):
+    if mode == 'uniform':
+        assert num_frames is not None, "Number of frames must be provided for uniform sampling."
+        if duration <= num_frames:
+            return np.arange(duration).astype(int)
+        # NOTE: v1 version
+        # Calculate the size of each segment from which a frame will be extracted
+        # if duration <= num_frames:
+        #     return np.arange(duration).astype(int)
+        # seg_size = float(duration - 1) / num_frames
+
+        # frame_ids = []
+        # for i in range(num_frames):
+        #     # Calculate the start and end indices of each segment
+        #     start = seg_size * i
+        #     end   = seg_size * (i + 1)
+        #     # Append the middle index of the segment to the list
+        #     frame_ids.append((start + end) / 2)
+
+        # return np.round(np.array(frame_ids) + 1e-6).astype(int)
+        # NOTE: v0 version
+        return np.linspace(0, duration-1, num_frames, dtype=int)
+    elif mode == 'fps':
+        assert vid_fps is not None, "FPS must be provided for FPS sampling."
+        assert fps is not None, "FPS must be provided for FPS sampling."
+        segment_len = min(vid_fps // fps, duration)
+        return np.arange(segment_len // 2, duration, segment_len, dtype=int)
+    else:
+        raise ImportError(f'Unsupported frame sampling mode: {mode}')
+
+
+def load_video_from_ids(video_path, s=None, e=None, fps=None, max_frames=128, temporal_factor=1):
+    if s is not None and e is not None:
+        s = s if s >= 0. else 0.
+        e = e if e >= 0. else 0.
+        if s > e:
+            s, e = e, s
+        elif s == e:
+            e = s + 1
+
+    # 1. Loading Video
+    if os.path.isdir(video_path):
+        frame_files = sorted(os.listdir(video_path))
+
+        vid_fps = 3
+        num_frames_of_video = len(frame_files)
+    elif video_path.endswith('.gif'):
+        gif_reader = imageio.get_reader(video_path)
+
+        vid_fps = 25
+        num_frames_of_video = len(gif_reader)
+    else:
+        vreader = VideoReader(video_path, ctx=cpu(0), num_threads=2)
+        # vreader = VideoReader(video_path, ctx=cpu(0), num_threads=1)
+
+        vid_fps = vreader.get_avg_fps()
+        num_frames_of_video = len(vreader)
+
+    # 2. Determine frame range & Calculate frame indices
+    f_start = 0                       if s is None else max(int(s * vid_fps) - 1, 0)
+    f_end   = num_frames_of_video - 1 if e is None else min(int(e * vid_fps) - 1, num_frames_of_video - 1)
+    frame_indices = list(range(f_start, f_end + 1))
+
+    duration = len(frame_indices)
+    # 3. Sampling frame indices
+    if fps is not None and duration / vid_fps < max_frames:
+        sampled_frame_indices = [frame_indices[i] for i in frame_sample(duration, mode='fps', vid_fps=vid_fps, fps=fps)]
+    else:
+        sampled_frame_indices = [frame_indices[i] for i in frame_sample(duration, mode='uniform', num_frames=max_frames)]
+
+    # 4. Acquire frame data
+    if os.path.isdir(video_path):
+        frames = np.array([cv2.cvtColor(cv2.imread(os.path.join(video_path, frame_files[frame_idx])), cv2.COLOR_BGR2RGB) for frame_idx in sampled_frame_indices])
+    elif video_path.endswith('.gif'):
+        frames = np.array([cv2.cvtColor(frame, cv2.COLOR_RGBA2RGB) for idx, frame in enumerate(gif_reader) if idx in sampled_frame_indices])
+    else:
+        frames = vreader.get_batch(sampled_frame_indices).asnumpy()
+
+    frames = frames.transpose(0, 3, 1, 2)
+    timestamps = [x / vid_fps for x in sampled_frame_indices]
+
+    if temporal_factor > 1:
+        pad_length = temporal_factor - len(frames) % temporal_factor
+        frames = np.concatenate([frames, frames[-1:].repeat(pad_length, axis=0)])
+        [timestamps.append(timestamps[-1] + 1 / fps) for _ in range(pad_length)]
+
+    frames = [frame for frame in frames]
+
+    return frames, timestamps
+
+
+class ChatTemplateKwargs(TypedDict, total=False):
+
+    chat_template: Optional[str]
+    add_system_prompt: Optional[bool]
+    add_generation_prompt: Optional[bool]
+
+
+class Videollama3Qwen3ProcessorKwargs(ProcessingKwargs, ChatTemplateKwargs, total=False):
+
+    chat_template_kwargs: ChatTemplateKwargs = {
+        **ChatTemplateKwargs.__annotations__,
+    }
+
+    _defaults = {
+        "text_kwargs": {
+            "padding": False,
+        },
+        "image_kwargs": {
+            "merge_size": None,
+        },
+        "chat_template_kwargs": {
+            "chat_template": None,
+            "add_system_prompt": False,
+            "add_generation_prompt": False,
+        },
+    }
+
+
+class Videollama3Qwen3Processor(ProcessorMixin):
+
+    attributes = ["image_processor", "audio_processor", "tokenizer"]
+    image_processor_class = "SFLImageProcessor"
+    audio_processor_class = "WhisperFeatureExtractor"
+    tokenizer_class = ("Qwen2Tokenizer", "Qwen2TokenizerFast")
+    valid_kwargs = ["chat_template", "image_merge_size", "video_merge_size", "fps", "max_frames"]
+
+    def __init__(
+        self,
+        image_processor=None,
+        audio_processor=None,
+        tokenizer=None,
+        chat_template: str = None,
+        image_merge_size: int = 1,
+        video_merge_size: int = 2,
+        fps: Optional[int] = 1,
+        max_frames: Optional[int] = 128,
+    ):
+        self.image_processor = image_processor
+        self.audio_processor = audio_processor
+        self.tokenizer = tokenizer
+        if chat_template is None:
+            chat_template = self.tokenizer.chat_template
+        self.chat_template = chat_template
+
+        self.image_merge_size = image_merge_size
+        self.video_merge_size = video_merge_size
+        self.fps = fps
+        self.max_frames = max_frames
+
+        self.generation_prompt = self._infer_generation_prompt()
+        self.generation_prompt_ids = self.tokenizer.encode(self.generation_prompt, return_tensors="pt")
+        self.generation_prompt_length = len(self.generation_prompt_ids[0])
+        self.image_token_id = self.tokenizer.convert_tokens_to_ids(DEFAULT_IMAGE_TOKEN)
+        self.eos_token_id = self.tokenizer.eos_token_id
+
+    @classmethod
+    def _get_arguments_from_pretrained(cls, pretrained_model_name_or_path, **kwargs):
+        args = []
+        for attribute_name in cls.attributes:
+            class_name = getattr(cls, f"{attribute_name}_class")
+            if isinstance(class_name, tuple):
+                classes = tuple(_custom_import(n) if n is not None else None for n in class_name)
+                use_fast = kwargs.get("use_fast", True)
+                if use_fast and classes[1] is not None:
+                    attribute_class = classes[1]
+                else:
+                    attribute_class = classes[0]
+            else:
+                attribute_class = _custom_import(class_name)
+
+            args.append(attribute_class.from_pretrained(pretrained_model_name_or_path, **kwargs))
+        return args
+
+    def get_generation_prompt(self):
+        return self.generation_prompt
+
+    def get_generation_prompt_ids(self):
+        return self.generation_prompt_ids
+
+    def _infer_generation_prompt(self):
+        pseudo_message = [{"role": "user", "content": ""}]
+        instruction = self.apply_chat_template(pseudo_message, tokenize=False, add_generation_prompt=True)
+        conversation = self.apply_chat_template(pseudo_message, tokenize=False, add_generation_prompt=False)
+        return instruction.replace(conversation, "")
+
+    def _get_downsampled_grid_sizes(self, image_inputs: Dict[str, Any]):
+        grid_sizes = []
+        for grid_size, merge_size in zip(image_inputs.get("grid_sizes", []), image_inputs.get("merge_sizes", [])):
+            if not torch.all(grid_size[1:] % merge_size == 0):
+                warnings.warn(f"Grid size {grid_size} is not divisible by merge size. Some undesired errors may occur.")
+            if grid_size[0] == 1:
+                grid_sizes.append(grid_size[1:] / merge_size)
+            elif grid_size[0] > 1:
+                grid_sizes.extend([grid_size[1:] / merge_size] * grid_size[0])
+        return grid_sizes
+
+    def _get_visual_seq_len(self, grid_size: torch.Tensor):
+        num_tokens = int(grid_size.prod().item())
+        return num_tokens
+
+    def load_images(self, image_path: Union[str, List[str], Image.Image, List[Image.Image]]):
+        if isinstance(image_path, str) and os.path.isfile(image_path):
+            # images = [cv2.cvtColor(cv2.imread(image_path), cv2.COLOR_BGR2RGB)]
+            images = [Image.open(image_path).convert('RGB')]
+        elif isinstance(image_path, str) and os.path.isdir(image_path):
+            # images = [cv2.cvtColor(cv2.imread(os.path.join(image_path, f)), cv2.COLOR_BGR2RGB) for f in sorted(os.listdir(image_path))]
+            images = [Image.open(os.path.join(image_path, f)).convert('RGB') for f in sorted(os.listdir(image_path))]
+        elif isinstance(image_path, list) and isinstance(image_path[0], str):
+            # images = [cv2.cvtColor(cv2.imread(f), cv2.COLOR_BGR2RGB) for f in image_path]
+            images = [Image.open(f).convert('RGB') for f in image_path]
+        elif isinstance(image_path, list) and isinstance(image_path[0], Image.Image):
+            images = [np.array(x) for x in image_path]
+        elif isinstance(image_path, Image.Image):
+            images = [np.array(image_path)]
+        else:
+            raise ValueError(f"Unsupported image path type: {type(image_path)}")
+        return images
+
+    def load_video(
+        self,
+        video_path: str,
+        start_time: Optional[float] = None,
+        end_time: Optional[float] = None,
+        fps: Optional[float] = None,
+        max_frames: Optional[float] = None,
+        size: Optional[int] = None,
+        size_divisible: int = 1,
+        precise_time: bool = False,
+        verbose: bool = False,
+        temporal_factor: int = 1
+    ):
+        """
+        Load and process a video file and return the frames and the timestamps of each frame.
+
+        Args:
+            video_path (str): Path to the video file.
+            start_time (float, optional): Start time in seconds. Defaults to None.
+            end_time (float, optional): End time in seconds. Defaults to None.
+            fps (float, optional): Frames per second. Defaults to None.
+            num_frames (float, optional): Number of frames to sample. Defaults to None.
+            size (int, optional): Size of the shortest side. Defaults to None.
+            size_divisible (int, optional): Size divisible by this number. Defaults to 1.
+            precise_time (bool, optional): Whether to use precise time. Defaults to False.
+            verbose (bool, optional): Print ffmpeg output. Defaults to False.
+
+        Returns:
+            frames (List[PIL.Image]): List of frames.
+            timestamps (List[float]): List of timestamps.
+        """
+        fps = self.fps if fps is None else fps
+        max_frames = self.max_frames if max_frames is None else max_frames
+
+        if start_time is not None and end_time is not None and end_time - start_time < 1:
+            return load_video_from_ids(video_path, start_time, end_time, fps=fps, max_frames=max_frames)
+        if os.path.isdir(video_path):
+            return load_video_from_ids(video_path, start_time, end_time, fps=fps, max_frames=max_frames)
+        if video_path.endswith('.gif'):
+            return load_video_from_ids(video_path, start_time, end_time, fps=fps, max_frames=max_frames)
+        probe = ffmpeg.probe(video_path)
+        duration = float(probe['format']['duration'])
+        video_stream = next((stream for stream in probe['streams'] if stream['codec_type'] == 'video'), None)
+        w, h = int(video_stream['width']), int(video_stream['height'])
+
+        kwargs, input_kwargs, output_kwargs = {}, {}, {}
+        do_trim = start_time is not None or end_time is not None
+        if start_time is not None:
+            new_start_time = max(float(video_stream['start_time']), start_time)
+            duration -= new_start_time - start_time
+            start_time = new_start_time
+        else:
+            start_time = float(video_stream['start_time'])
+        if end_time is not None:
+            duration = min(duration, end_time - start_time)
+        else:
+            duration = duration
+        if do_trim:
+            kwargs = {'ss': start_time, 't': duration}
+        if precise_time:
+            output_kwargs.update(kwargs)
+        else:
+            input_kwargs.update(kwargs)
+
+        if size is not None:
+            scale_factor = size / min(w, h)
+            new_w, new_h = round(w * scale_factor), round(h * scale_factor)
+        else:
+            new_w, new_h = w, h
+        new_w = new_w // size_divisible * size_divisible
+        new_h = new_h // size_divisible * size_divisible
+
+        # NOTE: It may result in unexpected number of frames in ffmpeg
+        # if calculate the fps directly according to max_frames
+        # if max_frames is not None and (fps is None or duration * fps > 2 * max_frames):
+        #     fps = round(max_frames / duration * 2)
+
+        stream = ffmpeg.input(video_path, **input_kwargs)
+        if fps is not None:
+            stream = ffmpeg.filter(stream, "fps", fps=fps, round="down")
+        if new_w != w or new_h != h:
+            stream = ffmpeg.filter(stream, 'scale', new_w, new_h)
+        stream = ffmpeg.output(stream, "pipe:", format="rawvideo", pix_fmt="rgb24", **output_kwargs)
+        out, _ = ffmpeg.run(stream, capture_stdout=True, quiet=not verbose)
+
+        frames = np.frombuffer(out, np.uint8).reshape([-1, new_h, new_w, 3]).transpose([0, 3, 1, 2])
+
+        if fps is not None:
+            timestamps = np.arange(start_time, start_time + duration + 1 / fps, 1 / fps)[:len(frames)]
+        else:
+            timestamps = np.linspace(start_time, start_time + duration, len(frames))
+
+        if max_frames is not None and len(frames) > max_frames:
+            indices = np.linspace(0, len(frames) - 1, max_frames, dtype=int)
+            frames = frames[indices]
+            timestamps = timestamps[indices]
+
+        if temporal_factor > 1:
+            pad_length = temporal_factor - len(frames) % temporal_factor
+            frames = np.concatenate([frames, frames[-1:].repeat(pad_length, axis=0)])
+            timestamps = np.concatenate([timestamps, timestamps[-1:].repeat(pad_length) + np.arange(1, pad_length + 1) / fps])
+
+        frames = [frame for frame in frames]
+        timestamps = [timestamp for timestamp in timestamps]
+
+        return frames, timestamps
+    
+    def load_audio(
+        self,
+        audio_path: str,
+        start_time: Optional[float] = None,
+        end_time: Optional[float] = None,
+        verbose: bool = False,
+        sample_rate: int = 16000,
+    ):
+        """
+        Load and process an audio file and return the wave and the timestamps of each frame.
+
+        Args:
+            audio_path (str): Path to the audio file.
+            start_time (float, optional): Start time in seconds. Defaults to None.
+            end_time (float, optional): End time in seconds. Defaults to None.
+            verbose (bool, optional): Print ffmpeg output. Defaults to False.
+
+        Returns:
+            wave (List[PIL.Image]): List of wave.
+            timestamps (List[float]): List of timestamps.
+        """
+
+        audio_stream_ff = (
+            ffmpeg
+            .input(audio_path)
+            .output(
+                "pipe:",
+                format="s16le",
+                acodec="pcm_s16le",
+                ac=1,
+                ar=sample_rate,
+            )
+        )
+        audio_out, audio_err = ffmpeg.run(audio_stream_ff, capture_stdout=True, quiet=not verbose)
+        audio = np.frombuffer(audio_out, dtype=np.int16).astype(np.float32) / 32768.0
+        duration = len(audio) / sample_rate
+        if duration > 30:
+            audio = [audio[i*30*sample_rate: (i+1)*30*sample_rate] for i in range(int(duration // 30) + 1)]
+        else:
+            audio = [audio]
+        timestamps = [t for n, chunk in enumerate(audio) for t in range(n*30, n*30 + math.ceil(len(chunk) / sample_rate), 2)]
+        return audio, timestamps
+
+    def _load_multimodal_data(self, conversation: Conversation):
+        multimodal_info = defaultdict(list)
+        new_conversation = []
+        for message in conversation:
+            new_message = {"role": message["role"]}
+            if not isinstance(message["content"], (list, tuple)):
+                new_message["content"] = message["content"]
+                new_conversation.append(new_message)
+                continue
+
+            new_contents = []
+            for content in message["content"]:
+                if not isinstance(content, dict):
+                    new_contents.append(content)
+                    continue
+                assert "type" in content, "Content must have 'type' field."
+                if content["type"] in ["image", "video", "audio"] and content["type"] in content and isinstance(content[content["type"]], dict):
+                    # TODO: support other types which are not compatible with json
+                    load_args = content[content["type"]]
+                    data_id = json.dumps({k: v for k, v in load_args.items() if not k in ["start_time", "end_time"]})
+                    new_content = copy.deepcopy(content)
+                    multimodal_info[data_id].append(new_content)
+                    new_contents.append(new_content)
+                else:
+                    new_contents.append(content)
+
+            new_message["content"] = new_contents
+            new_conversation.append(new_message)
+
+        for data_id, contents in multimodal_info.items():
+            data_type = contents[0]["type"]
+            if data_type == "image":
+                image = self.load_images(contents[0][data_type]["image_path"])[0]
+                for content in contents:
+                    content["image"] = [image.copy()]
+
+            elif data_type == "video":
+                # TODO: start_time is None?
+                start_times = [content["video"].get("start_time", 0.) for content in contents]
+                end_times = [content["video"].get("end_time", float("inf")) for content in contents]
+
+                load_args = contents[0][data_type]
+                start_time, end_time = min(start_times), max(end_times)
+                if start_time > 0:
+                    load_args["start_time"] = start_time
+                if end_time < float("inf"):
+                    load_args["end_time"] = end_time
+                images, timestamps = self.load_video(**load_args)
+
+                for content, start_time, end_time in zip(contents, start_times, end_times):
+                    cur_images, cur_timestamps = [], []
+                    for image, timestamp in zip(images, timestamps):
+                        if start_time <= timestamp <= end_time:
+                            cur_images.append(image.copy())
+                            cur_timestamps.append(timestamp)
+
+                    content[data_type] = cur_images
+                    content["num_frames"] = len(cur_images)
+                    content["timestamps"] = cur_timestamps
+                    if contents[0].get("with_audio", False):
+                        _ = content.pop("with_audio")
+                        waves, audio_timestamps = self.load_audio(load_args["video_path"])
+                        content["audio"] = [wave.copy() for wave in waves]
+                        audio_split = [0] * len(timestamps)
+                        temp_count = 0
+                        for t in audio_timestamps:
+                            while temp_count < len(timestamps) - 1 and t >= timestamps[temp_count+1]:
+                                temp_count += 1
+                            audio_split[temp_count] += 1
+                        content["audio_split"] = audio_split
+            elif data_type == "audio":
+                waves, timestamps = self.load_audio(contents[0][data_type]["audio_path"])
+                for content in contents:
+                    content["audio"] = [wave.copy() for wave in waves]
+                    content["num_frames"] = len(timestamps)
+                    content["timestamps"] = timestamps
+
+        return new_conversation
+
+    def _gather_multimodal_data(self, conversation: Conversation):
+        images = []
+        audios = []
+        for message in conversation:
+            if not isinstance(message["content"], (list, tuple)):
+                continue
+            for content in message["content"]:
+                if not isinstance(content, dict):
+                    continue
+                if content["type"] == "video":
+                    video = content["video"]
+                    assert is_valid_video(video), f"Invalid video data: {video}."
+                    images.append(("video", video))
+                    if "audio" in content:
+                        audio = content["audio"]
+                        audios.append(("audio", audio))
+                if content["type"] == "image":
+                    image = content["image"]
+                    images.append(("image", image))
+                if content["type"] == "audio":
+                    audio = content["audio"]
+                    audios.append(("audio", audio))
+        images = images if len(images) > 0 else None
+        audios = audios if len(audios) > 0 else None
+        return images, audios
+
+    def _process_conversation_with_label(
+        self,
+        conversation: Conversation,
+        image_inputs: Dict[str, Any],
+        audio_inputs: Dict[str, Any],
+        **kwargs,
+    ):
+        assert kwargs.pop("return_tensors", "pt") == "pt", "Only PyTorch tensors are supported when return_labels=True."
+        assert not "add_generation_prompt" in kwargs, "'add_generation_prompt' argument is not supported when return_labels=True."
+
+        output_kwargs = self._merge_kwargs(
+            Videollama3Qwen3ProcessorKwargs,
+            tokenizer_init_kwargs=self.tokenizer.init_kwargs,
+            **kwargs,
+        )
+        output_kwargs["chat_template_kwargs"].pop("add_generation_prompt")
+
+        grid_sizes = self._get_downsampled_grid_sizes(image_inputs)
+        text_inputs = {"input_ids": [], "labels": []}
+        sample_types_list = []
+        image_idx = 0
+
+        for message_idx, message in enumerate(conversation):
+            prompt = self.apply_chat_template(
+                [message],
+                tokenize=False,
+                add_generation_prompt=False,
+                **output_kwargs["chat_template_kwargs"],
+            )
+            prompt_chunks = prompt.split(DEFAULT_IMAGE_TOKEN)
+            prompt = []
+            for chunk_idx in range(len(prompt_chunks) - 1):
+                prompt.append(prompt_chunks[chunk_idx])
+                num_tokens = self._get_visual_seq_len(grid_sizes[image_idx])
+                prompt.append(DEFAULT_IMAGE_TOKEN * num_tokens)
+                image_idx += 1
+            prompt.append(prompt_chunks[-1])
+            prompt = "".join(prompt)
+
+            # TODO: support attention_mask, position_ids, etc.
+            input_ids = self.tokenizer.encode(prompt, return_tensors="pt", **output_kwargs["text_kwargs"])[0]
+            text_inputs["input_ids"].append(input_ids)
+
+            targets = torch.full_like(input_ids, IGNORE_INDEX)
+            sample_types = torch.full_like(input_ids, IGNORE_INDEX)
+            if message["role"] == "assistant":
+                targets[self.generation_prompt_length:-1] = input_ids[self.generation_prompt_length:-1].clone()
+            # elif message["role"] == "stream":
+            #     diff = torch.diff((input_ids == self.image_token_id).float())
+            #     image_end_indices = torch.nonzero(diff < 0)[:, 0]
+            #     targets[image_end_indices + 1] = input_ids[image_end_indices + 1]
+            #     sample_types = targets.clone()
+            #     sample_types[torch.logical_and(sample_types > 0, sample_types != self.eos_token_id)] = 0
+            #     targets[-2] = input_ids[-2]    # <|im_end|>
+
+            if message_idx > 0 and conversation[message_idx - 1]["role"] == "stream":
+                targets[0] = input_ids[0]
+                # TODO: consider non-special tokens
+                sample_types[0] = input_ids[0]
+
+            text_inputs["labels"].append(targets)
+            sample_types_list.append(sample_types)
+
+        # Negative sampling for streaming data
+        text_inputs = {k: torch.cat(v) for k, v in text_inputs.items()}
+        sample_types = torch.cat(sample_types_list)
+        types, counts = torch.unique(sample_types[sample_types > -1], return_counts=True)
+
+        if len(types) > 0:
+            target_num_samples = counts.amin()
+            for type_id, type_count in zip(types, counts):
+                if type_count > target_num_samples:
+                    indices = torch.nonzero(sample_types == type_id)[:, 0]
+                    random_selector = torch.randperm(indices.size(0))[:-target_num_samples]
+                    text_inputs["labels"][indices[random_selector]] = IGNORE_INDEX
+                    # sample_types[indices[random_selector]] = -1
+
+        assert len(grid_sizes) == image_idx, "Number of images does not match the number of image tokens in the text."
+
+        return text_inputs
+
+    def _process_conversation_without_label(
+        self,
+        conversation: Conversation,
+        image_inputs: Dict[str, Any],
+        audio_inputs: Dict[str, Any],
+        add_think_prompt: bool = False,
+        **kwargs,
+    ):
+        output_kwargs = self._merge_kwargs(
+            Videollama3Qwen3ProcessorKwargs,
+            tokenizer_init_kwargs=self.tokenizer.init_kwargs,
+            **kwargs,
+        )
+        prompt = self.apply_chat_template(
+            conversation,
+            tokenize=False,
+            add_think_prompt=add_think_prompt,
+            **output_kwargs["chat_template_kwargs"],
+        )
+        return self.process_text(prompt, image_inputs, audio_inputs, **output_kwargs["text_kwargs"])
+
+    def _process_conversation(
+        self,
+        conversation: Conversation,
+        images: Optional[Union[BatchedImage, BatchedNamedImage]] = None,
+        audios = None,
+        return_labels: bool = False,
+        add_think_prompt: bool = False,
+        **kwargs: Unpack[Videollama3Qwen3ProcessorKwargs],
+    ) -> BatchFeature:
+        assert isinstance(conversation, list), "Conversation must be a list of messages."
+
+        if images is None or audios is None:
+            conversation = self._load_multimodal_data(conversation)
+            images, audios = self._gather_multimodal_data(conversation)
+
+        output_kwargs = self._merge_kwargs(
+            Videollama3Qwen3ProcessorKwargs,
+            tokenizer_init_kwargs=self.tokenizer.init_kwargs,
+            **kwargs,
+        )
+
+        if images is not None:
+            image_inputs = self.process_images(images, **output_kwargs["images_kwargs"])
+        else:
+            image_inputs = {}
+
+        if audios is not None:
+            audio_inputs = self.process_audios(audios)
+            audio_inputs["audio_input_features"] = audio_inputs.pop("input_features")
+            audio_inputs["audio_attention_mask"] = audio_inputs.pop("attention_mask")
+        else:
+            audio_inputs = {}
+        # print(image_inputs, audio_inputs)
+
+        if return_labels:
+            text_inputs = self._process_conversation_with_label(conversation, image_inputs, audio_inputs, **kwargs)
+        else:
+            text_inputs = self._process_conversation_without_label(conversation, image_inputs, audio_inputs, add_think_prompt, **kwargs)
+
+        return BatchFeature(data={**text_inputs, **image_inputs, **audio_inputs})
+
+    def _process_plain(
+        self,
+        text: Union[TextInput, PreTokenizedInput] = None,
+        images: Optional[Union[BatchedImage, BatchedNamedImage]] = None,
+        audios = None,
+        return_labels: bool = False,
+        **kwargs: Unpack[Videollama3Qwen3ProcessorKwargs],
+    ) -> BatchFeature:
+        if text is None:
+            raise ValueError("You must provide 'text' or 'message'.")
+        if return_labels:
+            raise ValueError("return_labels is not supported for plain text processing.")
+
+        output_kwargs = self._merge_kwargs(
+            Videollama3Qwen3ProcessorKwargs,
+            tokenizer_init_kwargs=self.tokenizer.init_kwargs,
+            **kwargs,
+        )
+
+        if images is not None:
+            image_inputs = self.process_images(images, **output_kwargs["images_kwargs"])
+        else:
+            image_inputs = {}
+
+        text_inputs = self.process_text(text, image_inputs, **output_kwargs["text_kwargs"])
+
+        return BatchFeature(data={**text_inputs, **image_inputs})
+
+    def process_images(self, images: Union[BatchedImage, BatchedNamedImage], **kwargs):
+        modals, images = make_batched_images(images)
+        if not "merge_size" in kwargs:
+            kwargs["merge_size"] = [
+                self.image_merge_size if modal == "image" else self.video_merge_size
+                for modal in modals
+            ]
+        image_inputs = self.image_processor(images=images, **kwargs)
+        image_inputs["modals"] = modals
+        return image_inputs
+    
+    def process_audios(
+        self,
+        audios = None,
+        **kwargs,
+    ):
+        if audios is None:
+            return {}
+        audios = [a[1] for a in audios]
+        audios = sum(audios, [])
+        audio_inputs = self.audio_processor(raw_speech=audios, sampling_rate=16000, truncation=False, return_attention_mask=True, return_tensors='pt', **kwargs)
+        return audio_inputs
+
+    def process_text(
+        self,
+        text: TextInput,
+        image_inputs: Dict[str, Any],
+        audio_inputs: Dict[str, Any],
+        **kwargs,
+    ):
+        grid_sizes = self._get_downsampled_grid_sizes(image_inputs)
+
+        kwargs.pop("padding")
+        kwargs.pop("padding_side")
+
+        image_idx = 0
+        while DEFAULT_IMAGE_TOKEN in text:
+            num_tokens = self._get_visual_seq_len(grid_sizes[image_idx])
+            text = text.replace(DEFAULT_IMAGE_TOKEN, "<placeholder>" * num_tokens, 1)
+            image_idx += 1
+        text = text.replace("<placeholder>", DEFAULT_IMAGE_TOKEN)
+ 
+        assert len(grid_sizes) == image_idx, "Number of images does not match the number of image tokens in the text."
+
+        text = text.replace(DEFAULT_AUDIO_TOKEN, DEFAULT_AUDIO_TOKEN*12)
+        # print(text)
+
+        text_inputs = self.tokenizer(text, **kwargs)
+        return text_inputs
+
+    def __call__(
+        self,
+        text: Optional[TextInput] = None,
+        conversation: Optional[Conversation] = None,
+        images: Optional[Union[BatchedImage, BatchedNamedImage]] = None,
+        audios = None,
+        return_labels: bool = False,
+        add_think_prompt: bool = False,
+        **kwargs: Unpack[Videollama3Qwen3ProcessorKwargs],
+    ) -> BatchFeature:
+        if conversation is not None:
+            if text is not None:
+                raise ValueError("You cannot provide 'message' with 'text'.")
+            return self._process_conversation(conversation, images, audios, return_labels, add_think_prompt, **kwargs)
+        return self._process_plain(text, images, audios, return_labels, **kwargs)
+
+    def batch_decode(self, *args, **kwargs):
+        return self.tokenizer.batch_decode(*args, **kwargs)
+
+    def decode(self, *args, **kwargs):
+        return self.tokenizer.decode(*args, **kwargs)
+
+    def apply_chat_template(
+        self,
+        conversation: Conversation,
+        chat_template: Optional[str] = None,
+        tokenize: bool = False,
+        add_system_prompt: bool = False,
+        add_generation_prompt: bool = True,
+        add_think_prompt: bool = False,
+        image_token: Optional[str] = DEFAULT_IMAGE_TOKEN,
+        audio_token: Optional[str] = DEFAULT_AUDIO_TOKEN,
+        **kwargs,
+    ) -> str:
+        """
+        Similar to the `apply_chat_template` method on tokenizers, this method applies a Jinja template to input
+        conversations to turn them into a single tokenizable string.
+
+        Args:
+            conversation (`List[Dict, str, str]`):
+                The conversation to format.
+            chat_template (`Optional[str]`, *optional*):
+                The Jinja template to use for formatting the conversation. If not provided, the tokenizer's
+                chat template is used.
+            tokenize (`bool`, *optional*, defaults to `False`):
+                Whether to tokenize the output or not.
+            add_system_prompt (`bool`, *optional*, defaults to `False`):
+                Whether to add the system prompt to the output or not.
+            add_generation_prompt (`bool`, *optional*, defaults to `False`):
+                Whether to add the generation prompt to the output or not.
+            image_token (`Optional[str]`, *optional*, defaults to `<image>`):
+                The token to use for indicating images in the conversation.
+            **kwargs:
+                Additional keyword arguments
+        """
+
+        if chat_template is None:
+            if self.chat_template is not None:
+                chat_template = self.chat_template
+            else:
+                raise ValueError(
+                    "No chat template is set for this processor. Please either set the `chat_template` attribute, "
+                    "or provide a chat template as an argument. See "
+                    "https://huggingface.co/docs/transformers/main/en/chat_templating for more information."
+                )
+        return self.tokenizer.apply_chat_template(
+            conversation,
+            chat_template=chat_template,
+            tokenize=tokenize,
+            add_system_prompt=add_system_prompt,
+            add_generation_prompt=add_generation_prompt,
+            add_think_prompt=add_think_prompt,
+            image_token=image_token,
+            audio_token=audio_token,
+            **kwargs
+        )
+
+    @property
+    def model_input_names(self):
+        tokenizer_input_names = self.tokenizer.model_input_names
+        image_processor_input_names = self.image_processor.model_input_names
+        return list(dict.fromkeys(tokenizer_input_names + image_processor_input_names)) + ["modals"]
+
+    # modified from transformers.ProcessorMixin
+    def _merge_kwargs(
+        self,
+        ModelProcessorKwargs: ProcessingKwargs,
+        tokenizer_init_kwargs: Optional[Dict] = None,
+        **kwargs,
+    ) -> Dict[str, Dict]:
+        """
+        Method to merge dictionaries of kwargs cleanly separated by modality within a Processor instance.
+        The order of operations is as follows:
+            1) kwargs passed as before have highest priority to preserve BC.
+                ```python
+                high_priority_kwargs = {"crop_size" = {"height": 222, "width": 222}, "padding" = "max_length"}
+                processor(..., **high_priority_kwargs)
+                ```
+            2) kwargs passed as modality-specific kwargs have second priority. This is the recommended API.
+                ```python
+                processor(..., text_kwargs={"padding": "max_length"}, images_kwargs={"crop_size": {"height": 222, "width": 222}}})
+                ```
+            3) kwargs passed during instantiation of a modality processor have fourth priority.
+                ```python
+                tokenizer = tokenizer_class(..., {"padding": "max_length"})
+                image_processor = image_processor_class(...)
+                processor(tokenizer, image_processor) # will pass max_length unless overriden by kwargs at call
+                ```
+            4) defaults kwargs specified at processor level have lowest priority.
+                ```python
+                class MyProcessingKwargs(ProcessingKwargs, CommonKwargs, TextKwargs, ImagesKwargs, total=False):
+                    _defaults = {
+                        "text_kwargs": {
+                            "padding": "max_length",
+                            "max_length": 64,
+                        },
+                    }
+                ```
+        Args:
+            ModelProcessorKwargs (`ProcessingKwargs`):
+                Typed dictionary of kwargs specifically required by the model passed.
+            tokenizer_init_kwargs (`Dict`, *optional*):
+                Dictionary of kwargs the tokenizer was instantiated with and need to take precedence over defaults.
+
+        Returns:
+            output_kwargs (`Dict`):
+                Dictionary of per-modality kwargs to be passed to each modality-specific processor.
+
+        """
+        # Initialize dictionaries
+        output_kwargs = {
+            "text_kwargs": {},
+            "images_kwargs": {},
+            "audio_kwargs": {},
+            "videos_kwargs": {},
+            "chat_template_kwargs": {},
+            "common_kwargs": {},
+        }
+
+        default_kwargs = {
+            "text_kwargs": {},
+            "images_kwargs": {},
+            "audio_kwargs": {},
+            "videos_kwargs": {},
+            "chat_template_kwargs": {},
+            "common_kwargs": {},
+        }
+
+        used_keys = set()
+
+        # get defaults from set model processor kwargs if they exist
+        for modality in default_kwargs:
+            default_kwargs[modality] = ModelProcessorKwargs._defaults.get(modality, {}).copy()
+            # update defaults with arguments from tokenizer init
+            for modality_key in ModelProcessorKwargs.__annotations__[modality].__annotations__.keys():
+                # init with tokenizer init kwargs if necessary
+                if modality_key in tokenizer_init_kwargs:
+                    value = (
+                        getattr(self.tokenizer, modality_key)
+                        if hasattr(self.tokenizer, modality_key)
+                        else tokenizer_init_kwargs[modality_key]
+                    )
+                    default_kwargs[modality][modality_key] = value
+        # now defaults kwargs are updated with the tokenizers defaults.
+        # pass defaults to output dictionary
+        output_kwargs.update(default_kwargs)
+
+        # update modality kwargs with passed kwargs
+        non_modality_kwargs = set(kwargs) - set(output_kwargs)
+        for modality in output_kwargs:
+            for modality_key in ModelProcessorKwargs.__annotations__[modality].__annotations__.keys():
+                # check if we received a structured kwarg dict or not to handle it correctly
+                if modality in kwargs:
+                    kwarg_value = kwargs[modality].pop(modality_key, "__empty__")
+                    # check if this key was passed as a flat kwarg.
+                    if kwarg_value != "__empty__" and modality_key in non_modality_kwargs:
+                        raise ValueError(
+                            f"Keyword argument {modality_key} was passed two times:\n"
+                            f"in a dictionary for {modality} and as a **kwarg."
+                        )
+                elif modality_key in kwargs:
+                    # we get a modality_key instead of popping it because modality-specific processors
+                    # can have overlapping kwargs
+                    kwarg_value = kwargs.get(modality_key, "__empty__")
+                else:
+                    kwarg_value = "__empty__"
+                if kwarg_value != "__empty__":
+                    output_kwargs[modality][modality_key] = kwarg_value
+                    used_keys.add(modality_key)
+
+        # Determine if kwargs is a flat dictionary or contains nested dictionaries
+        if any(key in default_kwargs for key in kwargs):
+            # kwargs is dictionary-based, and some keys match modality names
+            for modality, subdict in kwargs.items():
+                if modality in default_kwargs:
+                    for subkey, subvalue in subdict.items():
+                        if subkey not in used_keys:
+                            output_kwargs[modality][subkey] = subvalue
+                            used_keys.add(subkey)
+        else:
+            # kwargs is a flat dictionary
+            for key in kwargs:
+                if key not in used_keys:
+                    output_kwargs["common_kwargs"][key] = kwargs[key]
+
+        # all modality-specific kwargs are updated with common kwargs
+        for modality in output_kwargs:
+            output_kwargs[modality].update(output_kwargs["common_kwargs"])
+        return output_kwargs

processor_config.json

@@ -0,0 +1,10 @@
+{
+  "auto_map": {
+    "AutoProcessor": "processing_videollama3_qwen3.Videollama3Qwen3Processor"
+  },
+  "fps": 1,
+  "image_merge_size": 1,
+  "max_frames": 180,
+  "processor_class": "Videollama3Qwen3Processor",
+  "video_merge_size": 2
+}

special_tokens_map.json

@@ -0,0 +1,31 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3de4265d6c1499ee2f7f7c2ec71004f59d8676ce0373cd32cbad37d40b945cbd
+size 11423788

tokenizer_config.json

@@ -0,0 +1,290 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151665": {
+      "content": "<tool_response>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151666": {
+      "content": "</tool_response>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151667": {
+      "content": "<think>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151668": {
+      "content": "</think>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151669": {
+      "content": "<image>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151670": {
+      "content": "<|stream_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151671": {
+      "content": "<|stream_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151672": {
+      "content": "<|audio|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151673": {
+      "content": "<|audio_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151674": {
+      "content": "<|audio_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "chat_template": "\n{%- set identifier = 'im' %}\n{% for message in messages %}\n    {% if message['role'] == 'stream' %}\n        {% set identifier = 'stream' %}\n    {% else %}\n        {% set identifier = 'im' %}\n    {% endif %}\n    {% if message['role'] is not none %}\n        {{- '<|' + identifier + '_start|>' + message['role'] + '\n' -}}\n    {% endif %}\n    {% if message['content'] is string %}\n        {{- message['content'] + '<|' + identifier + '_end|>\n' -}}\n    {% else %}\n        {% for content in message['content'] %}\n            {% if content['type'] == 'image' or 'image' in content or 'image_url' in content %}\n                {% if 'time' in content %}\n                    {{- 'Time ' + content['time'] | round(1) | string + 's: ' -}}\n                {% endif %}\n                {{- image_token + '\n' -}}\n            {% elif content['type'] == 'video' or 'video' in content or 'video_url' in content %}\n                {% for i in range(content['num_frames']) %}\n                    {% if 'timestamps' in content %}\n                        {{- 'Time ' + content['timestamps'][i] | round(1) | string + 's:' -}}\n                    {% endif %}\n                    {% if i < content['num_frames'] - 1 %}\n                        {{- image_token + ',' -}}\n                        {% if 'audio_split' in content and content['audio_split'][i] > 0 %}\n                            {{- '<|audio_start|>' + audio_token * content['audio_split'][i] + '<|audio_end|>,' -}}\n                        {% endif %}\n                    {% else %}\n                        {{- image_token -}}\n                        {% if 'audio_split' in content and content['audio_split'][i] > 0 %}\n                            {{- ',<|audio_start|>' + audio_token * content['audio_split'][i] + '<|audio_end|>\n' -}}\n                        {% else %}\n                            {{- '\n' -}}\n                        {% endif %}\n                    {% endif %}\n                {% endfor %}\n            {% elif content['type'] == 'audio' or 'audio' in content or 'audio_url' in content %}\n                {% for i in range(content['num_frames']) %}\n                    {% if 'timestamps' in content %}\n                        {{- 'Time ' + content['timestamps'][i] | round(1) | string + 's:' -}}\n                    {% endif %}\n                    {% if i < content['num_frames'] - 1 %}\n                        {{- '<|audio_start|>' + audio_token + '<|audio_end|>,' -}}\n                    {% else %}\n                        {{- '<|audio_start|>' + audio_token + '<|audio_end|>\n' -}}\n                    {% endif %}\n                {% endfor %}\n            {% elif content['type'] == 'text' or 'text' in content %}\n                {{- content['text'] -}}\n            {% endif %}\n        {% endfor %}\n        {% if message['role'] is not none %}\n            {{- '<|' + identifier + '_end|>\n' -}}\n        {% endif %}\n    {% endif %}\n{% endfor %}\n{% if add_generation_prompt %}\n    {{- '<|im_start|>assistant\n' -}}\n    {% if not add_think_prompt %}\n        {{- '<think>\n\n</think>\n\n' -}}\n    {% endif %}\n{% endif %}\n",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 16384,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "right",
+  "processor_class": "Videollama3Qwen3Processor",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

trainer_state.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eeae0aa430b05cbcf2d1820d2fd7421aead1e6cbb402df12048617552171fa33
+size 16188911