Upload folder using huggingface_hub

chat_template.json

@@ -0,0 +1,4 @@
+{
+    "chat_template": "{{- bos_token }}\n{%- if custom_tools is defined %}\n    {%- set tools = custom_tools %}\n{%- endif %}\n{%- if not tools_in_user_message is defined %}\n    {%- set tools_in_user_message = true %}\n{%- endif %}\n{%- if not date_string is defined %}\n    {%- if strftime_now is defined %}\n        {%- set date_string = strftime_now(\"%d %b %Y\") %}\n    {%- else %}\n        {%- set date_string = \"26 Jul 2024\" %}\n    {%- endif %}\n{%- endif %}\n{%- if not tools is defined %}\n    {%- set tools = none %}\n{%- endif %}\n\n{#- This block extracts the system message. #}\n{%- if messages[0]['role'] == 'system' %}\n    {%- set system_message = messages[0]['content']|trim %}\n    {%- set messages = messages[1:] %}\n    {%- set user_supplied_system_message = true %}\n{%- else %}\n    {%- set system_message = none %}\n    {%- set user_supplied_system_message = false %}\n{%- endif %}\n\n{#- Find out if there are any images, not used here but kept for consistency #}\n{% set image_ns = namespace(has_images=false) %}\n{%- for message in messages %}\n    {%- if message['content'] is iterable and message['content'] is not string %}\n        {%- for content in message['content'] %}\n            {%- if content['type'] == 'image' %}\n                {%- set image_ns.has_images = true %}\n            {%- endif %}\n        {%- endfor %}\n    {%- endif %}\n{%- endfor %}\n\n{#- System message printing logic: This block now ensures complete replacement. #}\n{{- \"<|start_header_id|>system<|end_header_id|>\\n\\n\" }}\n{%- if user_supplied_system_message %}\n    {{- system_message }}\n{%- else %}\n    {{- \"You are a helpful AI assistant. Respond to the user's request based on the provided text and/or images.\" }}\n{%- endif %}\n{{- \"<|eot_id|>\" }}\n\n{#- Custom tools are passed in a user message with some extra guidance #}\n{%- if tools_in_user_message and not tools is none %}\n    {#- Extract the first user message so we can plug it in here #}\n    {%- if messages | length != 0 %}\n        {%- set first_user_message = messages[0]['content']|trim %}\n        {%- set messages = messages[1:] %}\n    {%- else %}\n        {{- raise_exception(\"Cannot put tools in the first user message when there's no first user message!\") }}\n    {%- endif %}\n    {{- '<|start_header_id|>user<|end_header_id|>\\n\\n' -}}\n    {{- \"Given the following functions, please respond with a JSON for a function call \" }}\n    {{- \"with its proper arguments that best answers the given prompt.\\n\\n\" }}\n    {{- 'Respond in the format {\\\"name\\\": function name, \\\"parameters\\\": dictionary of argument name and its value}.' }}\n    {{- \"Do not use variables.\\n\\n\" }}\n    {%- for t in tools %}\n        {{- t | tojson(indent=4) }}\n        {{- \"\\n\\n\" }}\n    {%- endfor %}\n    {{- first_user_message + \"<|eot_id|>\"}}\n{%- endif %}\n\n{%- for message in messages %}\n    {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}\n    {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\\n\\n' }}\n        {%- if message['content'] is string %}\n            {{- message['content'] }}\n        {%- else %}\n            {%- for content in message['content'] %}\n                {%- if content['type'] == 'image' %}\n                    {{- '<|image|>' }}\n                {%- elif content['type'] == 'video' %}\n                    {{- '<|video|>' }}\n                {%- elif content['type'] == 'text' %}\n                    {{- content['text'] }}\n                {%- endif %}\n            {%- endfor %}\n        {%- endif %}\n        {{- '<|eot_id|>' }}\n    {%- elif 'tool_calls' in message %}\n        {%- if not message.tool_calls|length == 1 %}\n            {{- raise_exception(\"This model only supports single tool-calls at once!\") }}\n        {%- endif %}\n        {%- set tool_call = message.tool_calls[0].function %}\n        {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' -}}\n        {{- '{\\\"name\\\": \\\"' + tool_call.name + '\\\", ' }}\n        {{- '\\\"parameters\\\": ' }}\n        {{- tool_call.arguments | tojson }}\n        {{- \"}\" }}\n        {{- \"<|eot_id|>\" }}\n    {%- elif message.role == \"tool\" or message.role == \"ipython\" %}\n        {{- \"<|start_header_id|>ipython<|end_header_id|>\\n\\n\" }}\n        {%- if message.content is mapping or message.content is iterable %}\n            {{- message.content | tojson }}\n        {%- else %}\n            {{- message.content }}\n        {%- endif %}\n        {{- \"<|eot_id|>\" }}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|start_header_id|>assistant<|end_header_id|>\\n\\n' }}\n{%- endif %}\n"
+  }
+  

config.json

@@ -1,18 +1,87 @@
 {
-  "_name_or_path": "/inspire/hdd/project/embodied-multimodality/public/pywang/Streaming-Video-LLM/saves/streaming_video_pretrain_1_5_stage/32_node_20250702_mllm_pretrain_stage_1_123_image470k_video1369k",
   "architectures": [
     "VideoMllamaForConditionalGeneration"
   ],
-  "hidden_size": 4096,
   "image_token_index": 128256,
-  "model_type": "mllama",
+  "model_type": "video_mllama",
+  "auto_map": {
+    "AutoConfig": "configuration_video_mllama.VideoMllamaConfig",
+    "AutoModelForCausalLM": "modeling_video_mllama.VideoMllamaForConditionalGeneration",
+    "AutoModel": "modeling_video_mllama.VideoMllamaForConditionalGeneration",
+    "AutoProcessor": "processing_video_mllama.VideoMllamaProcessor"
+  },
   "text_config": {
+    "_name_or_path": "",
+    "add_cross_attention": false,
+    "architectures": null,
+    "bad_words_ids": null,
+    "begin_suppress_tokens": null,
+    "bos_token_id": 128000,
+    "chunk_size_feed_forward": 0,
+    "cross_attention_hidden_size": null,
+    "cross_attention_layers": [
+      3,
+      8,
+      13,
+      18,
+      23,
+      28,
+      33,
+      38
+    ],
+    "decoder_start_token_id": null,
+    "diversity_penalty": 0.0,
+    "do_sample": false,
+    "dropout": 0,
+    "early_stopping": false,
+    "encoder_no_repeat_ngram_size": 0,
     "eos_token_id": [
       128001,
       128008,
       128009
     ],
-    "model_type": "mllama_text_model",
+    "exponential_decay_length_penalty": null,
+    "finetuning_task": null,
+    "forced_bos_token_id": null,
+    "forced_eos_token_id": null,
+    "hidden_act": "silu",
+    "hidden_size": 4096,
+    "id2label": {
+      "0": "LABEL_0",
+      "1": "LABEL_1"
+    },
+    "initializer_range": 0.02,
+    "intermediate_size": 14336,
+    "is_decoder": false,
+    "is_encoder_decoder": false,
+    "label2id": {
+      "LABEL_0": 0,
+      "LABEL_1": 1
+    },
+    "length_penalty": 1.0,
+    "max_length": 20,
+    "max_position_embeddings": 131072,
+    "min_length": 0,
+    "model_type": "video_mllama_text_model",
+    "no_repeat_ngram_size": 0,
+    "num_attention_heads": 32,
+    "num_beam_groups": 1,
+    "num_beams": 1,
+    "num_hidden_layers": 40,
+    "num_key_value_heads": 8,
+    "num_return_sequences": 1,
+    "output_attentions": false,
+    "output_hidden_states": false,
+    "output_scores": false,
+    "pad_token_id": 128004,
+    "prefix": null,
+    "problem_type": null,
+    "pruned_heads": {},
+    "remove_invalid_values": false,
+    "repetition_penalty": 1.0,
+    "return_dict": true,
+    "return_dict_in_generate": false,
+    "rms_norm_eps": 1e-05,
     "rope_scaling": {
       "factor": 8.0,
       "high_freq_factor": 4.0,
@@ -20,14 +89,143 @@
       "original_max_position_embeddings": 8192,
       "rope_type": "llama3"
     },
-    "torch_dtype": "bfloat16"
+    "rope_theta": 500000.0,
+    "sep_token_id": null,
+    "suppress_tokens": null,
+    "task_specific_params": null,
+    "temperature": 1.0,
+    "tf_legacy_loss": false,
+    "tie_encoder_decoder": false,
+    "tie_word_embeddings": false,
+    "tokenizer_class": null,
+    "top_k": 50,
+    "top_p": 1.0,
+    "torch_dtype": "bfloat16",
+    "torchscript": false,
+    "typical_p": 1.0,
+    "use_bfloat16": false,
+    "use_cache": true,
+    "vocab_size": 128256
   },
   "torch_dtype": "bfloat16",
   "transformers_version": "4.47.1",
-  "use_cache": false,
   "vision_config": {
+    "_name_or_path": "",
+    "add_cross_attention": false,
+    "vision_ignore_attention_mask": true,
+    "merge_size": 4,
+    "merge_mode": "average",
+    "architectures": null,
+    "attention_heads": 16,
+    "bad_words_ids": null,
+    "begin_suppress_tokens": null,
+    "bos_token_id": null,
+    "chunk_size_feed_forward": 0,
+    "cross_attention_hidden_size": null,
+    "decoder_start_token_id": null,
+    "diversity_penalty": 0.0,
+    "do_sample": false,
+    "early_stopping": false,
+    "encoder_no_repeat_ngram_size": 0,
+    "eos_token_id": null,
+    "exponential_decay_length_penalty": null,
+    "finetuning_task": null,
+    "forced_bos_token_id": null,
+    "forced_eos_token_id": null,
+    "hidden_act": "gelu",
+    "hidden_size": 1280,
+    "id2label": {
+      "0": "LABEL_0",
+      "1": "LABEL_1"
+    },
     "image_size": 560,
-    "model_type": "mllama_vision_model",
-    "torch_dtype": "bfloat16"
+    "intermediate_layers_indices": [
+      3,
+      7,
+      15,
+      23,
+      30
+    ],
+    "intermediate_size": 5120,
+    "is_decoder": false,
+    "is_encoder_decoder": false,
+    "label2id": {
+      "LABEL_0": 0,
+      "LABEL_1": 1
+    },
+    "length_penalty": 1.0,
+    "max_length": 20,
+    "max_num_tiles": 4,
+    "min_length": 0,
+    "model_type": "video_mllama_vision_model",
+    "no_repeat_ngram_size": 0,
+    "norm_eps": 1e-05,
+    "num_beam_groups": 1,
+    "num_beams": 1,
+    "num_channels": 3,
+    "num_global_layers": 8,
+    "num_hidden_layers": 32,
+    "num_return_sequences": 1,
+    "output_attentions": false,
+    "output_hidden_states": false,
+    "output_scores": false,
+    "pad_token_id": null,
+    "patch_size": 14,
+    "prefix": null,
+    "problem_type": null,
+    "pruned_heads": {},
+    "remove_invalid_values": false,
+    "repetition_penalty": 1.0,
+    "return_dict": true,
+    "return_dict_in_generate": false,
+    "sep_token_id": null,
+    "supported_aspect_ratios": [
+      [
+        1,
+        1
+      ],
+      [
+        1,
+        2
+      ],
+      [
+        1,
+        3
+      ],
+      [
+        1,
+        4
+      ],
+      [
+        2,
+        1
+      ],
+      [
+        2,
+        2
+      ],
+      [
+        3,
+        1
+      ],
+      [
+        4,
+        1
+      ]
+    ],
+    "suppress_tokens": null,
+    "task_specific_params": null,
+    "temperature": 1.0,
+    "tf_legacy_loss": false,
+    "tie_encoder_decoder": false,
+    "tie_word_embeddings": true,
+    "tokenizer_class": null,
+    "top_k": 50,
+    "top_p": 1.0,
+    "torch_dtype": "bfloat16",
+    "torchscript": false,
+    "typical_p": 1.0,
+    "use_bfloat16": false,
+    "vision_output_dim": 7680
   }
 }

configuration_video_mllama.py

@@ -0,0 +1,375 @@
+# coding=utf-8
+# Copyright 2024 HuggingFace Inc. team. All rights reserved.
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""VideoMllama model configuration"""
+
+from typing import Dict, List, Optional
+
+from transformers.configuration_utils import PretrainedConfig
+from transformers.modeling_rope_utils import rope_config_validation
+from transformers.utils import logging
+
+
+logger = logging.get_logger(__name__)
+
+
+class VideoMllamaVisionConfig(PretrainedConfig):
+    r"""
+    This is the configuration class to store the configuration of a [`VideoMllamaVisionModel`]. It is used to instantiate an
+    VideoMllama vision model according to the specified arguments, defining the model architecture. Instantiating a configuration
+    with the defaults will yield a similar configuration to that of the VideoMllama-11B.
+
+    e.g. [meta-llama/Llama-3.2-11B-Vision](https://huggingface.co/meta-llama/Llama-3.2-11B-Vision)
+
+    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+    documentation from [`PretrainedConfig`] for more information.
+
+    Args:
+        hidden_size (`int`, *optional*, defaults to 1280):
+            Dimensionality of the encoder layers and the pooler layer.
+        hidden_act (`str` or `function`, *optional*, defaults to `"gelu"`):
+            The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
+            `"relu"`, `"selu"` and `"gelu_new"` `"quick_gelu"` are supported.
+        num_hidden_layers (`int`, *optional*, defaults to 32):
+            Number of hidden layers in the Transformer encoder.
+        num_global_layers (`int`, *optional*, defaults to 8):
+            Number of global layers in the Transformer encoder.
+            Vision model has a second transformer encoder, called global.
+        num_attention_heads (`int`, *optional*, defaults to 16):
+            Number of attention heads for each attention layer in the Transformer encoder.
+        num_channels (`int`, *optional*, defaults to 3):
+            Number of channels in the input image.
+        intermediate_size (`int`, *optional*, defaults to 5120):
+            Dimensionality of the "intermediate" (often named feed-forward) layer in the Transformer encoder.
+        vision_output_dim (`int`, *optional*, defaults to 7680):
+            Dimensionality of the vision model output. Includes output of transformer
+            encoder with intermediate layers and global transformer encoder.
+        image_size (`int`, *optional*, defaults to 448):
+            The size (resolution) of each image *tile*.
+        patch_size (`int`, *optional*, defaults to 14):
+            The size (resolution) of each patch.
+        norm_eps (`float`, *optional*, defaults to 1e-05):
+            The epsilon used by the layer normalization layers.
+        max_num_tiles (`int`, *optional*, defaults to 4):
+            Maximum number of tiles for image splitting.
+        intermediate_layers_indices (`List[int]`, *optional*, defaults to [3, 7, 15, 23, 30]):
+            Indices of intermediate layers of transformer encoder from which to extract and output features.
+            These output features are concatenated with final hidden state of transformer encoder.
+        supported_aspect_ratios (`List[List[int]]`, *optional*):
+            List of supported aspect ratios for image splitting. If not specified, the default supported aspect ratios
+            are [[1, 1], [1, 2], [1, 3], [1, 4], [2, 1], [2, 2], [3, 1], [4, 1]] for `max_num_tiles=4`.
+        initializer_range (`float`, *optional*, defaults to 0.02):
+            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
+
+    Example:
+
+    ```python
+    >>> from transformers import VideoMllamaVisionConfig, VideoMllamaVisionModel
+
+    >>> # Initializing a Llama config
+    >>> config = VideoMllamaVisionConfig()
+
+    >>> # Initializing a vision model from the VideoMllama-11b style configuration
+    >>> model = VideoMllamaVisionModel(config)
+
+    >>> # Accessing the model configuration
+    >>> configuration = model.config
+    ```"""
+
+    model_type = "video_mllama_vision_model"
+    base_config_key = "vision_config"
+
+    def __init__(
+        self,
+        hidden_size: int = 1280,
+        hidden_act: str = "gelu",
+        num_hidden_layers: int = 32,
+        num_global_layers: int = 8,
+        num_attention_heads: int = 16,
+        num_channels: int = 3,
+        intermediate_size: int = 5120,
+        vision_output_dim: int = 7680,
+        image_size: int = 448,
+        patch_size: int = 14,
+        norm_eps: float = 1e-5,
+        max_num_tiles: int = 4,
+        intermediate_layers_indices: Optional[List[int]] = None,
+        supported_aspect_ratios: Optional[List[List[int]]] = None,
+        initializer_range: float = 0.02,
+        vision_ignore_attention_mask: bool = True,
+        merge_size: int = 4,
+        merge_mode: str = "average",
+        **kwargs,
+    ):
+        if supported_aspect_ratios is None:
+            if max_num_tiles != 4:
+                raise ValueError("max_num_tiles must be 4 for default supported aspect ratios")
+            supported_aspect_ratios = [[1, 1], [1, 2], [1, 3], [1, 4], [2, 1], [2, 2], [3, 1], [4, 1]]
+
+        if intermediate_layers_indices is None:
+            intermediate_layers_indices = [3, 7, 15, 23, 30]
+
+        self.hidden_size = hidden_size
+        self.hidden_act = hidden_act
+        self.num_hidden_layers = num_hidden_layers
+        self.num_channels = num_channels
+        self.intermediate_size = intermediate_size
+        self.image_size = image_size
+        self.vision_output_dim = vision_output_dim
+        self.patch_size = patch_size
+        self.intermediate_layers_indices = intermediate_layers_indices
+        self.num_global_layers = num_global_layers
+        self.max_num_tiles = max_num_tiles
+        self.norm_eps = norm_eps
+        self.attention_heads = num_attention_heads
+        self.supported_aspect_ratios = supported_aspect_ratios
+        self.initializer_range = initializer_range
+        self.vision_ignore_attention_mask = vision_ignore_attention_mask
+        self.merge_size = merge_size
+        self.merge_mode = merge_mode
+
+
+        super().__init__(**kwargs)
+
+    @property
+    def max_aspect_ratio_id(self) -> int:
+        return len(self.supported_aspect_ratios)
+
+
+class VideoMllamaTextConfig(PretrainedConfig):
+    r"""
+    This is the configuration class to store the configuration of a [`VideoMllamaTextModel`]. It is used to instantiate an
+    VideoMllama text model according to the specified arguments, defining the model architecture. Instantiating a configuration
+    with the defaults will yield a similar configuration to that of the VideoMllama-11B.
+
+    e.g. [meta-llama/Llama-3.2-11B-Vision](https://huggingface.co/meta-llama/Llama-3.2-11B-Vision)
+
+    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+    documentation from [`PretrainedConfig`] for more information.
+
+    Args:
+        vocab_size (`int`, *optional*, defaults to 128256):
+            Vocabulary size of the VideoMllama text model. Defines the maximum number of different tokens that can be represented
+            by the `inputs_ids` passed when calling [`VideoMllamaTextModel`].
+        hidden_size (`int`, *optional*, defaults to 4096):
+            Dimensionality of the embeddings and hidden states.
+        hidden_act (`str` or `Callable`, *optional*, defaults to `"silu"`):
+            The non-linear activation function (function or string) in the encoder and pooler.
+        num_hidden_layers (`int`, *optional*, defaults to 40):
+            Number of hidden layers in the Transformer encoder.
+        num_attention_heads (`int`, *optional*, defaults to 32):
+            Number of attention heads for each attention layer in the Transformer encoder.
+        num_key_value_heads (`int`, *optional*, defaults to 8):
+            This is the number of key_value heads that should be used to implement Grouped Query Attention. If not
+            specified, will default to `num_attention_heads`.
+        intermediate_size (`int`, *optional*, defaults to 14336):
+            Dimensionality of the "intermediate" (often named feed-forward) layer in the Transformer encoder.
+        rope_theta (`float`, *optional*, defaults to `500000.0`):
+            The base period of the RoPE embeddings.
+        rope_scaling (`Dict`, *optional*):
+            Dictionary containing the scaling configuration for the RoPE embeddings. NOTE: if you apply new rope type
+            and you expect the model to work on longer `max_position_embeddings`, we recommend you to update this value
+            accordingly.
+            Expected contents:
+                `rope_type` (`str`):
+                    The sub-variant of RoPE to use. Can be one of ['default', 'linear', 'dynamic', 'yarn', 'longrope',
+                    'llama3'], with 'default' being the original RoPE implementation.
+                `factor` (`float`, *optional*):
+                    Used with all rope types except 'default'. The scaling factor to apply to the RoPE embeddings. In
+                    most scaling types, a `factor` of x will enable the model to handle sequences of length x *
+                    original maximum pre-trained length.
+                `original_max_position_embeddings` (`int`, *optional*):
+                    Used with 'dynamic', 'longrope' and 'llama3'. The original max position embeddings used during
+                    pretraining.
+                `attention_factor` (`float`, *optional*):
+                    Used with 'yarn' and 'longrope'. The scaling factor to be applied on the attention
+                    computation. If unspecified, it defaults to value recommended by the implementation, using the
+                    `factor` field to infer the suggested value.
+                `beta_fast` (`float`, *optional*):
+                    Only used with 'yarn'. Parameter to set the boundary for extrapolation (only) in the linear
+                    ramp function. If unspecified, it defaults to 32.
+                `beta_slow` (`float`, *optional*):
+                    Only used with 'yarn'. Parameter to set the boundary for interpolation (only) in the linear
+                    ramp function. If unspecified, it defaults to 1.
+                `short_factor` (`List[float]`, *optional*):
+                    Only used with 'longrope'. The scaling factor to be applied to short contexts (<
+                    `original_max_position_embeddings`). Must be a list of numbers with the same length as the hidden
+                    size divided by the number of attention heads divided by 2
+                `long_factor` (`List[float]`, *optional*):
+                    Only used with 'longrope'. The scaling factor to be applied to long contexts (<
+                    `original_max_position_embeddings`). Must be a list of numbers with the same length as the hidden
+                    size divided by the number of attention heads divided by 2
+                `low_freq_factor` (`float`, *optional*):
+                    Only used with 'llama3'. Scaling factor applied to low frequency components of the RoPE
+                `high_freq_factor` (`float`, *optional*):
+                    Only used with 'llama3'. Scaling factor applied to high frequency components of the RoPE
+        rms_norm_eps (`float`, *optional*, defaults to 1e-05):
+            The epsilon used by the rms normalization layers.
+        max_position_embeddings (`int`, *optional*, defaults to 131072):
+            The maximum sequence length that this model might ever be used with.
+        initializer_range (`float`, *optional*, defaults to 0.02):
+            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
+        use_cache (`bool`, *optional*, defaults to `True`):
+            Whether or not the model should return the last key/values attentions.
+        tie_word_embeddings (`bool`, *optional*, defaults to `False`):
+            Whether to tie weight embeddings
+        cross_attention_layers (`List[int]`, *optional*):
+            Indices of the cross attention layers. If not specified, will default to [3, 8, 13, 18, 23, 28, 33, 38].
+        dropout (`float`, *optional*, defaults to 0):
+            The dropout probability for self- and cross-attention layers.
+        bos_token_id (`int`, *optional*, defaults to 128000):
+            The id of the beginning of sentence token.
+        eos_token_id (`int`, *optional*, defaults to 128001):
+            The id of the end of sentence token.
+        pad_token_id (`int`, *optional*, defaults to 128004):
+            The id of the padding token.
+
+    Example:
+
+    ```python
+    >>> from transformers import VideoMllamaTextModel, VideoMllamaTextConfig
+
+    >>> # Initializing a VideoMllama text config
+    >>> config = VideoMllamaTextConfig()
+
+    >>> # Initializing a model from the VideoMllama text configuration
+    >>> model = VideoMllamaTextModel(config)
+
+    >>> # Accessing the model configuration
+    >>> configuration = model.config
+    ```"""
+
+    model_type = "video_mllama_text_model"
+    base_config_key = "text_config"
+
+    def __init__(
+        self,
+        vocab_size: int = 128256,
+        hidden_size: int = 4096,
+        hidden_act: str = "silu",
+        num_hidden_layers: int = 40,
+        num_attention_heads: int = 32,
+        num_key_value_heads: int = 8,
+        intermediate_size: int = 14_336,
+        rope_theta: float = 500_000,
+        rope_scaling: Optional[Dict] = None,
+        rms_norm_eps: float = 1e-5,
+        max_position_embeddings: int = 131_072,
+        initializer_range: float = 0.02,
+        use_cache: bool = True,
+        tie_word_embeddings: bool = False,
+        cross_attention_layers: Optional[List[int]] = None,
+        dropout: float = 0,
+        bos_token_id: int = 128000,
+        eos_token_id: int = 128001,
+        pad_token_id: Optional[int] = 128004,
+        **kwargs,
+    ):
+        if cross_attention_layers is None:
+            cross_attention_layers = [3, 8, 13, 18, 23, 28, 33, 38]
+
+        self.vocab_size = vocab_size
+        self.num_hidden_layers = num_hidden_layers
+        self.cross_attention_layers = cross_attention_layers
+        self.hidden_size = hidden_size
+        self.num_attention_heads = num_attention_heads
+        self.num_key_value_heads = num_key_value_heads
+        self.initializer_range = initializer_range
+        self.use_cache = use_cache
+        self.rope_theta = rope_theta
+        self.rms_norm_eps = rms_norm_eps
+        self.intermediate_size = intermediate_size
+        self.dropout = dropout
+        self.hidden_act = hidden_act
+        self.rope_scaling = rope_scaling
+        self.max_position_embeddings = max_position_embeddings
+        rope_config_validation(self)
+
+        super().__init__(
+            pad_token_id=pad_token_id,
+            bos_token_id=bos_token_id,
+            eos_token_id=eos_token_id,
+            tie_word_embeddings=tie_word_embeddings,
+            **kwargs,
+        )
+
+
+class VideoMllamaConfig(PretrainedConfig):
+    r"""
+    This is the configuration class to store the configuration of a [`VideoMllamaForConditionalGeneration`]. It is used to instantiate an
+    VideoMllama model according to the specified arguments, defining the model architecture. Instantiating a configuration
+    with the defaults will yield a similar configuration to that of the VideoMllama-9B.
+
+    e.g. [meta-llama/Llama-3.2-11B-Vision](https://huggingface.co/meta-llama/Llama-3.2-11B-Vision)
+
+    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+    documentation from [`PretrainedConfig`] for more information.
+
+    Args:
+        vision_config (`Union[AutoConfig, dict]`, *optional*, defaults to `VideoMllamaVisionConfig`):
+            The config object or dictionary of the vision backbone.
+        text_config (`Union[AutoConfig, dict]`, *optional*, defaults to `VideoMllamaTextConfig`):
+            The config object or dictionary of the text backbone.
+        image_token_index (`int`, *optional*, defaults to 128256):
+            The image token index to encode the image prompt.
+
+    Example:
+
+    ```python
+    >>> from transformers import VideoMllamaForConditionalGeneration, VideoMllamaConfig, VideoMllamaVisionConfig, VideoMllamaTextConfig
+
+    >>> # Initializing a CLIP-vision config
+    >>> vision_config = VideoMllamaVisionConfig()
+
+    >>> # Initializing a Llama config
+    >>> text_config = VideoMllamaTextConfig()
+
+    >>> # Initializing a VideoMllama-11b style configuration
+    >>> configuration = VideoMllamaConfig(vision_config, text_config)
+
+    >>> # Initializing a model from the VideoMllama-11b style configuration
+    >>> model = VideoMllamaForConditionalGeneration(configuration)
+
+    >>> # Accessing the model configuration
+    >>> configuration = model.config
+    ```"""
+
+    model_type = "video_mllama"
+    sub_configs = {"text_config": VideoMllamaTextConfig, "vision_config": VideoMllamaVisionConfig}
+
+    def __init__(
+        self,
+        vision_config=None,
+        text_config=None,
+        image_token_index=128256,
+        **kwargs,
+    ):
+        if vision_config is None:
+            self.vision_config = VideoMllamaVisionConfig()
+            logger.info("vision_config is None, using default VideoMllama vision config")
+        elif isinstance(vision_config, dict):
+            self.vision_config = VideoMllamaVisionConfig(**vision_config)
+        elif isinstance(vision_config, VideoMllamaVisionConfig):
+            self.vision_config = vision_config
+
+        self.image_token_index = image_token_index
+
+        if text_config is None:
+            self.text_config = VideoMllamaTextConfig()
+            logger.info("text_config is None, using default VideoMllama text config")
+        elif isinstance(text_config, dict):
+            self.text_config = VideoMllamaTextConfig(**text_config)
+        elif isinstance(text_config, VideoMllamaTextConfig):
+            self.text_config = text_config
+
+        super().__init__(**kwargs)

image_processing_video_mllama.py

@@ -0,0 +1,930 @@
+# coding=utf-8
+# Copyright 2024 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import math
+from functools import lru_cache
+from typing import Dict, List, Optional, Tuple, Union
+
+import numpy as np
+
+from transformers.image_processing_utils import BaseImageProcessor, BatchFeature
+from transformers.image_transforms import (
+    PaddingMode,
+    get_image_size,
+    pad,
+    resize,
+)
+from transformers.image_utils import (
+    IMAGENET_STANDARD_MEAN,
+    IMAGENET_STANDARD_STD,
+    ChannelDimension,
+    ImageInput,
+    PILImageResampling,
+    infer_channel_dimension_format,
+    is_valid_image,
+    is_vision_available,
+    to_numpy_array,
+    validate_preprocess_arguments,
+)
+from transformers.utils import TensorType, logging
+
+
+if is_vision_available():
+    import PIL
+    from PIL import Image
+
+
+logger = logging.get_logger(__name__)
+
+
+@lru_cache(maxsize=10)
+def get_all_supported_aspect_ratios(max_image_tiles: int) -> List[Tuple[int, int]]:
+    """
+    Computes all allowed aspect ratios for a given maximum number of input tiles.
+
+    This function calculates all possible arrangements of tiles that can be formed
+    within the constraint of the maximum number of tiles. Each arrangement is
+    represented by its aspect ratio (width/height) and the corresponding tile configuration.
+
+    Args:
+        max_image_tiles (`int`):
+            The maximum number of tiles allowed.
+
+    Returns:
+        `List[Tuple[int, int]]`: A list of tuples, each tuple representing a valid (width, height)
+        configuration in terms of number of tiles.
+
+    Example:
+        >>> get_all_supported_aspect_ratios(4)
+        [(1, 1), (1, 2), (1, 3), (1, 4), (2, 1), (2, 2), (3, 1), (4, 1)]
+
+    """
+    aspect_ratios = []
+    for width in range(1, max_image_tiles + 1):
+        for height in range(1, max_image_tiles + 1):
+            if width * height <= max_image_tiles:
+                aspect_ratios.append((width, height))
+    return aspect_ratios
+
+
+def get_image_size_fit_to_canvas(
+    image_height: int,
+    image_width: int,
+    canvas_height: int,
+    canvas_width: int,
+    tile_size: int,
+) -> Tuple[int, int]:
+    """
+    Calculates the new size of an image to fit within a canvas while maintaining aspect ratio.
+
+    This function calculates the optimal size for an image to fit within a canvas defined by
+    canvas_height and canvas_width, while ensuring that the image dimensions are not smaller than
+    tile_size. If the image is larger than the canvas, the returned size will fit within the canvas.
+    If the image already fits within the canvas, the size remains unchanged.
+    The aspect ratio of the original image is preserved.
+
+    Args:
+        image_height (`int`):
+            The height of the original image.
+        image_width (`int`):
+            The width of the original image.
+        canvas_height (`int`):
+            The height of the canvas.
+        canvas_width (`int`):
+            The width of the canvas.
+        tile_size (`int`):
+            The tile size.
+
+    Returns:
+        `Tuple[int, int]`: A tuple containing the new height and width of the image.
+
+    """
+    # Set target image size in between `tile_size` and canvas_size
+    target_width = np.clip(image_width, tile_size, canvas_width)
+    target_height = np.clip(image_height, tile_size, canvas_height)
+
+    scale_h = target_height / image_height
+    scale_w = target_width / image_width
+
+    if scale_w < scale_h:
+        new_width = target_width
+        new_height = min(math.floor(image_height * scale_w), target_height)
+    else:
+        new_height = target_height
+        new_width = min(math.floor(image_width * scale_h), target_width)
+
+    return new_height, new_width
+
+
+@lru_cache(maxsize=100)
+def get_optimal_tiled_canvas(
+    image_height: int,
+    image_width: int,
+    max_image_tiles: int,
+    tile_size: int,
+) -> Tuple[int, int]:
+    r"""
+    Determines the best canvas based on image and tile size and maximum number of tiles.
+
+    First, calculates possible resolutions based on the maximum number of tiles and tile size.
+    For example for max_image_tiles=2, tile_size=100, possible tile arrangements are:
+    [(1, 1), (1, 2), (2, 1)] and corresponding canvas sizes are:
+    [(100, 100), (100, 200), (200, 100)]
+
+    For each possible resolution, calculates the scaling factors for
+    width and height, and selects the smallest one, which is the limiting side.
+    E.g. to match the canvas you can upscale height by 2x, and width by 1.5x,
+    therefore, the maximum upscaling you can do is min(2, 1.5) = 1.5.
+
+    If upscaling is possible (any of the scaling factors is greater than 1),
+    then picks the smallest upscaling factor > 1.
+
+    If upscaling is not possible, then picks the largest scaling factor <= 1, i.e.
+    reduce downscaling as much as possible.
+
+    If there are multiple resolutions with the same max scale, we pick the one with the lowest area,
+    to minimize padding. E.g., the same image can be upscaled to 224x224 and 224x448, but the latter
+    has more padding.
+
+    Example of canvases made from tiles:
+
+    To visualize how the image can fit onto different tile grids, let's try fitting an ASCII cat into the tiles.
+
+    Here's an ASCII cat image you want to fit into the tiles:
+
+       /\_/\
+      ( o.o )
+       > ^ <
+
+    If `num_tiles=6`, possible tile grids would look like this:
+
+    **2x3 Canvas (2 tiles wide, 3 tiles tall)**: -> total of 6 tiles
+    +-------+-------+
+    | /\_/\ |   0   |   <- Cat image split across two tiles horizontally
+    +-------+-------+
+    | > ^ < |   0   |   <- Remaining part of the cat occupies the left tile
+    +-------+-------+
+    |( o.o )|   0   |
+    +-------+-------+
+
+    **3x2 Canvas (3 tiles wide, 2 tiles tall)**: -> total of 6 tiles
+    +-------+-------+-------+
+    | /\_/\ |( o.o )|   0   |   <- Cat image occupies the first two tiles, 1 tile remains empty
+    +-------+-------+-------+
+    | > ^ < |   0   |   0   |   <- Remaining part of the cat occupies the left tile
+    +-------+-------+-------+
+
+    **1x6 Canvas (1 tile wide, 6 tiles tall)**: -> total of 6 tiles
+    +-------+
+    | /\_/\ |   <- Top part of the cat
+    +-------+
+    |( o.o )|   <- Middle part of the cat
+    +-------+
+    | > ^ < |   <- Bottom part of the cat
+    +-------+
+    |   0   |
+    +-------+
+    |   0   |
+    +-------+
+    |   0   |
+    +-------+
+
+    Given that the tiles you get depend on the chosen aspect ratio, you have to add
+    embedding in the modeling code to help it know if it got a 3x2 or a 1x6 or a 2x3
+    aspect ratio.
+
+    The function tests these arrangements to find the smallest canvas where the image fits.
+    If multiple canvases fit, it selects the one where the dimensions are closest to the image size.
+
+    In this case the first canvas is the closest to the original image.
+
+    You then feed all of the tiles to the model:
+
+        +-------+-------+-------+-------+-------+-------+
+    -   | /\_/\ |( o.o )| > ^ < |   0   |   0   |   0   |  <- Last canvas
+        +-------+-------+-------+-------+-------+-------+
+
+        +-------+-------+-------+-------+-------+-------+
+    -   | /\_/\ | 0     |( o.o )|   0   | > ^ < |   0   | <- First canvas
+        +-------+-------+-------+-------+-------+-------+
+
+        +-------+-------+-------+-------+-------+-------+
+    -   | /\_/\ |( o.o )|   0   | > ^ < |   0   |   0   | <- second canvas
+        +-------+-------+-------+-------+-------+-------+
+
+    For each tile, you have num_channels (usually RGB so 3), tile_width, tile_height
+
+    Args:
+        image_height (`int`):
+            The height of the image.
+        image_width (`int`):
+            The width of the image.
+        max_image_tiles (`int`):
+            The maximum number of tiles any image can be split into.
+        tile_size (`int`):
+            The tile size.
+
+    Returns:
+        `Tuple[int, int]`: The best canvas resolution [height, width] for the given image.
+    """
+    possible_tile_arrangements = get_all_supported_aspect_ratios(max_image_tiles)
+    possible_canvas_sizes = np.array(possible_tile_arrangements) * tile_size
+
+    # get all possible resolutions heights/widths
+    target_heights, target_widths = np.array(possible_canvas_sizes).T
+
+    # get scaling factors to resize the image without distortion
+    scale_h = target_heights / image_height
+    scale_w = target_widths / image_width
+
+    # get the min scale between width and height (limiting side -> no distortion)
+    scales = np.where(scale_w > scale_h, scale_h, scale_w)
+
+    # filter only scales that allow upscaling
+    upscaling_options = scales[scales >= 1]
+    if len(upscaling_options) > 0:
+        selected_scale = np.min(upscaling_options)
+    else:
+        # no upscaling possible,
+        # get the minimum downscaling (max scale for scales<1)
+        downscaling_options = scales[scales < 1]
+        selected_scale = np.max(downscaling_options)
+
+    # get all resolutions that support this scaling factor,
+    # e.g. you can upscale to 224x224, 224x448, 224x672 without distortion
+    chosen_canvas = possible_canvas_sizes[scales == selected_scale]
+
+    # if there are multiple resolutions,
+    # get the one with minimum area to reduce padding
+    if len(chosen_canvas) > 1:
+        areas = chosen_canvas[:, 0] * chosen_canvas[:, 1]
+        optimal_idx = np.argmin(areas)
+        optimal_canvas = chosen_canvas[optimal_idx]
+    else:
+        optimal_canvas = chosen_canvas[0]
+
+    return optimal_canvas
+
+
+def split_to_tiles(image: np.ndarray, num_tiles_height: int, num_tiles_width: int) -> np.ndarray:
+    """
+    Split an image into a specified number of tiles along its width and height dimensions.
+
+    Args:
+        image (`np.ndarray`):
+            Input image with shape (num_channels, height, width).
+        num_tiles_height (`int`):
+            Number of tiles to split the image into along its height.
+        num_tiles_width (`int`):
+            Number of tiles to split the image into along its width.
+
+    Returns:
+        `np.ndarray`:
+            Array of image tiles with shape (num_tiles_width * num_tiles_height, num_channels, tile_height, tile_width).
+    """
+    num_channels, height, width = image.shape
+    tile_height = height // num_tiles_height
+    tile_width = width // num_tiles_width
+
+    image = image.reshape(num_channels, num_tiles_height, tile_height, num_tiles_width, tile_width)
+
+    # Permute to (num_tiles_height, num_tiles_width, num_channels, tile_height, tile_width)
+    image = image.transpose(1, 3, 0, 2, 4)
+
+    # Reshape into the desired output shape (num_tiles_width * num_tiles_height, num_channels, tile_height, tile_width)
+    image = image.reshape(num_tiles_width * num_tiles_height, num_channels, tile_height, tile_width)
+
+    return np.ascontiguousarray(image)
+
+
+def build_aspect_ratio_mask(aspect_ratios: List[List[Tuple[int, int]]], max_image_tiles: int) -> np.ndarray:
+    """
+    Builds a mask for the aspect ratios of the images.
+
+    Args:
+        aspect_ratios (`List[List[Tuple[int, int]]]`):
+            A list of lists containing aspect ratios for each image in the batch.
+            Each aspect ratio is represented as a tuple of (width, height) in terms of number of tiles.
+        max_image_tiles (`int`):
+            The maximum number of tiles any image can be split into.
+
+    Returns:
+        `np.ndarray`: A 3D numpy array of shape (batch_size, max_num_images, max_image_tiles).
+            The mask contains 1s for valid tiles and 0s for padding.
+    """
+    batch_size = len(aspect_ratios)
+    max_num_images = max([len(row) for row in aspect_ratios])
+
+    aspect_ratio_mask = np.zeros((batch_size, max_num_images, max_image_tiles), dtype=np.int64)
+
+    # Set the first tile to 1 for all aspect ratios
+    # because in original implementation aspect ratios are padded with (1, 1),
+    # but original code examples are not built to handle batches, so we might remove it later
+    aspect_ratio_mask[:, :, 0] = 1
+
+    # Set the aspect ratio mask for the rest of the tiles
+    for i, sample_aspect_ratios in enumerate(aspect_ratios):
+        for j, (num_tiles_w, num_tiles_h) in enumerate(sample_aspect_ratios):
+            aspect_ratio_mask[i, j, : num_tiles_w * num_tiles_h] = 1
+
+    return aspect_ratio_mask
+
+
+def pack_images(
+    batch_images: List[List[np.ndarray]],
+    max_image_tiles: int,
+) -> Tuple[np.ndarray, List[List[int]]]:
+    """
+    Stack a list of lists of images with variable lengths into a numpy array, applying zero padding as needed.
+    Each list in the input represents a batch sample, and each image within a list is expected to be
+    pre-split into tiles. The resulting array will have a shape of
+    (batch_size, max_num_images, max_image_tiles, channels, tile_height, tile_width).
+
+    Args:
+        batch_images (`List[List[np.ndarray]]`):
+            A list of lists of image tiles. Each inner list represents
+            a batch sample containing multiple images, where each image is pre-split into tiles.
+            The shape of each tile array is (num_tiles, channels, tile_height, tile_width).
+        max_image_tiles (int):
+            The maximum number of tiles any image was potantially split.
+
+    Returns:
+        `Tuple[np.ndarray, List[List[int]]]`: A tuple containing:
+            - stacked_images (`np.ndarray`):
+                A numpy array of stacked images with shape
+                (batch_size, max_num_images, max_image_tiles, channels, tile_height, tile_width).
+            - all_num_tiles (`List[List[int]]`):
+                A list of lists containing the number of tiles
+                for each image in each batch sample.
+    """
+
+    # Determine output shape
+    batch_size = len(batch_images)
+    max_num_images = max([len(images) for images in batch_images])
+    shapes = [image.shape for images in batch_images for image in images]
+    _, channels, tile_height, tile_width = shapes[0]
+
+    # Initialize the stacked images array with zeros
+    stacked_images = np.zeros(
+        (batch_size, max_num_images, max_image_tiles, channels, tile_height, tile_width),
+        dtype=np.float32,
+    )
+
+    # Fill the stacked images array with the tiled images from the batch
+    all_num_tiles = []
+    for i, images in enumerate(batch_images):
+        num_sample_tiles = []
+        for j, image in enumerate(images):
+            num_tiles = image.shape[0]
+            stacked_images[i, j, :num_tiles] = image
+            num_sample_tiles.append(num_tiles)
+        all_num_tiles.append(num_sample_tiles)
+
+    return stacked_images, all_num_tiles
+
+
+def pack_aspect_ratios(aspect_ratios: List[List[Tuple[int, int]]], pad_value: int = 1) -> np.ndarray:
+    """
+    Stack a list of aspect ratios into a numpy array.
+
+    Args:
+        aspect_ratios (`List[List[Tuple[int, int]]]`):
+            A list of aspect ratios.
+        pad_value (`int`, *optional*, defaults to 1):
+            The value to pad the aspect ratios with.
+
+    Returns:
+        `np.ndarray`:
+            The aspect ratios stacked into a numpy array with shape (batch_size, max_num_images, 2).
+    """
+    batch_size = len(aspect_ratios)
+    max_num_images = max([len(row) for row in aspect_ratios])
+
+    aspect_ratios_stacked = np.full((batch_size, max_num_images, 2), pad_value, dtype=np.int64)
+    for i, row in enumerate(aspect_ratios):
+        if len(row) > 0:
+            aspect_ratios_stacked[i, : len(row)] = np.array(row)
+    return aspect_ratios_stacked
+
+
+def convert_aspect_ratios_to_ids(aspect_ratios: List[List[Tuple[int, int]]], max_image_tiles: int) -> np.ndarray:
+    """
+    Convert aspect ratio tuples to unique ids.
+
+    For batch padding we use 0, because there might be different number of images in each batch.
+    The aspect ratio ids start from 1, with 1 corresponding to the first supported aspect ratio.
+
+    Args:
+        aspect_ratios (`List[List[Tuple[int, int]]]`):
+            A list of aspect ratios for each image in the batch.
+        max_image_tiles (`int`):
+            The maximum number of tiles any image can be split into.
+
+    Returns:
+        `np.ndarray`:
+            The aspect ratios ids as a numpy array with shape (batch_size, max_num_images).
+            Each id corresponds to the index of the aspect ratio in the list of supported aspect ratios,
+            offset by 1 (so 0 can be used for padding).
+    """
+
+    batch_size = len(aspect_ratios)
+    max_num_images = max([len(row) for row in aspect_ratios])
+    supported_aspect_ratios = get_all_supported_aspect_ratios(max_image_tiles)
+
+    aspect_ratios_ids = np.zeros((batch_size, max_num_images), dtype=np.int64)
+    for i, sample_aspect_ratios in enumerate(aspect_ratios):
+        for j, (num_tiles_h, num_tiles_w) in enumerate(sample_aspect_ratios):
+            aspect_ratios_ids[i, j] = supported_aspect_ratios.index((num_tiles_h, num_tiles_w)) + 1
+    return aspect_ratios_ids
+
+
+def to_channel_dimension_format(
+    image: np.ndarray,
+    channel_dim: Union[ChannelDimension, str],
+    input_channel_dim: Optional[Union[ChannelDimension, str]] = None,
+) -> np.ndarray:
+    """
+    Converts `image` to the channel dimension format specified by `channel_dim`.
+
+    Args:
+        image (`numpy.ndarray`):
+            The image to have its channel dimension set.
+        channel_dim (`ChannelDimension`):
+            The channel dimension format to use.
+        input_channel_dim (`ChannelDimension`, *optional*):
+            The channel dimension format of the input image. If not provided, it will be inferred from the input image.
+
+    Returns:
+        `np.ndarray`:
+            The image with the channel dimension set to `channel_dim`.
+    """
+    if not isinstance(image, np.ndarray):
+        raise ValueError(f"Input image must be of type np.ndarray, got {type(image)}")
+
+    if input_channel_dim is None:
+        input_channel_dim = infer_channel_dimension_format(image)
+
+    target_channel_dim = ChannelDimension(channel_dim)
+    if input_channel_dim == target_channel_dim:
+        return image
+
+    if target_channel_dim == ChannelDimension.FIRST:
+        image = image.transpose((2, 0, 1))
+    elif target_channel_dim == ChannelDimension.LAST:
+        image = image.transpose((1, 2, 0))
+    else:
+        raise ValueError("Unsupported channel dimension format: {}".format(channel_dim))
+
+    return image
+
+
+# Copied from transformers.models.idefics2.image_processing_idefics2.convert_to_rgb
+def convert_to_rgb(image: ImageInput) -> ImageInput:
+    """
+    Converts an image to RGB format. Only converts if the image is of type PIL.Image.Image, otherwise returns the image
+    as is.
+    Args:
+        image (Image):
+            The image to convert.
+    """
+    if not isinstance(image, PIL.Image.Image):
+        return image
+
+    # `image.convert("RGB")` would only work for .jpg images, as it creates a wrong background
+    # for transparent images. The call to `alpha_composite` handles this case
+    if image.mode == "RGB":
+        return image
+
+    image_rgba = image.convert("RGBA")
+    background = Image.new("RGBA", image_rgba.size, (255, 255, 255))
+    alpha_composite = Image.alpha_composite(background, image_rgba)
+    alpha_composite = alpha_composite.convert("RGB")
+    return alpha_composite
+
+
+# Modified from transformers.models.idefics2.image_processing_idefics2.make_list_of_images
+def make_list_of_images(images: ImageInput) -> List[List[Optional[np.ndarray]]]:
+    """
+    Convert a single image or a list of images to a list of numpy arrays.
+
+    Args:
+        images (`ImageInput`):
+            A single image or a list of images.
+
+    Returns:
+        A list of numpy arrays.
+    """
+    # If it's a single image, convert it to a list of lists
+    if is_valid_image(images):
+        output_images = [[images]]
+    # If it's a list of images, it's a single batch, so convert it to a list of lists
+    elif isinstance(images, (list, tuple)) and is_valid_list_of_images(images):
+        output_images = [images]
+    # If it's a list of batches, it's already in the right format
+    elif (
+        isinstance(images, (list, tuple))
+        and all(isinstance(images_i, (list, tuple)) for images_i in images)
+        and any(is_valid_list_of_images(images_i) for images_i in images)
+    ):
+        output_images = images
+    else:
+        raise ValueError(
+            "Invalid input type. Must be a single image, a list of images, or a list of batches of images."
+        )
+    return output_images
+
+
+def is_valid_list_of_images(images: List):
+    return images and all(is_valid_image(image) for image in images)
+
+
+def _validate_size(size: Dict[str, int]) -> None:
+    if not ("height" in size and "width" in size):
+        raise ValueError(f"Argument `size` must be a dictionary with keys 'height' and 'width'. Got: {size}")
+    if size["height"] != size["width"]:
+        raise ValueError(f"Argument `size` must have the same height and width, got {size}")
+
+
+def _validate_mllama_preprocess_arguments(do_resize, size, do_pad, max_image_tiles):
+    if not do_pad:
+        raise ValueError("VideoMllamaImageProcessor doesn't support `do_pad=False` mode.")
+    if not do_resize:
+        raise ValueError("VideoMllamaImageProcessor doesn't support `do_resize=False` mode.")
+    if max_image_tiles is None or max_image_tiles <= 0:
+        raise ValueError(f"VideoMllamaImageProcessor `max_image_tiles` must be a positive integer, got {max_image_tiles}.")
+    _validate_size(size)
+
+
+class VideoMllamaImageProcessor(BaseImageProcessor):
+    """
+    Constructs a VideoMllama image processor.
+
+    Args:
+        do_convert_rgb (`bool`, *optional*, defaults to `True`):
+            Whether to convert the image to RGB. This is useful if the input image is of a different format e.g. RGBA.
+            Only has an effect if the input image is in the PIL format.
+        do_resize (`bool`, *optional*, defaults to `True`):
+            Whether to resize the image.
+        size (`Dict[str, int]`, *optional*, defaults to `self.size`):
+            Size of the image tile. Should be a dictionary containing 'height' and 'width' keys, both with integer values.
+            The height and width values should be equal.
+        resample (`int`, *optional*, defaults to `Resampling.BILINEAR`):
+            Resampling filter to use if resizing the image. This can be one of the enum `PILImageResampling`. Only
+            has an effect if `do_resize` is set to `True`.
+        do_rescale (`bool`, *optional*, defaults to `True`):
+            Whether to rescale the image.
+        rescale_factor (`float`, *optional*, defaults to 0.0):
+            Rescale factor to rescale the image by if `do_rescale` is set to `True`.
+        do_normalize (`bool`, *optional*, defaults to `True`):
+            Whether to normalize the image.
+        image_mean (`float` or `List[float]`, *optional*, defaults to `self.image_mean`):
+            Image mean to use for normalization. Only has an effect if `do_normalize` is set to `True`.
+        image_std (`float` or `List[float]`, *optional*, defaults to `self.image_std`):
+            Image standard deviation to use for normalization. Only has an effect if `do_normalize` is set to
+            `True`.
+        do_pad (`bool`, *optional*, defaults to `True`):
+            Whether or not to pad the images to the largest height and width in the batch.
+        max_image_tiles (`int`, *optional*, defaults to 4):
+            The maximum number of tiles to split the image into.
+    """
+
+    model_input_names = ["pixel_values", "num_tiles", "aspect_ratio_ids", "aspect_ratio_mask"]
+
+    def __init__(
+        self,
+        do_convert_rgb: bool = True,
+        do_resize: bool = True,
+        size: Optional[Dict[str, int]] = None,
+        resample: PILImageResampling = PILImageResampling.BILINEAR,
+        do_rescale: bool = True,
+        rescale_factor: float = 1 / 255,
+        do_normalize: bool = True,
+        image_mean: Optional[Union[float, List[float]]] = None,
+        image_std: Optional[Union[float, List[float]]] = None,
+        do_pad: bool = True,
+        max_image_tiles: int = 4,
+        **kwargs,
+    ) -> None:
+        super().__init__(**kwargs)
+        self.do_convert_rgb = do_convert_rgb
+        self.do_resize = do_resize
+        self.size = size if size is not None else {"height": 224, "width": 224}
+        self.resample = resample
+        self.do_rescale = do_rescale
+        self.rescale_factor = rescale_factor
+        self.do_normalize = do_normalize
+        self.image_mean = image_mean if image_mean is not None else IMAGENET_STANDARD_MEAN
+        self.image_std = image_std if image_std is not None else IMAGENET_STANDARD_STD
+        self.do_pad = do_pad
+        self.max_image_tiles = max_image_tiles
+
+        _validate_mllama_preprocess_arguments(self.do_resize, self.size, self.do_pad, self.max_image_tiles)
+
+    def preprocess(
+        self,
+        images: ImageInput,
+        do_convert_rgb: Optional[bool] = None,
+        do_resize: Optional[bool] = None,
+        size: Optional[Dict[str, int]] = None,
+        resample: Optional[PILImageResampling] = None,
+        do_rescale: Optional[bool] = None,
+        rescale_factor: Optional[float] = None,
+        do_normalize: Optional[bool] = None,
+        image_mean: Optional[Union[float, List[float]]] = None,
+        image_std: Optional[Union[float, List[float]]] = None,
+        do_pad: Optional[bool] = None,
+        max_image_tiles: Optional[int] = None,
+        input_data_format: Optional[Union[str, ChannelDimension]] = None,
+        return_tensors: Optional[Union[str, TensorType]] = None,
+    ):
+        """
+        Preprocess a batch of images.
+
+        Args:
+            images (`ImageInput`):
+                A list of images to preprocess.
+            do_convert_rgb (`bool`, *optional*, defaults to `self.do_convert_rgb`):
+                Whether to convert the image to RGB.
+            do_resize (`bool`, *optional*, defaults to `self.do_resize`):
+                Whether to resize the image.
+            size (`Dict[str, int]`, *optional*, defaults to `self.size`):
+                Size of the image tile. Should be a dictionary containing 'height' and 'width' keys, both with integer values.
+                The height and width values should be equal.
+            resample (`int`, *optional*, defaults to `self.resample`):
+                Resampling filter to use if resizing the image. This can be one of the enum `PILImageResampling`. Only
+                has an effect if `do_resize` is set to `True`.
+            do_rescale (`bool`, *optional*, defaults to `self.do_rescale`):
+                Whether to rescale the image.
+            rescale_factor (`float`, *optional*, defaults to `self.rescale_factor`):
+                Rescale factor to rescale the image by if `do_rescale` is set to `True`.
+            do_normalize (`bool`, *optional*, defaults to `self.do_normalize`):
+                Whether to normalize the image.
+            image_mean (`float` or `List[float]`, *optional*, defaults to `self.image_mean`):
+                Image mean to use for normalization. Only has an effect if `do_normalize` is set to `True`.
+            image_std (`float` or `List[float]`, *optional*, defaults to `self.image_std`):
+                Image standard deviation to use for normalization. Only has an effect if `do_normalize` is set to
+                `True`.
+            do_pad (`bool`, *optional*, defaults to `self.do_pad`):
+                Whether or not to pad the images to the largest height and width in the batch.
+            max_image_tiles (`int`, *optional*, defaults to `self.max_image_tiles`):
+                The maximum number of tiles to split the image into.
+            input_data_format (`ChannelDimension` or `str`, *optional*):
+                The channel dimension format for the input image. If unset, the channel dimension format is inferred
+                from the input image. Can be one of:
+                - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format.
+                - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format.
+                - `"none"` or `ChannelDimension.NONE`: image in (height, width) format.
+            return_tensors (`str` or `TensorType`, *optional*):
+                The type of tensors to return. Can be one of:
+                - Unset: Return a list of `np.ndarray`.
+                - `TensorType.TENSORFLOW` or `'tf'`: Return a batch of type `tf.Tensor`.
+                - `TensorType.PYTORCH` or `'pt'`: Return a batch of type `torch.Tensor`.
+                - `TensorType.NUMPY` or `'np'`: Return a batch of type `np.ndarray`.
+                - `TensorType.JAX` or `'jax'`: Return a batch of type `jax.numpy.ndarray`.
+
+        Returns:
+            `BatchFeature` of the following structure:
+                - **pixel_values** (`TensorType`): The preprocessed pixel values.
+                - **aspect_ratio_ids** (`TensorType`): The aspect ratio ids of the images.
+                - **num_tiles** (`List[List[int]]`): The number of tiles for each image in the batch.
+        """
+        do_convert_rgb = do_convert_rgb if do_convert_rgb is not None else self.do_convert_rgb
+        do_resize = do_resize if do_resize is not None else self.do_resize
+        size = size if size is not None else self.size
+        resample = resample if resample is not None else self.resample
+        do_rescale = do_rescale if do_rescale is not None else self.do_rescale
+        rescale_factor = rescale_factor if rescale_factor is not None else self.rescale_factor
+        do_normalize = do_normalize if do_normalize is not None else self.do_normalize
+        image_mean = image_mean if image_mean is not None else self.image_mean
+        image_std = image_std if image_std is not None else self.image_std
+        do_pad = do_pad if do_pad is not None else self.do_pad
+        max_image_tiles = max_image_tiles if max_image_tiles is not None else self.max_image_tiles
+
+        validate_preprocess_arguments(
+            do_rescale=do_rescale,
+            rescale_factor=rescale_factor,
+            do_normalize=do_normalize,
+            image_mean=image_mean,
+            image_std=image_std,
+            do_resize=do_resize,
+            size=size,
+            resample=resample,
+        )
+
+        # extra validation
+        _validate_mllama_preprocess_arguments(do_resize, size, do_pad, max_image_tiles)
+
+        images_list = make_list_of_images(images)
+
+        if self.do_convert_rgb:
+            images_list = [[convert_to_rgb(image) for image in images] for images in images_list]
+
+        images_list = [[to_numpy_array(image) for image in images] for images in images_list]
+
+        batch_images = []
+        batch_aspect_ratios = []
+
+        # iterate over batch samples
+        for images in images_list:
+            sample_images = []
+            sample_aspect_ratios = []
+
+            # iterate over images in a batch sample
+            for image in images:
+                # convert images to channels first format for faster processing
+                # LAST is slower for `pad` and not supported by `split_to_tiles`
+                data_format = ChannelDimension.FIRST
+                image = to_channel_dimension_format(image, data_format, input_channel_dim=input_data_format)
+
+                # do_resize=False is not supported, validated
+                image, aspect_ratio = self.resize(
+                    image=image,
+                    size=size,
+                    resample=resample,
+                    max_image_tiles=max_image_tiles,
+                    input_data_format=data_format,
+                    data_format=data_format,
+                )
+
+                # do_pad=False is not supported, validated
+                image = self.pad(
+                    image=image,
+                    size=size,
+                    aspect_ratio=aspect_ratio,
+                    input_data_format=data_format,
+                    data_format=data_format,
+                )
+
+                if do_rescale:
+                    image = self.rescale(
+                        image=image,
+                        scale=rescale_factor,
+                        input_data_format=input_data_format,
+                        data_format=data_format,
+                    )
+
+                if do_normalize:
+                    image = self.normalize(
+                        image=image,
+                        mean=image_mean,
+                        std=image_std,
+                        input_data_format=input_data_format,
+                        data_format=data_format,
+                    )
+
+                num_tiles_height, num_tiles_width = aspect_ratio
+                image = split_to_tiles(image, num_tiles_height, num_tiles_width)
+
+                sample_images.append(image)
+                sample_aspect_ratios.append((num_tiles_height, num_tiles_width))
+
+            batch_images.append(sample_images)
+            batch_aspect_ratios.append(sample_aspect_ratios)
+
+        images, num_tiles = pack_images(batch_images, max_image_tiles)
+
+        aspect_ratio_ids = convert_aspect_ratios_to_ids(batch_aspect_ratios, max_image_tiles=max_image_tiles)
+        aspect_ratio_mask = build_aspect_ratio_mask(batch_aspect_ratios, max_image_tiles=max_image_tiles)
+
+        # images (np.ndarray) with shape (batch_size, max_num_images, max_image_tiles, channels, tile_height, tile_width)
+        # aspect_ratio_ids (np.ndarray) with shape (batch_size, max_num_images) - aspect ratio ids for each image, padded to max_num_images with 0
+        # num_tiles (List[List[int]]) with (batch_size, num_images_in_batch) - real number of tiles for each image, not padded
+        # aspect_ratio_mask (np.ndarray) with shape (batch_size, max_num_images, max_image_tiles) - number of tiles for each image, padded to max_num_images with 0
+        encoded_inputs = BatchFeature(
+            data={
+                "pixel_values": images,
+                "aspect_ratio_ids": aspect_ratio_ids,
+                "aspect_ratio_mask": aspect_ratio_mask,
+            },
+            tensor_type=return_tensors,
+        )
+        encoded_inputs["num_tiles"] = num_tiles
+
+        return encoded_inputs
+
+    def pad(
+        self,
+        image: np.ndarray,
+        size: Dict[str, int],
+        aspect_ratio: Tuple[int, int],
+        data_format: Optional[Union[str, ChannelDimension]] = None,
+        input_data_format: Optional[Union[str, ChannelDimension]] = None,
+    ) -> np.ndarray:
+        """
+        Pad an image to the `size` x `aspect_ratio`. For example, if size is {height: 224, width: 224} and aspect ratio is
+        (1, 2), the image will be padded to 224x448.
+
+        Args:
+            image (`np.ndarray`):
+                Image to resize.
+            size (`Dict[str, int]`):
+                Size of the output image.
+            aspect_ratio (`Tuple[int, int]`):
+                The aspect ratio of the image.
+            data_format (`str` or `ChannelDimension`, *optional*):
+                The channel dimension format of the image. If not provided, it will be the same as the input image.
+            input_data_format (`ChannelDimension` or `str`, *optional*):
+                The channel dimension format of the input image. If not provided, it will be inferred.
+
+        Returns:
+            `np.ndarray`: The padded image.
+        """
+
+        _validate_size(size)
+
+        image_height, image_width = get_image_size(image, channel_dim=input_data_format)
+        num_tiles_height, num_tiles_width = aspect_ratio
+        padded_height = num_tiles_height * size["height"]
+        padded_width = num_tiles_width * size["width"]
+        pad_size = ((0, padded_height - image_height), (0, padded_width - image_width))
+
+        image = pad(
+            image,
+            pad_size,
+            mode=PaddingMode.CONSTANT,
+            constant_values=0,
+            data_format=data_format,
+            input_data_format=input_data_format,
+        )
+
+        return image
+
+    def resize(
+        self,
+        image: np.ndarray,
+        size: Dict[str, int],
+        max_image_tiles: int,
+        resample: PILImageResampling = PILImageResampling.BILINEAR,
+        data_format: Optional[Union[str, ChannelDimension]] = None,
+        input_data_format: Optional[Union[str, ChannelDimension]] = None,
+    ) -> Union[np.ndarray, Tuple[int, int]]:
+        """
+        Resizes an image to fit within a tiled canvas while maintaining its aspect ratio.
+        The optimal canvas size is calculated based on the maximum number of tiles and the tile size.
+
+        The function first determines the best tile arrangement for the image, then resizes the image
+        to fit within this canvas. The resized image and the number of tiles along the height and width
+        dimensions are returned.
+
+        Args:
+            image (`np.ndarray`):
+                Image to resize.
+            size (`Dict[str, int]`):
+                Size of the output image.
+            max_image_tiles (`int`):
+                The maximum number of tiles to split the image into.
+            resample (`PILImageResampling`, *optional*, defaults to `PILImageResampling.BICUBIC`):
+                Resampling filter to use when resizing the image.
+            data_format (`str` or `ChannelDimension`, *optional*):
+                The channel dimension format of the image. If not provided, it will be the same as the input image.
+            input_data_format (`ChannelDimension` or `str`, *optional*):
+                The channel dimension format of the input image. If not provided, it will be inferred.
+
+        Returns:
+            `Union[np.ndarray, Tuple[int, int]]`: The resized image and a tuple containing the number of tiles
+            along the height and width dimensions.
+        """
+
+        _validate_size(size)
+
+        image_height, image_width = get_image_size(image, channel_dim=input_data_format)
+        tile_size = size["height"]
+
+        canvas_height, canvas_width = get_optimal_tiled_canvas(
+            image_height=image_height,
+            image_width=image_width,
+            max_image_tiles=max_image_tiles,
+            tile_size=tile_size,
+        )
+        num_tiles_height = canvas_height // tile_size
+        num_tiles_width = canvas_width // tile_size
+
+        new_height, new_width = get_image_size_fit_to_canvas(
+            image_height=image_height,
+            image_width=image_width,
+            canvas_height=canvas_height,
+            canvas_width=canvas_width,
+            tile_size=tile_size,
+        )
+
+        image = resize(
+            image,
+            (new_height, new_width),
+            resample=resample,
+            data_format=data_format,
+            input_data_format=input_data_format,
+        )
+
+        return image, (num_tiles_height, num_tiles_width)

processing_video_mllama.py

@@ -0,0 +1,836 @@
+# coding=utf-8
+# Copyright 2024 The HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Processor class for VideoMllama."""
+
+import av
+import cv2
+import math
+import numpy as np
+import concurrent.futures
+
+from PIL import Image
+from typing import List, Optional, Union, Tuple
+
+from transformers.feature_extraction_utils import BatchFeature
+from transformers.image_utils import ImageInput, to_numpy_array
+from transformers.processing_utils import ImagesKwargs, ProcessingKwargs, ProcessorMixin, Unpack
+from transformers.tokenization_utils_base import (
+    PreTokenizedInput,
+    TextInput,
+)
+
+from .image_processing_video_mllama import make_list_of_images
+
+
+
+
+
+class VideoMllamaImagesKwargs(ImagesKwargs, total=False):
+    max_image_tiles: Optional[int]
+
+
+class VideoMllamaProcessorKwargs(ProcessingKwargs, total=False):
+    images_kwargs: VideoMllamaImagesKwargs
+    add_video_position_encoding: Optional[bool]
+
+    _defaults = {
+        "image_kwargs": {
+            "max_image_tiles": 1,
+        },
+        "add_video_position_encoding": True,
+    }
+
+
+# --- Start of new video sampling functions (adapted from streaming/mm_plugin.py) ---
+
+
+def validate_frame_sampling(sample_indices, frames, max_missing_frames=2, max_missing_ratio=0.1):
+    """
+    Validate the completeness of sampled frames.
+    """
+    expected_count = len(sample_indices)
+    actual_count = len(frames)
+    missing_count = expected_count - actual_count
+
+    if missing_count <= 0:
+        return
+
+    missing_ratio = missing_count / expected_count
+
+    if missing_count > max_missing_frames and missing_ratio > max_missing_ratio:
+        raise ValueError(
+            f"Too many frames missing: {missing_count}/{expected_count} "
+            f"({missing_ratio:.1%}) frames missing, exceeding "
+            f"{max_missing_ratio:.0%} threshold."
+        )
+
+
+def _get_video_sample_frames(video_stream, total_frames: int = 0, **kwargs) -> np.ndarray:
+    """
+    Core logic to compute video sample frame indices.
+    """
+    video_fps: float = kwargs.get("video_fps", 1.0)
+    video_minlen: int = kwargs.get("video_minlen", 8)
+    video_maxlen: int = kwargs.get("video_maxlen", 256)
+
+    obtained_total_frames = int(video_stream.frames)
+
+    duration = float(video_stream.duration * video_stream.time_base)
+    frame_rate = float(video_stream.average_rate)
+    calculated_total_frames = round(duration * frame_rate)
+    assert video_fps <= frame_rate, f"Sampling frequency ({video_fps}) must be less than or equal to video frame rate ({frame_rate})"
+
+    total_frames_num = [x for x in [total_frames, obtained_total_frames, calculated_total_frames] if x > 0]
+    final_total_frames = min(total_frames_num) if total_frames_num else 0
+    if final_total_frames == 0:
+        raise AttributeError("Unable to obtain or calculate the total number of frames in the video.")
+
+    target_total_frames = int(math.ceil(duration * video_fps - 1e-6))
+    sample_frames = max(target_total_frames, video_minlen)
+    sample_frames = min(sample_frames, video_maxlen, final_total_frames)
+
+    if target_total_frames == sample_frames and video_fps > 0 and frame_rate > 0:
+        sample_indices = np.arange(target_total_frames, dtype=np.int32)
+        sample_indices = (sample_indices * frame_rate / video_fps).astype(np.int32)
+    else:
+        sample_indices = np.linspace(0, final_total_frames - 1, sample_frames).astype(np.int32)
+
+    return sample_indices
+
+
+def _get_cv2_video_sample_frames(video_path: str, total_frames: int = 0, **kwargs) -> np.ndarray:
+    container = av.open(video_path, "r")
+    video_stream = next(stream for stream in container.streams if stream.type == "video")
+    sample_indices = _get_video_sample_frames(video_stream, total_frames=total_frames, **kwargs)
+    return sample_indices
+
+
+def get_video_sample_frames_av(video_path: str, **kwargs) -> List[Image.Image]:
+    container = av.open(video_path, "r")
+    video_stream = next(stream for stream in container.streams if stream.type == "video")
+
+    sample_indices = _get_video_sample_frames(video_stream, **kwargs)
+    sample_indices_set = set(sample_indices)
+
+    frames: List[Image.Image] = []
+
+    container.seek(0)
+    for frame_idx, frame in enumerate(container.decode(video_stream)):
+        if frame_idx in sample_indices_set:
+            frames.append(frame.to_image())
+        if len(frames) == len(sample_indices):
+            break
+
+    validate_frame_sampling(sample_indices, frames)
+    
+    return frames
+
+
+def get_cv2_video_sample_frames_multithread(video_path: str, **kwargs) -> List[Image.Image]:
+    num_threads: int = kwargs.get("frame_extract_num_threads", 4)
+    num_threads = int(num_threads)
+
+    cap = cv2.VideoCapture(video_path)
+    if not cap.isOpened():
+        raise ValueError(f"Unable to open video file: {video_path}")
+
+    total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
+    cap.release()
+
+    frame_indices = _get_cv2_video_sample_frames(video_path, total_frames=total_frames, **kwargs)
+
+    unique_frames: List[Optional[np.ndarray]] = [None] * len(frame_indices)
+    index_map = {idx: pos for pos, idx in enumerate(frame_indices)}
+
+    chunks = np.array_split(frame_indices, min(num_threads, len(frame_indices)))
+
+    def worker(chunk_indices):
+        local_cap = cv2.VideoCapture(video_path)
+        if not local_cap.isOpened():
+            return
+
+        if chunk_indices[0] > 0:
+            local_cap.set(cv2.CAP_PROP_POS_FRAMES, chunk_indices[0])
+
+        frame_idx_cursor = chunk_indices[0]
+        chunk_cursor = 0
+
+        while chunk_cursor < len(chunk_indices):
+            target_idx = chunk_indices[chunk_cursor]
+            ok = local_cap.grab()
+            if not ok:
+                break
+
+            if frame_idx_cursor == target_idx:
+                ret, frame = local_cap.retrieve()
+                if ret:
+                    unique_pos = index_map[target_idx]
+                    unique_frames[unique_pos] = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
+                chunk_cursor += 1
+            frame_idx_cursor += 1
+        local_cap.release()
+
+    with concurrent.futures.ThreadPoolExecutor(max_workers=num_threads) as executor:
+        list(executor.map(worker, [chunk for chunk in chunks if len(chunk) > 0]))
+
+    pil_frames = [Image.fromarray(frame) for frame in unique_frames if frame is not None]
+
+    validate_frame_sampling(frame_indices, pil_frames)
+
+    if not pil_frames:
+        return get_video_sample_frames_av(video_path, **kwargs)
+
+    return pil_frames
+
+
+# --- End of new video sampling functions ---
+
+
+def get_cross_attention_token_mask(
+    input_ids: List[int],
+    attention_mask: List[int],
+    image_token_id: int,
+    video_token_id: int,
+    frame_num_per_video: List[int],
+    cross_attention_token_mask_pad_token_id: int = -100,
+) -> Tuple[List[int], List[int], List[int]]:
+    """
+    Generate a cross-attention-token-mask for each input_tokens in the input sequence.
+    This function implements a causal attention logic:
+    - A text token can see all image tokens that appeared before it.
+    - An image token can see itself and all image tokens that appeared before it.
+    """
+    # 1. Convert video tokens to image tokens
+    input_ids_np = np.array(input_ids, dtype=np.int64)
+    if video_token_id in input_ids_np:
+        total_vid_num = np.sum(input_ids_np == video_token_id)
+        f_num_per_vid = frame_num_per_video[:total_vid_num]
+
+        convert_input_ids_list = []
+        convert_attention_mask_list = []
+        vid_idx = 0
+        for token_id, mask_val in zip(input_ids_np, attention_mask):
+            if token_id == video_token_id:
+                vid_len = f_num_per_vid[vid_idx]
+                vid_idx += 1
+                convert_input_ids_list.extend([image_token_id] * vid_len)
+                convert_attention_mask_list.extend([mask_val] * vid_len)
+            else:
+                convert_input_ids_list.append(token_id)
+                convert_attention_mask_list.append(mask_val)
+        convert_input_ids = np.array(convert_input_ids_list, dtype=np.int64)
+        convert_attention_mask = np.array(convert_attention_mask_list, dtype=np.int64)
+    else:
+        convert_input_ids = input_ids_np
+        convert_attention_mask = np.array(attention_mask, dtype=np.int64)
+
+    # 2. Generate the sparse attention mask based on causal visibility
+    is_image = convert_input_ids == image_token_id
+    # Cumulative count of images up to and including the current position
+    image_count_cumulative = np.cumsum(is_image)
+    # Cumulative count of images up to the previous position
+    image_count_before = np.pad(image_count_cumulative[:-1], (1, 0), "constant", constant_values=0)
+
+    # For text tokens, num_seen = image_count_before.
+    # For image tokens, num_seen = image_count_cumulative (sees itself).
+    num_images_seen = np.where(is_image, image_count_cumulative, image_count_before)
+
+    # Convert num_seen to sparse mask value (num_seen - 1).
+    vision_masks = np.full(len(convert_input_ids), cross_attention_token_mask_pad_token_id, dtype=np.int64)
+    valid_mask = num_images_seen > 0
+    vision_masks[valid_mask] = num_images_seen[valid_mask] - 1
+
+    return vision_masks.tolist(), convert_input_ids.tolist(), convert_attention_mask.tolist()
+
+
+def convert_sparse_cross_attention_mask_to_dense(
+    cross_attention_token_masks: np.ndarray,
+    num_tiles: List[List[int]],
+    max_num_tiles: int,
+    cross_attention_token_mask_pad_token_id: int = -100,
+) -> np.ndarray:
+    """
+    Convert the cross attention mask indices to a cross attention mask 4D array.
+
+    This function takes a sparse representation of cross attention masks and converts it to a dense 4D numpy array.
+    The sparse representation is a tensor that defines [the range of images that can be seen] for [each input token].
+    """
+    batch_size, length = cross_attention_token_masks.shape
+    max_num_images = max([len(n_tiles) for n_tiles in num_tiles]) if num_tiles else 0
+
+    cross_attention_mask = np.zeros(
+        shape=(batch_size, length, max_num_images, max_num_tiles),
+        dtype=np.int64,
+    )
+
+    if max_num_images == 0:
+        return cross_attention_mask
+
+    for batch_idx, (sparse_mask, n_tiles) in enumerate(zip(cross_attention_token_masks, num_tiles)):
+        # For each image, find all text tokens that are allowed to see it.
+        # A token with sparse_mask value N can see all images with index i <= N.
+        for image_idx, mask_n_tiles in enumerate(n_tiles):
+            # Find all token positions where the sparse mask value is >= the current image's index.
+            # This correctly implements the causal logic.
+            visible_token_indices = (sparse_mask >= image_idx) & (sparse_mask != cross_attention_token_mask_pad_token_id)
+            # Set the attention mask to 1 for these tokens and the current image.
+            cross_attention_mask[batch_idx, visible_token_indices, image_idx, :mask_n_tiles] = 1
+
+    return cross_attention_mask
+
+
+def build_string_from_input(prompt: str, bos_token: str, image_token: str, video_token: str) -> str:
+    """
+    Builds a string from the input prompt by adding `bos_token` if not already present.
+    It handles prompts starting with image or video tokens.
+    """
+
+    if bos_token in prompt:
+        return prompt
+
+    num_media_tokens_on_start = 0
+    media_tokens = []
+    
+    while prompt.startswith(image_token) or prompt.startswith(video_token):
+        if prompt.startswith(image_token):
+            prompt = prompt[len(image_token) :]
+            media_tokens.append(image_token)
+        elif prompt.startswith(video_token):
+            prompt = prompt[len(video_token) :]
+            media_tokens.append(video_token)
+        num_media_tokens_on_start += 1
+    
+    print(f"No bos_token `{bos_token}` in prompt, so it is added after the {num_media_tokens_on_start} media tokens at the start of the prompt.")
+
+    return f"{''.join(media_tokens)}{bos_token}{prompt}"
+
+
+VIDEO_MLLAMA_PROCESSOR_PAD_POSITION_ID = 0
+VIDEO_MLLAMA_PROCESSOR_CROSS_ATTENTION_TOKEN_MASK_PAD_TOKEN_ID = -100
+
+class VideoMllamaProcessor(ProcessorMixin):
+    r"""
+    Constructs a VideoMllama processor which wraps [`VideoMllamaImageProcessor`] and
+    [`PretrainedTokenizerFast`] into a single processor that inherits both the image processor and
+    tokenizer functionalities. See the [`~VideoMllamaProcessor.__call__`] and [`~OwlViTProcessor.decode`] for more
+    information.
+    The preferred way of passing kwargs is as a dictionary per modality, see usage example below.
+        ```python
+        from transformers import VideoMllamaProcessor
+        from PIL import Image
+
+        processor = VideoMllamaProcessor.from_pretrained("meta-llama/Llama-3.2-11B-Vision")
+
+        processor(
+            images=your_pil_image,
+            text=["<|image|>If I had to write a haiku for this one"],
+            images_kwargs = {"size": {"height": 448, "width": 448}},
+            text_kwargs = {"padding": "right"},
+            common_kwargs = {"return_tensors": "pt"},
+        )
+        ```
+
+    Args:
+        image_processor ([`VideoMllamaImageProcessor`]):
+            The image processor is a required input.
+        tokenizer ([`PreTrainedTokenizer`, `PreTrainedTokenizerFast`]):
+            The tokenizer is a required input.
+
+    """
+
+    attributes = ["image_processor", "tokenizer"]
+    image_processor_class = "AutoImageProcessor"
+    tokenizer_class = "PreTrainedTokenizerFast"
+
+    _defaults = {
+        "image_kwargs": {
+            "max_image_tiles": 1,
+        },
+        "add_video_position_encoding": True,
+    }
+
+    def __init__(self, image_processor, tokenizer, video_fps = None, video_minlen = None, video_maxlen = None,frame_extract_num_threads = None, extract_frame_func = None, max_image_tiles: Optional[int] = None, **kwargs):
+        # User-facing placeholders
+        self.image_placeholder = "<image>"
+        self.video_placeholder = "<video>"
+        self.tokenizer = tokenizer
+
+        super().__init__(image_processor, tokenizer)
+        if max_image_tiles is not None:
+            self.image_processor.max_image_tiles = max_image_tiles
+            
+        if not hasattr(self.tokenizer, "image_token"):
+            self.image_token = "<|image|>"
+            self.image_token_id = self.tokenizer.convert_tokens_to_ids(self.image_token)
+        else:
+            self.image_token = self.tokenizer.image_token
+            self.image_token_id = self.tokenizer.image_token_id
+
+        if not hasattr(self.tokenizer, "video_token"):
+            self.video_token = "<|video|>"
+            self.video_token_id = self.tokenizer.convert_tokens_to_ids(self.video_token)
+        else:
+            self.video_token = self.tokenizer.video_token
+            self.video_token_id = self.tokenizer.video_token_id
+
+        self.add_video_position_encoding = self._defaults["add_video_position_encoding"]
+        
+        self.pad_position_id = VIDEO_MLLAMA_PROCESSOR_PAD_POSITION_ID
+        self.cross_attention_token_mask_pad_token_id = VIDEO_MLLAMA_PROCESSOR_CROSS_ATTENTION_TOKEN_MASK_PAD_TOKEN_ID
+
+        # New configuration for video frame sampling
+        if video_fps is None:
+            self.video_fps = getattr(image_processor, "video_fps", 1.0)
+        if video_minlen is None:
+            self.video_minlen =  getattr(image_processor, "video_minlen", 8)
+        if video_maxlen is None:
+            self.video_maxlen = getattr(image_processor, "video_maxlen", 256)
+        if frame_extract_num_threads is None:
+            self.frame_extract_num_threads = getattr(image_processor, "frame_extract_num_threads", 4)
+        if extract_frame_func is None:
+            self.extract_frame_func = getattr(image_processor, "extract_frame_func", "cv2")  # Options: "av", "cv2"
+
+        self.bos_token = self.tokenizer.bos_token
+        self.chat_template = self.tokenizer.chat_template
+
+    def _pad_sequences(
+        self,
+        sequences: List[List[int]],
+        pad_value: int,
+        max_length: Optional[int] = None,
+        padding_side: str = "right",
+        pad_to_multiple_of: Optional[int] = None,
+    ) -> np.ndarray:
+        if not sequences:
+            return np.array([], dtype=np.int64)
+
+        if max_length is None:
+            max_length = max(len(seq) for seq in sequences)
+
+        if pad_to_multiple_of is not None and (max_length % pad_to_multiple_of != 0):
+            max_length = ((max_length // pad_to_multiple_of) + 1) * pad_to_multiple_of
+
+        padded_sequences = np.full((len(sequences), max_length), pad_value, dtype=np.int64)
+        
+        for i, seq in enumerate(sequences):
+            length = len(seq)
+            if padding_side == "right":
+                padded_sequences[i, :length] = seq
+            else:  # left
+                padded_sequences[i, -length:] = seq
+        return padded_sequences
+
+
+    def __call__(
+        self,
+        text: Optional[Union[TextInput, PreTokenizedInput, List[TextInput], List[PreTokenizedInput]]] = None,
+        images: Optional[Union[ImageInput, str, List[str], List[List[str]]]] = None,
+        videos: Optional[Union[str, List[str], List[List[str]]]] = None,
+        **kwargs: Unpack[VideoMllamaProcessorKwargs],
+    ) -> BatchFeature:
+        """
+        Main method to prepare text(s), image(s) and video(s) to be fed as input to the model. This method forwards the `text`
+        arguments to PreTrainedTokenizerFast's [`~PreTrainedTokenizerFast.__call__`] if `text` is not `None` to encode
+        the text. To prepare the image(s), this method forwards the `images` arguments to
+        VideoMllamaImageProcessor's [`~VideoMllamaImageProcessor.__call__`] if `images` is not `None`. Videos are first
+        processed into frames and then handled as images.
+
+        Args:
+            text (`str`, `List[str]`, `List[List[str]]`):
+                The sequence or batch of sequences to be encoded. Each sequence can be a string or a list of strings
+                (pretokenized string). If the sequences are provided as list of strings (pretokenized), you must set
+                `is_split_into_words=True` (to lift the ambiguity with a batch of sequences).
+            images (`str`, `PIL.Image.Image`, `np.ndarray`, `torch.Tensor`, `List[str]`, `List[PIL.Image.Image]`, `List[np.ndarray]`, `List[torch.Tensor]`):
+                The image or batch of images to be prepared. Can be a single image path, a single PIL image, or a
+                list of paths or PIL images. When a list of single images is passed, it's treated as a batch of samples.
+            videos (`str`, `List[str]`, `List[List[str]]`):
+                 The video or batch of videos to be prepared. Can be a single video path, a list of video paths (which
+                 is treated as a batch), or a list of lists of video paths (already batched).
+            return_tensors (`str` or [`~utils.TensorType`], *optional*):
+                If set, will return tensors of a particular framework. Acceptable values are:
+                    - `'tf'`: Return TensorFlow `tf.constant` objects.
+                    - `'pt'`: Return PyTorch `torch.Tensor` objects.
+                    - `'np'`: Return NumPy `np.ndarray` objects.
+                    - `'jax'`: Return JAX `jnp.ndarray` objects.
+        Returns:
+            [`BatchFeature`]: A [`BatchFeature`] with the following fields:
+
+            - **input_ids** -- List of token ids to be fed to a model. Returned when `text` is not `None`.
+            - **attention_mask** -- List of indices specifying which tokens should be attended to by the model (when
+              `return_attention_mask=True` or if *"attention_mask"* is in `self.model_input_names` and if `text` is not
+              `None`).
+            - **pixel_values** -- Pixel values to be fed to a model. Returned when `images` is not `None`.
+        """
+        # Step 0: Sanity checks
+        # Step 0.1: Convert empty lists to None
+        if images is not None and isinstance(images, list) and len(images) == 0:
+            images = None
+        if videos is not None and isinstance(videos, list) and len(videos) == 0:
+            videos = None
+            
+        # Step 0.2: If no text, images, or videos are provided, raise an error
+        if text is None and images is None and videos is None:
+            raise ValueError("You have to specify either `text`, `images` or `videos`.")
+        # Step 0.3: If no text is provided, at least one of images or videos must be present
+        if text is None and not (images or videos):
+            raise ValueError("If no text is provided, at least one of images or videos must be present.")
+
+        # Step 1: Pop video-specific kwargs to allow overriding class-level settings
+        video_fps = kwargs.pop("video_fps", self.video_fps)
+        video_minlen = kwargs.pop("video_minlen", self.video_minlen)
+        video_maxlen = kwargs.pop("video_maxlen", self.video_maxlen)
+        frame_extract_num_threads = kwargs.pop("frame_extract_num_threads", self.frame_extract_num_threads)
+        extract_frame_func = kwargs.pop("extract_frame_func", self.extract_frame_func)
+        max_image_tiles = kwargs.pop("max_image_tiles", self.image_processor.max_image_tiles)
+
+        # Step 2: Standardize inputs
+        # Step 2.1: Standardize text
+        # Step 2.1.1: If text is a string, convert it to a list of strings
+        if text is not None and isinstance(text, str):
+            text = [text]
+        # Step 2.1.2: Replace user-facing placeholders with tokenizer's special tokens
+        if text is not None:
+            processed_text: List[str] = []
+            for t in text:
+                # Can be List[str] or str
+                if isinstance(t, str):
+                    t = t.replace(self.image_placeholder, self.image_token)
+                    t = t.replace(self.video_placeholder, self.video_token)
+                    processed_text.append(t)
+            text = processed_text
+
+        # Step 2.2: Normalize images
+        # If the input `images` is a string or a list of strings, open them into PIL Images first.
+        # Then, let `make_list_of_images` handle the rest.
+        loaded_images: Optional[ImageInput] = None
+        if images is not None:
+            if isinstance(images, str):
+                loaded_images = Image.open(images)
+            elif isinstance(images, list) and all(isinstance(i, str) for i in images):
+                loaded_images = [Image.open(i) for i in images]
+            elif isinstance(images, list) and all(isinstance(i, list) for i in images) and all(isinstance(j, str) for i in images for j in i):
+                loaded_images = [[Image.open(j) for j in i] for i in images]
+            else:
+                # If it's already a PIL Image, a list of PIL Images, or other formats, keep it as is.
+                loaded_images = images
+        images_list = make_list_of_images(loaded_images) if loaded_images is not None else None
+
+        # Step 2.3: Normalize videos
+        # If the input `videos` is a string (video_path), convert it to a list of strings
+        videos_list: Optional[List[List[str]]] = None
+        if videos is not None:
+            if isinstance(videos, str):
+                videos_list = [[videos]]  # Batch of 1, sample with 1 video
+            elif isinstance(videos, list) and all(isinstance(i, str) for i in videos):
+                videos_list = [videos]  # Batch of 1, sample with multiple videos
+            elif isinstance(videos, list) and all(isinstance(i, list) for i in videos) and all(isinstance(j, str) for i in videos for j in i):
+                videos_list = videos  # Already in correct batch format
+            else:
+                raise ValueError(f"Invalid video input type: {type(videos)}")
+
+        # Step 3: Initialize the output kwargs
+        # Step 3.1: Initialize the output kwargs with `default values` and `kwargs` provided by the user
+        output_kwargs = self._merge_kwargs(
+            VideoMllamaProcessorKwargs,
+            tokenizer_init_kwargs=self.tokenizer.init_kwargs,
+            **kwargs,
+        )
+        # Step 3.2: Pop the kwargs from the output kwargs to get `common_kwargs`, `text_kwargs`, and `images_kwargs`
+        common_kwargs = output_kwargs["common_kwargs"]
+        text_kwargs = output_kwargs["text_kwargs"]
+        images_kwargs = output_kwargs["images_kwargs"]
+
+        # [** Final data to be returned **]
+        data = {}
+
+        # Step 4: Text processing
+        original_encoding = {}
+        if text is not None:
+            batch_size = len(text)
+            # Step 4.1: Check n_images_in_text and n_videos_in_text to make sure they match the number of images and videos provided
+            if images_list is not None and len(images_list) != batch_size:
+                raise ValueError(
+                    f"The number of samples in `text` ({batch_size}) and `images` ({len(images_list)}) do not match."
+                )
+            if videos_list is not None and len(videos_list) != batch_size:
+                raise ValueError(
+                    f"The number of samples in `text` ({batch_size}) and `videos` ({len(videos_list)}) do not match."
+                )
+
+            n_images_in_text = [t.count(self.image_token) for t in text]
+            n_videos_in_text = [t.count(self.video_token) for t in text]
+
+            # Early validation for media token and provided media counts
+            n_images_provided = [len(img) for img in images_list] if images_list else [0] * len(text)
+            if any(img1 != img2 for img1, img2 in zip(n_images_in_text, n_images_provided)):
+                raise ValueError(
+                    "Number of image tokens does not match number of images provided. "
+                    f"Found {n_images_in_text} image tokens and {n_images_provided} images."
+                )
+
+            n_videos_provided = [len(vid) for vid in videos_list] if videos_list else [0] * len(text)
+            if any(vid1 != vid2 for vid1, vid2 in zip(n_videos_in_text, n_videos_provided)):
+                raise ValueError(
+                    "Number of video tokens does not match number of videos provided. "
+                    f"Found {n_videos_in_text} video tokens and {n_videos_provided} videos."
+                )
+
+            # Step 4.2: Build the text to be encoded. Specifically, if `bos_token` not exist, add it to the beginning of the text.
+            # text_to_encode = [
+            #     build_string_from_input(text_item, self.bos_token, self.image_token, self.video_token)
+            #     for text_item in text
+            # ]
+            
+            # Step 4.2: Use the original text instead of the processed text
+            text_to_encode = text
+            # Step 4.3: Encode the text
+            original_encoding = self.tokenizer(text_to_encode, **text_kwargs)
+        
+        # Step 4.4: Update the final data with the processed text encoding
+        if original_encoding:
+            data.update(original_encoding)
+
+        # Step 5: Image and Video processing
+        # This part handles image and video loading, frame extraction, and ordering.
+        batch_media = []
+        frame_num_per_video_batch = []
+        # Step 5.1: Determine the number of samples to process.
+        num_samples = 0
+        if text is not None:
+            num_samples = len(text)
+        else:
+            # If text is not provided, determine sample count from the longer of images/videos lists.
+            # We've already validated that at least one of them is present.
+            num_img_samples = len(images_list) if images_list else 0
+            num_vid_samples = len(videos_list) if videos_list else 0
+            num_samples = max(num_img_samples, num_vid_samples)
+            
+            # No text provided, process media in default order (images first, then videos).
+            print("No text provided, processing media in default order (images first, then videos).")
+
+        for i in range(num_samples):
+            sample_images = images_list[i] if images_list and i < len(images_list) else []
+            sample_videos = videos_list[i] if videos_list and i < len(videos_list) else []
+
+            # Step 5.2: Determine the media order
+            media_order = []
+            if text is not None:
+                # Determine media order from text tokens
+                txt = text[i]
+                img_tokens_in_text = [(pos, "image") for pos, char in enumerate(txt) if txt.startswith(self.image_token, pos)]
+                vid_tokens_in_text = [(pos, "video") for pos, char in enumerate(txt) if txt.startswith(self.video_token, pos)]
+                # sort by position and get only the media type
+                media_order = [media_type for _, media_type in sorted(img_tokens_in_text + vid_tokens_in_text)]
+            else:
+                # Default order: all images, then all videos
+                media_order.extend(["image"] * len(sample_images))
+                media_order.extend(["video"] * len(sample_videos))
+
+            # Step 5.3: Add images and videos to the batch in the order of `media_order`
+            media_for_sample = []
+            num_frames_per_video_sample = []
+            image_idx = 0
+            video_idx = 0
+
+            for media_type in media_order:
+                if media_type == "image":
+                    media_for_sample.append(sample_images[image_idx])
+                    image_idx += 1
+                elif media_type == "video":
+                    video_path = sample_videos[video_idx]
+
+                    sampling_kwargs = {
+                        "video_fps": video_fps,
+                        "video_minlen": video_minlen,
+                        "video_maxlen": video_maxlen,
+                        "frame_extract_num_threads": frame_extract_num_threads,
+                    }
+                    try:
+                        if extract_frame_func == "cv2":
+                            frames = get_cv2_video_sample_frames_multithread(video_path, **sampling_kwargs)
+                        else:  # "av"
+                            frames = get_video_sample_frames_av(video_path, **sampling_kwargs)
+                    except Exception as e:
+                        raise ValueError(f"This video format is not supported.\nvideo processing failed: {e}")
+
+                    media_for_sample.extend(frames)
+                    num_frames_per_video_sample.append(len(frames))
+                    video_idx += 1
+
+            batch_media.append(media_for_sample)
+            frame_num_per_video_batch.append(num_frames_per_video_sample)
+
+        # Step 5.4: Process images with image_processor
+        num_tiles_batch = []
+        if any(batch_media):
+            images_kwargs["max_image_tiles"] = max_image_tiles
+            image_features = self.image_processor(batch_media, **images_kwargs)
+            num_tiles_batch = image_features.pop("num_tiles")
+            data.update(image_features)
+
+        # Step 6: Calculate cross attention token masks
+        if original_encoding:
+            cross_attention_token_masks = []
+            final_input_ids = []
+            final_attention_mask = []
+
+            # Step 6.1: Check if the attention mask is provided
+            has_attention_mask = "attention_mask" in original_encoding
+
+            for i, token_ids in enumerate(original_encoding["input_ids"]):
+                # Step 6.2: Get the attention mask for the sample
+                attention_mask_for_sample = (
+                    original_encoding["attention_mask"][i] if has_attention_mask else [1] * len(token_ids)
+                )
+                # Step 6.3: Get the cross attention token masks for the sample
+                mask, converted_ids, converted_attn_mask = get_cross_attention_token_mask(
+                    token_ids,
+                    attention_mask_for_sample,
+                    self.image_token_id,
+                    self.video_token_id,
+                    frame_num_per_video_batch[i],
+                    self.cross_attention_token_mask_pad_token_id,
+                )
+                # Step 6.4: Append the `cross_attention_token_masks`, converted `final_input_ids`, and converted `converted_attn_mask` for the sample
+                cross_attention_token_masks.append(np.array(mask))
+                final_input_ids.append(converted_ids)
+                final_attention_mask.append(converted_attn_mask)
+
+        # Step 7: Calculate position_ids and vision_position_ids
+        if original_encoding:
+            batch_position_ids = []
+            batch_vision_position_ids = []
+
+            # NOTE: The logic is now changed to KEEP the <|image|> tokens in the input_ids.
+            # `final_input_ids` already contains the full sequence, and we will not filter it.
+            for i, ids in enumerate(final_input_ids):
+                # Step 7.1: Calculate `attention_mask_arr` and `image_mask` for `position_ids`
+                ids_arr = np.array(ids, dtype=np.int64)
+                attention_mask_arr = np.array(final_attention_mask[i], dtype=np.int64)
+                image_mask = ids_arr == self.image_token_id
+
+                # Step 7.2: Calculate `position_ids`
+                # Always calculate `position_ids`.
+                position_ids = np.cumsum(attention_mask_arr, dtype=np.int64) - 1
+                position_ids[attention_mask_arr == 0] = self.pad_position_id
+                batch_position_ids.append(position_ids)
+
+                # Step 7.3: Calculate `vision_position_ids`
+                # `vision_position_ids` are only calculated if add_video_position_encoding is enabled and we have media input
+                if self.add_video_position_encoding and any(batch_media):
+                    batch_vision_position_ids.append(position_ids[image_mask])
+
+        # Step 8: Padding
+        if original_encoding:
+            # Step 8.1: Pad `input_ids`
+            data["input_ids"] = self._pad_sequences(
+                final_input_ids,
+                self.tokenizer.pad_token_id,
+                padding_side=self.tokenizer.padding_side,
+                pad_to_multiple_of=text_kwargs.get("pad_to_multiple_of"),
+            )
+            
+            # Step 8.2: Pad `attention_mask`
+            data["attention_mask"] = self._pad_sequences(
+                final_attention_mask,
+                0,
+                padding_side=self.tokenizer.padding_side,
+                pad_to_multiple_of=text_kwargs.get("pad_to_multiple_of"),
+            )
+            
+            # Step 8.3: Pad `padded_cross_attention_token_masks`
+            padded_cross_attention_token_masks = self._pad_sequences(
+                cross_attention_token_masks,
+                self.cross_attention_token_mask_pad_token_id,
+                padding_side=self.tokenizer.padding_side,
+                pad_to_multiple_of=text_kwargs.get("pad_to_multiple_of"),
+            )
+        
+            # Step 8.4: Pad `position_ids`
+            data["position_ids"] = self._pad_sequences(
+                batch_position_ids,
+                self.pad_position_id,
+                padding_side=self.tokenizer.padding_side,
+                pad_to_multiple_of=text_kwargs.get("pad_to_multiple_of"),
+            )
+            
+            # Step 8.5: Pad `vision_position_ids`
+            if self.add_video_position_encoding and any(batch_media):
+                # For vision_position_ids, we don't pad to multiple of.
+                # It must be right-padded to align with multi-modal feature processing.
+                data["vision_position_ids"] = self._pad_sequences(
+                    batch_vision_position_ids,
+                    self.pad_position_id,
+                    padding_side="right",
+                )
+
+        # Step 9: Convert sparse cross attention token masks to dense
+        # original_encoding is not empty means: input_text is not empty
+        # num_tiles_batch is not empty means: images or videos are provided
+        if original_encoding and num_tiles_batch:
+            # shape is (batch_size, seq_len, max_num_images, max_num_tiles)
+            cross_attention_mask = convert_sparse_cross_attention_mask_to_dense(
+                padded_cross_attention_token_masks,
+                num_tiles=num_tiles_batch,
+                max_num_tiles=max_image_tiles,
+                cross_attention_token_mask_pad_token_id=self.cross_attention_token_mask_pad_token_id,
+            )
+            data["cross_attention_mask"] = cross_attention_mask
+
+        # Step 10: Return the batch feature
+        return_tensors = common_kwargs.pop("return_tensors", None)
+        batch_feature = BatchFeature(data=data, tensor_type=return_tensors)
+
+        return batch_feature
+
+    def batch_decode(self, *args, **kwargs):
+        """
+        This method forwards all its arguments to PreTrainedTokenizerFast's [`~PreTrainedTokenizer.batch_decode`]. Please
+        refer to the docstring of this method for more information.
+        """
+        return self.tokenizer.batch_decode(*args, **kwargs)
+
+    def decode(self, *args, **kwargs):
+        """
+        This method forwards all its arguments to PreTrainedTokenizerFast's [`~PreTrainedTokenizer.decode`]. Please refer to
+        the docstring of this method for more information.
+        """
+        return self.tokenizer.decode(*args, **kwargs)
+
+    def post_process_image_text_to_text(self, generated_outputs):
+        """
+        Post-process the output of the model to decode the text.
+
+        Args:
+            generated_outputs (`torch.Tensor` or `np.ndarray`):
+                The output of the model `generate` function. The output is expected to be a tensor of shape `(batch_size, sequence_length)`
+                or `(sequence_length,)`.
+
+        Returns:
+            `List[str]`: The decoded text.
+        """
+        return self.tokenizer.batch_decode(
+            generated_outputs, skip_special_tokens=True, clean_up_tokenization_spaces=False
+        )
+
+    @property
+    def model_input_names(self):
+        tokenizer_input_names = self.tokenizer.model_input_names
+        image_processor_input_names = self.image_processor.model_input_names
+        model_inputs = list(tokenizer_input_names + image_processor_input_names + ["cross_attention_mask"])
+        if self.add_video_position_encoding:
+            model_inputs.extend(["position_ids", "vision_position_ids"])
+        return model_inputs