Upload files with `vila-upload`.

Upload tokenizer_config.json
Upload config.json
Upload configuration_vila.py
Upload generation_config.json
Upload chat_template.jinja
Upload processing_vila.py
Upload processor_config.json
Upload modeling_vila.py

chat_template.jinja

@@ -0,0 +1 @@
+{% for message in messages %}{% if loop.first and message['role'] != 'system' %}{{ '<|im_start|>system\nYou are a helpful assistant<|im_end|>\n' }}{% endif %}{{ '<|im_start|>' + message['role'] + '\n' }}{% if message['content'] is string %}{{ message['content'] + '<|im_end|>\n' }}{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' or 'image' in content or 'image_url' in content %}{{ '<image>' }}{% elif content['type'] == 'video' or 'video' in content %}{{ '<video>' }}{% elif 'text' in content %}{{ content['text'] }}{% endif %}{% endfor %}{{ '<|im_end|>\n' }}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}

config.json

@@ -10,7 +10,6 @@
     "AutoModelForVision2Seq": "modeling_vila.VILAForConditionalGeneration"
   },
   "hidden_size": 3584,
-  "image_end_token_id": 198,
   "image_token_id": 151648,
   "mm_hidden_size": 1152,
   "mm_projector_type": "mlp_downsample_3x3_fix",
@@ -47,7 +46,7 @@
     "vocab_size": 151648
   },
   "torch_dtype": "bfloat16",
-  "transformers_version": "4.51.3",
+  "transformers_version": "4.52.3",
   "video_token_id": 151649,
   "vision_config": {
     "architectures": [

configuration_vila.py

@@ -21,7 +21,6 @@ class VILAConfig(PretrainedConfig):
     # Model configuration.
     hidden_size: int
     image_token_id: int
-    image_end_token_id: int
     mm_hidden_size: int
     mm_projector_type: str
     mm_vision_select_feature: str
@@ -30,17 +29,16 @@ class VILAConfig(PretrainedConfig):
 
     def __init__(
         self,
-        *,
         text_config: Optional[Dict[str, Any]] = None,
         vision_config: Optional[Dict[str, Any]] = None,
-        hidden_size: Optional[int] = None,
-        image_token_id: Optional[int] = None,
-        image_end_token_id: Optional[int] = None,
-        mm_hidden_size: Optional[int] = None,
-        mm_projector_type: Optional[str] = None,
-        mm_vision_select_feature: Optional[str] = None,
-        mm_vision_select_layer: Optional[int] = None,
-        video_token_id: Optional[int] = None,
+        *,
+        hidden_size: int = 1536,
+        image_token_id: int = 151649,
+        mm_hidden_size: int = 1152,
+        mm_projector_type: str = "mlp_downsample_3x3_fix",
+        mm_vision_select_feature: str = "cls_patch",
+        mm_vision_select_layer: int = -2,
+        video_token_id: int = 151650,
         **kwargs,
     ):
         super().__init__(**kwargs)
@@ -48,14 +46,10 @@ class VILAConfig(PretrainedConfig):
         self.text_config = Qwen2Config(**text_config) if text_config else Qwen2Config()
         self.vision_config = SiglipVisionConfig(**vision_config) if vision_config else SiglipVisionConfig()
 
-        # By default, we use settings from NVILA-Lite.
-        self.hidden_size = hidden_size if hidden_size is not None else 1536
-        self.image_token_id = image_token_id if image_token_id is not None else 151649
-        self.image_end_token_id = image_end_token_id if image_end_token_id is not None else 198  # "\n"
-        self.mm_hidden_size = mm_hidden_size if mm_hidden_size is not None else 1152
-        self.mm_projector_type = mm_projector_type if mm_projector_type is not None else "mlp_downsample_3x3_fix"
-        self.mm_vision_select_feature = (
-            mm_vision_select_feature if mm_vision_select_feature is not None else "cls_patch"
-        )
-        self.mm_vision_select_layer = mm_vision_select_layer if mm_vision_select_layer is not None else -2
-        self.video_token_id = video_token_id if video_token_id is not None else 151650
+        self.hidden_size = hidden_size
+        self.image_token_id = image_token_id
+        self.mm_hidden_size = mm_hidden_size
+        self.mm_projector_type = mm_projector_type
+        self.mm_vision_select_feature = mm_vision_select_feature
+        self.mm_vision_select_layer = mm_vision_select_layer
+        self.video_token_id = video_token_id

generation_config.json

@@ -3,5 +3,5 @@
   "bos_token_id": 151643,
   "eos_token_id": 151645,
   "pad_token_id": 151643,
-  "transformers_version": "4.51.3"
+  "transformers_version": "4.52.3"
 }

modeling_vila.py

@@ -2,6 +2,7 @@ from typing import List, Optional, Type
 
 import torch
 import torch.nn as nn
+import torch.nn.functional as F
 from torch import Tensor
 from transformers.configuration_utils import PretrainedConfig
 from transformers.generation.utils import GenerationMixin
@@ -13,68 +14,34 @@ from transformers.models.siglip.modeling_siglip import SiglipVisionModel
 from .configuration_vila import VILAConfig
 
 
-class DownSampleBlock(nn.Module):
-    @staticmethod
-    def flat_square(x: Tensor) -> Tensor:
-        n, w, h, c = x.size()
-        if w % 2 == 1:
-            x = torch.concat([x, torch.zeros((n, 1, h, c), device=x.device, dtype=x.dtype)], dim=1).contiguous()
-            n, w, h, c = x.size()
-        if h % 2 == 1:
-            x = torch.concat([x, torch.zeros((n, w, 1, c), device=x.device, dtype=x.dtype)], dim=2).contiguous()
-            n, w, h, c = x.size()
-        x = x.contiguous()
-        x = x.view(n, w, int(h / 2), int(c * 2))
-        x = x.permute(0, 2, 1, 3).contiguous()
-        x = x.view(n, int(h / 2), int(w / 2), int(c * 4))
-        x = x.permute(0, 2, 1, 3).contiguous()
-        return x
-
+class DownSample3x3BlockFix(nn.Module):
     def forward(self, x: Tensor) -> Tensor:
-        vit_embeds = x
-        h = w = int(vit_embeds.shape[1] ** 0.5)
-        vit_embeds = vit_embeds.reshape(vit_embeds.shape[0], h, w, -1)
-        vit_embeds = self.flat_square(vit_embeds)
-        vit_embeds = vit_embeds.reshape(vit_embeds.shape[0], -1, vit_embeds.shape[-1])
-        return vit_embeds
+        """
+        Args:
+            x: The input tensor of shape (batch_size, sequence_length, mm_hidden_size).
 
+        Returns:
+            The output tensor of shape (batch_size, image_pad_len, mm_hidden_size * 9).
+        """
 
-class DownSample3x3BlockFix(nn.Module):
-    @staticmethod
-    def flat_square_3x3(x: Tensor) -> Tensor:
-        n, w, h, c = x.size()
-        if w % 3 != 0:
-            x = torch.concat(
-                [
-                    x,
-                    torch.zeros((n, 3 - (w % 3), h, c), device=x.device, dtype=x.dtype),
-                ],
-                dim=1,
-            ).contiguous()
-            n, w, h, c = x.size()
-        x = x.contiguous()
-        if h % 3 != 0:
-            x = torch.concat(
-                [
-                    x,
-                    torch.zeros((n, w, 3 - (h % 3), c), device=x.device, dtype=x.dtype),
-                ],
-                dim=2,
-            ).contiguous()
-            n, w, h, c = x.size()
-        x = x.view(n, w, int(h / 3), int(c * 3))
-        x = x.permute(0, 2, 1, 3).contiguous()
-        x = x.view(n, int(h / 3), int(w / 3), int(c * 9))
-        x = x.permute(0, 2, 1, 3).contiguous()
-        return x
+        batch_size, sequence_length, hidden_size = x.shape
 
-    def forward(self, x: Tensor) -> Tensor:
-        vit_embeds = x
-        h = w = int(vit_embeds.shape[1] ** 0.5)
-        vit_embeds = vit_embeds.reshape(vit_embeds.shape[0], h, w, -1)
-        vit_embeds = self.flat_square_3x3(vit_embeds)
-        vit_embeds = vit_embeds.reshape(vit_embeds.shape[0], -1, vit_embeds.shape[-1])
-        return vit_embeds
+        feat_size = int(sequence_length**0.5)
+        if feat_size**2 != sequence_length:
+            raise ValueError(f"Cannot take square root: sequence_length {sequence_length} is not a perfect square")
+
+        features = x.reshape(batch_size, feat_size, feat_size, hidden_size)
+
+        pad_after = (3 - feat_size % 3) % 3
+        if pad_after > 0:
+            features = F.pad(features, (0, 0, 0, pad_after, 0, pad_after))
+            feat_size = feat_size + pad_after
+
+        features = features.reshape(batch_size, feat_size // 3, 3, feat_size // 3, 3, hidden_size)
+        features = features.permute(0, 1, 3, 2, 4, 5).contiguous()
+        features = features.reshape(batch_size, -1, 9 * hidden_size)
+
+        return features
 
 
 class MultimodalProjector(nn.Module):
@@ -89,18 +56,6 @@ class MultimodalProjector(nn.Module):
         super().__init__(*args, **kwargs)
 
         match config.mm_projector_type:
-            case "linear":
-                self.layers = nn.Sequential(
-                    nn.Linear(config.vision_config.hidden_size, config.hidden_size),
-                )
-            case "mlp_downsample":
-                self.layers = nn.Sequential(
-                    DownSampleBlock(),
-                    nn.LayerNorm(config.mm_hidden_size * 4),
-                    nn.Linear(config.mm_hidden_size * 4, config.hidden_size),
-                    nn.GELU(),
-                    nn.Linear(config.hidden_size, config.hidden_size),
-                )
             case "mlp_downsample_3x3_fix":
                 self.layers = nn.Sequential(
                     DownSample3x3BlockFix(),
@@ -116,9 +71,9 @@ class MultimodalProjector(nn.Module):
                     nn.Linear(config.hidden_size, config.hidden_size),
                 )
             case _:
-                raise NotImplementedError(f"mm_projector_type={config.mm_projector_type} not implemented.")
+                raise NotImplementedError(f"Unsupported mm_projector_type: {config.mm_projector_type}")
 
-        self.layers.to(dtype=config.torch_dtype)
+        self.layers.type(config.torch_dtype)
 
     @property
     def device(self) -> torch.device:
@@ -129,7 +84,15 @@ class MultimodalProjector(nn.Module):
         return next(self.parameters()).dtype
 
     def forward(self, x: Tensor) -> Tensor:
-        return self.layers(x)
+        """
+        Args:
+            x: The input tensor of shape (batch_size, sequence_length, mm_hidden_size).
+
+        Returns:
+            The output tensor of shape (batch_size, image_pad_len, hidden_size).
+        """
+
+        return self.layers(x.to(device=self.device, dtype=self.dtype))
 
 
 class VILAForConditionalGeneration(PreTrainedModel, GenerationMixin):
@@ -156,9 +119,9 @@ class VILAForConditionalGeneration(PreTrainedModel, GenerationMixin):
     ):
         super().__init__(config, *args, **kwargs)
 
-        self.llm = Qwen2ForCausalLM(config.text_config, *args, **kwargs)
+        self.llm = Qwen2ForCausalLM._from_config(config.text_config, *args, **kwargs)
         self.mm_projector = MultimodalProjector(config)
-        self.vision_tower = SiglipVisionModel(config.vision_config, *args, **kwargs)
+        self.vision_tower = SiglipVisionModel._from_config(config.vision_config, *args, **kwargs)
 
         self.post_init()
 
@@ -175,29 +138,15 @@ class VILAForConditionalGeneration(PreTrainedModel, GenerationMixin):
         if kwargs.get("past_key_values", None) is not None:
             pixel_values = None
 
-        inputs_embeds = inputs_embeds.to(dtype=self.dtype) if inputs_embeds is not None else None
-        pixel_values = pixel_values.to(dtype=self.dtype) if pixel_values is not None else None
-
         if inputs_embeds is None:
-            assert input_ids is not None
+            if input_ids is None:
+                raise ValueError("input_ids is required when inputs_embeds is None")
 
             inputs_embeds = self._embed(input_ids, pixel_values)
-        else:
-            assert input_ids is None
-            assert pixel_values is None
 
         outputs = self.llm.__call__(
-            inputs_embeds=inputs_embeds.to(
-                device=self.llm.device,
-                dtype=self.llm.dtype,
-            ),
-            attention_mask=(
-                attention_mask.to(
-                    device=self.llm.device,
-                )
-                if attention_mask is not None
-                else None
-            ),
+            inputs_embeds=inputs_embeds.to(device=self.llm.device, dtype=self.llm.dtype),
+            attention_mask=(attention_mask.to(device=self.llm.device) if attention_mask is not None else None),
             **kwargs,
         )
 
@@ -221,8 +170,6 @@ class VILAForConditionalGeneration(PreTrainedModel, GenerationMixin):
             The embedding of the input ids and pixel values.
         """
 
-        # Video tokens should be removed during preprocessing, so there must not be any video
-        # tokens in the input ids.
         if torch.any(input_ids == self.config.video_token_id):
             raise ValueError("Video token ids should not be present in the input ids.")
 
@@ -233,56 +180,38 @@ class VILAForConditionalGeneration(PreTrainedModel, GenerationMixin):
         if pixel_values is None:
             return text_embedding
 
-        image_features: BaseModelOutputWithPooling = self.vision_tower.__call__(
-            pixel_values.to(
-                device=self.vision_tower.device,
-                dtype=self.vision_tower.dtype,
-            ),
+        vision_tower_output: BaseModelOutputWithPooling = self.vision_tower.__call__(
+            pixel_values.to(device=self.vision_tower.device, dtype=self.vision_tower.dtype),
             output_hidden_states=True,
         )
-        assert image_features.hidden_states is not None
 
-        # Select image feature.
-        selected_layer_output = image_features.hidden_states[self.config.mm_vision_select_layer]
-        match self.config.mm_vision_select_feature:
-            case "cls_patch":
-                selected_feature = selected_layer_output
-            case _:
-                raise NotImplementedError(
-                    f"mm_vision_select_feature={self.config.mm_vision_select_feature} not implemented."
-                )
-
-        # TODO: Support dynamic_s2.
+        mm_projector_input = self._vision_tower_output_to_mm_projector_input(vision_tower_output)
 
         image_embedding: Tensor = self.mm_projector.__call__(
-            selected_feature.to(
-                device=self.mm_projector.device,
-                dtype=self.mm_projector.dtype,
-            )
+            mm_projector_input.to(device=self.mm_projector.device, dtype=self.mm_projector.dtype)
         )
 
-        # Append image end token to every image embedding.
-        image_end_token_embedding: Tensor = self.llm.get_input_embeddings().__call__(
-            torch.tensor(
-                self.config.image_end_token_id,
-                device=text_embedding.device,
-                dtype=torch.long,
-            ).view(1, -1)
-        )  # Shape: (1, 1, dim_feature)
-        image_end_token_embedding = image_end_token_embedding.expand(
-            image_embedding.shape[0], 1, -1
-        )  # Shape: (n_images, 1, dim_feature)
-        image_embedding = torch.concat(
-            [
-                image_embedding.to(device=text_embedding.device),
-                image_end_token_embedding,
-            ],
-            dim=1,
+        image_embedding = image_embedding.reshape(-1, image_embedding.shape[-1])
+
+        text_embedding.masked_scatter_(
+            image_token_mask.to(device=text_embedding.device, dtype=torch.bool).unsqueeze(-1),
+            image_embedding.to(device=text_embedding.device, dtype=text_embedding.dtype).flatten(),
         )
 
-        n_images, n_feature, dim_feature = image_embedding.shape
-        image_embedding = image_embedding.view(n_images * n_feature, dim_feature)
+        return text_embedding
 
-        text_embedding[image_token_mask.to(device=text_embedding.device)] = image_embedding
+    def _vision_tower_output_to_mm_projector_input(
+        self,
+        vision_tower_output: BaseModelOutputWithPooling,
+    ) -> Tensor:
+        assert vision_tower_output.hidden_states is not None
 
-        return text_embedding
+        selected_layer_hidden_states = vision_tower_output.hidden_states[self.config.mm_vision_select_layer]
+
+        match self.config.mm_vision_select_feature:
+            case "cls_patch":
+                return selected_layer_hidden_states
+            case _:
+                raise NotImplementedError(
+                    f"Unsupported mm_vision_select_feature: {self.config.mm_vision_select_feature}"
+                )

processing_vila.py

@@ -3,17 +3,19 @@ from typing import List, Optional, Tuple, cast
 import transformers.image_transforms as image_transforms
 import transformers.image_utils as image_utils
 import transformers.utils.logging
+import transformers.video_utils as video_utils
 from PIL.Image import Image
 from torch import Tensor
 from transformers.feature_extraction_utils import BatchFeature
 from transformers.image_processing_utils import BaseImageProcessor
 from transformers.image_processing_utils_fast import BaseImageProcessorFast
-from transformers.image_utils import ImageInput, VideoInput
+from transformers.image_utils import ImageInput
 from transformers.models.siglip.image_processing_siglip import SiglipImageProcessor
 from transformers.models.siglip.image_processing_siglip_fast import SiglipImageProcessorFast
-from transformers.processing_utils import ImagesKwargs, ProcessingKwargs, ProcessorMixin, Unpack
+from transformers.processing_utils import ProcessingKwargs, ProcessorMixin, Unpack
 from transformers.tokenization_utils import PreTrainedTokenizer
 from transformers.tokenization_utils_base import PreTrainedTokenizerBase, TextInput
+from transformers.video_utils import VideoInput
 
 logger = transformers.utils.logging.get_logger(__name__)
 
@@ -41,6 +43,7 @@ class VILAProcessor(ProcessorMixin):
         "image_pad_len",
         "max_tiles",
         "min_tiles",
+        "video_max_tiles",
     ]
 
     # Attributes.
@@ -51,15 +54,17 @@ class VILAProcessor(ProcessorMixin):
     image_pad_len: int
     max_tiles: int
     min_tiles: int
+    video_max_tiles: int
 
     def __init__(
         self,
         image_processor: BaseImageProcessor,
         tokenizer: PreTrainedTokenizer,
         *,
-        image_pad_len: Optional[int] = None,
-        max_tiles: Optional[int] = None,
-        min_tiles: Optional[int] = None,
+        image_pad_len: int = 121,
+        max_tiles: int = 12,
+        min_tiles: int = 1,
+        video_max_tiles: int = 1,
         **kwargs,
     ):
         super().__init__(
@@ -68,9 +73,10 @@ class VILAProcessor(ProcessorMixin):
             **kwargs,
         )
 
-        self.image_pad_len = image_pad_len if image_pad_len is not None else 122
-        self.max_tiles = max_tiles if max_tiles is not None else 12
-        self.min_tiles = min_tiles if min_tiles is not None else 1
+        self.image_pad_len = image_pad_len
+        self.max_tiles = max_tiles
+        self.min_tiles = min_tiles
+        self.video_max_tiles = video_max_tiles
 
     def __call__(
         self,
@@ -78,7 +84,7 @@ class VILAProcessor(ProcessorMixin):
         images: Optional[ImageInput] = None,
         videos: Optional[VideoInput] = None,
         audio: None = None,
-        **kwargs: Unpack[VILAProcessorProcessingKwargs],
+        **kwargs: Unpack[ProcessingKwargs],
     ) -> VILAProcessorOutput:
         """Preprocesses inputs for VILA.
 
@@ -99,39 +105,59 @@ class VILAProcessor(ProcessorMixin):
             **kwargs,
         )
 
-        text, images, videos = self._prepare_inputs(
+        prepared_text, prepared_images, prepared_videos = self._prepare_inputs(
             text=text,
             images=images,
             videos=videos,
         )
 
         # Process videos.
-        text, images, video_flags = self._treat_videos_as_image_seqs(
-            text=text,
-            images=images,
-            videos=videos,
+        prepared_text, prepared_images, video_flags = self._treat_videos_as_image_seqs(
+            text=prepared_text,
+            images=prepared_images,
+            videos=prepared_videos,
         )
 
         # Process images.
         image_inputs, num_cropped_images = self._process_images(
-            images=images,
+            images=prepared_images,
+            video_flags=video_flags,
             **merged_kwargs["images_kwargs"],
         )
 
         # Process text.
-        text = self._pad_image_tokens_by_num_crops(
-            text,
+        prepared_text = self._pad_image_tokens_by_num_crops(
+            prepared_text,
             num_cropped_images=num_cropped_images,
             video_flags=video_flags,
         )
 
-        text = self._pad_image_tokens_by_num_embeddings(text)
+        prepared_text = self._pad_image_tokens_by_num_embeddings(prepared_text)
 
         text_inputs = self.tokenizer.__call__(
-            text,
+            prepared_text,
             **merged_kwargs["text_kwargs"],
         )
 
+        # Find the last image token of each image tile and replace to "\n".
+        lf_token_id = self.tokenizer.encode("\n")[0]
+        image_token_id = self.tokenizer.image_token_id
+
+        for i in range(len(text_inputs.input_ids)):
+            input_ids = text_inputs.input_ids[i]
+
+            idx = 0
+            while idx < len(input_ids):
+                if input_ids[idx] != image_token_id:
+                    idx += 1
+                    continue
+
+                if idx + self.image_pad_len < len(input_ids):
+                    input_ids[idx + self.image_pad_len] = lf_token_id
+                    idx += self.image_pad_len + 1
+                else:
+                    break
+
         return VILAProcessorOutput(
             data={
                 **text_inputs,
@@ -142,6 +168,8 @@ class VILAProcessor(ProcessorMixin):
     def _crop_image(
         self,
         image: Image,
+        *,
+        is_video_frame: bool,
     ) -> List[Image]:
         """Crops the image into multiple tiles.
 
@@ -162,7 +190,7 @@ class VILAProcessor(ProcessorMixin):
         cropped_images: List[Image] = dynamic_preprocess(
             image,
             min_num=self.min_tiles,
-            max_num=self.max_tiles,
+            max_num=self.max_tiles if not is_video_frame else self.video_max_tiles,
             image_size=cropped_size,
         )
 
@@ -240,12 +268,9 @@ class VILAProcessor(ProcessorMixin):
             The padded text.
         """
 
-        return [
-            text_item.replace(
-                cast(str, self.tokenizer.image_token), cast(str, self.tokenizer.image_token) * self.image_pad_len
-            )
-            for text_item in text
-        ]
+        image_token = cast(str, self.tokenizer.image_token)
+
+        return [text_item.replace(image_token, image_token * (self.image_pad_len + 1)) for text_item in text]
 
     @staticmethod
     def _prepare_inputs(
@@ -253,35 +278,36 @@ class VILAProcessor(ProcessorMixin):
         images: Optional[ImageInput],
         videos: Optional[VideoInput],
     ) -> Tuple[List[str], List[Image], List[List[Image]]]:
-        # Prepare text.
-        text = text if isinstance(text, list) else [text]
+        prepared_text = text if isinstance(text, list) else [text]
 
-        # Prepare images.
         if images is not None:
             image_list = cast(List, image_utils.make_flat_list_of_images(images))
-            images = [image_transforms.to_pil_image(image) for image in image_list]
+            prepared_images = [cast(Image, image_transforms.to_pil_image(image)) for image in image_list]
         else:
-            images = cast(List[Image], [])
+            prepared_images = []
 
-        # Prepare videos.
         if videos is not None:
-            video_list = cast(List[List], image_utils.make_batched_videos(videos))
-            videos = [[image_transforms.to_pil_image(image) for image in video] for video in video_list]
+            video_list = cast(List[List], video_utils.make_batched_videos(videos))
+            prepared_videos = [
+                [cast(Image, image_transforms.to_pil_image(image)) for image in video] for video in video_list
+            ]
         else:
-            videos = cast(List[List[Image]], [])
+            prepared_videos = []
 
-        return text, images, videos
+        return prepared_text, prepared_images, prepared_videos
 
     def _process_images(
         self,
         images: List[Image],
-        **kwargs: Unpack[ImagesKwargs],
+        *,
+        video_flags: List[bool],
+        **kwargs,
     ) -> Tuple[BatchFeature, List[int]]:
         cropped_images: List[Image] = []
         num_cropped_images: List[int] = []
 
-        for image in images:
-            single_cropped_images = self._crop_image(image)
+        for image, video_flag in zip(images, video_flags):
+            single_cropped_images = self._crop_image(image, is_video_frame=video_flag)
 
             cropped_images.extend(single_cropped_images)
             num_cropped_images.append(len(single_cropped_images))

processor_config.json

@@ -2,8 +2,9 @@
   "auto_map": {
     "AutoProcessor": "processing_vila.VILAProcessor"
   },
-  "image_pad_len": 122,
+  "image_pad_len": 121,
   "max_tiles": 12,
   "min_tiles": 1,
-  "processor_class": "VILAProcessor"
+  "processor_class": "VILAProcessor",
+  "video_max_tiles": 1
 }

tokenizer_config.json

@@ -66,7 +66,6 @@
     "AutoProcessor": "processing_vila.VILAProcessor"
   },
   "bos_token": "[BOS]",
-  "chat_template": null,
   "clean_up_tokenization_spaces": false,
   "eos_token": "<|im_end|>",
   "errors": "replace",