| |
| |
| |
| |
| |
|
|
| import os |
| from typing import Union |
|
|
| from transformers.configuration_utils import PretrainedConfig |
| from transformers.utils import logging |
| from .configuration_siglip import SiglipVisionConfig |
| logger = logging.get_logger(__name__) |
|
|
|
|
| class MultiBackboneChannelConcatenationVisionModelConfig(PretrainedConfig): |
| r""" |
| This is the configuration class to store the configuration of a [`MultiBackboneChannelConcatenationVisionModelConfig`]. It is used to |
| instantiate a vision encoder according to the specified arguments, defining the model architecture. |
| |
| Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the |
| documentation from [`PretrainedConfig`] for more information. |
| |
| Args: |
| vision_path (str): Path to the vision model or its configuration. |
| mm_vision_select_layer (int, optional): The layer to select from the vision model |
| for multi-modal processing. Defaults to -2. |
| grid_size (int, optional): The size of the grid for vision processing. Defaults to 32. |
| **kwargs: Additional keyword arguments to be passed to the parent PretrainedConfig. |
| |
| """ |
|
|
| model_type = 'MOB' |
|
|
| def __init__( |
| self, |
| vision_path, |
| mm_vision_select_layer=-2, |
| grid_size=32, |
| input_image_size=1024, |
| hidden_size='lazy_calculation', |
| image_size=1024, |
| freeze_backbones=None, |
| moe_version_type=None, |
| delay_load=False, |
| convnext_img_size=1024, |
| vision_tower_siglip_path=None, |
| vision_tower_convnext_path='convnext_xxlarge.clip_laion2b_soup', |
| normalize_type='siglip', |
| **kwargs, |
| ): |
| super().__init__(**kwargs) |
|
|
| self.normalize_type = normalize_type |
| self.vision_path = vision_path |
| self.mm_vision_select_layer = mm_vision_select_layer |
| self.grid_size = grid_size |
| self.input_image_size = input_image_size |
| self.image_size = image_size |
| self.hidden_size = hidden_size |
| self.freeze_backbones = freeze_backbones |
| self.moe_version_type = moe_version_type |
| self.delay_load = delay_load |
| self.convnext_img_size = convnext_img_size |
| |
| self.vision_tower_siglip_path = vision_tower_siglip_path |
| self.vision_tower_convnext_path = vision_tower_convnext_path |
| self.vision_tower = self.vision_path[4:] |
|
|
| |
| assert image_size == input_image_size, f"input_image_size ({input_image_size}) != image_size ({image_size})" |
|
|
| @classmethod |
| def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> 'PretrainedConfig': |
| config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs) |
|
|
| if 'vision_config' in config_dict: |
| config_dict = config_dict['vision_config'] |
|
|
| if 'model_type' in config_dict and hasattr(cls, 'model_type') and config_dict['model_type'] != cls.model_type: |
| logger.warning( |
| f"You are using a model of type {config_dict['model_type']} to instantiate a model of type " |
| f'{cls.model_type}. This is not supported for all configurations of models and can yield errors.' |
| ) |
|
|
| return cls.from_dict(config_dict, **kwargs) |
|
|
