configuration_locateanything.py

# --------------------------------------------------------
# InternVL
# Copyright (c) 2023 OpenGVLab
# Licensed under The MIT License [see LICENSE for details]
# --------------------------------------------------------

import copy

from transformers.models.qwen2.configuration_qwen2 import Qwen2Config
from transformers.models.qwen3.configuration_qwen3 import Qwen3Config
from transformers.configuration_utils import PretrainedConfig
from transformers.utils import logging
logger = logging.get_logger(__name__)

class MoonViTConfig(PretrainedConfig):
    model_type = "moonvit"

    def __init__(
        self,
        patch_size: int = 14,
        init_pos_emb_height: int = 64,
        init_pos_emb_width: int = 64,
        num_attention_heads: int = 16,
        num_hidden_layers: int = 27,
        hidden_size: int = 1152,
        intermediate_size: int = 4304,
        merge_kernel_size: tuple[int, int] = (2, 2),
        **kwargs,
    ):
        super().__init__(**kwargs)
        self.patch_size = patch_size
        # Positional embedding config
        self.init_pos_emb_height = init_pos_emb_height
        self.init_pos_emb_width = init_pos_emb_width
        # Transformer config
        self.num_hidden_layers = num_hidden_layers
        self.num_attention_heads = num_attention_heads
        self.hidden_size = hidden_size
        self.intermediate_size = intermediate_size
        # Patch merger config
        self.merge_kernel_size = merge_kernel_size


class LocateAnythingConfig(PretrainedConfig):
    model_type = 'locateanything'
    is_composition = True
    sub_configs = {"vision_config": MoonViTConfig, "text_config": Qwen2Config}
    def __init__(
            self,
            vision_config=None,
            text_config=None,
            use_backbone_lora=0,
            use_llm_lora=0,
            downsample_ratio=0.5,
            template=None,
            loss_version='v1',
            mlp_checkpoint=False,
            image_token_index=151667,
            box_start_token_id=151668,
            box_end_token_id=151669,
            coord_start_token_id=151677,
            coord_end_token_id=152677,
            ref_start_token_id=151672,
            ref_end_token_id=151673,
            none_token_id=4064,
            **kwargs):
        super().__init__(**kwargs)

        if vision_config is None:
            vision_config = {'model_type': 'moonvit'}
            logger.info('vision_config is None. Initializing the MoonViTConfig with default values.')

        if text_config is None:
            text_config = {'architectures': ['Qwen2ForCausalLM']}
            logger.info('text_config is None. Initializing the Qwen2Config config with default values.')

        if vision_config['model_type'] == 'moonvit':
            self.vision_config = MoonViTConfig(**vision_config)
        else:
            raise ValueError('Unsupported model_type: {}. Only moonvit is supported.'.format(vision_config['model_type']))


        if text_config['architectures'][0] == 'Qwen2ForCausalLM':
            self.text_config = Qwen2Config(**text_config)
        elif text_config['architectures'][0] == 'Qwen3ForCausalLM':
            self.text_config = Qwen3Config(**text_config)
        else:
            raise ValueError('Unsupported architecture: {}. Only Qwen2ForCausalLM and Qwen3ForCausalLM are supported.'.format(text_config['architectures'][0]))
        self.use_backbone_lora = use_backbone_lora
        self.use_llm_lora = use_llm_lora
        self.mlp_checkpoint = mlp_checkpoint
        self.downsample_ratio = downsample_ratio
        self.template = template
        self.loss_version = loss_version
        self.tie_word_embeddings = self.text_config.tie_word_embeddings
        self.image_token_index = image_token_index
        self.box_start_token_id = box_start_token_id
        self.box_end_token_id = box_end_token_id
        self.coord_start_token_id = coord_start_token_id
        self.coord_end_token_id = coord_end_token_id
        self.ref_start_token_id = ref_start_token_id
        self.ref_end_token_id = ref_end_token_id
        self.none_token_id = none_token_id

    def to_dict(self):
        """
        Serializes this instance to a Python dictionary. Override the default [`~PretrainedConfig.to_dict`].

        Returns:
            `Dict[str, any]`: Dictionary of all the attributes that make up this configuration instance,
        """
        output = copy.deepcopy(self.__dict__)
        output['vision_config'] = self.vision_config.to_dict()
        output['text_config'] = self.text_config.to_dict()
        output['model_type'] = self.__class__.model_type
        output['use_backbone_lora'] = self.use_backbone_lora
        output['use_llm_lora'] = self.use_llm_lora
        output['downsample_ratio'] = self.downsample_ratio
        output['template'] = self.template
        output['image_token_index'] = self.image_token_index
        output['box_start_token_id'] = self.box_start_token_id
        output['box_end_token_id'] = self.box_end_token_id
        output['coord_start_token_id'] = self.coord_start_token_id
        output['coord_end_token_id'] = self.coord_end_token_id
        output['ref_start_token_id'] = self.ref_start_token_id
        output['ref_end_token_id'] = self.ref_end_token_id
        output['none_token_id'] = self.none_token_id
        output['_attn_implementation'] = self._attn_implementation
        if hasattr(self, '_attn_implementation_autoset'):
            output['_attn_implementation_autoset'] = self._attn_implementation_autoset
        return output