| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| |
| """ vision-encoder-language-decoder-t5 model configuration""" |
| import copy |
|
|
| from transformers.configuration_utils import PretrainedConfig |
| from transformers.utils import logging |
| from transformers.models.auto.configuration_auto import AutoConfig |
| from transformers import T5Config, ViTConfig |
|
|
|
|
| logger = logging.get_logger(__name__) |
|
|
| class VELDConfig(PretrainedConfig): |
| r""" |
| [`VELDConfig`] is the configuration class to store the configuration of a |
| [`VELDConfig`]. It is used to instantiate a Vision-Encoder-Text-Decoder model according to the |
| specified arguments, defining the encoder and decoder configs. |
| |
| Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the |
| documentation from [`PretrainedConfig`] for more information. |
| |
| Args: |
| kwargs (*optional*): |
| Dictionary of keyword arguments. Notably: |
| |
| - **encoder** ([`PretrainedConfig`], *optional*) -- An instance of a configuration object that defines |
| the encoder config. |
| - **decoder** ([`PretrainedConfig`], *optional*) -- An instance of a configuration object that defines |
| the decoder config. |
| |
| Examples: |
| |
| ```python |
| >>> from transformers import T5Config, ViTConfig |
| >>> from configuration_veld import VELDConfig |
| >>> from modeling_veld import VELDModel |
| |
| >>> # Initializing a ViT & T5 style configuration |
| >>> config_encoder = ViTConfig() |
| >>> config_decoder = T5Config() |
| |
| >>> config = VELDConfig.from_encoder_decoder_configs(config_encoder, config_decoder) |
| |
| >>> # Initializing a ViTBert model from a ViT & bert-base-uncased style configurations |
| >>> model = VELDModel(config=config) |
| |
| >>> # Accessing the model configuration |
| >>> config_encoder = model.config.encoder |
| >>> config_decoder = model.config.decoder |
| >>> # set decoder config to causal lm |
| >>> config_decoder.is_decoder = True |
| >>> config_decoder.add_cross_attention = True |
| |
| >>> # Saving the model, including its configuration |
| >>> model.save_pretrained("my-model") |
| |
| >>> # loading model and config from pretrained folder |
| >>> encoder_decoder_config = VELDConfig.from_pretrained("my-model") |
| >>> model = VELDModel.from_pretrained("my-model", config=encoder_decoder_config) |
| ```""" |
| model_type = "veld" |
| is_composition = True |
|
|
| def __init__(self, **kwargs): |
| super().__init__(**kwargs) |
| if "encoder" not in kwargs or "decoder" not in kwargs: |
| raise ValueError( |
| f"A configuraton of type {self.model_type} cannot be instantiated because " |
| f"not both `encoder` and `decoder` sub-configurations are passed, but only {kwargs}" |
| ) |
|
|
| encoder_config = kwargs.pop("encoder") |
| encoder_model_type = encoder_config.pop("model_type") |
| decoder_config = kwargs.pop("decoder") |
| decoder_model_type = decoder_config.pop("model_type") |
|
|
| self.encoder = ViTConfig(**encoder_config) |
| self.decoder = T5Config(**decoder_config) |
| self.is_encoder_decoder = True |
|
|
| self.pad_token_id=self.decoder.pad_token_id |
| self.eos_token_id=self.decoder.eos_token_id |
|
|
| self.num_queries_global = getattr(kwargs, "num_queries_global", 1) |
| self.num_queries_local = getattr(kwargs, "num_queries_local", 256) |
|
|
|
|
| @classmethod |
| def from_encoder_decoder_configs( |
| cls, encoder_config: PretrainedConfig, decoder_config: T5Config, **kwargs |
| ) -> PretrainedConfig: |
| r""" |
| Instantiate a [`VELDConfig`] (or a derived class) from a pre-trained encoder model |
| configuration and decoder model configuration. |
| |
| Returns: |
| [`VELDConfig`]: An instance of a configuration object |
| """ |
| logger.info("Setting `config.is_decoder=True` and `config.is_encoder_decoder=False` for decoder_config") |
| decoder_config.is_decoder = True |
| decoder_config.is_encoder_decoder = False |
|
|
| return cls(encoder=encoder_config.to_dict(), decoder=decoder_config.to_dict(), **kwargs) |
|
|
| def to_dict(self): |
| """ |
| Serializes this instance to a Python dictionary. Override the default *to_dict()* from *PretrainedConfig*. |
| |
| Returns: |
| `Dict[str, any]`: Dictionary of all the attributes that make up this configuration instance, |
| """ |
| output = copy.deepcopy(self.__dict__) |
| output["encoder"] = self.encoder.to_dict() |
| output["decoder"] = self.decoder.to_dict() |
| output["model_type"] = self.__class__.model_type |
| return output |
|
|
