|
|
"""MarkupDM configuration""" |
|
|
|
|
|
from transformers import AutoConfig, PretrainedConfig |
|
|
from transformers.utils import logging |
|
|
|
|
|
logger = logging.get_logger(__name__) |
|
|
|
|
|
|
|
|
class MarkupDMConfig(PretrainedConfig): |
|
|
model_type = "markupdm" |
|
|
is_composition = True |
|
|
has_no_defaults_at_init = True |
|
|
|
|
|
def __init__( |
|
|
self, |
|
|
vocab_size: int = 49156, |
|
|
image_size: int = 256, |
|
|
image_pos_size: int = 4, |
|
|
image_pos_sigma: float = 10.0, |
|
|
image_loss_weight: float = 1.0, |
|
|
freeze_text_embeddings: bool = False, |
|
|
**kwargs, |
|
|
) -> None: |
|
|
super().__init__(**kwargs) |
|
|
|
|
|
if "text_model" not in kwargs or "vision_model" not in kwargs: |
|
|
raise ValueError( |
|
|
f"A configuraton of type {self.model_type} cannot be" |
|
|
"instantiated because not both `text_model` and `vision_model`" |
|
|
f"sub-configurations are passed, but only {kwargs}" |
|
|
) |
|
|
|
|
|
self.vocab_size = vocab_size |
|
|
self.image_size = image_size |
|
|
self.image_pos_size = image_pos_size |
|
|
self.image_pos_sigma = image_pos_sigma |
|
|
self.loss_type = "WeightedCausalLMLoss" |
|
|
self.image_loss_weight = image_loss_weight |
|
|
self.freeze_text_embeddings = freeze_text_embeddings |
|
|
|
|
|
text_config = kwargs.pop("text_model") |
|
|
vision_config = kwargs.pop("vision_model") |
|
|
|
|
|
if isinstance(text_config, PretrainedConfig): |
|
|
self.text_model = text_config |
|
|
else: |
|
|
path = text_config.pop("_name_or_path") |
|
|
self.text_model = AutoConfig.from_pretrained( |
|
|
path, |
|
|
**text_config, |
|
|
) |
|
|
|
|
|
if isinstance(vision_config, PretrainedConfig): |
|
|
self.vision_model = vision_config |
|
|
else: |
|
|
path = vision_config.pop("_name_or_path") |
|
|
self.vision_model = AutoConfig.from_pretrained( |
|
|
path, |
|
|
trust_remote_code=True, |
|
|
**vision_config, |
|
|
) |
|
|
|
|
|
|
|
|
self.initializer_range = self.text_model.initializer_range |
|
|
self.num_hidden_layers = self.text_model.num_hidden_layers |
|
|
self.is_decoder = True |
|
|
|
