File size: 2,194 Bytes
599f7fa d3e05e9 599f7fa 6d6c47a 599f7fa | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 | """MarkupDM configuration"""
from transformers import AutoConfig, PretrainedConfig
from transformers.utils import logging
logger = logging.get_logger(__name__)
class MarkupDMConfig(PretrainedConfig):
model_type = "markupdm"
is_composition = True
has_no_defaults_at_init = True
def __init__(
self,
vocab_size: int = 49156,
image_size: int = 256,
image_pos_size: int = 4,
image_pos_sigma: float = 10.0,
image_loss_weight: float = 1.0,
freeze_text_embeddings: bool = False,
**kwargs,
) -> None:
super().__init__(**kwargs)
if "text_model" not in kwargs or "vision_model" not in kwargs:
raise ValueError(
f"A configuraton of type {self.model_type} cannot be"
"instantiated because not both `text_model` and `vision_model`"
f"sub-configurations are passed, but only {kwargs}"
)
self.vocab_size = vocab_size
self.image_size = image_size
self.image_pos_size = image_pos_size
self.image_pos_sigma = image_pos_sigma
self.loss_type = "WeightedCausalLMLoss"
self.image_loss_weight = image_loss_weight
self.freeze_text_embeddings = freeze_text_embeddings
text_config = kwargs.pop("text_model")
vision_config = kwargs.pop("vision_model")
if isinstance(text_config, PretrainedConfig):
self.text_model = text_config
else:
path = text_config.pop("_name_or_path")
self.text_model = AutoConfig.from_pretrained(
path,
**text_config,
)
if isinstance(vision_config, PretrainedConfig):
self.vision_model = vision_config
else:
path = vision_config.pop("_name_or_path")
self.vision_model = AutoConfig.from_pretrained(
path,
trust_remote_code=True,
**vision_config,
)
# Update config
self.initializer_range = self.text_model.initializer_range
self.num_hidden_layers = self.text_model.num_hidden_layers
self.is_decoder = True
|