| |
|
|
| from transformers.configuration_utils import PretrainedConfig |
| from open_clip import get_model_config |
| from configuration_phi import PhiConfig |
|
|
|
|
| class LlavaConfig(PretrainedConfig): |
| model_type = "llava" |
| is_composition = False |
|
|
| def __init__( |
| self, |
| text_config=None, |
| vision_tower_name="ViT-SO400M-14-SigLIP-384", |
| ignore_index=-100, |
| image_token_index=50297, |
| projector_hidden_act="gelu", |
| projector_tokens_num=1, |
| vocab_size=51200, |
| **kwargs, |
| ): |
| self.ignore_index = ignore_index |
| self.image_token_index = image_token_index |
| self.projector_hidden_act = projector_hidden_act |
| self.projector_tokens_num = projector_tokens_num |
| self.vocab_size = vocab_size |
|
|
| self.vision_tower_name = vision_tower_name |
| vision_config = get_model_config(vision_tower_name) |
| self.vision_embed_dim = vision_config["embed_dim"] |
|
|
| self.vocab_size = self.vocab_size |
|
|
| self.text_config = text_config |
| if isinstance(self.text_config, dict): |
| text_config["model_type"] = text_config["model_type"] if "model_type" in text_config else "llama" |
| self.text_config = PhiConfig(**text_config) |
| self.vocab_size = self.text_config.vocab_size |
|
|
| super().__init__(**kwargs) |
