Auto commit at $(date '+m-H:S')

configuration.py

@@ -0,0 +1,125 @@
+import logging
+
+from transformers.configuration_utils import PretrainedConfig
+from transformers.models.llama.configuration_llama import LlamaConfig
+from transformers.utils.constants import OPENAI_CLIP_MEAN, OPENAI_CLIP_STD
+
+logger = logging.getLogger("kanana-1.5-v")
+
+
+class KananaVVisionConfig(PretrainedConfig):
+    model_type = "kanana-1.5-v-visual-encoder"
+    base_config_key = "vision_config"
+
+    def __init__(
+        self,
+        depth=32,
+        embed_dim=1280,
+        mlp_ratio=4,
+        num_heads=16,
+        in_chans=3,
+        hidden_size=1280,
+        patch_size=14,
+        spatial_merge_size=2,
+        spatial_patch_size=14,
+        temporal_patch_size=2,
+        initializer_range=0.02,
+        image_size="dynamic",
+        image_mean=OPENAI_CLIP_MEAN,
+        image_std=OPENAI_CLIP_STD,
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+
+        self.depth = depth
+        self.embed_dim = embed_dim
+        self.mlp_ratio = mlp_ratio
+        self.num_heads = num_heads
+        self.in_chans = in_chans
+        self.hidden_size = hidden_size
+        self.patch_size = patch_size
+        self.spatial_merge_size = spatial_merge_size
+        self.spatial_patch_size = spatial_patch_size
+        self.temporal_patch_size = temporal_patch_size
+        self.initializer_range = initializer_range
+        self.image_size = image_size
+        self.image_mean = image_mean
+        self.image_std = image_std
+
+
+class KananaVVisualProjectorConfig(PretrainedConfig):
+    model_type = "kanana-1.5-v-visual_projector"
+    base_config_key = "projector_config"
+
+    def __init__(
+        self,
+        depth=2,
+        encoder_hidden_size=1280,
+        feature_layer_index=-1,
+        hidden_size=1024,
+        merge_size=2,
+        mlp_depth=2,
+        num_eos_tokens=0,
+        output_hidden_size=2048,
+        pos_emb=True,
+        pos_emb_size=576,
+        prenorm=False,
+        projector_type="dynamic-c-abs",
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+
+        self.depth = depth
+        self.encoder_hidden_size = encoder_hidden_size
+        self.feature_layer_index = feature_layer_index
+        self.hidden_size = hidden_size
+        self.merge_size = merge_size
+        self.mlp_depth = mlp_depth
+        self.num_eos_tokens = num_eos_tokens
+        self.output_hidden_size = output_hidden_size
+        self.pos_emb = pos_emb
+        self.pos_emb_size = pos_emb_size
+        self.prenorm = prenorm
+        self.projector_type = projector_type
+
+
+class KananaLanguageConfig(LlamaConfig):
+    model_type = "kanana-1.5-3b-instruct"
+    base_config_key = "text_config"
+
+    def __init__(
+        self,
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+
+
+class KananaVConfig(PretrainedConfig):
+    model_type = "kanana-1.5-v"
+    is_composition = True
+
+    def __init__(
+        self,
+        vision_config: dict = {},
+        projector_config: dict = {},
+        text_config: dict = {},
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+
+        # Vision config
+        self.vision_config = KananaVVisionConfig(**vision_config)
+
+        # Visual projector config
+        self.projector_config = KananaVVisualProjectorConfig(**projector_config)
+
+        # Language model config
+        self.text_config = KananaLanguageConfig(**text_config)
+
+    @property
+    def num_visual_tokens(self):
+        return "dynamic"
+
+    @property
+    def hidden_size(self):
+        return self.text_config.hidden_size