first commit

config.json

@@ -0,0 +1,176 @@
+{
+  "_name_or_path": "JaronTHU/Video-CCAM-4B-v1.2",
+  "architectures": [
+    "VideoCCAM"
+  ],
+  "auto_map": {
+    "AutoConfig": "configuration_videoccam.VideoCCAMConfig",
+    "AutoModel": "modeling_videoccam.VideoCCAM"
+  },
+  "image_token_id": 32011,
+  "model_type": "videoccam",
+  "projector_config": {
+    "attention_bias": true,
+    "attention_dropout": 0.1,
+    "cross_hidden_size": 1152,
+    "dropout": 0.1,
+    "hidden_act": "swiglu",
+    "hidden_size": 1024,
+    "intermediate_size": 4096,
+    "layer_norm_eps": 1e-05,
+    "mlp_bias": true,
+    "num_heads": 16,
+    "num_key_value_heads": 16,
+    "num_query": 1024,
+    "output_size": 3072
+  },
+  "text_config": {
+    "_name_or_path": "Phi-3.5-mini-instruct",
+    "architectures": [
+      "Phi3ForCausalLM"
+    ],
+    "attention_dropout": 0.0,
+    "bos_token_id": 1,
+    "embd_pdrop": 0.0,
+    "eos_token_id": [
+        32007,
+        32001,
+        32000
+    ],
+    "hidden_act": "silu",
+    "hidden_size": 3072,
+    "initializer_range": 0.02,
+    "intermediate_size": 8192,
+    "max_position_embeddings": 131072,
+    "model_type": "phi3",
+    "num_attention_heads": 32,
+    "num_hidden_layers": 32,
+    "num_key_value_heads": 32,
+    "original_max_position_embeddings": 4096,
+    "pad_token_id": 32000,
+    "resid_pdrop": 0.0,
+    "rms_norm_eps": 1e-05,
+    "rope_scaling": {
+      "long_factor": [
+        1.0800000429153442,
+        1.1100000143051147,
+        1.1399999856948853,
+        1.340000033378601,
+        1.5899999141693115,
+        1.600000023841858,
+        1.6200000047683716,
+        2.620000123977661,
+        3.2300000190734863,
+        3.2300000190734863,
+        4.789999961853027,
+        7.400000095367432,
+        7.700000286102295,
+        9.09000015258789,
+        12.199999809265137,
+        17.670000076293945,
+        24.46000099182129,
+        28.57000160217285,
+        30.420001983642578,
+        30.840002059936523,
+        32.590003967285156,
+        32.93000411987305,
+        42.320003509521484,
+        44.96000289916992,
+        50.340003967285156,
+        50.45000457763672,
+        57.55000305175781,
+        57.93000411987305,
+        58.21000289916992,
+        60.1400032043457,
+        62.61000442504883,
+        62.62000274658203,
+        62.71000289916992,
+        63.1400032043457,
+        63.1400032043457,
+        63.77000427246094,
+        63.93000411987305,
+        63.96000289916992,
+        63.970001220703125,
+        64.02999877929688,
+        64.06999969482422,
+        64.08000183105469,
+        64.12000274658203,
+        64.41000366210938,
+        64.4800033569336,
+        64.51000213623047,
+        64.52999877929688,
+        64.83999633789062
+      ],
+      "short_factor": [
+        1.0,
+        1.0199999809265137,
+        1.0299999713897705,
+        1.0299999713897705,
+        1.0499999523162842,
+        1.0499999523162842,
+        1.0499999523162842,
+        1.0499999523162842,
+        1.0499999523162842,
+        1.0699999332427979,
+        1.0999999046325684,
+        1.1099998950958252,
+        1.1599998474121094,
+        1.1599998474121094,
+        1.1699998378753662,
+        1.2899998426437378,
+        1.339999794960022,
+        1.679999828338623,
+        1.7899998426437378,
+        1.8199998140335083,
+        1.8499997854232788,
+        1.8799997568130493,
+        1.9099997282028198,
+        1.9399996995925903,
+        1.9899996519088745,
+        2.0199997425079346,
+        2.0199997425079346,
+        2.0199997425079346,
+        2.0199997425079346,
+        2.0199997425079346,
+        2.0199997425079346,
+        2.0299997329711914,
+        2.0299997329711914,
+        2.0299997329711914,
+        2.0299997329711914,
+        2.0299997329711914,
+        2.0299997329711914,
+        2.0299997329711914,
+        2.0299997329711914,
+        2.0299997329711914,
+        2.0799996852874756,
+        2.0899996757507324,
+        2.189999580383301,
+        2.2199995517730713,
+        2.5899994373321533,
+        2.729999542236328,
+        2.749999523162842,
+        2.8399994373321533
+      ],
+      "type": "longrope"
+    },
+    "rope_theta": 10000.0,
+    "sliding_window": 262144,
+    "tie_word_embeddings": false,
+    "use_cache": true,
+    "attention_bias": false,
+    "vocab_size": 32064
+  },
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.44.2",
+  "video_token_id": 32012,
+  "vision_config": {
+    "hidden_size": 1152,
+    "image_size": 384,
+    "intermediate_size": 4304,
+    "model_type": "siglip_vision_model",
+    "num_attention_heads": 16,
+    "num_hidden_layers": 26,
+    "patch_size": 14,
+    "vision_use_head": false
+  }
+}

configuration_videoccam.py

@@ -0,0 +1,81 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+================================================
+@author: Jaron
+@time: 2024/08/21 17:51:45
+@email: fjjth98@163.com
+@description:
+================================================
+"""
+from typing import Union
+
+from transformers import PretrainedConfig
+from transformers.models.auto import CONFIG_MAPPING
+
+
+class CCAMConfig(PretrainedConfig):
+
+    def __init__(
+        self,
+        num_query: int = 1024,
+        num_heads: int = 16,
+        hidden_size: int = 1024,
+        intermediate_size: int = 4096,
+        num_key_value_heads: int = 16,
+        dropout: float = 0.1,
+        mlp_bias: bool = True,
+        hidden_act: str = 'swiglu',
+        output_size: int = None,            # inferred from llm
+        attention_bias: bool = True,
+        layer_norm_eps: float = 1e-5,
+        cross_hidden_size: int = None,      # inferred from vision encoder
+        attention_dropout: float = 0.1,
+        _attn_implementation: str = 'flash_attention_2',
+        **kwargs
+    ):
+        super().__init__(**kwargs)
+        self.dropout = dropout
+        self.mlp_bias = mlp_bias
+        self.num_query = num_query
+        self.num_heads = num_heads
+        self.hidden_act = hidden_act
+        self.hidden_size = hidden_size
+        self.output_size = output_size
+        self.layer_norm_eps = layer_norm_eps
+        self.attention_bias = attention_bias
+        self.intermediate_size = intermediate_size
+        self.cross_hidden_size = cross_hidden_size
+        self.attention_dropout = attention_dropout
+        self.num_key_value_heads = num_key_value_heads
+        self._attn_implementation = _attn_implementation
+
+
+class VideoCCAMConfig(PretrainedConfig):
+    model_type = 'videoccam'
+    _auto_class = 'AutoConfig'
+
+    def __init__(
+        self,
+        vision_config: Union[dict, PretrainedConfig] = None,
+        text_config: Union[dict, PretrainedConfig] = None,
+        projector_config: dict = None,
+        image_token_id: int = None,
+        video_token_id: int = None,
+        **kwargs
+    ):
+        super().__init__(**kwargs)
+        if isinstance(vision_config, dict):
+            self.vision_config = CONFIG_MAPPING[vision_config['model_type']](**vision_config)
+        else:
+            self.vision_config = vision_config
+        if isinstance(text_config, dict):
+            self.text_config = CONFIG_MAPPING[text_config['model_type']](**text_config)
+        else:
+            self.text_config = text_config
+        if isinstance(projector_config, dict):
+            self.projector_config = CCAMConfig(**projector_config)
+        else:
+            self.projector_config = projector_config
+        self.image_token_id = image_token_id
+        self.video_token_id = video_token_id

generation_config.json

@@ -0,0 +1,11 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": [
+      32007,
+      32001,
+      32000
+  ],
+  "pad_token_id": 32000,
+  "transformers_version": "4.44.2"
+}

model-00001-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fe6b04d55bbc51bbf119d109f5ff4ba7a2eb36307005594956041e69d81e8b5b
+size 4971634728

model-00002-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e530c6d8a0b74b6fe0a73af1957ce5540426da02646e5f627f929518901df445
+size 3519091320

model.safetensors.index.json

@@ -0,0 +1,638 @@
+{
+  "metadata": {
+    "total_size": 8490641984
+  },
+  "weight_map": {
+    "llm.lm_head.weight": "model-00002-of-00002.safetensors",
+    "llm.model.embed_tokens.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.0.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.0.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.0.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.1.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.1.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.1.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.10.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.10.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.10.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.10.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.10.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.11.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.11.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.11.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.11.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.12.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.12.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.12.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.12.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.12.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.12.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.13.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.13.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.13.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.13.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.13.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.14.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.14.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.14.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.14.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.14.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.14.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.15.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.15.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.15.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.15.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.15.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.15.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.16.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.16.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.16.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.16.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.16.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.16.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.17.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "llm.model.layers.17.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "llm.model.layers.17.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "llm.model.layers.17.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "llm.model.layers.17.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.17.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.18.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "llm.model.layers.18.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "llm.model.layers.18.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "llm.model.layers.18.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "llm.model.layers.18.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "llm.model.layers.18.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "llm.model.layers.19.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "llm.model.layers.19.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "llm.model.layers.19.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "llm.model.layers.19.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "llm.model.layers.19.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "llm.model.layers.19.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "llm.model.layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.2.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.2.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.2.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.20.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "llm.model.layers.20.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "llm.model.layers.20.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "llm.model.layers.20.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "llm.model.layers.20.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "llm.model.layers.20.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "llm.model.layers.21.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "llm.model.layers.21.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "llm.model.layers.21.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "llm.model.layers.21.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "llm.model.layers.21.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "llm.model.layers.21.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "llm.model.layers.22.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "llm.model.layers.22.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "llm.model.layers.22.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "llm.model.layers.22.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "llm.model.layers.22.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "llm.model.layers.22.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "llm.model.layers.23.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "llm.model.layers.23.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "llm.model.layers.23.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "llm.model.layers.23.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "llm.model.layers.23.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "llm.model.layers.23.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "llm.model.layers.24.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "llm.model.layers.24.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "llm.model.layers.24.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "llm.model.layers.24.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "llm.model.layers.24.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "llm.model.layers.24.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "llm.model.layers.25.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "llm.model.layers.25.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "llm.model.layers.25.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "llm.model.layers.25.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "llm.model.layers.25.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "llm.model.layers.25.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "llm.model.layers.26.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "llm.model.layers.26.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "llm.model.layers.26.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "llm.model.layers.26.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "llm.model.layers.26.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "llm.model.layers.26.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "llm.model.layers.27.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "llm.model.layers.27.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "llm.model.layers.27.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "llm.model.layers.27.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "llm.model.layers.27.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "llm.model.layers.27.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "llm.model.layers.28.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "llm.model.layers.28.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "llm.model.layers.28.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "llm.model.layers.28.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "llm.model.layers.28.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "llm.model.layers.28.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "llm.model.layers.29.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "llm.model.layers.29.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "llm.model.layers.29.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "llm.model.layers.29.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "llm.model.layers.29.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "llm.model.layers.29.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "llm.model.layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.3.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.3.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.30.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "llm.model.layers.30.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "llm.model.layers.30.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "llm.model.layers.30.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "llm.model.layers.30.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "llm.model.layers.30.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "llm.model.layers.31.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "llm.model.layers.31.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "llm.model.layers.31.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "llm.model.layers.31.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "llm.model.layers.31.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "llm.model.layers.31.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "llm.model.layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.4.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.4.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.4.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.5.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.5.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.5.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.6.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.6.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.6.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.7.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.7.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.8.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.8.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.8.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.8.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.8.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.9.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.9.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.9.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "llm.model.layers.9.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "llm.model.norm.weight": "model-00002-of-00002.safetensors",
+    "projector.ccam.k_proj.bias": "model-00001-of-00002.safetensors",
+    "projector.ccam.k_proj.weight": "model-00001-of-00002.safetensors",
+    "projector.ccam.o_proj.bias": "model-00001-of-00002.safetensors",
+    "projector.ccam.o_proj.weight": "model-00001-of-00002.safetensors",
+    "projector.ccam.q_proj.bias": "model-00001-of-00002.safetensors",
+    "projector.ccam.q_proj.weight": "model-00001-of-00002.safetensors",
+    "projector.ccam.v_proj.bias": "model-00001-of-00002.safetensors",
+    "projector.ccam.v_proj.weight": "model-00001-of-00002.safetensors",
+    "projector.post_ccam.0.bias": "model-00001-of-00002.safetensors",
+    "projector.post_ccam.0.weight": "model-00001-of-00002.safetensors",
+    "projector.post_ccam.2.fc1.bias": "model-00001-of-00002.safetensors",
+    "projector.post_ccam.2.fc1.weight": "model-00001-of-00002.safetensors",
+    "projector.post_ccam.2.fc2.bias": "model-00001-of-00002.safetensors",
+    "projector.post_ccam.2.fc2.weight": "model-00001-of-00002.safetensors",
+    "projector.pre_ccam.0.bias": "model-00001-of-00002.safetensors",
+    "projector.pre_ccam.0.weight": "model-00001-of-00002.safetensors",
+    "projector.query": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.embeddings.patch_embedding.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.embeddings.patch_embedding.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.embeddings.position_embedding.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.0.layer_norm1.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.0.layer_norm1.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.0.layer_norm2.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.0.layer_norm2.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.0.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.0.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.0.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.0.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.0.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.0.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.0.self_attn.out_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.0.self_attn.out_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.0.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.0.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.0.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.0.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.1.layer_norm1.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.1.layer_norm1.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.1.layer_norm2.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.1.layer_norm2.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.1.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.1.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.1.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.1.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.1.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.1.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.1.self_attn.out_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.1.self_attn.out_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.1.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.1.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.1.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.1.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.10.layer_norm1.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.10.layer_norm1.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.10.layer_norm2.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.10.layer_norm2.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.10.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.10.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.10.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.10.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.10.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.10.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.10.self_attn.out_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.10.self_attn.out_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.10.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.10.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.10.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.10.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.11.layer_norm1.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.11.layer_norm1.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.11.layer_norm2.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.11.layer_norm2.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.11.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.11.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.11.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.11.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.11.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.11.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.11.self_attn.out_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.11.self_attn.out_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.11.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.11.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.11.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.11.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.12.layer_norm1.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.12.layer_norm1.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.12.layer_norm2.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.12.layer_norm2.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.12.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.12.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.12.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.12.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.12.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.12.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.12.self_attn.out_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.12.self_attn.out_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.12.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.12.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.12.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.12.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.13.layer_norm1.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.13.layer_norm1.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.13.layer_norm2.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.13.layer_norm2.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.13.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.13.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.13.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.13.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.13.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.13.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.13.self_attn.out_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.13.self_attn.out_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.13.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.13.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.13.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.13.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.14.layer_norm1.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.14.layer_norm1.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.14.layer_norm2.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.14.layer_norm2.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.14.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.14.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.14.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.14.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.14.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.14.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.14.self_attn.out_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.14.self_attn.out_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.14.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.14.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.14.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.14.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.15.layer_norm1.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.15.layer_norm1.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.15.layer_norm2.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.15.layer_norm2.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.15.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.15.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.15.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.15.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.15.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.15.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.15.self_attn.out_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.15.self_attn.out_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.15.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.15.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.15.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.15.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.16.layer_norm1.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.16.layer_norm1.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.16.layer_norm2.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.16.layer_norm2.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.16.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.16.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.16.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.16.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.16.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.16.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.16.self_attn.out_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.16.self_attn.out_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.16.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.16.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.16.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.16.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.17.layer_norm1.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.17.layer_norm1.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.17.layer_norm2.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.17.layer_norm2.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.17.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.17.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.17.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.17.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.17.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.17.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.17.self_attn.out_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.17.self_attn.out_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.17.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.17.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.17.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.17.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.18.layer_norm1.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.18.layer_norm1.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.18.layer_norm2.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.18.layer_norm2.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.18.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.18.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.18.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.18.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.18.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.18.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.18.self_attn.out_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.18.self_attn.out_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.18.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.18.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.18.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.18.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.19.layer_norm1.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.19.layer_norm1.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.19.layer_norm2.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.19.layer_norm2.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.19.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.19.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.19.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.19.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.19.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.19.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.19.self_attn.out_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.19.self_attn.out_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.19.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.19.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.19.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.19.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.2.layer_norm1.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.2.layer_norm1.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.2.layer_norm2.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.2.layer_norm2.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.2.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.2.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.2.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.2.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.2.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.2.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.2.self_attn.out_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.2.self_attn.out_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.2.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.2.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.2.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.2.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.20.layer_norm1.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.20.layer_norm1.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.20.layer_norm2.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.20.layer_norm2.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.20.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.20.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.20.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.20.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.20.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.20.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.20.self_attn.out_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.20.self_attn.out_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.20.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.20.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.20.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.20.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.21.layer_norm1.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.21.layer_norm1.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.21.layer_norm2.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.21.layer_norm2.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.21.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.21.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.21.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.21.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.21.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.21.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.21.self_attn.out_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.21.self_attn.out_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.21.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.21.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.21.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.21.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.22.layer_norm1.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.22.layer_norm1.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.22.layer_norm2.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.22.layer_norm2.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.22.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.22.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.22.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.22.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.22.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.22.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.22.self_attn.out_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.22.self_attn.out_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.22.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.22.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.22.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.22.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.23.layer_norm1.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.23.layer_norm1.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.23.layer_norm2.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.23.layer_norm2.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.23.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.23.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.23.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.23.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.23.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.23.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.23.self_attn.out_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.23.self_attn.out_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.23.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.23.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.23.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.23.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.24.layer_norm1.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.24.layer_norm1.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.24.layer_norm2.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.24.layer_norm2.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.24.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.24.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.24.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.24.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.24.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.24.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.24.self_attn.out_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.24.self_attn.out_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.24.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.24.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.24.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.24.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.25.layer_norm1.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.25.layer_norm1.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.25.layer_norm2.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.25.layer_norm2.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.25.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.25.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.25.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.25.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.25.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.25.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.25.self_attn.out_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.25.self_attn.out_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.25.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.25.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.25.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.25.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.3.layer_norm1.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.3.layer_norm1.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.3.layer_norm2.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.3.layer_norm2.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.3.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.3.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.3.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.3.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.3.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.3.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.3.self_attn.out_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.3.self_attn.out_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.3.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.3.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.3.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.3.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.4.layer_norm1.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.4.layer_norm1.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.4.layer_norm2.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.4.layer_norm2.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.4.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.4.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.4.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.4.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.4.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.4.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.4.self_attn.out_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.4.self_attn.out_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.4.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.4.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.4.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.4.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.5.layer_norm1.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.5.layer_norm1.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.5.layer_norm2.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.5.layer_norm2.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.5.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.5.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.5.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.5.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.5.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.5.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.5.self_attn.out_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.5.self_attn.out_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.5.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.5.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.5.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.5.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.6.layer_norm1.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.6.layer_norm1.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.6.layer_norm2.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.6.layer_norm2.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.6.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.6.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.6.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.6.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.6.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.6.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.6.self_attn.out_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.6.self_attn.out_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.6.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.6.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.6.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.6.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.7.layer_norm1.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.7.layer_norm1.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.7.layer_norm2.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.7.layer_norm2.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.7.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.7.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.7.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.7.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.7.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.7.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.7.self_attn.out_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.7.self_attn.out_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.7.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.7.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.7.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.7.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.8.layer_norm1.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.8.layer_norm1.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.8.layer_norm2.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.8.layer_norm2.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.8.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.8.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.8.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.8.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.8.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.8.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.8.self_attn.out_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.8.self_attn.out_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.8.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.8.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.8.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.8.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.9.layer_norm1.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.9.layer_norm1.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.9.layer_norm2.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.9.layer_norm2.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.9.mlp.fc1.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.9.mlp.fc1.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.9.mlp.fc2.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.9.mlp.fc2.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.9.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.9.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.9.self_attn.out_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.9.self_attn.out_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.9.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.9.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.9.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "vision_encoder.vision_model.encoder.layers.9.self_attn.v_proj.weight": "model-00001-of-00002.safetensors"
+  }
+}

modeling_videoccam.py

@@ -0,0 +1,377 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+"""
+================================================
+@author: Jaron
+@time: 2024/08/21 17:41:52
+@email: fjjth98@163.com
+@description: Video-CCAM
+================================================
+"""
+from typing import Optional, Union
+
+import torch
+from PIL import Image
+from torch import nn
+from torch.nn import functional as F
+from transformers import (AutoImageProcessor, AutoModel, AutoModelForCausalLM,
+                          AutoTokenizer, Cache, DynamicCache, GenerationConfig,
+                          PreTrainedModel)
+from transformers.activations import ACT2FN
+
+from .configuration_videoccam import CCAMConfig, VideoCCAMConfig
+
+
+class CCAMMLP(nn.Module):
+
+    def __init__(self, config):
+        super().__init__()
+        self.hidden_act = config.hidden_act
+        self.hidden_size = config.hidden_size
+        self.intermediate_size = config.intermediate_size
+        self.output_size = config.output_size
+        if self.hidden_act == 'swiglu':
+            self.fc1 = nn.Linear(self.hidden_size, self.intermediate_size * 2, bias=config.mlp_bias)
+            self.act_fn = ACT2FN['silu']
+        else:
+            self.fc1 = nn.Linear(self.hidden_size, self.intermediate_size, bias=config.mlp_bias)
+            self.act_fn = ACT2FN[self.hidden_act]
+        self.fc2 = nn.Linear(self.intermediate_size, self.output_size, bias=config.mlp_bias)
+
+    def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+        hidden_states = self.fc1(hidden_states)
+        if self.hidden_act == 'swiglu':
+            gate, up = hidden_states.chunk(2, dim=-1)
+            hidden_states = self.act_fn(gate) * up
+        else:
+            hidden_states = self.act_fn(hidden_states)
+        hidden_states = self.fc2(hidden_states)
+        return hidden_states
+
+
+class CCAMCrossAttention(nn.Module):
+    """Cross-attention layer of the CCAM projector.
+
+    Flash Attention 2 is not supported since the mask may be neither full nor causal. Only support `attn_implementation` as `eager` and `sdpa`.
+    """
+
+    def __init__(self, config):
+        super().__init__()
+        self.num_heads = config.num_heads
+        self.hidden_size = config.hidden_size
+        self.attention_bias = config.attention_bias
+        self.attention_dropout = config.attention_dropout
+        self.cross_hidden_size = config.cross_hidden_size
+        self.num_key_value_heads = config.num_key_value_heads
+        self.attn_implementation = config._attn_implementation
+        self.head_dim = self.hidden_size // self.num_heads
+        self.num_key_value_groups = self.num_heads // self.num_key_value_heads
+
+        assert self.head_dim * self.num_heads == self.hidden_size, f'hidden_size must be divisible by num_heads (got `hidden_size`: {self.hidden_size} and `num_heads`: {self.num_heads}).'
+
+        self.q_proj = nn.Linear(self.hidden_size, self.hidden_size, bias=self.attention_bias)
+        self.k_proj = nn.Linear(self.cross_hidden_size, self.num_key_value_heads * self.head_dim, bias=self.attention_bias)
+        self.v_proj = nn.Linear(self.cross_hidden_size, self.num_key_value_heads * self.head_dim, bias=self.attention_bias)
+        self.o_proj = nn.Linear(self.hidden_size, self.hidden_size, bias=self.attention_bias)
+
+    def forward(
+        self,
+        hidden_states: torch.Tensor,                # (B, Q, C)
+        cross_hidden_states: torch.Tensor,          # (B, L, C')
+        attention_mask: torch.Tensor = None         # (Q, L), '-inf' means masked, 0 means not masked
+    ) -> torch.Tensor:      # (B, Q, C)
+        B, Q, C = hidden_states.size()
+        query_states = self.q_proj(hidden_states)   # (B, Q, C)
+        key_states = self.k_proj(cross_hidden_states)
+        value_states = self.v_proj(cross_hidden_states)
+
+        L = key_states.size(1)
+        query_states = query_states.view(B, Q, self.num_heads, self.head_dim).transpose(1, 2)
+        key_states = key_states.view(B, L, self.num_key_value_heads, self.head_dim).transpose(1, 2)
+        value_states = value_states.view(B, L, self.num_key_value_heads, self.head_dim).transpose(1, 2)
+        if self.num_key_value_groups > 1:
+            key_states = key_states.repeat_interleave(repeats=self.num_key_value_groups, dim=1)
+            value_states = value_states.repeat_interleave(repeats=self.num_key_value_groups, dim=1)
+
+        if self.attn_implementation == 'eager':
+            attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / self.head_dim ** 0.5    # (B, num_heads, Q, L)
+            if attention_mask is not None:
+                attn_weights = attn_weights + attention_mask.view(1, 1, Q, L)
+            # upcast attention to fp32
+            attn_weights = F.softmax(attn_weights, dim=-1, dtype=torch.float32).to(query_states.dtype)
+            attn_weights = F.dropout(attn_weights, p=self.attention_dropout, training=self.training)
+            attn_output = torch.matmul(attn_weights, value_states)      # (B, num_heads, Q, head_dim)
+        else:           # 'sdpa'
+            # there are bugs in torch <=2.1.0, requiring qkv as contiguous(), be careful
+            attn_output = F.scaled_dot_product_attention(
+                query_states,
+                key_states,
+                value_states,
+                attn_mask=attention_mask,
+                dropout_p=self.attention_dropout if self.training else 0.0
+            )
+        attn_output = attn_output.transpose(1, 2).reshape(B, Q, C)          # (B, Q, C)
+        attn_output = self.o_proj(attn_output)
+
+        return attn_output
+
+
+class CCAMModel(PreTrainedModel):
+    config_class = CCAMConfig
+    _no_split_modules = ['CCAMCrossAttention']
+    _supports_flash_attn_2 = True   # actually flash_attention_2 is not supported in the projector, manually convert it to sdpa
+    _supports_sdpa = True
+
+    def __init__(self, config: CCAMConfig):
+        super().__init__(config)
+        self.num_query = config.num_query
+        self.hidden_size = config.hidden_size
+        self.output_size = config.output_size
+        self.cross_hidden_size = config.cross_hidden_size
+
+        self.query = nn.Parameter(torch.empty(1, self.num_query, self.hidden_size).normal_(mean=.0, std=.02))
+        self.pre_ccam = nn.Sequential(
+            nn.LayerNorm(self.hidden_size, eps=config.layer_norm_eps),
+            nn.Dropout(config.dropout)
+        )
+        self.ccam = CCAMCrossAttention(config)
+        self.post_ccam = nn.Sequential(
+            nn.LayerNorm(self.hidden_size, eps=config.layer_norm_eps),
+            nn.Dropout(config.dropout),
+            CCAMMLP(config)
+        )
+
+    def get_ccam(self, vision_hidden_state: torch.Tensor) -> torch.Tensor:      # (Q, T*L)
+        """Compute CCAM Mask for vision hidden state
+
+        Args:
+            vision_hidden_state (torch.Tensor): (T, L, C)
+
+        Returns:
+            torch.Tensor: (Q, T*L) -inf means masked
+        """
+        T, L, _ = vision_hidden_state.size()
+        dtype, device = vision_hidden_state.dtype, vision_hidden_state.device
+        base_mask = torch.zeros(T, T, dtype=dtype, device=device)
+        t = torch.arange(T, device=device)
+        base_mask.masked_fill_(t > t[:, None], float('-inf'))
+        attention_mask = torch.zeros(self.num_query, T * L, dtype=dtype, device=device)
+        attention_mask[:self.num_query // T * T] = torch.kron(base_mask, torch.ones(self.num_query // T, L, dtype=dtype, device=device))
+        return attention_mask
+
+    def forward(self, vision_hidden_states: list[torch.Tensor]) -> torch.Tensor:      # (B, Q, C)
+        """Forward function, do not collect batch due to the support of zero3
+
+        Args:
+            vision_hidden_states (list[torch.Tensor]): [(t0, L, C), (t1, L, C), ...]
+
+        Returns:
+            torch.Tensor: (B, Q, C)
+        """
+        output = []
+        for hidden_states in vision_hidden_states:
+            # reshape inputs and construct ccam masks
+            attention_mask = self.get_ccam(hidden_states)    # (Q, ti * L)
+            # forward
+            x = self.pre_ccam(self.query)       # (1, Q, C)
+            x = self.ccam(
+                hidden_states=x,                # (1, Q, C)
+                cross_hidden_states=hidden_states.flatten(0, 1)[None],      # (1, ti * L, C')
+                attention_mask=attention_mask[None]     # (1, Q, ti * L)
+            ) + x
+            x = self.post_ccam(x)
+            output.append(x)
+        output = torch.cat(output, dim=0)
+        return output
+
+
+# Modified from transformers.models.llava_next.modeling_llava_next.py
+class VideoCCAM(PreTrainedModel):
+    config_class = VideoCCAMConfig
+    _auto_class = 'AutoModel'
+    _supports_flash_attn_2 = True
+
+    def __init__(self, config: VideoCCAMConfig):
+        super().__init__(config)
+        # the following only works for SiglipVisionModel
+        self.vision_encoder = AutoModel.from_config(config.vision_config, torch_dtype=config.torch_dtype, attn_implementation=config._attn_implementation)
+        self.vision_encoder.vision_model.post_layernorm = nn.Identity()
+        self.projector = CCAMModel._from_config(config.projector_config, torch_dtype=config.torch_dtype, attn_implementation=config._attn_implementation)
+        self.llm = AutoModelForCausalLM.from_config(config.text_config, torch_dtype=config.torch_dtype, attn_implementation=config._attn_implementation)
+        self.post_init()
+
+    # copied from transformers.models.llava_next.modeling_llava_next
+    def _init_weights(self, module, std=.02):
+        if isinstance(module, (nn.Linear, nn.Conv2d)):
+            module.weight.data.normal_(mean=0.0, std=std)
+            if module.bias is not None:
+                module.bias.data.zero_()
+        elif isinstance(module, nn.Embedding):
+            module.weight.data.normal_(mean=0.0, std=std)
+            if module.padding_idx is not None:
+                module.weight.data[module.padding_idx].zero_()
+
+    @property
+    def _supports_sdpa(self):
+        """
+        Retrieve language_model's attribute to check whether the model supports
+        SDPA or not.
+        """
+        return self.llm._supports_sdpa
+
+    @property
+    def _no_split_modules(self):
+        """
+        Retrieve language_model's attribute to check whether the model supports
+        SDPA or not.
+        """
+        return self.vision_encoder._no_split_modules + self.projector._no_split_modules + self.llm._no_split_modules
+
+    @torch.inference_mode
+    def generate(
+        self,
+        input_ids: list[list[int]] = None,              # [(l_0,), (l_1,), ...]
+        pixel_values: torch.FloatTensor = None,         # (t_0+t_1+..., 3, H, W)
+        vision_split_sizes: list[int] = None,           # [t_0, t_1, ...]
+        past_key_values: Union[tuple, Cache] = None,
+        batch_generation: bool = False,
+        generation_config: GenerationConfig = None,
+        **kwargs
+    ) -> tuple[torch.LongTensor, Optional[Cache]]:
+        """Generation for multi-modal inputs
+
+        Args:
+            input_ids (list[list[int]]): input token indices, use list[int] for efficient embeddings concatenation.
+            pixel_values (torch.FloatTensor): input image/video (processed) pixel values.
+            vision_split_sizes (list[int]): for each vision token (<image>, <video>), how many frames are required.
+            past_key_values (Union[tuple, Cache]): past_key_values for efficient generation, only used for multi-turn dialogue and single inputs. If this argument is not None, new past_key_values will also be returned.
+            batch_generation (bool, optional): whether left padding for batch inputs. Defaults to False.
+            generation_config (GenerationConfig, optional): _description_. Defaults to None.
+
+        Returns:
+            torch.LongTensor: _description_
+        """
+        if past_key_values is not None and len(input_ids) != 1:
+            raise ValueError(f'`past_key_values` is only supported when there is only 1 `input_ids`.')
+        # compute text embeddings
+        device = self.llm.get_input_embeddings().weight.device
+        _input_ids, text_split_pos = [], [0]
+        for ids in input_ids:
+            _input_ids += ids
+            text_split_pos.append(text_split_pos[-1] + len(ids))
+        _input_ids = torch.tensor(_input_ids, dtype=torch.long, device=device)
+        vision_pos = torch.where((_input_ids == self.config.image_token_id) | (_input_ids == self.config.video_token_id))[0].tolist()
+        _inputs_embeds = self.llm.get_input_embeddings()(_input_ids)
+
+        # compute vision embeddings
+        if pixel_values is not None:
+            assert len(vision_pos) == len(vision_split_sizes), f'The number of visual tokens ({len(vision_pos)}) should be equal to the number of visual features ({len(vision_split_sizes)}).'
+            vision_embeds = self.vision_encoder(pixel_values, output_hidden_states=False).last_hidden_state
+            vision_embeds = self.projector(vision_embeds.split(vision_split_sizes, dim=0))
+
+        # insert vision embeddings among text embeddings
+        inputs_embeds_len, inputs_embeds, idx = [], [], 0
+        for i in range(1, len(text_split_pos)):
+            start, cur_inputs_embeds = text_split_pos[i-1], []
+            while idx < len(vision_pos) and vision_pos[idx] < text_split_pos[i]:
+                cur_inputs_embeds.append(_inputs_embeds[start:vision_pos[idx]])
+                cur_inputs_embeds.append(vision_embeds[idx])
+                start, idx = vision_pos[idx] + 1, idx + 1
+            if start < text_split_pos[i]:
+                cur_inputs_embeds.append(_inputs_embeds[start:text_split_pos[i]])
+            inputs_embeds_len.append(sum(i.size(0) for i in cur_inputs_embeds))
+            inputs_embeds.append(cur_inputs_embeds)
+
+        # batch processing is only supported only if no `past_key_values` is provided
+        if past_key_values is None:
+            # left padding for batch generation
+            if batch_generation:
+                B, L = len(input_ids), max(inputs_embeds_len)
+                padded_inputs_embeds, attention_mask = [], []
+                pad_embeds = self.llm.get_input_embeddings()(torch.tensor([self.config.text_config.pad_token_id], dtype=torch.long, device=device))   # (1, C')
+                for l, embeds in zip(inputs_embeds_len, inputs_embeds):
+                    padded_inputs_embeds.append(pad_embeds.expand(L - l, -1))
+                    padded_inputs_embeds += embeds
+                    attention_mask += [0] * (L- l) + [1] * l
+                padded_inputs_embeds = torch.cat(padded_inputs_embeds, dim=0).view(B, L, -1)
+                attention_mask = torch.tensor(attention_mask, dtype=torch.long, device=device).view(B, L)
+                output_ids = self.llm.generate(
+                    inputs_embeds=padded_inputs_embeds,
+                    attention_mask=attention_mask,
+                    generation_config=generation_config,
+                    **kwargs
+                )
+            else:
+                output_ids = []
+                for l, embeds in zip(inputs_embeds_len, inputs_embeds):
+                    output_ids += self.llm.generate(
+                        inputs_embeds=torch.cat(embeds, dim=0)[None],
+                        attention_mask=torch.ones(1, l, dtype=torch.long, device=device),
+                        generation_config=generation_config,
+                        **kwargs
+                    )
+            return output_ids
+        else:
+            inputs_embeds = torch.cat(inputs_embeds[0], dim=0)
+            if not isinstance(past_key_values, Cache):
+                past_key_values = DynamicCache.from_legacy_cache(past_key_values)
+            # use inputs_embeds and input past_key_values to compute output past_key_values (manually prefill)
+            past_key_values = self.llm(
+                inputs_embeds=inputs_embeds[None, :-1],
+                past_key_values=past_key_values,
+                return_dict=True
+            ).past_key_values
+            # here pseudo_input_ids means the prefix ids are just placeholders
+            pseudo_input_ids_len = past_key_values.get_seq_length() + 1
+            pseudo_input_ids = torch.zeros(1, pseudo_input_ids_len, dtype=torch.long, device=device)
+            pseudo_input_ids[0, -1] = _input_ids[-1]
+            output = self.llm.generate(
+                input_ids=pseudo_input_ids,
+                past_key_values=past_key_values,
+                generation_config=generation_config,
+                return_dict_in_generate=True,
+                **kwargs
+            )
+            return output.sequences[0, pseudo_input_ids_len:], output.past_key_values
+
+    def chat(
+        self,
+        messages: list[list[dict]],
+        images: list[list[Image.Image]] = None,
+        tokenizer: AutoTokenizer = None,
+        image_processor: AutoImageProcessor = None,
+        batch_generation: bool = False,
+        generation_config = None,
+        **kwargs
+    ) -> list[str]:
+        # images
+        pixel_values, vision_split_sizes = [], []
+        for image in images:
+            pixel_values += image
+            vision_split_sizes.append(len(image))
+        if len(pixel_values) > 0:
+            pixel_values = image_processor(pixel_values, return_tensors='pt')['pixel_values'].to(
+                dtype=self.vision_encoder.get_input_embeddings().weight.dtype,
+                device=self.vision_encoder.get_input_embeddings().weight.device
+            )
+        else:
+            pixel_values = None
+        # texts
+        input_ids = tokenizer.apply_chat_template(
+            messages,
+            tokenize=True,
+            add_generation_prompt=True,
+            return_dict=False
+        )
+        # generation
+        output_ids = self.generate(
+            input_ids=input_ids,
+            pixel_values=pixel_values,
+            vision_split_sizes=vision_split_sizes,
+            batch_generation=batch_generation,
+            generation_config=generation_config,
+            **kwargs
+        )
+        # decoding
+        prediction = tokenizer.batch_decode(output_ids, skip_special_tokens=True)
+        return prediction

preprocessor_config.json

@@ -0,0 +1,24 @@
+{
+  "do_convert_rgb": null,
+  "do_normalize": true,
+  "do_rescale": true,
+  "do_resize": true,
+  "image_mean": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "image_processor_type": "SiglipImageProcessor",
+  "image_std": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "processor_class": "SiglipProcessor",
+  "resample": 3,
+  "rescale_factor": 0.00392156862745098,
+  "size": {
+    "height": 384,
+    "width": 384
+  }
+}

special_tokens_map.json

@@ -0,0 +1,34 @@
+{
+  "additional_special_tokens": [
+    "<image>",
+    "<video>"
+  ],
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,151 @@
+{
+  "add_bos_token": false,
+  "add_eos_token": false,
+  "add_prefix_space": null,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": false
+    },
+    "32000": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32001": {
+      "content": "<|assistant|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32002": {
+      "content": "<|placeholder1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32003": {
+      "content": "<|placeholder2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32004": {
+      "content": "<|placeholder3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32005": {
+      "content": "<|placeholder4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32006": {
+      "content": "<|system|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32007": {
+      "content": "<|end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32008": {
+      "content": "<|placeholder5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32009": {
+      "content": "<|placeholder6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32010": {
+      "content": "<|user|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32011": {
+      "content": "<image>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32012": {
+      "content": "<video>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<image>",
+    "<video>"
+  ],
+  "bos_token": "<s>",
+  "chat_template": "{% for message in messages %}{% if message['role'] == 'system' and message['content'] %}{{'<|system|>\n' + message['content'] + '<|end|>\n'}}{% elif message['role'] == 'user' %}{{'<|user|>\n' + message['content'] + '<|end|>\n'}}{% elif message['role'] == 'assistant' %}{{'<|assistant|>\n' + message['content'] + '<|end|>\n'}}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '<|assistant|>\n' }}{% else %}{{ eos_token }}{% endif %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "legacy": false,
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "left",
+  "sp_model_kwargs": {},
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}