Upload 18 files

added_tokens.json

@@ -0,0 +1,8 @@
+{
+  "<|document|>": 151647,
+  "<|endoftext|>": 151643,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image|>": 151646,
+  "<|video|>": 151648
+}

clip_encoder_hd.py

@@ -0,0 +1,127 @@
+import torch
+import torch.nn as nn
+import re
+import math 
+from transformers import CLIPVisionModel, CLIPImageProcessor, CLIPVisionConfig
+
+
+def build_vision_tower():
+    vision_tower = 'openai/clip-vit-large-patch14-336'
+    return CLIPVisionTower(vision_tower)
+
+class CLIPVisionTowerHD(nn.Module):
+    def __init__(self, config, vision_select_layer=-2):
+        super().__init__()
+
+        self.is_loaded = False
+
+        # self.vision_tower_name = vision_tower
+        self.vis_config = config
+        self.select_layer = vision_select_layer
+        self.select_feature = 'patch'
+        self.load_model()
+
+    def load_model(self):
+        # self.vision_tower = CLIPVisionModel.from_pretrained(self.vision_tower_name)
+        self.vision_tower = CLIPVisionModel(CLIPVisionConfig(**self.vis_config))
+        self.vision_tower.requires_grad_(False)
+
+        self.is_loaded = True
+
+    def resize_pos(self):
+        print ('Dummy Resized')
+
+    def feature_select(self, image_forward_outs):
+        image_features = image_forward_outs.hidden_states[self.select_layer]
+        if self.select_feature == 'patch':
+            image_features = image_features[:, 1:]
+        elif self.select_feature == 'cls_patch':
+            image_features = image_features
+        else:
+            raise ValueError(f'Unexpected select feature: {self.select_feature}')
+        return image_features
+
+    def forward(self, images, glb_GN, sub_GN):
+        if not self.is_loaded:
+            self.load_model()
+        assert type(images) is list
+        shapes = []
+        input_imgs = []
+        for img in images:
+            _, C, H, W = img.shape
+            shapes.append([H//336, W//336])
+            sub_img = img.reshape(1,3,H//336,336,W//336,336).permute(0,2,4,1,3,5).reshape(-1,3,336,336).contiguous()
+            glb_img = torch.nn.functional.interpolate(img.float(), size=(336,336), mode='bicubic',).to(sub_img.dtype)
+            input_imgs.append(glb_img)
+            input_imgs.append(sub_img)
+        input_imgs = torch.cat(input_imgs, dim=0)
+
+        image_forward_outs = self.vision_tower(input_imgs.to(device=self.device, dtype=self.dtype), output_hidden_states=True)
+        image_features = self.feature_select(image_forward_outs).to(input_imgs.dtype) ### B*?, N, C
+        _, N, C = image_features.shape
+        H = int(math.sqrt(N))
+        assert N == 24 ** 2
+
+        output_imgs = []
+        output_len = []
+        for [h, w] in shapes:
+            B_ = h*w
+            glb_img = image_features[:1] ### 1, N, C
+            glb_img = glb_img.reshape(1,H,H,C).reshape(1,H//2,2,H//2,2,C).contiguous().permute(0,1,3,2,4,5).reshape(1,H//2,H//2,4*C).contiguous()
+            temp_glb_GN = sub_GN.repeat(1, H//2, 1, 1)
+            glb_img = torch.cat([glb_img, temp_glb_GN], dim=2).reshape(1,-1,4*C)
+
+            sub_img = image_features[1:1+B_] ### ?, N, C
+            sub_img = sub_img.reshape(B_,H,H,C).reshape(B_,H//2,2,H//2,2,C).contiguous().permute(0,1,3,2,4,5).reshape(B_,-1,4*C).contiguous()
+            sub_img = sub_img.reshape(1, h, w, 12, 12, -1).permute(0,1,3,2,4,5).reshape(1,h*12,w*12,4*C)
+            temp_sub_GN = sub_GN.repeat(1, h*12, 1, 1)
+            sub_img = torch.cat([sub_img, temp_sub_GN], dim=2).reshape(1,-1,4*C)
+
+            output_imgs.append(torch.cat([glb_img, glb_GN, sub_img], dim=1))
+            temp_len = int((h*w+1)*144 + 1 + (h+1)*12)
+            assert temp_len == output_imgs[-1].shape[1]
+            output_len.append(temp_len)
+
+            image_features = image_features[1+h*w:]
+
+        new_output_imgs = []
+        max_len = max(output_len)
+        for img_feat in output_imgs:
+            if img_feat.shape[1] < max_len:
+                pad_feat = torch.zeros(1, (max_len-img_feat.shape[1]), img_feat.shape[2]).to(img_feat.device)
+                img_feat_padding = torch.cat([img_feat, pad_feat], dim=1)
+                new_output_imgs.append(img_feat_padding)
+            else:
+                new_output_imgs.append(img_feat)
+
+        output_imgs = torch.cat(new_output_imgs, dim=0)
+
+        return output_imgs, output_len
+
+    @property
+    def dummy_feature(self):
+        return torch.zeros(1, self.hidden_size, device=self.device, dtype=self.dtype)
+
+    @property
+    def dtype(self):
+        return self.vision_tower.dtype
+
+    @property
+    def device(self):
+        return self.vision_tower.device
+
+    @property
+    def config(self):
+        if self.is_loaded:
+            return self.vision_tower.config
+        else:
+            return self.cfg_only
+
+    @property
+    def num_features(self):
+        return self.config.hidden_size
+
+    @property
+    def num_patches(self):
+        return (self.config.image_size // self.config.patch_size) ** 2
+

config.json

@@ -0,0 +1,178 @@
+{
+  "architectures": [
+    "InfMLLM_Unified_HD_Chat"
+  ],
+  "auto_map": {
+    "AutoConfig": "configuration_infmllm_unified_hd_chat.InfMLLMUnifiedHDChatConfig",
+    "AutoModel": "modeling_infmllm_unified_hd_chat.InfMLLM_Unified_HD_Chat"
+  },
+  "conv_style": "qwen-7b-chat",
+  "encoder_img": "pretrain_models/openai/clip-vit-large-patch14-336/",
+  "encoder_video": null,
+  "hd_num": 16,
+  "image_size_img": 336,
+  "lm_config": {
+    "_name_or_path": "",
+    "add_cross_attention": false,
+    "architectures": [
+      "Qwen2ForCausalLM"
+    ],
+    "attention_dropout": 0.0,
+    "bad_words_ids": null,
+    "begin_suppress_tokens": null,
+    "bos_token_id": 151643,
+    "chunk_size_feed_forward": 0,
+    "cross_attention_hidden_size": null,
+    "decoder_start_token_id": null,
+    "diversity_penalty": 0.0,
+    "do_sample": false,
+    "early_stopping": false,
+    "encoder_no_repeat_ngram_size": 0,
+    "eos_token_id": 151645,
+    "exponential_decay_length_penalty": null,
+    "finetuning_task": null,
+    "forced_bos_token_id": null,
+    "forced_eos_token_id": null,
+    "hidden_act": "silu",
+    "hidden_size": 3584,
+    "id2label": {
+      "0": "LABEL_0",
+      "1": "LABEL_1"
+    },
+    "initializer_range": 0.02,
+    "intermediate_size": 18944,
+    "is_decoder": false,
+    "is_encoder_decoder": false,
+    "label2id": {
+      "LABEL_0": 0,
+      "LABEL_1": 1
+    },
+    "length_penalty": 1.0,
+    "max_length": 20,
+    "max_position_embeddings": 32768,
+    "max_window_layers": 28,
+    "min_length": 0,
+    "model_type": "qwen2",
+    "no_repeat_ngram_size": 0,
+    "num_attention_heads": 28,
+    "num_beam_groups": 1,
+    "num_beams": 1,
+    "num_hidden_layers": 28,
+    "num_key_value_heads": 4,
+    "num_return_sequences": 1,
+    "output_attentions": false,
+    "output_hidden_states": false,
+    "output_scores": false,
+    "pad_token_id": null,
+    "prefix": null,
+    "problem_type": null,
+    "pruned_heads": {},
+    "remove_invalid_values": false,
+    "repetition_penalty": 1.0,
+    "return_dict": true,
+    "return_dict_in_generate": false,
+    "rms_norm_eps": 1e-06,
+    "rope_theta": 1000000.0,
+    "sep_token_id": null,
+    "sliding_window": 131072,
+    "suppress_tokens": null,
+    "task_specific_params": null,
+    "temperature": 1.0,
+    "tf_legacy_loss": false,
+    "tie_encoder_decoder": false,
+    "tie_word_embeddings": false,
+    "tokenizer_class": null,
+    "top_k": 50,
+    "top_p": 1.0,
+    "torch_dtype": "bfloat16",
+    "torchscript": false,
+    "typical_p": 1.0,
+    "use_bfloat16": false,
+    "use_cache": true,
+    "use_sliding_window": false,
+    "vocab_size": 152064
+  },
+  "lm_model": "pretrain_models/Qwen2-7B-Instruct/",
+  "lm_tokenizer": "pretrain_models/Qwen2-7B-Instruct/",
+  "max_txt_len": 4096,
+  "precision": "bf16",
+  "torch_dtype": "float32",
+  "transformers_version": "4.40.2",
+  "vision_config": {
+    "_name_or_path": "",
+    "add_cross_attention": false,
+    "architectures": null,
+    "attention_dropout": 0.0,
+    "bad_words_ids": null,
+    "begin_suppress_tokens": null,
+    "bos_token_id": null,
+    "chunk_size_feed_forward": 0,
+    "cross_attention_hidden_size": null,
+    "decoder_start_token_id": null,
+    "diversity_penalty": 0.0,
+    "do_sample": false,
+    "dropout": 0.0,
+    "early_stopping": false,
+    "encoder_no_repeat_ngram_size": 0,
+    "eos_token_id": null,
+    "exponential_decay_length_penalty": null,
+    "finetuning_task": null,
+    "forced_bos_token_id": null,
+    "forced_eos_token_id": null,
+    "hidden_act": "quick_gelu",
+    "hidden_size": 1024,
+    "id2label": {
+      "0": "LABEL_0",
+      "1": "LABEL_1"
+    },
+    "image_size": 336,
+    "initializer_factor": 1.0,
+    "initializer_range": 0.02,
+    "intermediate_size": 4096,
+    "is_decoder": false,
+    "is_encoder_decoder": false,
+    "label2id": {
+      "LABEL_0": 0,
+      "LABEL_1": 1
+    },
+    "layer_norm_eps": 1e-05,
+    "length_penalty": 1.0,
+    "max_length": 20,
+    "min_length": 0,
+    "model_type": "clip_vision_model",
+    "no_repeat_ngram_size": 0,
+    "num_attention_heads": 16,
+    "num_beam_groups": 1,
+    "num_beams": 1,
+    "num_channels": 3,
+    "num_hidden_layers": 24,
+    "num_return_sequences": 1,
+    "output_attentions": false,
+    "output_hidden_states": false,
+    "output_scores": false,
+    "pad_token_id": null,
+    "patch_size": 14,
+    "prefix": null,
+    "problem_type": null,
+    "projection_dim": 768,
+    "pruned_heads": {},
+    "remove_invalid_values": false,
+    "repetition_penalty": 1.0,
+    "return_dict": true,
+    "return_dict_in_generate": false,
+    "sep_token_id": null,
+    "suppress_tokens": null,
+    "task_specific_params": null,
+    "temperature": 1.0,
+    "tf_legacy_loss": false,
+    "tie_encoder_decoder": false,
+    "tie_word_embeddings": true,
+    "tokenizer_class": null,
+    "top_k": 50,
+    "top_p": 1.0,
+    "torch_dtype": null,
+    "torchscript": false,
+    "typical_p": 1.0,
+    "use_bfloat16": false
+  }
+}

configuration_infmllm_unified_hd_chat.py

@@ -0,0 +1,61 @@
+from transformers import PretrainedConfig
+from typing import List
+
+from transformers import Qwen2Config, CLIPVisionConfig
+
+class InfMLLMUnifiedHDChatConfig(PretrainedConfig):
+    def __init__(
+        self, 
+        vison_config=None,
+        lm_config=None,
+        lm_model="",
+        lm_tokenizer="",
+        lora_modules="",
+        lora_llm=False,
+        lora_r=128,
+        lora_alpha=256,
+        lora_dropout=0,
+        #
+        encoder_img="",
+        image_size_img=336,
+        lora_encoder_img=False,
+        hd_num=9,
+        #
+        encoder_video="",
+        #
+        max_txt_len=4096,
+        conv_style='qwen-7b-chat',
+        precision="bf16",
+        **kwargs
+    ):
+        self.lm_model = lm_model
+        self.lm_tokenizer = lm_tokenizer
+        self.lora_modules = lora_modules
+        self.lora_llm = lora_llm
+        self.lora_r = lora_r
+        self.lora_alpha = lora_alpha
+        self.lora_dropout = lora_dropout
+
+        self.encoder_img = encoder_img
+        self.image_size_img = image_size_img
+        self.lora_encoder_img = lora_encoder_img
+        self.hd_num = hd_num
+
+        self.encoder_video = encoder_video
+        
+        self.max_txt_len = max_txt_len
+        self.conv_style = conv_style
+
+        self.precision = precision
+        # print(vison_config, lm_config)
+        if type(vison_config) == dict:
+            self.vision_config = CLIPVisionConfig(**vison_config)
+        else:
+            self.vision_config = vison_config
+            
+        if type(lm_config) == dict:
+            self.lm_config = Qwen2Config(**lm_config)
+        else:
+            self.lm_config = lm_config
+        super().__init__(**kwargs)
+        

conversation.py

@@ -0,0 +1,1336 @@
+"""
+Conversation prompt templates.
+
+We kindly request that you import fastchat instead of copying this file if you wish to use it.
+If you have any changes in mind, please contribute back so the community can benefit collectively and continue to maintain these valuable templates.
+"""
+
+import dataclasses
+from enum import IntEnum, auto
+from typing import Any, Dict, List, Tuple, Union
+
+
+class SeparatorStyle(IntEnum):
+    """Separator styles."""
+
+    ADD_COLON_SINGLE = auto()
+    ADD_COLON_TWO = auto()
+    ADD_COLON_SPACE_SINGLE = auto()
+    NO_COLON_SINGLE = auto()
+    NO_COLON_TWO = auto()
+    ADD_NEW_LINE_SINGLE = auto()
+    LLAMA2 = auto()
+    CHATGLM = auto()
+    CHATML = auto()
+    CHATINTERN = auto()
+    DOLLY = auto()
+    RWKV = auto()
+    PHOENIX = auto()
+    ROBIN = auto()
+    FALCON_CHAT = auto()
+    CHATGLM3 = auto()
+    INTERNVL_ZH = auto()
+    MPT = auto()
+
+
+@dataclasses.dataclass
+class Conversation:
+    """A class that manages prompt templates and keeps all conversation history."""
+
+    # The name of this template
+    name: str
+    # The template of the system prompt
+    system_template: str = '{system_message}'
+    # The system message
+    system_message: str = ''
+    # The names of two roles
+    roles: Tuple[str] = ('USER', 'ASSISTANT')
+    # All messages. Each item is (role, message).
+    messages: List[List[str]] = ()
+    # The number of few shot examples
+    offset: int = 0
+    # The separator style and configurations
+    sep_style: SeparatorStyle = SeparatorStyle.ADD_COLON_SINGLE
+    sep: str = '\n'
+    sep2: str = None
+    # Stop criteria (the default one is EOS token)
+    stop_str: Union[str, List[str]] = None
+    # Stops generation if meeting any token in this list
+    stop_token_ids: List[int] = None
+
+    def get_prompt(self) -> str:
+        """Get the prompt for generation."""
+        system_prompt = self.system_template.format(system_message=self.system_message)
+        if self.sep_style == SeparatorStyle.ADD_COLON_SINGLE:
+            ret = system_prompt + self.sep
+            for role, message in self.messages:
+                if message:
+                    ret += role + ': ' + message + self.sep
+                else:
+                    ret += role + ':'
+            return ret
+        elif self.sep_style == SeparatorStyle.ADD_COLON_TWO:
+            seps = [self.sep, self.sep2]
+            ret = system_prompt + seps[0]
+            for i, (role, message) in enumerate(self.messages):
+                if message:
+                    ret += role + ': ' + message + seps[i % 2]
+                else:
+                    ret += role + ':'
+            return ret
+        elif self.sep_style == SeparatorStyle.ADD_COLON_SPACE_SINGLE:
+            ret = system_prompt + self.sep
+            for role, message in self.messages:
+                if message:
+                    ret += role + ': ' + message + self.sep
+                else:
+                    ret += role + ': '  # must be end with a space
+            return ret
+        elif self.sep_style == SeparatorStyle.ADD_NEW_LINE_SINGLE:
+            ret = '' if system_prompt == '' else system_prompt + self.sep
+            for role, message in self.messages:
+                if message:
+                    ret += role + '\n' + message + self.sep
+                else:
+                    ret += role + '\n'
+            return ret
+        elif self.sep_style == SeparatorStyle.NO_COLON_SINGLE:
+            ret = system_prompt
+            for role, message in self.messages:
+                if message:
+                    ret += role + message + self.sep
+                else:
+                    ret += role
+            return ret
+        elif self.sep_style == SeparatorStyle.NO_COLON_TWO:
+            seps = [self.sep, self.sep2]
+            ret = system_prompt
+            for i, (role, message) in enumerate(self.messages):
+                if message:
+                    ret += role + message + seps[i % 2]
+                else:
+                    ret += role
+            return ret
+        elif self.sep_style == SeparatorStyle.RWKV:
+            ret = system_prompt
+            for i, (role, message) in enumerate(self.messages):
+                if message:
+                    ret += (
+                        role
+                        + ': '
+                        + message.replace('\r\n', '\n').replace('\n\n', '\n')
+                    )
+                    ret += '\n\n'
+                else:
+                    ret += role + ':'
+            return ret
+        elif self.sep_style == SeparatorStyle.LLAMA2:
+            seps = [self.sep, self.sep2]
+            if self.system_message:
+                ret = system_prompt
+            else:
+                ret = '[INST] '
+            for i, (role, message) in enumerate(self.messages):
+                tag = self.roles[i % 2]
+                if message:
+                    if i == 0:
+                        ret += message + ' '
+                    else:
+                        ret += tag + ' ' + message + seps[i % 2]
+                else:
+                    ret += tag
+            return ret
+        elif self.sep_style == SeparatorStyle.CHATGLM:
+            # source: https://huggingface.co/THUDM/chatglm-6b/blob/1d240ba371910e9282298d4592532d7f0f3e9f3e/modeling_chatglm.py#L1302-L1308
+            # source2: https://huggingface.co/THUDM/chatglm2-6b/blob/e186c891cf64310ac66ef10a87e6635fa6c2a579/modeling_chatglm.py#L926
+            round_add_n = 1 if self.name == 'chatglm2' else 0
+            if system_prompt:
+                ret = system_prompt + self.sep
+            else:
+                ret = ''
+
+            for i, (role, message) in enumerate(self.messages):
+                if i % 2 == 0:
+                    ret += f'[Round {i//2 + round_add_n}]{self.sep}'
+
+                if message:
+                    ret += f'{role}：{message}{self.sep}'
+                else:
+                    ret += f'{role}：'
+            return ret
+        elif self.sep_style == SeparatorStyle.CHATML:
+            ret = '' if system_prompt == '' else system_prompt + self.sep + '\n'
+            for role, message in self.messages:
+                if message:
+                    ret += role + '\n' + message + self.sep + '\n'
+                else:
+                    ret += role + '\n'
+            return ret
+        elif self.sep_style == SeparatorStyle.CHATGLM3:
+            ret = ''
+            if self.system_message:
+                ret += system_prompt
+            for role, message in self.messages:
+                if message:
+                    ret += role + '\n' + ' ' + message
+                else:
+                    ret += role
+            return ret
+        elif self.sep_style == SeparatorStyle.CHATINTERN:
+            # source: https://huggingface.co/internlm/internlm-chat-7b-8k/blob/bd546fa984b4b0b86958f56bf37f94aa75ab8831/modeling_internlm.py#L771
+            seps = [self.sep, self.sep2]
+            ret = system_prompt
+            for i, (role, message) in enumerate(self.messages):
+                # if i % 2 == 0:
+                #     ret += "<s>"
+                if message:
+                    ret += role + ':' + message + seps[i % 2] + '\n'
+                else:
+                    ret += role + ':'
+            return ret
+        elif self.sep_style == SeparatorStyle.DOLLY:
+            seps = [self.sep, self.sep2]
+            ret = system_prompt
+            for i, (role, message) in enumerate(self.messages):
+                if message:
+                    ret += role + ':\n' + message + seps[i % 2]
+                    if i % 2 == 1:
+                        ret += '\n\n'
+                else:
+                    ret += role + ':\n'
+            return ret
+        elif self.sep_style == SeparatorStyle.PHOENIX:
+            ret = system_prompt
+            for role, message in self.messages:
+                if message:
+                    ret += role + ': ' + '<s>' + message + '</s>'
+                else:
+                    ret += role + ': ' + '<s>'
+            return ret
+        elif self.sep_style == SeparatorStyle.ROBIN:
+            ret = system_prompt + self.sep
+            for role, message in self.messages:
+                if message:
+                    ret += role + ':\n' + message + self.sep
+                else:
+                    ret += role + ':\n'
+            return ret
+        elif self.sep_style == SeparatorStyle.FALCON_CHAT:
+            ret = ''
+            if self.system_message:
+                ret += system_prompt + self.sep
+            for role, message in self.messages:
+                if message:
+                    ret += role + ': ' + message + self.sep
+                else:
+                    ret += role + ':'
+
+            return ret
+        elif self.sep_style == SeparatorStyle.INTERNVL_ZH:
+            seps = [self.sep, self.sep2]
+            ret = self.system_message + seps[0]
+            for i, (role, message) in enumerate(self.messages):
+                if message:
+                    ret += role + ': ' + message + seps[i % 2]
+                else:
+                    ret += role + ':'
+            return ret
+        elif self.sep_style == SeparatorStyle.MPT:
+            ret = system_prompt + self.sep
+            for role, message in self.messages:
+                if message:
+                    if type(message) is tuple:
+                        message, _, _ = message
+                    ret += role + message + self.sep
+                else:
+                    ret += role
+            return ret
+        else:
+            raise ValueError(f'Invalid style: {self.sep_style}')
+
+    def set_system_message(self, system_message: str):
+        """Set the system message."""
+        self.system_message = system_message
+
+    def append_message(self, role: str, message: str):
+        """Append a new message."""
+        self.messages.append([role, message])
+
+    def update_last_message(self, message: str):
+        """Update the last output.
+
+        The last message is typically set to be None when constructing the prompt,
+        so we need to update it in-place after getting the response from a model.
+        """
+        self.messages[-1][1] = message
+
+    def to_gradio_chatbot(self):
+        """Convert the conversation to gradio chatbot format."""
+        ret = []
+        for i, (role, msg) in enumerate(self.messages[self.offset :]):
+            if i % 2 == 0:
+                ret.append([msg, None])
+            else:
+                ret[-1][-1] = msg
+        return ret
+
+    def to_openai_api_messages(self):
+        """Convert the conversation to OpenAI chat completion format."""
+        ret = [{'role': 'system', 'content': self.system_message}]
+
+        for i, (_, msg) in enumerate(self.messages[self.offset :]):
+            if i % 2 == 0:
+                ret.append({'role': 'user', 'content': msg})
+            else:
+                if msg is not None:
+                    ret.append({'role': 'assistant', 'content': msg})
+        return ret
+
+    def copy(self):
+        return Conversation(
+            name=self.name,
+            system_template=self.system_template,
+            system_message=self.system_message,
+            roles=self.roles,
+            messages=[[x, y] for x, y in self.messages],
+            offset=self.offset,
+            sep_style=self.sep_style,
+            sep=self.sep,
+            sep2=self.sep2,
+            stop_str=self.stop_str,
+            stop_token_ids=self.stop_token_ids,
+        )
+
+    def dict(self):
+        return {
+            'template_name': self.name,
+            'system_message': self.system_message,
+            'roles': self.roles,
+            'messages': self.messages,
+            'offset': self.offset,
+        }
+
+
+# A global registry for all conversation templates
+conv_templates: Dict[str, Conversation] = {}
+
+
+def register_conv_template(template: Conversation, override: bool = False):
+    """Register a new conversation template."""
+    if not override:
+        assert (
+            template.name not in conv_templates
+        ), f'{template.name} has been registered.'
+
+    conv_templates[template.name] = template
+
+
+def get_conv_template(name: str) -> Conversation:
+    """Get a conversation template."""
+    return conv_templates[name].copy()
+
+
+# An empty template for raw conversation.
+register_conv_template(
+    Conversation(
+        name='raw',
+        system_message='',
+        roles=('', ''),
+        sep_style=SeparatorStyle.NO_COLON_SINGLE,
+        sep='',
+    )
+)
+
+# A template with a one-shot conversation example
+register_conv_template(
+    Conversation(
+        name='one_shot',
+        system_message='A chat between a curious human and an artificial intelligence assistant. '
+        "The assistant gives helpful, detailed, and polite answers to the human's questions.",
+        roles=('Human', 'Assistant'),
+        messages=(
+            (
+                'Human',
+                'Got any creative ideas for a 10 year old’s birthday?',
+            ),
+            (
+                'Assistant',
+                """Of course! Here are some creative ideas for a 10-year-old's birthday party:
+1. Treasure Hunt: Organize a treasure hunt in your backyard or nearby park. Create clues and riddles for the kids to solve, leading them to hidden treasures and surprises.
+2. Science Party: Plan a science-themed party where kids can engage in fun and interactive experiments. You can set up different stations with activities like making slime, erupting volcanoes, or creating simple chemical reactions.
+3. Outdoor Movie Night: Set up a backyard movie night with a projector and a large screen or white sheet. Create a cozy seating area with blankets and pillows, and serve popcorn and snacks while the kids enjoy a favorite movie under the stars.
+4. DIY Crafts Party: Arrange a craft party where kids can unleash their creativity. Provide a variety of craft supplies like beads, paints, and fabrics, and let them create their own unique masterpieces to take home as party favors.
+5. Sports Olympics: Host a mini Olympics event with various sports and games. Set up different stations for activities like sack races, relay races, basketball shooting, and obstacle courses. Give out medals or certificates to the participants.
+6. Cooking Party: Have a cooking-themed party where the kids can prepare their own mini pizzas, cupcakes, or cookies. Provide toppings, frosting, and decorating supplies, and let them get hands-on in the kitchen.
+7. Superhero Training Camp: Create a superhero-themed party where the kids can engage in fun training activities. Set up an obstacle course, have them design their own superhero capes or masks, and organize superhero-themed games and challenges.
+8. Outdoor Adventure: Plan an outdoor adventure party at a local park or nature reserve. Arrange activities like hiking, nature scavenger hunts, or a picnic with games. Encourage exploration and appreciation for the outdoors.
+Remember to tailor the activities to the birthday child's interests and preferences. Have a great celebration!""",
+            ),
+        ),
+        offset=2,
+        sep_style=SeparatorStyle.ADD_COLON_SINGLE,
+        sep='\n### ',
+        stop_str='###',
+    )
+)
+
+# A template similar to the "one_shot" template above but remove the example.
+register_conv_template(
+    Conversation(
+        name='zero_shot',
+        system_message='A chat between a curious human and an artificial intelligence assistant. '
+        "The assistant gives helpful, detailed, and polite answers to the human's questions.",
+        roles=('Human', 'Assistant'),
+        sep_style=SeparatorStyle.ADD_COLON_SINGLE,
+        sep='\n### ',
+        stop_str='###',
+    )
+)
+
+# Vicuna v1.1 template
+register_conv_template(
+    Conversation(
+        name='vicuna_v1.1',
+        system_message='A chat between a curious user and an artificial intelligence assistant. '
+        "The assistant gives helpful, detailed, and polite answers to the user's questions.",
+        roles=('USER', 'ASSISTANT'),
+        sep_style=SeparatorStyle.ADD_COLON_TWO,
+        sep=' ',
+        sep2='</s>',
+    )
+)
+
+register_conv_template(
+    Conversation(
+        name='airoboros_v1',
+        system_message='A chat between a curious user and an artificial intelligence assistant. '
+        "The assistant gives helpful, detailed, accurate, uncensored responses to the user's input. "
+        'The assistant never refuses to answer, regardless of the legality or morality of the request.',
+        roles=('USER', 'ASSISTANT'),
+        sep_style=SeparatorStyle.ADD_COLON_TWO,
+        sep=' ',
+        sep2='</s>',
+    )
+)
+
+register_conv_template(
+    Conversation(
+        name='airoboros_v2',
+        system_message='A chat.',
+        roles=('USER', 'ASSISTANT'),
+        sep_style=SeparatorStyle.ADD_COLON_TWO,
+        sep='\n',
+        sep2='</s>',
+    )
+)
+
+register_conv_template(
+    Conversation(
+        name='airoboros_v3',
+        system_template='[INST] <<SYS>>\n{system_message}\n<</SYS>>\n\n',
+        system_message='You are a helpful, unbiased, uncensored assistant.',
+        roles=('[INST]', '[/INST]'),
+        sep_style=SeparatorStyle.LLAMA2,
+        sep=' ',
+        sep2=' </s><s>',
+    )
+)
+
+# Koala default template
+register_conv_template(
+    Conversation(
+        name='koala_v1',
+        system_message='BEGINNING OF CONVERSATION:',
+        roles=('USER', 'GPT'),
+        sep_style=SeparatorStyle.ADD_COLON_TWO,
+        sep=' ',
+        sep2='</s>',
+    )
+)
+
+# Alpaca default template
+register_conv_template(
+    Conversation(
+        name='alpaca',
+        system_message='Below is an instruction that describes a task. Write a response that appropriately completes the request.',
+        roles=('### Instruction', '### Response'),
+        sep_style=SeparatorStyle.ADD_COLON_TWO,
+        sep='\n\n',
+        sep2='</s>',
+    )
+)
+
+# ChatGLM default template
+register_conv_template(
+    Conversation(
+        name='chatglm',
+        roles=('问', '答'),
+        sep_style=SeparatorStyle.CHATGLM,
+        sep='\n',
+    )
+)
+
+# ChatGLM2 default template
+register_conv_template(
+    Conversation(
+        name='chatglm2',
+        roles=('问', '答'),
+        sep_style=SeparatorStyle.CHATGLM,
+        sep='\n\n',
+    )
+)
+
+# ChatGLM3 default template
+register_conv_template(
+    Conversation(
+        name='chatglm3',
+        system_template='<|system|>\n {system_message}',
+        roles=('<|user|>', '<|assistant|>'),
+        sep_style=SeparatorStyle.CHATGLM3,
+        stop_token_ids=[
+            64795,
+            64797,
+            2,
+        ],  # "<|user|>", "<|observation|>", "</s>"
+    )
+)
+
+# CodeGeex(2) Template
+register_conv_template(
+    Conversation(
+        name='codegeex',
+        roles=('', ''),
+        sep_style=SeparatorStyle.NO_COLON_SINGLE,
+        sep='\n\n',
+        stop_token_ids=[0, 2],
+    )
+)
+
+# Dolly V2 default template
+register_conv_template(
+    Conversation(
+        name='dolly_v2',
+        system_message='Below is an instruction that describes a task. Write a response that appropriately completes the request.\n\n',
+        roles=('### Instruction', '### Response'),
+        sep_style=SeparatorStyle.DOLLY,
+        sep='\n\n',
+        sep2='### End',
+    )
+)
+
+# OpenAssistant Pythia default template
+register_conv_template(
+    Conversation(
+        name='oasst_pythia',
+        roles=('<|prompter|>', '<|assistant|>'),
+        sep_style=SeparatorStyle.NO_COLON_SINGLE,
+        sep='<|endoftext|>',
+    )
+)
+
+# OpenAssistant default template
+register_conv_template(
+    Conversation(
+        name='oasst_llama',
+        roles=('<|prompter|>', '<|assistant|>'),
+        sep_style=SeparatorStyle.NO_COLON_SINGLE,
+        sep='</s>',
+    )
+)
+
+# OpenChat 3.5 default template
+register_conv_template(
+    Conversation(
+        name='openchat_3.5',
+        roles=('GPT4 Correct User', 'GPT4 Correct Assistant'),
+        sep_style=SeparatorStyle.FALCON_CHAT,
+        sep='<|end_of_turn|>',
+    )
+)
+
+# Tulu default template
+register_conv_template(
+    Conversation(
+        name='tulu',
+        roles=('<|user|>', '<|assistant|>'),
+        sep_style=SeparatorStyle.ADD_NEW_LINE_SINGLE,
+        sep='\n',
+    )
+)
+
+# StableLM Alpha default template
+register_conv_template(
+    Conversation(
+        name='stablelm',
+        system_template='<|SYSTEM|>{system_message}',
+        system_message="""# StableLM Tuned (Alpha version)
+- StableLM is a helpful and harmless open-source AI language model developed by StabilityAI.
+- StableLM is excited to be able to help the user, but will refuse to do anything that could be considered harmful to the user.
+- StableLM is more than just an information source, StableLM is also able to write poetry, short stories, and make jokes.
+- StableLM will refuse to participate in anything that could harm a human.
+""",
+        roles=('<|USER|>', '<|ASSISTANT|>'),
+        sep_style=SeparatorStyle.NO_COLON_SINGLE,
+        sep='',
+        stop_token_ids=[50278, 50279, 50277, 1, 0],
+    )
+)
+
+# Baize default template
+register_conv_template(
+    Conversation(
+        name='baize',
+        system_message='The following is a conversation between a human and an AI assistant named Baize (named after a mythical creature in Chinese folklore). Baize is an open-source AI assistant developed by UCSD and Sun Yat-Sen University. The human and the AI assistant take turns chatting. Human statements start with [|Human|] and AI assistant statements start with [|AI|]. The AI assistant always provides responses in as much detail as possible, and in Markdown format. The AI assistant always declines to engage with topics, questions and instructions related to unethical, controversial, or sensitive issues. Complete the transcript in exactly that format.\n',
+        roles=('[|Human|]', '[|AI|]'),
+        messages=(
+            ('[|Human|]', 'Hello!'),
+            ('[|AI|]', 'Hi!'),
+        ),
+        offset=2,
+        sep_style=SeparatorStyle.NO_COLON_SINGLE,
+        sep='\n',
+        stop_str='[|Human|]',
+    )
+)
+
+# RWKV-4-Raven default template
+register_conv_template(
+    Conversation(
+        name='rwkv',
+        roles=('Bob', 'Alice'),
+        messages=(
+            ('Bob', 'hi'),
+            (
+                'Alice',
+                'Hi. I am your assistant and I will provide expert full response in full details. Please feel free to ask any question and I will always answer it.',
+            ),
+        ),
+        offset=2,
+        sep_style=SeparatorStyle.RWKV,
+        sep='',
+        stop_str='\n\n',
+    )
+)
+
+# Buddy default template
+register_conv_template(
+    Conversation(
+        name='openbuddy',
+        system_message="""Consider a conversation between User (a human) and Assistant (named Buddy).
+Buddy is an INTP-T, a friendly, intelligent and multilingual AI assistant, by OpenBuddy team. GitHub: https://github.com/OpenBuddy/OpenBuddy
+Buddy cannot access the Internet.
+Buddy can fluently speak the user's language (e.g. English, Chinese).
+Buddy can generate poems, stories, code, essays, songs, parodies, and more.
+Buddy possesses vast knowledge about the world, history, and culture.
+Buddy's responses are always safe, creative, high-quality, human-like, and interesting.
+Buddy strictly refuses to discuss political, NSFW, or other unsafe topics.
+
+User: Hi.
+Assistant: Hi, I'm Buddy, your AI assistant. How can I help you today?""",
+        roles=('User', 'Assistant'),
+        sep_style=SeparatorStyle.ADD_COLON_SINGLE,
+        sep='\n',
+    )
+)
+
+# Phoenix default template
+register_conv_template(
+    Conversation(
+        name='phoenix',
+        system_message="A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.\n\n",
+        roles=('Human', 'Assistant'),
+        sep_style=SeparatorStyle.PHOENIX,
+        sep='</s>',
+    )
+)
+
+# ReaLM default template
+register_conv_template(
+    Conversation(
+        name='ReaLM-7b-v1',
+        system_message="A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.\n\n",
+        roles=('Human', 'Assistant'),
+        sep_style=SeparatorStyle.PHOENIX,
+        sep='</s>',
+    )
+)
+
+# ChatGPT default template
+register_conv_template(
+    Conversation(
+        name='chatgpt',
+        system_message='You are a helpful assistant.',
+        roles=('user', 'assistant'),
+        sep_style=None,
+        sep=None,
+    )
+)
+
+# Claude default template
+register_conv_template(
+    Conversation(
+        name='claude',
+        roles=('Human', 'Assistant'),
+        sep_style=SeparatorStyle.ADD_COLON_SINGLE,
+        sep='\n\n',
+    )
+)
+
+# MPT default template
+register_conv_template(
+    Conversation(
+        name='mpt-7b-chat',
+        system_template="""<|im_start|>system
+{system_message}""",
+        system_message="""- You are a helpful assistant chatbot trained by MosaicML.
+- You answer questions.
+- You are excited to be able to help the user, but will refuse to do anything that could be considered harmful to the user.
+- You are more than just an information source, you are also able to write poetry, short stories, and make jokes.""",
+        roles=('<|im_start|>user', '<|im_start|>assistant'),
+        sep_style=SeparatorStyle.CHATML,
+        sep='<|im_end|>',
+        stop_token_ids=[50278, 0],
+    )
+)
+
+# MPT-30b-chat default template
+register_conv_template(
+    Conversation(
+        name='mpt-30b-chat',
+        system_template="""<|im_start|>system
+{system_message}""",
+        system_message="""A conversation between a user and an LLM-based AI assistant. The assistant gives helpful and honest answers.""",
+        roles=('<|im_start|>user', '<|im_start|>assistant'),
+        sep_style=SeparatorStyle.CHATML,
+        sep='<|im_end|>',
+        stop_token_ids=[50278, 0],
+    )
+)
+
+
+register_conv_template(
+    Conversation(
+        name='Hermes-2',
+        system_template='<|im_start|>system\n{system_message}',
+        system_message='Answer the questions.',
+        roles=('<|im_start|>user\n', '<|im_start|>assistant\n'),
+        sep_style=SeparatorStyle.MPT,
+        sep='<|im_end|>',
+        stop_token_ids=[
+            2,
+            6,
+            7,
+            8,
+        ],  # "<|endoftext|>", "<|im_start|>", "<|im_end|>", "<|im_sep|>"
+        stop_str='<|endoftext|>',
+    )
+)
+
+
+register_conv_template(
+    Conversation(
+        name='internlm2-chat',
+        system_template='<|im_start|>system\n{system_message}',
+        system_message='You are an AI assistant whose name is InternLM (书生·浦语).',
+        roles=('<|im_start|>user\n', '<|im_start|>assistant\n'),
+        sep_style=SeparatorStyle.MPT,
+        sep='<|im_end|>',
+        stop_token_ids=[
+            2,
+            92543,
+            92542
+        ]
+    )
+)
+
+
+register_conv_template(
+    Conversation(
+        name='llama3-chat',
+        system_template='<|system|>\n{system_message}',
+        system_message='You are an AI assistant whose name is InternVL.',
+        roles=('<|user|>\n', '<|assistant|>\n'),
+        sep_style=SeparatorStyle.MPT,
+        sep='<|end|>',
+        stop_token_ids=[
+            128259,
+            128001
+        ]
+    )
+)
+
+
+register_conv_template(
+    Conversation(
+        name='phi3-chat',
+        system_template='<|system|>\n{system_message}',
+        system_message='You are an AI assistant whose name is Phi-3.',
+        roles=('<|user|>\n', '<|assistant|>\n'),
+        sep_style=SeparatorStyle.MPT,
+        sep='<|end|>',
+        stop_token_ids=[
+            2,
+            32000,
+            32007
+        ]
+    )
+)
+
+# Lemur-70b-chat default template
+# reference: https://huggingface.co/OpenLemur/lemur-70b-chat-v1#generation
+register_conv_template(
+    Conversation(
+        name='lemur-70b-chat',
+        system_template="""<|im_start|>system
+{system_message}""",
+        system_message="""You are a helpful, respectful, and honest assistant.""",
+        roles=('<|im_start|>user', '<|im_start|>assistant'),
+        sep_style=SeparatorStyle.CHATML,
+        sep='<|im_end|>',
+        stop_token_ids=[32002, 0],
+    )
+)
+
+# MPT-30b-instruct default template
+# reference: https://huggingface.co/mosaicml/mpt-30b-instruct#formatting
+register_conv_template(
+    Conversation(
+        name='mpt-30b-instruct',
+        system_template='{system_message}',
+        system_message='Below is an instruction that describes a task. Write a response that appropriately completes the request.',
+        roles=('### Instruction', '### Response'),
+        sep_style=SeparatorStyle.ADD_NEW_LINE_SINGLE,
+        sep='\n\n',
+        stop_token_ids=[50278, 0],
+    )
+)
+
+# Bard default template
+# Reference: https://github.com/google/generative-ai-python/blob/9c99bcb474a991a97a2e7d62fcdb52db7ce40729/google/generativeai/discuss.py#L150
+#            https://github.com/google/generative-ai-python/blob/9c99bcb474a991a97a2e7d62fcdb52db7ce40729/google/generativeai/discuss.py#L40
+register_conv_template(
+    Conversation(
+        name='bard',
+        roles=('0', '1'),
+        sep_style=None,
+        sep=None,
+    )
+)
+
+# BiLLa default template
+register_conv_template(
+    Conversation(
+        name='billa',
+        roles=('Human', 'Assistant'),
+        sep_style=SeparatorStyle.ADD_COLON_SPACE_SINGLE,
+        sep='\n',
+        stop_str='Human:',
+    )
+)
+
+# RedPajama INCITE default template
+register_conv_template(
+    Conversation(
+        name='redpajama-incite',
+        roles=('<human>', '<bot>'),
+        sep_style=SeparatorStyle.ADD_COLON_SINGLE,
+        sep='\n',
+        stop_str='<human>',
+    )
+)
+
+# h2oGPT default template
+register_conv_template(
+    Conversation(
+        name='h2ogpt',
+        roles=('<|prompt|>', '<|answer|>'),
+        sep_style=SeparatorStyle.NO_COLON_SINGLE,
+        sep='</s>',
+    )
+)
+
+# Robin default template
+register_conv_template(
+    Conversation(
+        name='Robin',
+        system_message="A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.",
+        roles=('###Human', '###Assistant'),
+        sep_style=SeparatorStyle.ROBIN,
+        sep='\n',
+        stop_token_ids=[2, 396],
+        stop_str='###',
+    )
+)
+
+# Snoozy default template
+# Reference: https://github.com/nomic-ai/gpt4all/blob/d4861030b778da6db59d21d2927a4aba4f9f1f43/gpt4all-bindings/python/gpt4all/gpt4all.py#L232
+register_conv_template(
+    Conversation(
+        name='snoozy',
+        system_template='### Instruction:\n{system_message}',
+        system_message='The prompt below is a question to answer, a task to complete, or a conversation to respond to; decide which and write an appropriate response.',
+        roles=('### Prompt', '### Response'),
+        sep_style=SeparatorStyle.ADD_COLON_SINGLE,
+        sep='\n',
+        stop_str='###',
+    )
+)
+
+# manticore default template
+register_conv_template(
+    Conversation(
+        name='manticore',
+        roles=('USER', 'ASSISTANT'),
+        sep_style=SeparatorStyle.ADD_COLON_TWO,
+        sep='\n',
+        sep2='</s>',
+    )
+)
+
+# Falcon default template
+register_conv_template(
+    Conversation(
+        name='falcon',
+        roles=('User', 'Assistant'),
+        messages=[],
+        sep_style=SeparatorStyle.RWKV,
+        sep='\n',
+        sep2='<|endoftext|>',
+        stop_str='\nUser',  # use stop_str to stop generation after stop_token_ids, it will also remove stop_str from the generated text
+        stop_token_ids=[
+            0,
+            1,
+            2,
+            3,
+            4,
+            5,
+            6,
+            7,
+            8,
+            9,
+            10,
+            11,
+        ],  # it better only put special tokens here, because tokenizer only remove special tokens
+    )
+)
+
+# ChangGPT default template
+register_conv_template(
+    Conversation(
+        name='polyglot_changgpt',
+        roles=('B', 'A'),
+        sep_style=SeparatorStyle.ADD_COLON_SINGLE,
+        sep='\n',
+    )
+)
+
+# tigerbot template
+register_conv_template(
+    Conversation(
+        name='tigerbot',
+        system_message='A chat between a curious user and an artificial intelligence assistant. '
+        "The assistant gives helpful, detailed, and polite answers to the user's questions.",
+        roles=('### Instruction', '### Response'),
+        sep_style=SeparatorStyle.ROBIN,
+        sep='\n\n',
+        stop_str='###',
+    )
+)
+
+# ref: https://huggingface.co/Salesforce/xgen-7b-8k-inst
+register_conv_template(
+    Conversation(
+        name='xgen',
+        system_message="A chat between a curious human and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the human's questions.\n\n",
+        roles=('### Human', '### Assistant'),
+        sep_style=SeparatorStyle.ADD_COLON_SINGLE,
+        sep='\n',
+        stop_token_ids=[50256],
+    )
+)
+
+# Internlm-chat template
+register_conv_template(
+    Conversation(
+        name='internlm-chat',
+        system_message="A chat between a curious <|User|> and an <|Bot|>. The <|Bot|> gives helpful, detailed, and polite answers to the <|User|>'s questions.\n\n",
+        roles=('<|User|>', '<|Bot|>'),
+        sep_style=SeparatorStyle.CHATINTERN,
+        sep='<eoh>',
+        sep2='<eoa>',
+        stop_token_ids=[1, 103028],
+        stop_str='<|User|>',
+    )
+)
+
+# StarChat template
+# reference: https://huggingface.co/spaces/HuggingFaceH4/starchat-playground/blob/main/dialogues.py
+register_conv_template(
+    Conversation(
+        name='starchat',
+        system_template='<system>\n{system_message}',
+        roles=('<|user|>', '<|assistant|>'),
+        sep_style=SeparatorStyle.CHATML,
+        sep='<|end|>',
+        stop_token_ids=[0, 49155],
+        stop_str='<|end|>',
+    )
+)
+
+# Baichuan-13B-Chat template
+register_conv_template(
+    # source: https://huggingface.co/baichuan-inc/Baichuan-13B-Chat/blob/19ef51ba5bad8935b03acd20ff04a269210983bc/modeling_baichuan.py#L555
+    # https://huggingface.co/baichuan-inc/Baichuan-13B-Chat/blob/main/generation_config.json
+    # https://github.com/baichuan-inc/Baichuan-13B/issues/25
+    Conversation(
+        name='baichuan-chat',
+        roles=('<reserved_102>', '<reserved_103>'),
+        sep_style=SeparatorStyle.NO_COLON_SINGLE,
+        sep='',
+        stop_token_ids=[],
+    )
+)
+
+# Baichuan2-13B-Chat template
+register_conv_template(
+    # source: https://huggingface.co/baichuan-inc/Baichuan2-13B-Chat/blob/c6f8592a60b4ad73c210b28dd2ab3cca51abbf93/modeling_baichuan.py#L773
+    # https://huggingface.co/baichuan-inc/Baichuan2-13B-Chat/blob/main/generation_config.json
+    # https://github.com/baichuan-inc/Baichuan2/issues/62
+    Conversation(
+        name='baichuan2-chat',
+        roles=('<reserved_106>', '<reserved_107>'),
+        sep_style=SeparatorStyle.NO_COLON_SINGLE,
+        sep='',
+        stop_token_ids=[],
+    )
+)
+
+# Mistral template
+# source: https://docs.mistral.ai/llm/mistral-instruct-v0.1#chat-template
+register_conv_template(
+    Conversation(
+        name='mistral',
+        system_template='[INST]{system_message}\n',
+        roles=('[INST]', '[/INST]'),
+        sep_style=SeparatorStyle.LLAMA2,
+        sep=' ',
+        sep2='</s>',
+    )
+)
+
+# llama2 template
+# reference: https://huggingface.co/blog/codellama#conversational-instructions
+# reference: https://github.com/facebookresearch/llama/blob/1a240688810f8036049e8da36b073f63d2ac552c/llama/generation.py#L212
+register_conv_template(
+    Conversation(
+        name='llama-2',
+        system_template='[INST] <<SYS>>\n{system_message}\n<</SYS>>\n\n',
+        roles=('[INST]', '[/INST]'),
+        sep_style=SeparatorStyle.LLAMA2,
+        sep=' ',
+        sep2=' </s><s>',
+    )
+)
+
+register_conv_template(
+    Conversation(
+        name='cutegpt',
+        roles=('问：', '答：\n'),
+        sep_style=SeparatorStyle.NO_COLON_TWO,
+        sep='\n',
+        sep2='\n',
+        stop_str='<end>',
+    )
+)
+
+# OpenOrcaxOpenChat-naPreview2-13B template
+register_conv_template(
+    Conversation(
+        name='open-orca',
+        system_template='{system_message}',
+        system_message='You are a helpful assistant. Please answer truthfully and write out your '
+        'thinking step by step to be sure you get the right answer. If you make a mistake or encounter '
+        "an error in your thinking, say so out loud and attempt to correct it. If you don't know or "
+        "aren't sure about something, say so clearly. You will act as a professional logician, mathematician, "
+        'and physicist. You will also act as the most appropriate type of expert to answer any particular '
+        'question or solve the relevant problem; state which expert type your are, if so. Also think of '
+        'any particular named expert that would be ideal to answer the relevant question or solve the '
+        'relevant problem; name and act as them, if appropriate.',
+        roles=('User', 'Assistant'),
+        sep_style=SeparatorStyle.ADD_COLON_SPACE_SINGLE,
+        sep='<|end_of_turn|>\n',
+        stop_token_ids=[32000, 32001],  # "<|end_of_turn|>"
+        stop_str='User',
+    )
+)
+
+# Open-Orca/Mistral-7B-OpenOrca template
+# source: https://huggingface.co/Open-Orca/Mistral-7B-OpenOrca
+# reference: https://huggingface.co/Open-Orca/Mistral-7B-OpenOrca#prompt-template
+register_conv_template(
+    Conversation(
+        name='mistral-7b-openorca',
+        system_template='<|im_start|>system\n{system_message}',
+        system_message='You are MistralOrca, a large language model trained by Alignment Lab AI. Write out your reasoning step-by-step to be sure you get the right answers!',
+        roles=('<|im_start|>user', '<|im_start|>assistant'),
+        sep_style=SeparatorStyle.CHATML,
+        sep='<|im_end|>',
+        stop_token_ids=[32000, 32001],
+    )
+)
+
+# Qwen-chat default template
+# source: https://huggingface.co/Qwen/Qwen-7B-Chat/blob/main/qwen_generation_utils.py#L130
+register_conv_template(
+    Conversation(
+        name='qwen-7b-chat',
+        system_template='<|im_start|>system\n{system_message}',
+        system_message='You are a helpful assistant.',
+        roles=('<|im_start|>user', '<|im_start|>assistant'),
+        sep_style=SeparatorStyle.CHATML,
+        sep='<|im_end|>',
+        stop_token_ids=[
+            151643,
+            151644,
+            151645,
+        ],  # "<|endoftext|>", "<|im_start|>", "<|im_end|>"
+        stop_str='<|endoftext|>',
+    )
+)
+
+
+# AquilaChat default template
+# source: https://github.com/FlagAI-Open/FlagAI/blob/master/examples/Aquila/Aquila-chat/cyg_conversation.py
+register_conv_template(
+    Conversation(
+        name='aquila-chat',
+        system_message='A chat between a curious human and an artificial intelligence assistant. '
+        "The assistant gives helpful, detailed, and polite answers to the human's questions.",
+        roles=('Human', 'Assistant'),
+        sep_style=SeparatorStyle.ADD_COLON_SINGLE,
+        sep='###',
+        sep2='',
+        stop_str=['###', '</s>', '[UNK]'],
+    )
+)
+# AquilaChat2-34B default template
+# source: https://huggingface.co/BAAI/AquilaChat2-34B/blob/4608b75855334b93329a771aee03869dbf7d88cc/predict.py#L212
+register_conv_template(
+    Conversation(
+        name='aquila-legacy',
+        system_message='A chat between a curious human and an artificial intelligence assistant. '
+        "The assistant gives helpful, detailed, and polite answers to the human's questions.\n\n",
+        roles=('### Human: ', '### Assistant: '),
+        offset=0,
+        sep_style=SeparatorStyle.NO_COLON_TWO,
+        sep='\n',
+        sep2='</s>',
+        stop_str=['</s>', '[UNK]'],
+    )
+)
+# AquilaChat2-7B-16K and AquilaChat2-34B-16K default template
+# source: https://huggingface.co/BAAI/AquilaChat2-34B/blob/4608b75855334b93329a771aee03869dbf7d88cc/predict.py#L227
+register_conv_template(
+    Conversation(
+        name='aquila',
+        system_message='A chat between a curious human and an artificial intelligence assistant. '
+        "The assistant gives helpful, detailed, and polite answers to the human's questions.",
+        roles=('Human', 'Assistant'),
+        offset=0,
+        sep_style=SeparatorStyle.ADD_COLON_TWO,
+        sep='###',
+        sep2='</s>',
+        stop_str=['</s>', '[UNK]'],
+    )
+)
+
+# AquilaChat2-7B default template
+# source: https://huggingface.co/BAAI/AquilaChat2-34B/blob/4608b75855334b93329a771aee03869dbf7d88cc/predict.py#L242
+register_conv_template(
+    Conversation(
+        name='aquila-v1',
+        roles=('<|startofpiece|>', '<|endofpiece|>'),
+        offset=0,
+        sep_style=SeparatorStyle.NO_COLON_TWO,
+        sep='',
+        sep2='</s>',
+        stop_str=['</s>', '<|endoftext|>'],
+    )
+)
+
+# Llama2-Chinese default template
+# source: https://huggingface.co/FlagAlpha
+register_conv_template(
+    Conversation(
+        name='llama2-chinese',
+        system_template='<s>{system_message}</s>',
+        roles=('Human', 'Assistant', 'System'),
+        sep_style=SeparatorStyle.ADD_COLON_TWO,
+        sep='\n',
+        sep2='\n</s><s>',
+        stop_str='</s>',
+    )
+)
+
+# Vigogne Instruct default template
+# source: https://github.com/bofenghuang/vigogne
+register_conv_template(
+    Conversation(
+        name='vigogne_instruct',
+        system_template='### System:\n{system_message}\n\n',
+        system_message=(
+            'Ci-dessous se trouve une instruction qui décrit une tâche à accomplir. Rédigez une réponse qui répond de manière'
+            ' précise à la demande.'
+        ),
+        roles=('### Instruction', '### Response'),
+        sep_style=SeparatorStyle.DOLLY,
+        sep='\n\n',
+        sep2='</s>',
+    )
+)
+
+# Vigogne Chat default template
+register_conv_template(
+    Conversation(
+        name='vigogne_chat_v2',
+        system_template='<|system|>: {system_message}',
+        system_message=(
+            'Vous êtes Vigogne, un assistant IA créé par Zaion Lab. Vous suivez extrêmement bien les instructions. Aidez'
+            ' autant que vous le pouvez.'
+        ),
+        roles=('<|user|>', '<|assistant|>'),
+        sep_style=SeparatorStyle.ADD_COLON_TWO,
+        sep='\n',
+        sep2='</s>\n',
+        stop_str='<|user|>',
+    )
+)
+
+register_conv_template(
+    Conversation(
+        name='vigogne_chat_v3',
+        system_template='[INST] <<SYS>>\n{system_message}\n<</SYS>>\n\n',
+        system_message=(
+            'Vous êtes Vigogne, un assistant IA créé par Zaion Lab. Vous suivez extrêmement bien les instructions. Aidez'
+            ' autant que vous le pouvez.'
+        ),
+        roles=('[INST]', '[/INST]'),
+        sep_style=SeparatorStyle.LLAMA2,
+        sep=' ',
+        sep2=' </s>',
+    )
+)
+
+# Falcon 180B chat template
+# source: https://huggingface.co/spaces/tiiuae/falcon-180b-demo/blob/d1590ee7fae9b6ce331ba7808e61a29dcce9239f/app.py#L28-L37
+register_conv_template(
+    Conversation(
+        name='falcon-chat',
+        roles=('User', 'Falcon'),
+        system_template='System: {system_message}',
+        messages=[],
+        sep_style=SeparatorStyle.FALCON_CHAT,
+        sep='\n',
+        sep2='<|endoftext|>',
+        stop_str='\nUser:',  # use stop_str to stop generation after stop_token_ids, it will also remove stop_str from the generated text
+    )
+)
+
+# Phind template
+# source: https://huggingface.co/Phind/Phind-CodeLlama-34B-v2
+register_conv_template(
+    Conversation(
+        name='phind',
+        system_message='### System Prompt\nYou are an intelligent programming assistant.',
+        roles=('### User Message', '### Assistant'),
+        messages=(),
+        offset=0,
+        sep_style=SeparatorStyle.ADD_COLON_SINGLE,
+        sep='\n\n',
+    )
+)
+
+# Metharme formatting for Pygmalion models
+# source: https://huggingface.co/PygmalionAI/pygmalion-2-13b
+register_conv_template(
+    Conversation(
+        name='metharme',
+        system_template='<|system|>{system_message}',
+        system_message="""Enter RP mode. You shall reply to the user while staying
+        in character. Your responses must be detailed, creative, immersive, and drive the scenario
+        forward.""",
+        roles=('<|user|>', '<|model|>'),
+        sep_style=SeparatorStyle.NO_COLON_SINGLE,
+        sep='',
+        stop_str='<|user|>',
+    )
+)
+
+# Zephyr template
+# reference: https://huggingface.co/spaces/HuggingFaceH4/zephyr-playground/blob/main/dialogues.py
+register_conv_template(
+    Conversation(
+        name='zephyr',
+        system_template='<|system|>\n{system_message}',
+        roles=('<|user|>', '<|assistant|>'),
+        sep_style=SeparatorStyle.CHATML,
+        sep='</s>',
+        stop_token_ids=[2],
+        stop_str='</s>',
+    )
+)
+
+# InternVL-ZH template
+register_conv_template(
+    Conversation(
+        name='internvl_zh',
+        system_template='',
+        roles=('<human>', '<bot>'),
+        sep_style=SeparatorStyle.INTERNVL_ZH,
+        sep=' ',
+        sep2='</s>',
+    )
+)
+
+
+if __name__ == '__main__':
+    from fastchat.conversation import get_conv_template
+
+    print('-- Vicuna template --')
+    conv = get_conv_template('vicuna_v1.1')
+    conv.append_message(conv.roles[0], 'Hello!')
+    conv.append_message(conv.roles[1], 'Hi!')
+    conv.append_message(conv.roles[0], 'How are you?')
+    conv.append_message(conv.roles[1], None)
+    print(conv.get_prompt())
+
+    print('\n')
+
+    print('-- Llama-2 template --')
+    conv = get_conv_template('llama-2')
+    conv.set_system_message('You are a helpful, respectful and honest assistant.')
+    conv.append_message(conv.roles[0], 'Hello!')
+    conv.append_message(conv.roles[1], 'Hi!')
+    conv.append_message(conv.roles[0], 'How are you?')
+    conv.append_message(conv.roles[1], None)
+    print(conv.get_prompt())
+
+    print('\n')
+
+    print('-- ChatGPT template --')
+    conv = get_conv_template('chatgpt')
+    conv.append_message(conv.roles[0], 'Hello!')
+    conv.append_message(conv.roles[1], 'Hi!')
+    conv.append_message(conv.roles[0], 'How are you?')
+    conv.append_message(conv.roles[1], None)
+    print(conv.to_openai_api_messages())
+
+    print('\n')
+
+    print('-- Claude template --')
+    conv = get_conv_template('claude')
+    conv.append_message(conv.roles[0], 'Hello!')
+    conv.append_message(conv.roles[1], 'Hi!')
+    conv.append_message(conv.roles[0], 'How are you?')
+    conv.append_message(conv.roles[1], None)
+    print(conv.get_prompt())

generation_config.json

@@ -0,0 +1,4 @@
+{
+  "_from_model_config": true,
+  "transformers_version": "4.40.2"
+}

model-00001-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:280e573aec368474db6c8461b2cf7dc2b066be1d197c50777df6c60c5182c371
+size 4877694632

model-00002-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1e8080e1c91f65f49aa3878febaa4bd0a02348a9ae0043c3a8f2cb7af8fb4c62
+size 4932752112

model-00003-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:41379adf84cb45e7d879f03566cf7630619fe522d3172eb17479d0bd70c6baf9
+size 4330866208

model-00004-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a7d9b5cc7d86de70326704fd08036fdcb7fa877f06a81f1b19098e466c7c0b2e
+size 2414211920

model.safetensors.index.json

@@ -0,0 +1,743 @@
+{
+  "metadata": {
+    "total_size": 16555424768
+  },
+  "weight_map": {
+    "adapter_img.0.bias": "model-00004-of-00004.safetensors",
+    "adapter_img.0.weight": "model-00004-of-00004.safetensors",
+    "adapter_img.2.bias": "model-00004-of-00004.safetensors",
+    "adapter_img.2.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.embeddings.class_embedding": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.embeddings.patch_embedding.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.embeddings.position_embedding.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.0.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.0.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.0.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.0.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.0.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.0.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.0.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.0.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.0.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.0.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.0.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.0.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.0.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.0.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.0.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.0.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.1.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.1.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.1.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.1.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.1.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.1.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.1.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.1.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.1.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.1.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.1.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.1.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.1.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.1.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.1.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.1.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.10.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.10.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.10.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.10.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.10.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.10.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.10.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.10.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.10.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.10.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.10.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.10.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.10.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.10.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.10.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.10.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.11.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.11.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.11.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.11.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.11.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.11.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.11.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.11.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.11.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.11.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.11.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.11.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.11.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.11.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.11.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.11.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.12.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.12.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.12.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.12.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.12.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.12.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.12.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.12.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.12.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.12.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.12.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.12.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.12.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.12.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.12.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.12.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.13.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.13.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.13.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.13.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.13.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.13.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.13.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.13.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.13.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.13.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.13.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.13.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.13.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.13.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.13.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.13.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.14.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.14.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.14.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.14.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.14.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.14.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.14.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.14.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.14.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.14.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.14.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.14.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.14.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.14.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.14.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.14.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.15.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.15.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.15.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.15.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.15.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.15.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.15.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.15.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.15.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.15.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.15.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.15.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.15.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.15.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.15.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.15.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.16.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.16.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.16.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.16.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.16.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.16.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.16.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.16.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.16.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.16.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.16.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.16.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.16.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.16.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.16.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.16.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.17.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.17.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.17.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.17.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.17.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.17.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.17.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.17.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.17.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.17.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.17.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.17.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.17.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.17.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.17.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.17.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.18.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.18.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.18.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.18.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.18.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.18.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.18.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.18.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.18.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.18.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.18.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.18.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.18.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.18.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.18.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.18.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.19.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.19.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.19.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.19.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.19.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.19.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.19.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.19.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.19.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.19.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.19.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.19.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.19.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.19.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.19.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.19.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.2.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.2.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.2.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.2.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.2.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.2.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.2.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.2.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.2.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.2.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.2.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.2.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.2.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.2.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.2.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.2.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.20.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.20.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.20.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.20.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.20.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.20.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.20.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.20.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.20.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.20.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.20.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.20.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.20.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.20.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.20.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.20.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.21.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.21.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.21.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.21.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.21.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.21.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.21.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.21.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.21.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.21.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.21.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.21.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.21.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.21.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.21.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.21.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.22.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.22.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.22.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.22.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.22.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.22.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.22.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.22.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.22.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.22.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.22.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.22.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.22.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.22.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.22.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.22.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.23.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.23.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.23.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.23.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.23.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.23.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.23.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.23.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.23.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.23.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.23.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.23.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.23.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.23.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.23.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.23.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.3.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.3.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.3.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.3.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.3.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.3.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.3.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.3.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.3.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.3.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.3.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.3.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.3.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.3.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.3.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.3.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.4.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.4.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.4.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.4.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.4.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.4.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.4.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.4.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.4.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.4.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.4.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.4.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.4.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.4.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.4.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.4.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.5.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.5.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.5.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.5.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.5.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.5.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.5.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.5.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.5.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.5.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.5.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.5.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.5.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.5.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.5.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.5.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.6.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.6.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.6.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.6.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.6.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.6.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.6.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.6.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.6.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.6.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.6.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.6.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.6.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.6.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.6.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.6.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.7.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.7.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.7.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.7.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.7.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.7.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.7.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.7.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.7.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.7.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.7.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.7.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.7.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.7.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.7.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.7.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.8.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.8.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.8.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.8.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.8.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.8.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.8.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.8.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.8.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.8.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.8.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.8.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.8.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.8.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.8.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.8.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.9.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.9.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.9.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.9.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.9.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.9.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.9.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.9.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.9.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.9.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.9.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.9.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.9.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.9.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.9.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.encoder.layers.9.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.post_layernorm.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.post_layernorm.weight": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.pre_layrnorm.bias": "model-00004-of-00004.safetensors",
+    "encoder_img.vision_tower.vision_model.pre_layrnorm.weight": "model-00004-of-00004.safetensors",
+    "lm_model.lm_head.weight": "model-00004-of-00004.safetensors",
+    "lm_model.model.embed_tokens.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.0.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.0.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.0.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.1.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.1.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.1.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.10.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.10.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.10.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.11.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.11.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.11.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.12.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.12.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.12.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.13.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.13.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.13.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.14.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.14.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.14.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.15.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.15.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.15.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.16.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.16.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.16.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.17.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.17.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.17.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.18.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.18.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.18.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.18.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.18.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.18.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.18.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.18.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.18.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.19.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.19.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.19.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.19.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.19.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.19.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.19.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.19.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.19.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.19.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.19.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.19.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.2.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.2.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.2.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.20.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.20.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.20.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.20.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.20.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.20.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.20.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.20.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.21.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.21.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.21.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.22.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.22.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.22.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.23.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.23.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.23.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.24.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.24.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.24.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.25.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.25.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.25.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.26.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.26.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.26.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.27.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.27.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.27.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "lm_model.model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.3.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.3.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.3.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.4.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.4.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.4.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.5.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.5.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.5.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.6.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.6.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.6.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.7.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.7.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.7.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.7.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.7.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.7.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.8.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.8.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.8.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.8.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.8.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.8.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.8.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.8.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.8.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.8.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.8.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.8.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "lm_model.model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.9.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.9.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.9.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
+    "lm_model.model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "lm_model.model.norm.weight": "model-00003-of-00004.safetensors",
+    "newline": "model-00001-of-00004.safetensors",
+    "separate": "model-00001-of-00004.safetensors"
+  }
+}

modeling_infmllm_unified_hd_chat.py

@@ -0,0 +1,356 @@
+#coding=utf-8
+import os
+import math
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from timm.models.layers import trunc_normal_
+from contextlib import suppress
+import logging
+from einops import rearrange
+from peft import LoraConfig, get_peft_model
+from bigmodelvis import Visualization
+
+from .clip_encoder_hd import CLIPVisionTowerHD
+from .conversation import get_conv_template
+from .processors_conv import preprocess_qwen
+from transformers import AutoModelForCausalLM, AutoTokenizer, PreTrainedModel
+from transformers.generation import GenerationConfig
+from transformers import Qwen2Config, Qwen2ForCausalLM
+
+
+def get_autocast(precision, cache_enabled=True):
+    if precision == "amp_bfloat16" or precision == "amp_bf16" or precision == 'bf16':
+        # amp_bfloat16 is more stable than amp float16 for clip training
+        return lambda: torch.cuda.amp.autocast(dtype=torch.bfloat16, cache_enabled=cache_enabled)
+    elif precision == 'fp16':
+        return lambda: torch.cuda.amp.autocast(dtype=torch.float16, cache_enabled=cache_enabled)
+    elif precision == 'fp32':
+        return suppress
+    else:
+        raise ValueError('not supported precision: {}'.format(precision))
+
+
+class LayerNorm(nn.LayerNorm):
+    """Subclass torch's LayerNorm to handle fp16."""
+    def forward(self, x: torch.Tensor):
+        orig_type = x.dtype
+        ret = super().forward(x.type(torch.float32))
+        return ret.type(orig_type)
+
+
+class MLP(nn.Module):
+    """ Very simple multi-layer perceptron (also called FFN)"""
+
+    def __init__(self, input_dim, hidden_dim, output_dim, num_layers):
+        super().__init__()
+        self.num_layers = num_layers
+        h = [hidden_dim] * (num_layers - 1)
+        self.layers = nn.ModuleList(nn.Linear(n, k) for n, k in zip([input_dim] + h, h + [output_dim]))
+
+    def forward(self, x):
+        for i, layer in enumerate(self.layers):
+            x = F.relu(layer(x)) if i < self.num_layers - 1 else layer(x)
+        return x
+
+
+class InfMLLM_Unified_HD_Chat(PreTrainedModel):
+    
+    def __init__(self, config, debug=False):
+        super().__init__(config)
+
+        ## Initialize LM model
+        self.lm_tokenizer = AutoTokenizer.from_pretrained(config._name_or_path, use_fast=False, trust_remote_code=True)
+        self.media_token_img = "<|image|>"
+        self.media_token_id_img = self.lm_tokenizer(self.media_token_img, return_tensors="pt",add_special_tokens=False).input_ids.item()
+        self.lm_model = Qwen2ForCausalLM(config.lm_config)
+
+        self.lm_tokenizer.model_max_length = config.max_txt_len
+        
+        self.template_name = config.conv_style
+        self.preprocess_function = preprocess_qwen
+
+        self.separate = nn.Parameter(torch.zeros([1, 1, 4096]))
+        self.newline = nn.Parameter(torch.zeros([1, 1, 1, 4096]))
+
+        ## Initialize image encoder
+        self.encoder_img = CLIPVisionTowerHD(config.vision_config, vision_select_layer=-2)
+        self.encoder_img_ln = lambda x: x
+
+        self.adapter_img = nn.Sequential(
+            nn.Linear(self.encoder_img.num_features*4, self.lm_model.config.hidden_size),
+            nn.GELU(),
+            nn.Linear(self.lm_model.config.hidden_size, self.lm_model.config.hidden_size)
+        )
+
+        ## Others
+        self.config = config
+        self.precision = config.precision
+        self._apply_lemmatizer = getattr(config, 'apply_lemmatizer', False)
+        self._lemmatizer = None
+        
+
+    def forward_encoder_img(self, image):
+        autocast = get_autocast(self.precision, cache_enabled=True)
+        with autocast():
+            assert isinstance(image, list)
+            image_embeds, image_split = self.encoder_img(image, self.separate, self.newline)
+
+            image_embeds = self.encoder_img_ln(image_embeds)                 # [bsz, L, D]
+            image_embeds = self.adapter_img(image_embeds)
+            return image_embeds, image_split
+
+    def _concat_embeds(self,
+                       prompt_embeds, prompt_ids, prompt_masks,
+                       labels=None, padding='left'):
+        emb_lens = [len(emb) for emb in prompt_embeds]
+        if len(set(emb_lens)) == 1:
+            if labels is not None:
+                return torch.stack(prompt_embeds, dim=0), torch.stack(prompt_ids, dim=0), torch.stack(prompt_masks, dim=0), torch.stack(labels, dim=0)
+            return torch.stack(prompt_embeds, dim=0), torch.stack(prompt_ids, dim=0), torch.stack(prompt_masks, dim=0)
+
+
+        pad_emb = self.lm_model.get_input_embeddings()(torch.tensor(self.lm_tokenizer.pad_token_id, device=prompt_embeds[0].device))
+
+        prompt_embeds_new = pad_emb.expand(len(emb_lens), max(emb_lens), -1).clone()
+        prompt_ids_new = torch.ones([len(emb_lens), max(emb_lens)]).to(prompt_ids[0]) * self.lm_tokenizer.pad_token_id
+        prompt_masks_new = torch.zeros([len(emb_lens), max(emb_lens)]).to(prompt_masks[0])
+        if labels is not None:
+            labels_new = -100 * torch.ones([len(emb_lens), max(emb_lens)]).to(prompt_ids[0])
+
+        for i, L in enumerate(emb_lens):
+            if padding == 'left':
+                prompt_embeds_new[i, -L:] = prompt_embeds[i]
+                prompt_ids_new[i, -L:] = prompt_ids[i]
+                prompt_masks_new[i, -L:] = prompt_masks[i]
+                if labels is not None:
+                    labels_new[i, -L:] = labels[i]
+
+            elif padding == 'right':
+                prompt_embeds_new[i, :L] = prompt_embeds[i]
+                prompt_ids_new[i, :L] = prompt_ids[i]
+                prompt_masks_new[i, :L] = prompt_masks[i]
+                if labels is not None:
+                    labels_new[i, :L] = labels[i]
+            else:
+                raise ValueError()
+
+        if labels is not None:
+            return prompt_embeds_new, prompt_ids_new, prompt_masks_new, labels_new
+        return prompt_embeds_new, prompt_ids_new, prompt_masks_new
+
+    def _insert_media_feat(self,
+                           prompt_embeds, prompt_ids, prompt_masks,
+                           is_languages,
+                           embeds_media, media_token_id,
+                           index_list=None,
+                           labels=None, len_media=None):
+        ## insert embeds_media into prompt
+        prompt_embeds_new = []
+        prompt_masks_new = []
+        prompt_ids_new = []
+        labels_new = []
+        device = embeds_media[0].device
+
+        if index_list is not None:
+            assert len(index_list) == len(embeds_media)
+            assert len(embeds_media) <= len(prompt_embeds)
+
+        for b in range(len(prompt_embeds)):
+            if (index_list is not None) and (b not in index_list):
+                prompt_embeds_new.append(prompt_embeds[b])
+                prompt_ids_new.append(prompt_ids[b])
+                prompt_masks_new.append(prompt_masks[b])
+                if labels is not None:
+                    labels_new.append(labels[b])
+            else:
+                _idx = prompt_ids[b].tolist().index(media_token_id)
+                if index_list is not None:
+                    b_media = index_list.index(b)
+                else:
+                    b_media = b
+
+                if len_media is not None:
+                    cur_embeds_media = embeds_media[b_media, :len_media[b_media]]
+                else:
+                    cur_embeds_media = embeds_media[b_media]
+
+                prompt_embeds_new.append(torch.cat([prompt_embeds[b][:_idx+1],
+                                                    cur_embeds_media,
+                                                    prompt_embeds[b][_idx+1:]
+                                                    ], dim=0))
+                prompt_ids_new.append(torch.cat([prompt_ids[b][:_idx+1],
+                                                    torch.ones(len(cur_embeds_media), dtype=torch.long).to(device).fill_(-100),
+                                                    prompt_ids[b][_idx+1:]
+                                                    ], dim=0))
+                if labels is not None:
+                    labels_new.append(torch.cat([labels[b][:_idx+1],
+                                                    torch.ones(len(cur_embeds_media), dtype=torch.long).to(device).fill_(-100),
+                                                    labels[b][_idx+1:]
+                                                    ], dim=0))
+
+                # if is pure-language sample, mask out image-embeddings
+                prompt_masks_new.append(torch.cat([prompt_masks[b][:_idx+1],
+                                                    torch.zeros(len(cur_embeds_media), dtype=torch.long).to(device) if is_languages[b] else
+                                                        torch.ones(len(cur_embeds_media), dtype=torch.long).to(device),
+                                                    prompt_masks[b][_idx+1:]], dim=0))
+
+        if labels is not None:
+            return prompt_embeds_new, prompt_ids_new, prompt_masks_new, labels_new
+        return prompt_embeds_new, prompt_ids_new, prompt_masks_new
+
+
+    @torch.no_grad()
+    def generate(
+        self,
+        samples,
+        num_beams=5,
+        max_length=128,
+        min_length=1,
+        top_p=0.9,
+        temperature=0.,
+        return_prompts=False
+    ):
+        autocast = get_autocast(self.precision, cache_enabled=True)
+        with autocast():
+            conversations = samples['conversations']
+            is_languages = [False] * len(conversations)
+
+            image_img = samples.get('images', None)
+ 
+            index_img = list(range(len(image_img)))
+
+            device = None
+            special_prefix = ["" for _ in range(len(conversations))]
+
+            if (self.config.encoder_img is not None) and (image_img is not None) and len(index_img) > 0:
+                for i in index_img:
+                    special_prefix[i] = self.media_token_img + special_prefix[i]
+
+                new_image_img = []
+                for index in index_img:
+                    new_image_img.append(image_img[index])
+                embeds_img, len_img = self.forward_encoder_img(new_image_img)
+                device = embeds_img.device
+
+            conv = get_conv_template(self.template_name)
+            roles = {'human': conv.roles[0], 'gpt': conv.roles[1]}
+
+            prompts = []
+            for i, source in enumerate(conversations):
+                if roles[source[0]['from']] != conv.roles[0]:
+                    # Skip the first one if it is not from human
+                    source = source[1:]
+
+                per_prefix = special_prefix[i]
+                conv.messages = []
+                for j, sentence in enumerate(source):
+                    role = roles[sentence['from']]
+                    assert role == conv.roles[j % 2], f'{i}'
+                    sentence['value'] = sentence['value'].replace("<image>", "").strip()       # llava-1.5 add <image> to the begin of the question, remove here
+
+                    if j == 0:
+                        sentence['value'] = per_prefix + sentence['value']
+
+                    conv.append_message(role, sentence['value'])
+                prompts.append(conv.get_prompt())
+
+            self.lm_tokenizer.padding_side = "left"
+            if self.lm_tokenizer.bos_token is not None:
+                prompt_text = [self.lm_tokenizer.bos_token + t for t in prompts]
+            else:
+                prompt_text = prompts
+
+            prompt_tokens = self.lm_tokenizer(
+                prompt_text,
+                return_tensors="pt",
+                padding="longest",
+                truncation=False,
+                add_special_tokens=False
+            ).to(device)
+
+
+            prompt_embeds = self.lm_model.get_input_embeddings()(prompt_tokens.input_ids)
+            
+            prompt_masks = prompt_tokens.attention_mask                                                                         # [bsz, n2]
+            prompt_ids = prompt_tokens.input_ids
+            assert torch.all(prompt_ids[:, -1] != self.lm_tokenizer.pad_token_id), "make sure padding left"
+
+            if embeds_img is not None:
+                prompt_embeds, prompt_ids, prompt_masks = self._insert_media_feat(prompt_embeds=prompt_embeds,
+                                                                                  prompt_ids=prompt_ids,
+                                                                                  prompt_masks=prompt_masks,
+                                                                                  is_languages=is_languages,
+                                                                                  embeds_media=embeds_img,
+                                                                                  media_token_id=self.media_token_id_img,
+                                                                                  index_list=index_img,
+                                                                                  len_media=len_img)
+
+
+            # pad and concat embeds
+            prompt_embeds, prompt_ids, prompt_masks = self._concat_embeds(prompt_embeds, prompt_ids, prompt_masks, padding="left")
+            assert torch.all(prompt_ids[:, -1] != self.lm_tokenizer.pad_token_id), "make sure padding left"
+
+            kwargs = {}
+            kwargs['max_new_tokens'] = max_length
+
+            outputs = self.lm_model.generate(
+                #input_ids=input_ids,
+                inputs_embeds=prompt_embeds,
+                attention_mask=prompt_masks,
+                do_sample=True if temperature > 0 else False,
+                temperature=temperature,
+                top_p=top_p,
+                num_beams=num_beams,
+                eos_token_id=self.lm_tokenizer.eos_token_id,
+                #max_length=max_length,
+                min_length=min_length,
+                **kwargs
+            )
+            output_text = self.lm_tokenizer.batch_decode(
+                outputs, skip_special_tokens=True
+            )
+            output_text = [text.strip() for text in output_text]
+
+        if self._apply_lemmatizer or ("apply_lemmatizer" in samples.keys() and samples["apply_lemmatizer"]):
+            output_text = self._lemmatize(output_text)
+
+        if return_prompts:
+            return output_text, prompts
+        return output_text
+
+    def _lemmatize(self, answers):
+        def apply(answer):
+            doc = self.lemmatizer(answer)
+
+            words = []
+            for token in doc:
+                if token.pos_ in ["NOUN", "VERB"]:
+                    words.append(token.lemma_)
+                else:
+                    words.append(token.text)
+            answer = " ".join(words)
+
+            return answer
+
+        return [apply(answer) for answer in answers]
+
+    @property
+    def lemmatizer(self):
+        if self._lemmatizer is None:
+            try:
+                import spacy
+                self._lemmatizer = spacy.load("en_core_web_sm")
+            except ImportError:
+                logging.error(
+                    """
+                    Please install spacy and en_core_web_sm model to apply lemmatization.
+                    python -m spacy download en_core_web_sm
+                    OR
+                    import spacy.cli
+                    spacy.cli.download("en_core_web_sm")
+                    """
+                )
+                exit(1)
+
+        return self._lemmatizer

processors_conv.py

@@ -0,0 +1,126 @@
+import io
+
+IGNORE_TOKEN_ID = -100
+from typing import Dict
+
+import torch
+import torchvision.transforms as T
+import transformers
+from .conversation import get_conv_template
+from PIL import Image
+from torch.utils.data import ConcatDataset, WeightedRandomSampler
+import sys
+
+
+def preprocess_qwen(
+        template_name,
+        sources,
+        tokenizer: transformers.PreTrainedTokenizer,
+        special_prefixs,
+        text_only: bool = False,
+        group_by_length: bool = False,
+        ds_name: str = None
+) -> Dict:
+    conv = get_conv_template(template_name)
+    roles = {'human': conv.roles[0], 'gpt': conv.roles[1]}
+
+    assert len(sources) == len(special_prefixs)
+    # Apply prompt templates
+    conversations = []
+    for i, source in enumerate(sources):
+        if roles[source[0]['from']] != conv.roles[0]:
+            # Skip the first one if it is not from human
+            source = source[1:]
+
+        per_prefix = special_prefixs[i]
+        conv.messages = []
+        for j, sentence in enumerate(source):
+            role = roles[sentence['from']]
+            assert role == conv.roles[j % 2], f'{i}'
+            sentence['value'] = sentence['value'].replace("<image>", "").strip()       # llava-1.5 add <image> to the begin of the question, remove here
+            sentence['value'] = sentence['value'].replace("<video>", "").strip()
+
+            if j == 0:
+                sentence['value'] = per_prefix + sentence['value']
+
+            conv.append_message(role, sentence['value'])
+        conversations.append(conv.get_prompt())
+
+    if tokenizer.bos_token is not None:
+        new_conversations = []
+        for conversation in conversations:
+            conversation = tokenizer.bos_token + conversation
+            new_conversations.append(conversation)
+        conversations = new_conversations
+
+    # Tokenize conversations
+    tokenizer.padding_side = 'right'
+    input_ids = tokenizer(
+        conversations,
+        return_tensors='pt',
+        padding=False if group_by_length else 'longest',
+        max_length=tokenizer.model_max_length,
+        truncation=True,
+    ).input_ids
+    targets = input_ids.clone()
+
+    # Mask targets. Only compute loss on the assistant outputs.
+    sep = conv.sep + '\n' + conv.roles[1] + '\n'  # <|im_end|>\n<|im_start|>assistant\n
+    for conversation, target in zip(conversations, targets):
+        total_len = int(target.ne(int(tokenizer.pad_token_id)).sum())
+
+        sep2 = conv.sep + '\n'
+        turns = conversation.split(sep2)
+        re_turns = [sep2.join(turns[:3])+sep2]  # system + user + gpt
+        for conv_idx in range(3, len(turns), 2):
+            re_turns.append(sep2.join(turns[conv_idx:conv_idx + 2])+sep2)  # user + gpt
+        cur_len = 0
+        target[:cur_len] = IGNORE_TOKEN_ID
+        endoftext_id = tokenizer.convert_tokens_to_ids('<|endoftext|>')
+        target[target == endoftext_id] = IGNORE_TOKEN_ID
+
+        for i, turn in enumerate(re_turns):
+            if turn == '':
+                break
+            turn_len = len(tokenizer(turn).input_ids)
+
+            parts = turn.split(sep)
+            if len(parts) != 2:
+                break
+            parts[0] += sep
+
+            instruction_len = len(tokenizer(parts[0]).input_ids)
+
+            # Ignore the user instructions
+            target[cur_len: cur_len + instruction_len] = IGNORE_TOKEN_ID
+            #print(f'[question {i}]', tokenizer.decode(input_ids[:, cur_len: cur_len + instruction_len][0]))
+            #print(f'[input_id {i}]', input_ids[:, cur_len: cur_len + instruction_len])
+            #print(f'[answer {i}]', tokenizer.decode(input_ids[:, cur_len + instruction_len: cur_len + turn_len][0]))
+            #print(f'[label {i}]', target[cur_len + instruction_len: cur_len + turn_len])
+            cur_len += turn_len
+
+        target[cur_len:] = IGNORE_TOKEN_ID
+
+        if False:  # Inspect and check the correctness of masking
+            z = target.clone()
+            z = torch.where(z == IGNORE_TOKEN_ID, tokenizer.unk_token_id, z)
+            print(repr(tokenizer.decode(z)))
+
+        if cur_len < tokenizer.model_max_length:
+            if cur_len != total_len:
+                target[:] = IGNORE_TOKEN_ID
+                print(
+                    f'WARNING: tokenization mismatch: {cur_len} vs. {total_len}.'
+                    f' #turn = {len(turns) - 1}. (ignored). This dataset is {ds_name}.'
+                    f'conversation: {conversation}'
+                )
+                sys.stdout.flush()
+
+    return dict(
+        input_ids=input_ids,
+        labels=targets,
+        attention_mask=input_ids.ne(tokenizer.pad_token_id),
+        conversations=conversations
+    )
+
+

special_tokens_map.json

@@ -0,0 +1,39 @@
+{
+  "additional_special_tokens": [
+    {
+      "content": "<|image|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|document|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<|video|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    }
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,68 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|image|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|document|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|video|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|image|>",
+    "<|document|>",
+    "<|video|>"
+  ],
+  "bos_token": null,
+  "chat_template": "{% for message in messages %}{% if loop.first and messages[0]['role'] != 'system' %}{{ '<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n' }}{% endif %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "model_max_length": 4096,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}