Delete model

model/added_tokens.json

@@ -1,24 +0,0 @@
-{
-  "</tool_call>": 151658,
-  "<tool_call>": 151657,
-  "<|box_end|>": 151649,
-  "<|box_start|>": 151648,
-  "<|endoftext|>": 151643,
-  "<|file_sep|>": 151664,
-  "<|fim_middle|>": 151660,
-  "<|fim_pad|>": 151662,
-  "<|fim_prefix|>": 151659,
-  "<|fim_suffix|>": 151661,
-  "<|im_end|>": 151645,
-  "<|im_start|>": 151644,
-  "<|image_pad|>": 151655,
-  "<|object_ref_end|>": 151647,
-  "<|object_ref_start|>": 151646,
-  "<|quad_end|>": 151651,
-  "<|quad_start|>": 151650,
-  "<|repo_name|>": 151663,
-  "<|video_pad|>": 151656,
-  "<|vision_end|>": 151653,
-  "<|vision_pad|>": 151654,
-  "<|vision_start|>": 151652
-}

model/config.json

@@ -1,197 +0,0 @@
-{
-  "_commit_hash": null,
-  "_name_or_path": "LlavaQw",
-  "architectures": [
-    "LlavaQwModel"
-  ],
-  "auto_map": {
-    "AutoConfig": "configuration_llavaqw.LlavaQwConfig",
-    "AutoModel": "modeling_llavaqw.LlavaQwModel",
-    "AutoModelForCausalLM": "modeling_llavaqw.LlavaQwModel"
-  },
-  "downsample_ratio": 0.5,
-  "dynamic_image_size": true,
-  "force_image_size": 448,
-  "llm_config": {
-    "_name_or_path": "Qwen2.5-7B-Instruct",
-    "add_cross_attention": false,
-    "architectures": [
-      "Qwen2ForCausalLM"
-    ],
-    "attention_dropout": 0.0,
-    "bad_words_ids": null,
-    "begin_suppress_tokens": null,
-    "bos_token_id": 151643,
-    "chunk_size_feed_forward": 0,
-    "cross_attention_hidden_size": null,
-    "decoder_start_token_id": null,
-    "diversity_penalty": 0.0,
-    "do_sample": false,
-    "early_stopping": false,
-    "encoder_no_repeat_ngram_size": 0,
-    "eos_token_id": 151645,
-    "exponential_decay_length_penalty": null,
-    "finetuning_task": null,
-    "forced_bos_token_id": null,
-    "forced_eos_token_id": null,
-    "hidden_act": "silu",
-    "hidden_size": 3584,
-    "id2label": {
-      "0": "LABEL_0",
-      "1": "LABEL_1"
-    },
-    "initializer_range": 0.02,
-    "intermediate_size": 18944,
-    "is_decoder": false,
-    "is_encoder_decoder": false,
-    "label2id": {
-      "LABEL_0": 0,
-      "LABEL_1": 1
-    },
-    "length_penalty": 1.0,
-    "max_length": 20,
-    "max_position_embeddings": 32768,
-    "max_window_layers": 28,
-    "min_length": 0,
-    "model_type": "qwen2",
-    "no_repeat_ngram_size": 0,
-    "num_attention_heads": 28,
-    "num_beam_groups": 1,
-    "num_beams": 1,
-    "num_hidden_layers": 28,
-    "num_key_value_heads": 4,
-    "num_return_sequences": 1,
-    "output_attentions": false,
-    "output_hidden_states": false,
-    "output_scores": false,
-    "pad_token_id": null,
-    "prefix": null,
-    "problem_type": null,
-    "pruned_heads": {},
-    "remove_invalid_values": false,
-    "repetition_penalty": 1.0,
-    "return_dict": true,
-    "return_dict_in_generate": false,
-    "rms_norm_eps": 1e-06,
-    "rope_scaling": null,
-    "rope_theta": 1000000.0,
-    "sep_token_id": null,
-    "sliding_window": null,
-    "suppress_tokens": null,
-    "task_specific_params": null,
-    "temperature": 1.0,
-    "tf_legacy_loss": false,
-    "tie_encoder_decoder": false,
-    "tie_word_embeddings": false,
-    "tokenizer_class": null,
-    "top_k": 50,
-    "top_p": 1.0,
-    "torch_dtype": "bfloat16",
-    "torchscript": false,
-    "transformers_version": "4.45.0",
-    "typical_p": 1.0,
-    "use_bfloat16": false,
-    "use_cache": true,
-    "use_sliding_window": false,
-    "vocab_size": 152064
-  },
-  "max_dynamic_patch": 6,
-  "min_dynamic_patch": 1,
-  "model_type": "LlavaQw",
-  "ps_version": "v2",
-  "select_layer": -1,
-  "template": "chatml",
-  "torch_dtype": "bfloat16",
-  "transformers_version": null,
-  "use_backbone_lora": 0,
-  "use_llm_lora": 0,
-  "use_thumbnail": true,
-  "vision_config": {
-    "_name_or_path": "",
-    "add_cross_attention": false,
-    "architectures": [
-      "InternVisionModel"
-    ],
-    "attention_dropout": 0.0,
-    "auto_map": {
-      "AutoConfig": "configuration_intern_vit.InternVisionConfig",
-      "AutoModel": "modeling_intern_vit.InternVisionModel"
-    },
-    "bad_words_ids": null,
-    "begin_suppress_tokens": null,
-    "bos_token_id": null,
-    "chunk_size_feed_forward": 0,
-    "cross_attention_hidden_size": null,
-    "decoder_start_token_id": null,
-    "diversity_penalty": 0.0,
-    "do_sample": false,
-    "drop_path_rate": 0.1,
-    "dropout": 0.0,
-    "early_stopping": false,
-    "encoder_no_repeat_ngram_size": 0,
-    "eos_token_id": null,
-    "exponential_decay_length_penalty": null,
-    "finetuning_task": null,
-    "forced_bos_token_id": null,
-    "forced_eos_token_id": null,
-    "hidden_act": "gelu",
-    "hidden_size": 1024,
-    "id2label": {
-      "0": "LABEL_0",
-      "1": "LABEL_1"
-    },
-    "image_size": 448,
-    "initializer_factor": 1.0,
-    "initializer_range": 0.02,
-    "intermediate_size": 4096,
-    "is_decoder": false,
-    "is_encoder_decoder": false,
-    "label2id": {
-      "LABEL_0": 0,
-      "LABEL_1": 1
-    },
-    "layer_norm_eps": 1e-06,
-    "length_penalty": 1.0,
-    "max_length": 20,
-    "min_length": 0,
-    "model_type": "intern_vit_6b",
-    "no_repeat_ngram_size": 0,
-    "norm_type": "layer_norm",
-    "num_attention_heads": 16,
-    "num_beam_groups": 1,
-    "num_beams": 1,
-    "num_channels": 3,
-    "num_hidden_layers": 24,
-    "num_return_sequences": 1,
-    "output_attentions": false,
-    "output_hidden_states": false,
-    "output_scores": false,
-    "pad_token_id": null,
-    "patch_size": 14,
-    "prefix": null,
-    "problem_type": null,
-    "pruned_heads": {},
-    "qk_normalization": false,
-    "qkv_bias": true,
-    "remove_invalid_values": false,
-    "repetition_penalty": 1.0,
-    "return_dict": true,
-    "return_dict_in_generate": false,
-    "sep_token_id": null,
-    "suppress_tokens": null,
-    "task_specific_params": null,
-    "temperature": 1.0,
-    "tf_legacy_loss": false,
-    "tie_encoder_decoder": false,
-    "tie_word_embeddings": true,
-    "tokenizer_class": null,
-    "top_k": 50,
-    "top_p": 1.0,
-    "torch_dtype": "bfloat16",
-    "torchscript": false,
-    "transformers_version": "4.45.0",
-    "typical_p": 1.0,
-    "use_bfloat16": false,
-    "use_flash_attn": true
-  }
-}

model/configuration_intern_vit.py

@@ -1,119 +0,0 @@
-# --------------------------------------------------------
-# InternVL
-# Copyright (c) 2023 OpenGVLab
-# Licensed under The MIT License [see LICENSE for details]
-# --------------------------------------------------------
-import os
-from typing import Union
-
-from transformers.configuration_utils import PretrainedConfig
-from transformers.utils import logging
-
-logger = logging.get_logger(__name__)
-
-
-class InternVisionConfig(PretrainedConfig):
-    r"""
-    This is the configuration class to store the configuration of a [`InternVisionModel`]. It is used to
-    instantiate a vision encoder according to the specified arguments, defining the model architecture.
-
-    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
-    documentation from [`PretrainedConfig`] for more information.
-
-    Args:
-        num_channels (`int`, *optional*, defaults to 3):
-            Number of color channels in the input images (e.g., 3 for RGB).
-        patch_size (`int`, *optional*, defaults to 14):
-            The size (resolution) of each patch.
-        image_size (`int`, *optional*, defaults to 224):
-            The size (resolution) of each image.
-        qkv_bias (`bool`, *optional*, defaults to `False`):
-            Whether to add a bias to the queries and values in the self-attention layers.
-        hidden_size (`int`, *optional*, defaults to 3200):
-            Dimensionality of the encoder layers and the pooler layer.
-        num_attention_heads (`int`, *optional*, defaults to 25):
-            Number of attention heads for each attention layer in the Transformer encoder.
-        intermediate_size (`int`, *optional*, defaults to 12800):
-            Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.
-        qk_normalization (`bool`, *optional*, defaults to `True`):
-            Whether to normalize the queries and keys in the self-attention layers.
-        num_hidden_layers (`int`, *optional*, defaults to 48):
-            Number of hidden layers in the Transformer encoder.
-        use_flash_attn (`bool`, *optional*, defaults to `True`):
-            Whether to use flash attention mechanism.
-        hidden_act (`str` or `function`, *optional*, defaults to `"gelu"`):
-            The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
-            `"relu"`, `"selu"` and `"gelu_new"` ``"gelu"` are supported.
-        layer_norm_eps (`float`, *optional*, defaults to 1e-6):
-            The epsilon used by the layer normalization layers.
-        dropout (`float`, *optional*, defaults to 0.0):
-            The dropout probability for all fully connected layers in the embeddings, encoder, and pooler.
-        drop_path_rate (`float`, *optional*, defaults to 0.0):
-            Dropout rate for stochastic depth.
-        attention_dropout (`float`, *optional*, defaults to 0.0):
-            The dropout ratio for the attention probabilities.
-        initializer_range (`float`, *optional*, defaults to 0.02):
-            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
-        initializer_factor (`float`, *optional*, defaults to 0.1):
-            A factor for layer scale.
-    """
-
-    model_type = 'intern_vit_6b'
-
-    def __init__(
-            self,
-            num_channels=3,
-            patch_size=14,
-            image_size=224,
-            qkv_bias=False,
-            hidden_size=3200,
-            num_attention_heads=25,
-            intermediate_size=12800,
-            qk_normalization=True,
-            num_hidden_layers=48,
-            use_flash_attn=True,
-            hidden_act='gelu',
-            norm_type='rms_norm',
-            layer_norm_eps=1e-6,
-            dropout=0.0,
-            drop_path_rate=0.0,
-            attention_dropout=0.0,
-            initializer_range=0.02,
-            initializer_factor=0.1,
-            **kwargs,
-    ):
-        super().__init__(**kwargs)
-
-        self.hidden_size = hidden_size
-        self.intermediate_size = intermediate_size
-        self.dropout = dropout
-        self.drop_path_rate = drop_path_rate
-        self.num_hidden_layers = num_hidden_layers
-        self.num_attention_heads = num_attention_heads
-        self.num_channels = num_channels
-        self.patch_size = patch_size
-        self.image_size = image_size
-        self.initializer_range = initializer_range
-        self.initializer_factor = initializer_factor
-        self.attention_dropout = attention_dropout
-        self.layer_norm_eps = layer_norm_eps
-        self.hidden_act = hidden_act
-        self.norm_type = norm_type
-        self.qkv_bias = qkv_bias
-        self.qk_normalization = qk_normalization
-        self.use_flash_attn = use_flash_attn
-
-    @classmethod
-    def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> 'PretrainedConfig':
-        config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs)
-
-        if 'vision_config' in config_dict:
-            config_dict = config_dict['vision_config']
-
-        if 'model_type' in config_dict and hasattr(cls, 'model_type') and config_dict['model_type'] != cls.model_type:
-            logger.warning(
-                f"You are using a model of type {config_dict['model_type']} to instantiate a model of type "
-                f'{cls.model_type}. This is not supported for all configurations of models and can yield errors.'
-            )
-
-        return cls.from_dict(config_dict, **kwargs)

model/configuration_llavaqw.py

@@ -1,99 +0,0 @@
-# --------------------------------------------------------
-# Adapted from https://huggingface.co/OpenGVLab/InternVL2-Llama3-76B under MIT License
-#     LICENSE is in incl_licenses directory.
-# --------------------------------------------------------
-
-import copy
-
-from transformers import AutoConfig, Qwen2Config
-from transformers.configuration_utils import PretrainedConfig
-from transformers.utils import logging
-
-from .configuration_intern_vit import InternVisionConfig
-
-logger = logging.get_logger(__name__)
-
-
-class LlavaQwConfig(PretrainedConfig):
-    model_type = 'LlavaQw'
-    is_composition = True
-
-    def __init__(
-            self,
-            vision_config=None,
-            llm_config=None,
-            use_backbone_lora=0,
-            use_llm_lora=0,
-            select_layer=-1,
-            force_image_size=None,
-            downsample_ratio=0.5,
-            template=None,
-            dynamic_image_size=False,
-            use_thumbnail=False,
-            ps_version='v1',
-            min_dynamic_patch=1,
-            max_dynamic_patch=6,
-            **kwargs
-    ):
-        super().__init__(**kwargs)
-
-        if vision_config is None:
-            vision_config = {'architectures': ['InternVisionModel']}
-            logger.info('vision_config is None. Initializing InternVisionConfig with default values.')
-
-        # Handle llm_config initialization
-        if llm_config is None:
-            llm_config = {'architectures': ['Qwen2ForCausalLM']}
-            logger.info('llm_config is None. Initializing LLM Config with default values.')
-
-        self.vision_config = InternVisionConfig(**vision_config)
-
-        # Check for supported architecture
-        if llm_config.get('architectures', [None])[0] == 'Qwen2ForCausalLM':
-            self.llm_config = Qwen2Config(**llm_config)
-        else:
-            raise ValueError(f"Unsupported architecture: {llm_config.get('architectures', [None])[0]}")
-
-        # Assign configuration values
-        self.use_backbone_lora = use_backbone_lora
-        self.use_llm_lora = use_llm_lora
-        self.select_layer = select_layer
-        self.force_image_size = force_image_size
-        self.downsample_ratio = downsample_ratio
-        self.template = template
-        self.dynamic_image_size = dynamic_image_size
-        self.use_thumbnail = use_thumbnail
-        self.ps_version = ps_version  # Pixel shuffle version
-        self.min_dynamic_patch = min_dynamic_patch
-        self.max_dynamic_patch = max_dynamic_patch
-
-        # Log important parameters
-        logger.info(f'vision_select_layer: {self.select_layer}')
-        logger.info(f'ps_version: {self.ps_version}')
-        logger.info(f'min_dynamic_patch: {self.min_dynamic_patch}')
-        logger.info(f'max_dynamic_patch: {self.max_dynamic_patch}')
-
-    def to_dict(self):
-        """
-        Serializes this instance to a Python dictionary. Overrides the default `PretrainedConfig.to_dict`.
-
-        Returns:
-            Dict[str, Any]: Dictionary of all the attributes that make up this configuration instance.
-        """
-        output = copy.deepcopy(self.__dict__)
-        output['vision_config'] = self.vision_config.to_dict()
-        output['llm_config'] = self.llm_config.to_dict()
-        output['model_type'] = self.model_type
-        output['use_backbone_lora'] = self.use_backbone_lora
-        output['use_llm_lora'] = self.use_llm_lora
-        output['select_layer'] = self.select_layer
-        output['force_image_size'] = self.force_image_size
-        output['downsample_ratio'] = self.downsample_ratio
-        output['template'] = self.template
-        output['dynamic_image_size'] = self.dynamic_image_size
-        output['use_thumbnail'] = self.use_thumbnail
-        output['ps_version'] = self.ps_version
-        output['min_dynamic_patch'] = self.min_dynamic_patch
-        output['max_dynamic_patch'] = self.max_dynamic_patch
-
-        return output

model/conversation.py

@@ -1,358 +0,0 @@
-"""
-Adapted from https://github.com/lm-sys/FastChat/blob/main/fastchat/conversation.py under the Apache License 2.0.
-     LICENSE is in incl_licenses directory.
-
-Conversation prompt templates.
-
-We kindly request that you import fastchat instead of copying this file if you wish to use it.
-If you have changes in mind, please contribute back so the community can benefit collectively and continue to maintain these valuable templates.
-"""
-
-import dataclasses
-from enum import IntEnum, auto
-from typing import Any, Dict, List, Tuple, Union
-
-
-class SeparatorStyle(IntEnum):
-    """Separator styles."""
-
-    ADD_COLON_SINGLE = auto()
-    ADD_COLON_TWO = auto()
-    ADD_COLON_SPACE_SINGLE = auto()
-    NO_COLON_SINGLE = auto()
-    NO_COLON_TWO = auto()
-    ADD_NEW_LINE_SINGLE = auto()
-    LLAMA2 = auto()
-    CHATGLM = auto()
-    CHATML = auto()
-    CHATINTERN = auto()
-    DOLLY = auto()
-    RWKV = auto()
-    PHOENIX = auto()
-    ROBIN = auto()
-    FALCON_CHAT = auto()
-    CHATGLM3 = auto()
-    INTERNVL_ZH = auto()
-    MPT = auto()
-
-
-@dataclasses.dataclass
-class Conversation:
-    """A class that manages prompt templates and keeps all conversation history."""
-
-    # The name of this template
-    name: str
-    # The template of the system prompt
-    system_template: str = '{system_message}'
-    # The system message
-    system_message: str = ''
-    # The names of two roles
-    roles: Tuple[str] = ('USER', 'ASSISTANT')
-    # All messages. Each item is (role, message).
-    messages: List[List[str]] = ()
-    # The number of few shot examples
-    offset: int = 0
-    # The separator style and configurations
-    sep_style: SeparatorStyle = SeparatorStyle.ADD_COLON_SINGLE
-    sep: str = '\n'
-    sep2: str = None
-    # Stop criteria (the default one is EOS token)
-    stop_str: Union[str, List[str]] = None
-    # Stops generation if meeting any token in this list
-    stop_token_ids: List[int] = None
-
-    def get_prompt(self) -> str:
-        """Get the prompt for generation."""
-        system_prompt = self.system_template.format(system_message=self.system_message)
-        if self.sep_style == SeparatorStyle.ADD_COLON_SINGLE:
-            ret = system_prompt + self.sep
-            for role, message in self.messages:
-                if message:
-                    ret += role + ': ' + message + self.sep
-                else:
-                    ret += role + ':'
-            return ret
-        elif self.sep_style == SeparatorStyle.ADD_COLON_TWO:
-            seps = [self.sep, self.sep2]
-            ret = system_prompt + seps[0]
-            for i, (role, message) in enumerate(self.messages):
-                if message:
-                    ret += role + ': ' + message + seps[i % 2]
-                else:
-                    ret += role + ':'
-            return ret
-        elif self.sep_style == SeparatorStyle.ADD_COLON_SPACE_SINGLE:
-            ret = system_prompt + self.sep
-            for role, message in self.messages:
-                if message:
-                    ret += role + ': ' + message + self.sep
-                else:
-                    ret += role + ': '  # must be end with a space
-            return ret
-        elif self.sep_style == SeparatorStyle.ADD_NEW_LINE_SINGLE:
-            ret = '' if system_prompt == '' else system_prompt + self.sep
-            for role, message in self.messages:
-                if message:
-                    ret += role + '\n' + message + self.sep
-                else:
-                    ret += role + '\n'
-            return ret
-        elif self.sep_style == SeparatorStyle.NO_COLON_SINGLE:
-            ret = system_prompt
-            for role, message in self.messages:
-                if message:
-                    ret += role + message + self.sep
-                else:
-                    ret += role
-            return ret
-        elif self.sep_style == SeparatorStyle.NO_COLON_TWO:
-            seps = [self.sep, self.sep2]
-            ret = system_prompt
-            for i, (role, message) in enumerate(self.messages):
-                if message:
-                    ret += role + message + seps[i % 2]
-                else:
-                    ret += role
-            return ret
-        elif self.sep_style == SeparatorStyle.RWKV:
-            ret = system_prompt
-            for i, (role, message) in enumerate(self.messages):
-                if message:
-                    ret += (
-                        role
-                        + ': '
-                        + message.replace('\r\n', '\n').replace('\n\n', '\n')
-                    )
-                    ret += '\n\n'
-                else:
-                    ret += role + ':'
-            return ret
-        elif self.sep_style == SeparatorStyle.LLAMA2:
-            seps = [self.sep, self.sep2]
-            if self.system_message:
-                ret = system_prompt
-            else:
-                ret = '[INST] '
-            for i, (role, message) in enumerate(self.messages):
-                tag = self.roles[i % 2]
-                if message:
-                    if i == 0:
-                        ret += message + ' '
-                    else:
-                        ret += tag + ' ' + message + seps[i % 2]
-                else:
-                    ret += tag
-            return ret
-        elif self.sep_style == SeparatorStyle.CHATGLM:
-            # source: https://huggingface.co/THUDM/chatglm-6b/blob/1d240ba371910e9282298d4592532d7f0f3e9f3e/modeling_chatglm.py#L1302-L1308
-            # source2: https://huggingface.co/THUDM/chatglm2-6b/blob/e186c891cf64310ac66ef10a87e6635fa6c2a579/modeling_chatglm.py#L926
-            round_add_n = 1 if self.name == 'chatglm2' else 0
-            if system_prompt:
-                ret = system_prompt + self.sep
-            else:
-                ret = ''
-
-            for i, (role, message) in enumerate(self.messages):
-                if i % 2 == 0:
-                    ret += f'[Round {i//2 + round_add_n}]{self.sep}'
-
-                if message:
-                    ret += f'{role}：{message}{self.sep}'
-                else:
-                    ret += f'{role}：'
-            return ret
-        elif self.sep_style == SeparatorStyle.CHATML:
-            ret = '' if system_prompt == '' else system_prompt + self.sep + '\n'
-            for role, message in self.messages:
-                if message:
-                    ret += role + '\n' + message + self.sep + '\n'
-                else:
-                    ret += role + '\n'
-            return ret
-        elif self.sep_style == SeparatorStyle.CHATGLM3:
-            ret = ''
-            if self.system_message:
-                ret += system_prompt
-            for role, message in self.messages:
-                if message:
-                    ret += role + '\n' + ' ' + message
-                else:
-                    ret += role
-            return ret
-        elif self.sep_style == SeparatorStyle.CHATINTERN:
-            # source: https://huggingface.co/internlm/internlm-chat-7b-8k/blob/bd546fa984b4b0b86958f56bf37f94aa75ab8831/modeling_internlm.py#L771
-            seps = [self.sep, self.sep2]
-            ret = system_prompt
-            for i, (role, message) in enumerate(self.messages):
-                # if i % 2 == 0:
-                #     ret += "<s>"
-                if message:
-                    ret += role + ':' + message + seps[i % 2] + '\n'
-                else:
-                    ret += role + ':'
-            return ret
-        elif self.sep_style == SeparatorStyle.DOLLY:
-            seps = [self.sep, self.sep2]
-            ret = system_prompt
-            for i, (role, message) in enumerate(self.messages):
-                if message:
-                    ret += role + ':\n' + message + seps[i % 2]
-                    if i % 2 == 1:
-                        ret += '\n\n'
-                else:
-                    ret += role + ':\n'
-            return ret
-        elif self.sep_style == SeparatorStyle.PHOENIX:
-            ret = system_prompt
-            for role, message in self.messages:
-                if message:
-                    ret += role + ': ' + '<s>' + message + '</s>'
-                else:
-                    ret += role + ': ' + '<s>'
-            return ret
-        elif self.sep_style == SeparatorStyle.ROBIN:
-            ret = system_prompt + self.sep
-            for role, message in self.messages:
-                if message:
-                    ret += role + ':\n' + message + self.sep
-                else:
-                    ret += role + ':\n'
-            return ret
-        elif self.sep_style == SeparatorStyle.FALCON_CHAT:
-            ret = ''
-            if self.system_message:
-                ret += system_prompt + self.sep
-            for role, message in self.messages:
-                if message:
-                    ret += role + ': ' + message + self.sep
-                else:
-                    ret += role + ':'
-
-            return ret
-        elif self.sep_style == SeparatorStyle.INTERNVL_ZH:
-            seps = [self.sep, self.sep2]
-            ret = self.system_message + seps[0]
-            for i, (role, message) in enumerate(self.messages):
-                if message:
-                    ret += role + ': ' + message + seps[i % 2]
-                else:
-                    ret += role + ':'
-            return ret
-        elif self.sep_style == SeparatorStyle.MPT:
-            ret = ''
-            for role, message in self.messages:
-                if message:
-                    if type(message) is tuple:
-                        message, _, _ = message
-                    ret += role + message + self.sep
-                else:
-                    ret += role
-            return ret
-        else:
-            raise ValueError(f'Invalid style: {self.sep_style}')
-
-    def set_system_message(self, system_message: str):
-        """Set the system message."""
-        self.system_message = system_message
-
-    def append_message(self, role: str, message: str):
-        """Append a new message."""
-        self.messages.append([role, message])
-
-    def update_last_message(self, message: str):
-        """Update the last output.
-
-        The last message is typically set to be None when constructing the prompt,
-        so we need to update it in-place after getting the response from a model.
-        """
-        self.messages[-1][1] = message
-
-    def to_gradio_chatbot(self):
-        """Convert the conversation to gradio chatbot format."""
-        ret = []
-        for i, (role, msg) in enumerate(self.messages[self.offset :]):
-            if i % 2 == 0:
-                ret.append([msg, None])
-            else:
-                ret[-1][-1] = msg
-        return ret
-
-    def to_openai_api_messages(self):
-        """Convert the conversation to OpenAI chat completion format."""
-        ret = [{'role': 'system', 'content': self.system_message}]
-
-        for i, (_, msg) in enumerate(self.messages[self.offset :]):
-            if i % 2 == 0:
-                ret.append({'role': 'user', 'content': msg})
-            else:
-                if msg is not None:
-                    ret.append({'role': 'assistant', 'content': msg})
-        return ret
-
-    def copy(self):
-        return Conversation(
-            name=self.name,
-            system_template=self.system_template,
-            system_message=self.system_message,
-            roles=self.roles,
-            messages=[[x, y] for x, y in self.messages],
-            offset=self.offset,
-            sep_style=self.sep_style,
-            sep=self.sep,
-            sep2=self.sep2,
-            stop_str=self.stop_str,
-            stop_token_ids=self.stop_token_ids,
-        )
-
-    def dict(self):
-        return {
-            'template_name': self.name,
-            'system_message': self.system_message,
-            'roles': self.roles,
-            'messages': self.messages,
-            'offset': self.offset,
-        }
-
-
-# A global registry for all conversation templates
-conv_templates: Dict[str, Conversation] = {}
-
-
-def register_conv_template(template: Conversation, override: bool = False):
-    """Register a new conversation template."""
-    if not override:
-        assert (
-            template.name not in conv_templates
-        ), f'{template.name} has been registered.'
-
-    conv_templates[template.name] = template
-
-
-def get_conv_template(name: str) -> Conversation:
-    """Get a conversation template."""
-    return conv_templates[name].copy()
-
-
-# Both Hermes-2 and internlm2-chat are chatml-format conversation templates. The difference
-# is that during training, the preprocessing function for the Hermes-2 template doesn't add
-# <s> at the beginning of the tokenized sequence, while the internlm2-chat template does.
-# Therefore, they are completely equivalent during inference.
-
-register_conv_template(
-    Conversation(
-        name='chatml',
-        system_template='<|im_start|>system\n{system_message}',
-        # note: The new system prompt was not used here to avoid changes in benchmark performance.
-        system_message='Answer the questions.',
-        roles=('<|im_start|>user\n', '<|im_start|>assistant\n'),
-        sep_style=SeparatorStyle.MPT,
-        sep='<|im_end|>',
-        stop_token_ids=[
-            2,
-            92543,
-            92542
-        ]
-    )
-)
-
-

model/flash_attention.py

@@ -1,75 +0,0 @@
-import torch
-import torch.nn as nn
-from einops import rearrange
-
-try:  # v1
-    from flash_attn.flash_attn_interface import \
-        flash_attn_unpadded_qkvpacked_func
-except:  # v2
-    from flash_attn.flash_attn_interface import flash_attn_varlen_qkvpacked_func as flash_attn_unpadded_qkvpacked_func
-
-from flash_attn.bert_padding import pad_input, unpad_input
-
-
-class FlashAttention(nn.Module):
-    """Implement the scaled dot product attention with softmax.
-    Arguments
-    ---------
-        softmax_scale: The temperature to use for the softmax attention.
-                      (default: 1/sqrt(d_keys) where d_keys is computed at
-                      runtime)
-        attention_dropout: The dropout rate to apply to the attention
-                           (default: 0.0)
-    """
-
-    def __init__(self, softmax_scale=None, attention_dropout=0.0, device=None, dtype=None):
-        super().__init__()
-        self.softmax_scale = softmax_scale
-        self.dropout_p = attention_dropout
-
-    def forward(self, qkv, key_padding_mask=None, causal=False, cu_seqlens=None,
-                max_s=None, need_weights=False):
-        """Implements the multihead softmax attention.
-        Arguments
-        ---------
-            qkv: The tensor containing the query, key, and value. (B, S, 3, H, D) if key_padding_mask is None
-                if unpadded: (nnz, 3, h, d)
-            key_padding_mask: a bool tensor of shape (B, S)
-        """
-        assert not need_weights
-        assert qkv.dtype in [torch.float16, torch.bfloat16]
-        assert qkv.is_cuda
-
-        if cu_seqlens is None:
-            batch_size = qkv.shape[0]
-            seqlen = qkv.shape[1]
-            if key_padding_mask is None:
-                qkv = rearrange(qkv, 'b s ... -> (b s) ...')
-                max_s = seqlen
-                cu_seqlens = torch.arange(0, (batch_size + 1) * seqlen, step=seqlen, dtype=torch.int32,
-                                          device=qkv.device)
-                output = flash_attn_unpadded_qkvpacked_func(
-                    qkv, cu_seqlens, max_s, self.dropout_p if self.training else 0.0,
-                    softmax_scale=self.softmax_scale, causal=causal
-                )
-                output = rearrange(output, '(b s) ... -> b s ...', b=batch_size)
-            else:
-                nheads = qkv.shape[-2]
-                x = rearrange(qkv, 'b s three h d -> b s (three h d)')
-                x_unpad, indices, cu_seqlens, max_s = unpad_input(x, key_padding_mask)
-                x_unpad = rearrange(x_unpad, 'nnz (three h d) -> nnz three h d', three=3, h=nheads)
-                output_unpad = flash_attn_unpadded_qkvpacked_func(
-                    x_unpad, cu_seqlens, max_s, self.dropout_p if self.training else 0.0,
-                    softmax_scale=self.softmax_scale, causal=causal
-                )
-                output = rearrange(pad_input(rearrange(output_unpad, 'nnz h d -> nnz (h d)'),
-                                             indices, batch_size, seqlen),
-                                   'b s (h d) -> b s h d', h=nheads)
-        else:
-            assert max_s is not None
-            output = flash_attn_unpadded_qkvpacked_func(
-                qkv, cu_seqlens, max_s, self.dropout_p if self.training else 0.0,
-                softmax_scale=self.softmax_scale, causal=causal
-            )
-
-        return output, None

model/generation_config.json

@@ -1,4 +0,0 @@
-{
-  "_from_model_config": true,
-  "transformers_version": "4.45.0"
-}

model/model-00004-of-00004.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:ca76e6a409549c80001b5af64a538530bbb785ce60105468a29f84f6ce2b43c0
-size 1380991480

model/model.safetensors.index.json

@@ -1,690 +0,0 @@
-{
-  "metadata": {
-    "total_size": 16130253824
-  },
-  "weight_map": {
-    "language_model.lm_head.weight": "model-00004-of-00004.safetensors",
-    "language_model.model.embed_tokens.weight": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.0.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.0.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.0.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.1.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.1.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.1.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.10.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.10.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.10.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.11.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.11.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.11.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.12.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.12.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.12.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.13.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.13.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.13.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.14.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.14.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.14.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.15.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.15.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.15.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.16.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.16.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.16.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.17.input_layernorm.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.17.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.17.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.17.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.17.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.17.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.18.input_layernorm.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.18.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.18.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.18.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.18.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.18.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.18.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.18.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.18.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.18.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.18.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.18.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.19.input_layernorm.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.19.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.19.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.19.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.19.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.19.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.19.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.19.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.19.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.19.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.19.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.19.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.2.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.2.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.2.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.20.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.20.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.20.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.20.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.20.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.20.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.20.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.20.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.21.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.21.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.21.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.22.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.22.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.22.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.23.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.23.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.23.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.24.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.24.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.24.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.25.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.25.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.25.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.26.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.26.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.26.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.27.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.27.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.27.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
-    "language_model.model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.3.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.3.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.3.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.4.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.4.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.4.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.5.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.5.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.5.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.6.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.6.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.6.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.7.input_layernorm.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.7.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.7.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.7.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.7.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.7.self_attn.k_proj.bias": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.7.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.7.self_attn.q_proj.bias": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.7.self_attn.v_proj.bias": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
-    "language_model.model.layers.8.input_layernorm.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.8.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.8.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.8.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.8.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.8.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.8.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.8.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.8.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.8.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.8.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.8.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.9.self_attn.k_proj.bias": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.9.self_attn.q_proj.bias": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.9.self_attn.v_proj.bias": "model-00002-of-00004.safetensors",
-    "language_model.model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
-    "language_model.model.norm.weight": "model-00003-of-00004.safetensors",
-    "mlp1.0.bias": "model-00004-of-00004.safetensors",
-    "mlp1.0.weight": "model-00004-of-00004.safetensors",
-    "mlp1.1.weight": "model-00004-of-00004.safetensors",
-    "mlp1.3.weight": "model-00004-of-00004.safetensors",
-    "vision_model.embeddings.class_embedding": "model-00001-of-00004.safetensors",
-    "vision_model.embeddings.patch_embedding.bias": "model-00001-of-00004.safetensors",
-    "vision_model.embeddings.patch_embedding.weight": "model-00001-of-00004.safetensors",
-    "vision_model.embeddings.position_embedding": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.0.attn.proj.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.0.attn.proj.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.0.attn.qkv.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.0.attn.qkv.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.0.ls1": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.0.ls2": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.0.mlp.fc1.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.0.mlp.fc1.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.0.mlp.fc2.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.0.mlp.fc2.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.0.norm1.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.0.norm1.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.0.norm2.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.0.norm2.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.1.attn.proj.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.1.attn.proj.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.1.attn.qkv.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.1.attn.qkv.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.1.ls1": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.1.ls2": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.1.mlp.fc1.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.1.mlp.fc1.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.1.mlp.fc2.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.1.mlp.fc2.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.1.norm1.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.1.norm1.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.1.norm2.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.1.norm2.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.10.attn.proj.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.10.attn.proj.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.10.attn.qkv.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.10.attn.qkv.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.10.ls1": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.10.ls2": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.10.mlp.fc1.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.10.mlp.fc1.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.10.mlp.fc2.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.10.mlp.fc2.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.10.norm1.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.10.norm1.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.10.norm2.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.10.norm2.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.11.attn.proj.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.11.attn.proj.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.11.attn.qkv.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.11.attn.qkv.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.11.ls1": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.11.ls2": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.11.mlp.fc1.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.11.mlp.fc1.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.11.mlp.fc2.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.11.mlp.fc2.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.11.norm1.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.11.norm1.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.11.norm2.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.11.norm2.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.12.attn.proj.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.12.attn.proj.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.12.attn.qkv.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.12.attn.qkv.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.12.ls1": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.12.ls2": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.12.mlp.fc1.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.12.mlp.fc1.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.12.mlp.fc2.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.12.mlp.fc2.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.12.norm1.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.12.norm1.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.12.norm2.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.12.norm2.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.13.attn.proj.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.13.attn.proj.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.13.attn.qkv.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.13.attn.qkv.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.13.ls1": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.13.ls2": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.13.mlp.fc1.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.13.mlp.fc1.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.13.mlp.fc2.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.13.mlp.fc2.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.13.norm1.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.13.norm1.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.13.norm2.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.13.norm2.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.14.attn.proj.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.14.attn.proj.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.14.attn.qkv.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.14.attn.qkv.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.14.ls1": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.14.ls2": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.14.mlp.fc1.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.14.mlp.fc1.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.14.mlp.fc2.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.14.mlp.fc2.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.14.norm1.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.14.norm1.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.14.norm2.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.14.norm2.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.15.attn.proj.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.15.attn.proj.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.15.attn.qkv.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.15.attn.qkv.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.15.ls1": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.15.ls2": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.15.mlp.fc1.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.15.mlp.fc1.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.15.mlp.fc2.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.15.mlp.fc2.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.15.norm1.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.15.norm1.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.15.norm2.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.15.norm2.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.16.attn.proj.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.16.attn.proj.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.16.attn.qkv.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.16.attn.qkv.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.16.ls1": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.16.ls2": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.16.mlp.fc1.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.16.mlp.fc1.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.16.mlp.fc2.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.16.mlp.fc2.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.16.norm1.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.16.norm1.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.16.norm2.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.16.norm2.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.17.attn.proj.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.17.attn.proj.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.17.attn.qkv.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.17.attn.qkv.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.17.ls1": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.17.ls2": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.17.mlp.fc1.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.17.mlp.fc1.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.17.mlp.fc2.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.17.mlp.fc2.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.17.norm1.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.17.norm1.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.17.norm2.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.17.norm2.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.18.attn.proj.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.18.attn.proj.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.18.attn.qkv.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.18.attn.qkv.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.18.ls1": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.18.ls2": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.18.mlp.fc1.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.18.mlp.fc1.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.18.mlp.fc2.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.18.mlp.fc2.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.18.norm1.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.18.norm1.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.18.norm2.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.18.norm2.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.19.attn.proj.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.19.attn.proj.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.19.attn.qkv.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.19.attn.qkv.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.19.ls1": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.19.ls2": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.19.mlp.fc1.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.19.mlp.fc1.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.19.mlp.fc2.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.19.mlp.fc2.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.19.norm1.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.19.norm1.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.19.norm2.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.19.norm2.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.2.attn.proj.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.2.attn.proj.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.2.attn.qkv.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.2.attn.qkv.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.2.ls1": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.2.ls2": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.2.mlp.fc1.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.2.mlp.fc1.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.2.mlp.fc2.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.2.mlp.fc2.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.2.norm1.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.2.norm1.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.2.norm2.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.2.norm2.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.20.attn.proj.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.20.attn.proj.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.20.attn.qkv.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.20.attn.qkv.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.20.ls1": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.20.ls2": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.20.mlp.fc1.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.20.mlp.fc1.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.20.mlp.fc2.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.20.mlp.fc2.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.20.norm1.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.20.norm1.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.20.norm2.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.20.norm2.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.21.attn.proj.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.21.attn.proj.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.21.attn.qkv.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.21.attn.qkv.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.21.ls1": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.21.ls2": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.21.mlp.fc1.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.21.mlp.fc1.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.21.mlp.fc2.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.21.mlp.fc2.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.21.norm1.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.21.norm1.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.21.norm2.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.21.norm2.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.22.attn.proj.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.22.attn.proj.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.22.attn.qkv.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.22.attn.qkv.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.22.ls1": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.22.ls2": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.22.mlp.fc1.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.22.mlp.fc1.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.22.mlp.fc2.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.22.mlp.fc2.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.22.norm1.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.22.norm1.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.22.norm2.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.22.norm2.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.23.attn.proj.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.23.attn.proj.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.23.attn.qkv.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.23.attn.qkv.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.23.ls1": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.23.ls2": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.23.mlp.fc1.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.23.mlp.fc1.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.23.mlp.fc2.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.23.mlp.fc2.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.23.norm1.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.23.norm1.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.23.norm2.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.23.norm2.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.3.attn.proj.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.3.attn.proj.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.3.attn.qkv.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.3.attn.qkv.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.3.ls1": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.3.ls2": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.3.mlp.fc1.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.3.mlp.fc1.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.3.mlp.fc2.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.3.mlp.fc2.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.3.norm1.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.3.norm1.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.3.norm2.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.3.norm2.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.4.attn.proj.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.4.attn.proj.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.4.attn.qkv.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.4.attn.qkv.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.4.ls1": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.4.ls2": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.4.mlp.fc1.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.4.mlp.fc1.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.4.mlp.fc2.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.4.mlp.fc2.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.4.norm1.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.4.norm1.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.4.norm2.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.4.norm2.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.5.attn.proj.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.5.attn.proj.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.5.attn.qkv.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.5.attn.qkv.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.5.ls1": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.5.ls2": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.5.mlp.fc1.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.5.mlp.fc1.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.5.mlp.fc2.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.5.mlp.fc2.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.5.norm1.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.5.norm1.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.5.norm2.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.5.norm2.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.6.attn.proj.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.6.attn.proj.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.6.attn.qkv.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.6.attn.qkv.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.6.ls1": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.6.ls2": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.6.mlp.fc1.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.6.mlp.fc1.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.6.mlp.fc2.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.6.mlp.fc2.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.6.norm1.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.6.norm1.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.6.norm2.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.6.norm2.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.7.attn.proj.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.7.attn.proj.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.7.attn.qkv.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.7.attn.qkv.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.7.ls1": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.7.ls2": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.7.mlp.fc1.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.7.mlp.fc1.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.7.mlp.fc2.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.7.mlp.fc2.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.7.norm1.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.7.norm1.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.7.norm2.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.7.norm2.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.8.attn.proj.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.8.attn.proj.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.8.attn.qkv.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.8.attn.qkv.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.8.ls1": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.8.ls2": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.8.mlp.fc1.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.8.mlp.fc1.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.8.mlp.fc2.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.8.mlp.fc2.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.8.norm1.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.8.norm1.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.8.norm2.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.8.norm2.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.9.attn.proj.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.9.attn.proj.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.9.attn.qkv.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.9.attn.qkv.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.9.ls1": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.9.ls2": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.9.mlp.fc1.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.9.mlp.fc1.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.9.mlp.fc2.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.9.mlp.fc2.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.9.norm1.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.9.norm1.weight": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.9.norm2.bias": "model-00001-of-00004.safetensors",
-    "vision_model.encoder.layers.9.norm2.weight": "model-00001-of-00004.safetensors"
-  }
-}

model/modeling_intern_vit.py

@@ -1,363 +0,0 @@
-# --------------------------------------------------------
-# InternVL
-# Copyright (c) 2023 OpenGVLab
-# Licensed under The MIT License [see LICENSE for details]
-# --------------------------------------------------------
-from typing import Optional, Tuple, Union
-
-import torch
-import torch.nn.functional as F
-import torch.utils.checkpoint
-from einops import rearrange
-from timm.models.layers import DropPath
-from torch import nn
-from transformers.activations import ACT2FN
-from transformers.modeling_outputs import (BaseModelOutput,
-                                           BaseModelOutputWithPooling)
-from transformers.modeling_utils import PreTrainedModel
-from transformers.utils import logging
-
-from .configuration_intern_vit import InternVisionConfig
-
-try:
-    from .flash_attention import FlashAttention
-    has_flash_attn = True
-except:
-    print('FlashAttention is not installed.')
-    has_flash_attn = False
-
-
-logger = logging.get_logger(__name__)
-
-
-class InternRMSNorm(nn.Module):
-    def __init__(self, hidden_size, eps=1e-6):
-        super().__init__()
-        self.weight = nn.Parameter(torch.ones(hidden_size))
-        self.variance_epsilon = eps
-
-    def forward(self, hidden_states):
-        input_dtype = hidden_states.dtype
-        hidden_states = hidden_states.to(torch.float32)
-        variance = hidden_states.pow(2).mean(-1, keepdim=True)
-        hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)
-        return self.weight * hidden_states.to(input_dtype)
-
-
-try:
-    from apex.normalization import FusedRMSNorm
-
-    InternRMSNorm = FusedRMSNorm  # noqa
-
-    logger.info('Discovered apex.normalization.FusedRMSNorm - will use it instead of InternRMSNorm')
-except ImportError:
-    # using the normal InternRMSNorm
-    pass
-except Exception:
-    logger.warning('discovered apex but it failed to load, falling back to InternRMSNorm')
-    pass
-
-
-NORM2FN = {
-    'rms_norm': InternRMSNorm,
-    'layer_norm': nn.LayerNorm,
-}
-
-
-class InternVisionEmbeddings(nn.Module):
-    def __init__(self, config: InternVisionConfig):
-        super().__init__()
-        self.config = config
-        self.embed_dim = config.hidden_size
-        self.image_size = config.image_size
-        self.patch_size = config.patch_size
-
-        self.class_embedding = nn.Parameter(
-            torch.randn(1, 1, self.embed_dim),
-        )
-
-        self.patch_embedding = nn.Conv2d(
-            in_channels=3, out_channels=self.embed_dim, kernel_size=self.patch_size, stride=self.patch_size
-        )
-
-        self.num_patches = (self.image_size // self.patch_size) ** 2
-        self.num_positions = self.num_patches + 1
-
-        self.position_embedding = nn.Parameter(torch.randn(1, self.num_positions, self.embed_dim))
-
-    def _get_pos_embed(self, pos_embed, H, W):
-        target_dtype = pos_embed.dtype
-        pos_embed = pos_embed.float().reshape(
-            1, self.image_size // self.patch_size, self.image_size // self.patch_size, -1).permute(0, 3, 1, 2)
-        pos_embed = F.interpolate(pos_embed, size=(H, W), mode='bicubic', align_corners=False).\
-            reshape(1, -1, H * W).permute(0, 2, 1).to(target_dtype)
-        return pos_embed
-
-    def forward(self, pixel_values: torch.FloatTensor) -> torch.Tensor:
-        target_dtype = self.patch_embedding.weight.dtype
-        patch_embeds = self.patch_embedding(pixel_values)  # shape = [*, channel, width, height]
-        batch_size, _, height, width = patch_embeds.shape
-        patch_embeds = patch_embeds.flatten(2).transpose(1, 2)
-        class_embeds = self.class_embedding.expand(batch_size, 1, -1).to(target_dtype)
-        embeddings = torch.cat([class_embeds, patch_embeds], dim=1)
-        position_embedding = torch.cat([
-            self.position_embedding[:, :1, :],
-            self._get_pos_embed(self.position_embedding[:, 1:, :], height, width)
-        ], dim=1)
-        embeddings = embeddings + position_embedding.to(target_dtype)
-        return embeddings
-
-
-class InternAttention(nn.Module):
-    """Multi-headed attention from 'Attention Is All You Need' paper"""
-
-    def __init__(self, config: InternVisionConfig):
-        super().__init__()
-        self.config = config
-        self.embed_dim = config.hidden_size
-        self.num_heads = config.num_attention_heads
-        self.use_flash_attn = config.use_flash_attn and has_flash_attn
-        if config.use_flash_attn and not has_flash_attn:
-            print('Warning: Flash Attention is not available, use_flash_attn is set to False.')
-        self.head_dim = self.embed_dim // self.num_heads
-        if self.head_dim * self.num_heads != self.embed_dim:
-            raise ValueError(
-                f'embed_dim must be divisible by num_heads (got `embed_dim`: {self.embed_dim} and `num_heads`:'
-                f' {self.num_heads}).'
-            )
-
-        self.scale = self.head_dim ** -0.5
-        self.qkv = nn.Linear(self.embed_dim, 3 * self.embed_dim, bias=config.qkv_bias)
-        self.attn_drop = nn.Dropout(config.attention_dropout)
-        self.proj_drop = nn.Dropout(config.dropout)
-
-        self.qk_normalization = config.qk_normalization
-
-        if self.qk_normalization:
-            self.q_norm = InternRMSNorm(self.embed_dim, eps=config.layer_norm_eps)
-            self.k_norm = InternRMSNorm(self.embed_dim, eps=config.layer_norm_eps)
-
-        if self.use_flash_attn:
-            self.inner_attn = FlashAttention(attention_dropout=config.attention_dropout)
-        self.proj = nn.Linear(self.embed_dim, self.embed_dim)
-
-    def _naive_attn(self, x):
-        B, N, C = x.shape
-        qkv = self.qkv(x).reshape(B, N, 3, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4)
-        q, k, v = qkv.unbind(0)  # make torchscript happy (cannot use tensor as tuple)
-
-        if self.qk_normalization:
-            B_, H_, N_, D_ = q.shape
-            q = self.q_norm(q.transpose(1, 2).flatten(-2, -1)).view(B_, N_, H_, D_).transpose(1, 2)
-            k = self.k_norm(k.transpose(1, 2).flatten(-2, -1)).view(B_, N_, H_, D_).transpose(1, 2)
-
-        attn = ((q * self.scale) @ k.transpose(-2, -1))
-        attn = attn.softmax(dim=-1)
-        attn = self.attn_drop(attn)
-
-        x = (attn @ v).transpose(1, 2).reshape(B, N, C)
-        x = self.proj(x)
-        x = self.proj_drop(x)
-        return x
-
-    def _flash_attn(self, x, key_padding_mask=None, need_weights=False):
-        qkv = self.qkv(x)
-        qkv = rearrange(qkv, 'b s (three h d) -> b s three h d', three=3, h=self.num_heads)
-
-        if self.qk_normalization:
-            q, k, v = qkv.unbind(2)
-            q = self.q_norm(q.flatten(-2, -1)).view(q.shape)
-            k = self.k_norm(k.flatten(-2, -1)).view(k.shape)
-            qkv = torch.stack([q, k, v], dim=2)
-
-        context, _ = self.inner_attn(
-            qkv, key_padding_mask=key_padding_mask, need_weights=need_weights, causal=False
-        )
-        outs = self.proj(rearrange(context, 'b s h d -> b s (h d)'))
-        outs = self.proj_drop(outs)
-        return outs
-
-    def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
-        x = self._naive_attn(hidden_states) if not self.use_flash_attn else self._flash_attn(hidden_states)
-        return x
-
-
-class InternMLP(nn.Module):
-    def __init__(self, config: InternVisionConfig):
-        super().__init__()
-        self.config = config
-        self.act = ACT2FN[config.hidden_act]
-        self.fc1 = nn.Linear(config.hidden_size, config.intermediate_size)
-        self.fc2 = nn.Linear(config.intermediate_size, config.hidden_size)
-
-    def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
-        hidden_states = self.fc1(hidden_states)
-        hidden_states = self.act(hidden_states)
-        hidden_states = self.fc2(hidden_states)
-        return hidden_states
-
-
-class InternVisionEncoderLayer(nn.Module):
-    def __init__(self, config: InternVisionConfig, drop_path_rate: float):
-        super().__init__()
-        self.embed_dim = config.hidden_size
-        self.intermediate_size = config.intermediate_size
-        self.norm_type = config.norm_type
-
-        self.attn = InternAttention(config)
-        self.mlp = InternMLP(config)
-        self.norm1 = NORM2FN[self.norm_type](self.embed_dim, eps=config.layer_norm_eps)
-        self.norm2 = NORM2FN[self.norm_type](self.embed_dim, eps=config.layer_norm_eps)
-
-        self.ls1 = nn.Parameter(config.initializer_factor * torch.ones(self.embed_dim))
-        self.ls2 = nn.Parameter(config.initializer_factor * torch.ones(self.embed_dim))
-        self.drop_path1 = DropPath(drop_path_rate) if drop_path_rate > 0. else nn.Identity()
-        self.drop_path2 = DropPath(drop_path_rate) if drop_path_rate > 0. else nn.Identity()
-
-    def forward(
-            self,
-            hidden_states: torch.Tensor,
-    ) -> Tuple[torch.FloatTensor, Optional[torch.FloatTensor], Optional[Tuple[torch.FloatTensor]]]:
-        """
-        Args:
-            hidden_states (`Tuple[torch.FloatTensor, Optional[torch.FloatTensor]]`): input to the layer of shape `(batch, seq_len, embed_dim)`
-        """
-        hidden_states = hidden_states + self.drop_path1(self.attn(self.norm1(hidden_states)) * self.ls1)
-
-        hidden_states = hidden_states + self.drop_path2(self.mlp(self.norm2(hidden_states)) * self.ls2)
-
-        return hidden_states
-
-
-class InternVisionEncoder(nn.Module):
-    """
-    Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a
-    [`InternEncoderLayer`].
-
-    Args:
-        config (`InternConfig`):
-            The corresponding vision configuration for the `InternEncoder`.
-    """
-
-    def __init__(self, config: InternVisionConfig):
-        super().__init__()
-        self.config = config
-        # stochastic depth decay rule
-        dpr = [x.item() for x in torch.linspace(0, config.drop_path_rate, config.num_hidden_layers)]
-        self.layers = nn.ModuleList([
-            InternVisionEncoderLayer(config, dpr[idx]) for idx in range(config.num_hidden_layers)])
-        self.gradient_checkpointing = True
-
-    def forward(
-            self,
-            inputs_embeds,
-            output_hidden_states: Optional[bool] = None,
-            return_dict: Optional[bool] = None,
-    ) -> Union[Tuple, BaseModelOutput]:
-        r"""
-        Args:
-            inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
-                Embedded representation of the inputs. Should be float, not int tokens.
-            output_hidden_states (`bool`, *optional*):
-                Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors
-                for more detail.
-            return_dict (`bool`, *optional*):
-                Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
-        """
-        output_hidden_states = (
-            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
-        )
-        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
-
-        encoder_states = () if output_hidden_states else None
-        hidden_states = inputs_embeds
-
-        for idx, encoder_layer in enumerate(self.layers):
-            if output_hidden_states:
-                encoder_states = encoder_states + (hidden_states,)
-            if self.gradient_checkpointing and self.training:
-                layer_outputs = torch.utils.checkpoint.checkpoint(
-                    encoder_layer,
-                    hidden_states)
-            else:
-                layer_outputs = encoder_layer(
-                    hidden_states,
-                )
-            hidden_states = layer_outputs
-
-        if output_hidden_states:
-            encoder_states = encoder_states + (hidden_states,)
-
-        if not return_dict:
-            return tuple(v for v in [hidden_states, encoder_states] if v is not None)
-        return BaseModelOutput(
-            last_hidden_state=hidden_states, hidden_states=encoder_states
-        )
-
-
-class InternVisionModel(PreTrainedModel):
-    main_input_name = 'pixel_values'
-    config_class = InternVisionConfig
-    _no_split_modules = ['InternVisionEncoderLayer']
-
-    def __init__(self, config: InternVisionConfig):
-        super().__init__(config)
-        self.config = config
-
-        self.embeddings = InternVisionEmbeddings(config)
-        self.encoder = InternVisionEncoder(config)
-
-    def resize_pos_embeddings(self, old_size, new_size, patch_size):
-        pos_emb = self.embeddings.position_embedding
-        _, num_positions, embed_dim = pos_emb.shape
-        cls_emb = pos_emb[:, :1, :]
-        pos_emb = pos_emb[:, 1:, :].reshape(1, old_size // patch_size, old_size // patch_size, -1).permute(0, 3, 1, 2)
-        pos_emb = F.interpolate(pos_emb.float(), size=new_size // patch_size, mode='bicubic', align_corners=False)
-        pos_emb = pos_emb.to(cls_emb.dtype).reshape(1, embed_dim, -1).permute(0, 2, 1)
-        pos_emb = torch.cat([cls_emb, pos_emb], dim=1)
-        self.embeddings.position_embedding = nn.Parameter(pos_emb)
-        self.embeddings.image_size = new_size
-        logger.info('Resized position embeddings from {} to {}'.format(old_size, new_size))
-
-    def get_input_embeddings(self):
-        return self.embeddings
-
-    def forward(
-            self,
-            pixel_values: Optional[torch.FloatTensor] = None,
-            output_hidden_states: Optional[bool] = None,
-            return_dict: Optional[bool] = None,
-            pixel_embeds: Optional[torch.FloatTensor] = None,
-    ) -> Union[Tuple, BaseModelOutputWithPooling]:
-        output_hidden_states = (
-            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
-        )
-        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
-
-        if pixel_values is None and pixel_embeds is None:
-            raise ValueError('You have to specify pixel_values or pixel_embeds')
-
-        if pixel_embeds is not None:
-            hidden_states = pixel_embeds
-        else:
-            if len(pixel_values.shape) == 4:
-                hidden_states = self.embeddings(pixel_values)
-            else:
-                raise ValueError(f'wrong pixel_values size: {pixel_values.shape}')
-        encoder_outputs = self.encoder(
-            inputs_embeds=hidden_states,
-            output_hidden_states=output_hidden_states,
-            return_dict=return_dict,
-        )
-        last_hidden_state = encoder_outputs.last_hidden_state
-        pooled_output = last_hidden_state[:, 0, :]
-
-        if not return_dict:
-            return (last_hidden_state, pooled_output) + encoder_outputs[1:]
-
-        return BaseModelOutputWithPooling(
-            last_hidden_state=last_hidden_state,
-            pooler_output=pooled_output,
-            hidden_states=encoder_outputs.hidden_states,
-            attentions=encoder_outputs.attentions,
-        )

model/modeling_llavaqw.py

@@ -1,573 +0,0 @@
-import warnings
-from typing import Any, List, Optional, Tuple, Union
-import torch.utils.checkpoint
-import transformers
-from torch import nn
-from torch.nn import CrossEntropyLoss
-from transformers import GenerationConfig
-from transformers import Qwen2ForCausalLM
-from transformers.modeling_outputs import CausalLMOutputWithPast
-from transformers.modeling_utils import PreTrainedModel
-from transformers.utils import ModelOutput, logging
-import torch.nn.functional as F
-
-from .configuration_llavaqw import LlavaQwConfig
-from .conversation import get_conv_template
-from .modeling_intern_vit import InternVisionModel, has_flash_attn
-
-logger = logging.get_logger(__name__)
-
-
-def version_cmp(v1, v2, op='eq'):
-    import operator
-
-    from packaging import version
-    op_func = getattr(operator, op)
-    return op_func(version.parse(v1), version.parse(v2))
-
-
-class LlavaQwModel(PreTrainedModel):
-    config_class = LlavaQwConfig
-    main_input_name = 'pixel_values'
-    _supports_flash_attn_2 = True
-    _no_split_modules = ['InternVisionModel', 'Qwen2DecoderLayer']
-
-    def __init__(self, config: LlavaQwConfig, vision_model=None, language_model=None, use_flash_attn=True):
-        super().__init__(config)
-
-        assert version_cmp(transformers.__version__, '4.44.2', 'ge')
-        image_size = config.force_image_size or config.vision_config.image_size
-        patch_size = config.vision_config.patch_size
-        self.patch_size = patch_size
-        self.select_layer = config.select_layer
-        self.llm_arch_name = config.llm_config.architectures[0]
-        self.template = config.template
-        self.num_image_token = int((image_size // patch_size) ** 2 * (config.downsample_ratio ** 2))
-        self.downsample_ratio = config.downsample_ratio
-        self.ps_version = config.ps_version
-        use_flash_attn = use_flash_attn if has_flash_attn else False
-        config.vision_config.use_flash_attn = True if use_flash_attn else False
-        config.llm_config._attn_implementation = 'flash_attention_2' if use_flash_attn else 'eager'
-
-        logger.info(f'num_image_token: {self.num_image_token}')
-        logger.info(f'ps_version: {self.ps_version}')
-        if vision_model is not None:
-            self.vision_model = vision_model
-        else:
-            self.vision_model = InternVisionModel(config.vision_config)
-        if language_model is not None:
-            self.language_model = language_model
-        else:
-            if config.llm_config.architectures[0] == 'Qwen2ForCausalLM':
-                self.language_model = Qwen2ForCausalLM(config.llm_config)
-            else:
-                raise NotImplementedError(f'{config.llm_config.architectures[0]} is not implemented.')
-
-        vit_hidden_size = config.vision_config.hidden_size
-        llm_intermediate_size = config.llm_config.intermediate_size
-        llm_hidden_size = config.llm_config.hidden_size
-
-        self.mlp1 = nn.Sequential(
-            nn.LayerNorm(vit_hidden_size * int(1 / self.downsample_ratio) ** 2),
-            nn.Linear(vit_hidden_size * int(1 / self.downsample_ratio) ** 2, llm_intermediate_size, bias=False),
-            nn.GELU(),
-            nn.Linear(llm_intermediate_size, llm_hidden_size, bias=False)
-        )
-
-        self.img_context_token_id = 151654
-        self.conv_template = get_conv_template(self.template)
-        self.system_message = self.conv_template.system_message
-
-    def forward(
-            self,
-            pixel_values: torch.FloatTensor,
-            input_ids: torch.LongTensor = None,
-            attention_mask: Optional[torch.Tensor] = None,
-            position_ids: Optional[torch.LongTensor] = None,
-            image_flags: Optional[torch.LongTensor] = None,
-            past_key_values: Optional[List[torch.FloatTensor]] = None,
-            labels: Optional[torch.LongTensor] = None,
-            use_cache: Optional[bool] = None,
-            output_attentions: Optional[bool] = None,
-            output_hidden_states: Optional[bool] = None,
-            return_dict: Optional[bool] = None,
-    ) -> Union[Tuple, CausalLMOutputWithPast]:
-        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
-
-        input_embeds = self.language_model.get_input_embeddings()(input_ids)
-
-        vit_embeds = self.extract_feature(pixel_values)
-        vit_batch_size = pixel_values.shape[0]
-
-        B, N, C = input_embeds.shape
-        input_embeds = input_embeds.reshape(B * N, C)
-
-        input_ids = input_ids.reshape(B * N)
-        selected = (input_ids == self.img_context_token_id)
-        try:
-            input_embeds[selected] = input_embeds[selected] * 0.0 + vit_embeds.reshape(-1, C)
-        except Exception as e:
-            vit_embeds = vit_embeds.reshape(-1, C)
-            print(f'warning: {e}, input_embeds[selected].shape={input_embeds[selected].shape}, '
-                  f'vit_embeds.shape={vit_embeds.shape}')
-            n_token = selected.sum()
-            input_embeds[selected] = input_embeds[selected] * 0.0 + vit_embeds[:n_token]
-
-        input_embeds = input_embeds.reshape(B, N, C)
-
-        outputs = self.language_model(
-            inputs_embeds=input_embeds,
-            attention_mask=attention_mask,
-            position_ids=position_ids,
-            past_key_values=past_key_values,
-            use_cache=use_cache,
-            output_attentions=output_attentions,
-            output_hidden_states=output_hidden_states,
-            return_dict=return_dict,
-        )
-        logits = outputs.logits
-
-        loss = None
-        if labels is not None:
-            # Shift so that tokens < n predict n
-            shift_logits = logits[..., :-1, :].contiguous()
-            shift_labels = labels[..., 1:].contiguous()
-            # Flatten the tokens
-            loss_fct = CrossEntropyLoss()
-            shift_logits = shift_logits.view(-1, self.language_model.config.vocab_size)
-            shift_labels = shift_labels.view(-1)
-
-            # Enable model parallelism
-            shift_labels = shift_labels.to(shift_logits.device)
-            loss = loss_fct(shift_logits, shift_labels)
-
-        if not return_dict:
-            output = (logits,) + outputs[1:]
-            return (loss,) + output if loss is not None else output
-
-        return CausalLMOutputWithPast(
-            loss=loss,
-            logits=logits,
-            past_key_values=outputs.past_key_values,
-            hidden_states=outputs.hidden_states,
-            attentions=outputs.attentions,
-        )
-
-    def pixel_shuffle(self, x, scale_factor=0.5):
-        n, w, h, c = x.size()
-        # N, W, H, C --> N, W, H * scale, C // scale
-        x = x.view(n, w, int(h * scale_factor), int(c / scale_factor))
-        # N, W, H * scale, C // scale --> N, H * scale, W, C // scale
-        x = x.permute(0, 2, 1, 3).contiguous()
-        # N, H * scale, W, C // scale --> N, H * scale, W * scale, C // (scale ** 2)
-        x = x.view(n, int(h * scale_factor), int(w * scale_factor),
-                   int(c / (scale_factor * scale_factor)))
-        if self.ps_version == 'v1':
-            warnings.warn("In ps_version 'v1', the height and width have not been swapped back, "
-                          'which results in a transposed image.')
-        else:
-            x = x.permute(0, 2, 1, 3).contiguous()
-        return x
-
-    def extract_feature(self, pixel_values):
-        if self.select_layer == -1:
-            vit_embeds = self.vision_model(
-                pixel_values=pixel_values,
-                output_hidden_states=False,
-                return_dict=True).last_hidden_state
-        else:
-            vit_embeds = self.vision_model(
-                pixel_values=pixel_values,
-                output_hidden_states=True,
-                return_dict=True).hidden_states[self.select_layer]
-        vit_embeds = vit_embeds[:, 1:, :]
-
-        h = w = int(vit_embeds.shape[1] ** 0.5)
-        vit_embeds = vit_embeds.reshape(vit_embeds.shape[0], h, w, -1)
-        vit_embeds = self.pixel_shuffle(vit_embeds, scale_factor=self.downsample_ratio)
-        vit_embeds = vit_embeds.reshape(vit_embeds.shape[0], -1, vit_embeds.shape[-1])
-        vit_embeds = self.mlp1(vit_embeds)
-        return vit_embeds
-
-    def chat(self, tokenizer, pixel_values, question, generation_config, history=None, return_history=False,
-             num_patches_list=None, IMG_START_TOKEN='<|vision_start|>', IMG_END_TOKEN='<|vision_end|>',
-             IMG_CONTEXT_TOKEN='<|vision_pad|>', verbose=False, visual_features=None):
-
-        if history is None and pixel_values is not None and '<image>' not in question:
-            question = '<image>\n' + question
-
-        if num_patches_list is None:
-            num_patches_list = [pixel_values.shape[0]] if pixel_values is not None else []
-        assert pixel_values is None or len(pixel_values) == sum(num_patches_list)
-
-        img_context_token_id = tokenizer.convert_tokens_to_ids(IMG_CONTEXT_TOKEN)
-        self.img_context_token_id = img_context_token_id
-
-        template = get_conv_template(self.template)
-        template.system_message = self.system_message
-        eos_token_id = tokenizer.convert_tokens_to_ids(template.sep)
-
-        history = [] if history is None else history
-        for (old_question, old_answer) in history:
-            template.append_message(template.roles[0], old_question)
-            template.append_message(template.roles[1], old_answer)
-        template.append_message(template.roles[0], question)
-        template.append_message(template.roles[1], None)
-        query = template.get_prompt()
-        # print('query:', query)
-
-        if verbose and pixel_values is not None:
-            image_bs = pixel_values.shape[0]
-            print(f'dynamic ViT batch size: {image_bs}')
-
-        for num_patches in num_patches_list:
-            tile_pos_identifiers = [f"<tile_{i}>" for i in range(1, num_patches)] + ["<tile_global_thumbnail>"]
-            image_tokens = ''
-            for tile_pos_identifier in tile_pos_identifiers:
-                image_tokens += tile_pos_identifier + IMG_CONTEXT_TOKEN * self.num_image_token
-            image_tokens = IMG_START_TOKEN + image_tokens + IMG_END_TOKEN
-            query = query.replace('<image>', image_tokens, 1)
-
-        model_inputs = tokenizer(query, return_tensors='pt')
-        input_ids = model_inputs['input_ids'].cuda()
-        attention_mask = model_inputs['attention_mask'].cuda()
-        generation_config['eos_token_id'] = eos_token_id
-        generation_output = self.generate(
-            pixel_values=pixel_values,
-            visual_features=visual_features,
-            input_ids=input_ids,
-            attention_mask=attention_mask,
-            **generation_config
-        )
-        response = tokenizer.batch_decode(generation_output, skip_special_tokens=True)[0]
-        response = response.split(template.sep)[0].strip()
-        history.append((question, response))
-        if return_history:
-            return response, history
-        else:
-            query_to_print = query.replace(IMG_CONTEXT_TOKEN, '')
-            query_to_print = query_to_print.replace(f'{IMG_START_TOKEN}{IMG_END_TOKEN}', '<image>')
-            if verbose:
-                print(query_to_print, response)
-            return response
-
-    def chat_without_sys_prompt(self, tokenizer, pixel_values, question, generation_config, history=None,
-                                return_history=False,
-                                num_patches_list=None, IMG_START_TOKEN='<|vision_start|>',
-                                IMG_END_TOKEN='<|vision_end|>',
-                                IMG_CONTEXT_TOKEN='<|vision_pad|>', verbose=False, visual_features=None):
-
-        if history is None and pixel_values is not None and '<image>' not in question:
-            question = '<image>\n' + question
-
-        if num_patches_list is None:
-            num_patches_list = [pixel_values.shape[0]] if pixel_values is not None else []
-        assert pixel_values is None or len(pixel_values) == sum(num_patches_list)
-
-        img_context_token_id = tokenizer.convert_tokens_to_ids(IMG_CONTEXT_TOKEN)
-        self.img_context_token_id = img_context_token_id
-
-        template = get_conv_template(self.template)
-        system_prompt = "<|im_start|>system\nYou are a helpful assistant.<|im_end|>"  # override dummy system prompt
-        template.system_message = system_prompt
-        eos_token_id = tokenizer.convert_tokens_to_ids(template.sep)
-
-        history = [] if history is None else history
-        for (old_question, old_answer) in history:
-            template.append_message(template.roles[0], old_question)
-            template.append_message(template.roles[1], old_answer)
-        template.append_message(template.roles[0], question)
-        template.append_message(template.roles[1], None)
-        query = template.get_prompt()
-
-        if verbose and pixel_values is not None:
-            image_bs = pixel_values.shape[0]
-            print(f'dynamic ViT batch size: {image_bs}')
-
-        query = query[len(system_prompt):]
-
-        for num_patches in num_patches_list:
-            tile_pos_identifiers = [f"<tile_{i}>" for i in range(1, num_patches)] + ["<tile_global_thumbnail>"]
-            image_tokens = ''
-            for tile_pos_identifier in tile_pos_identifiers:
-                image_tokens += tile_pos_identifier + IMG_CONTEXT_TOKEN * self.num_image_token
-            image_tokens = IMG_START_TOKEN + image_tokens + IMG_END_TOKEN
-            query = query.replace('<image>', image_tokens, 1)
-
-        model_inputs = tokenizer(query, return_tensors='pt')
-        input_ids = model_inputs['input_ids'].cuda()
-        attention_mask = model_inputs['attention_mask'].cuda()
-        generation_config['eos_token_id'] = eos_token_id
-        generation_output = self.generate(
-            pixel_values=pixel_values,
-            visual_features=visual_features,
-            input_ids=input_ids,
-            attention_mask=attention_mask,
-            **generation_config
-        )
-        response = tokenizer.batch_decode(generation_output, skip_special_tokens=True)[0]
-        response = response.split(template.sep)[0].strip()
-        history.append((question, response))
-        if return_history:
-            return response, history
-        else:
-            query_to_print = query.replace(IMG_CONTEXT_TOKEN, '')
-            query_to_print = query_to_print.replace(f'{IMG_START_TOKEN}{IMG_END_TOKEN}', '<image>')
-            if verbose:
-                print(query_to_print, response)
-            return response
-
-    def chat_without_chat_prompt(self, tokenizer, pixel_values, question, generation_config,
-                                 num_patches_list=None, IMG_START_TOKEN='<|vision_start|>',
-                                 IMG_END_TOKEN='<|vision_end|>',
-                                 IMG_CONTEXT_TOKEN='<|vision_pad|>', verbose=False, visual_features=None):
-
-        if pixel_values is not None and '<image>' not in question:
-            question = '<image>\n' + question
-
-        if num_patches_list is None:
-            num_patches_list = [pixel_values.shape[0]] if pixel_values is not None else []
-        assert pixel_values is None or len(pixel_values) == sum(num_patches_list)
-
-        img_context_token_id = tokenizer.convert_tokens_to_ids(IMG_CONTEXT_TOKEN)
-        self.img_context_token_id = img_context_token_id
-
-        template = get_conv_template(self.template)
-        template.system_message = self.system_message
-        eos_token_id = tokenizer.convert_tokens_to_ids(template.sep)
-
-        if verbose and pixel_values is not None:
-            image_bs = pixel_values.shape[0]
-            print(f'dynamic ViT batch size: {image_bs}')
-
-        query = question
-
-        for num_patches in num_patches_list:
-            tile_pos_identifiers = [f"<tile_{i}>" for i in range(1, num_patches)] + ["<tile_global_thumbnail>"]
-            image_tokens = ''
-            for tile_pos_identifier in tile_pos_identifiers:
-                image_tokens += tile_pos_identifier + IMG_CONTEXT_TOKEN * self.num_image_token
-            image_tokens = IMG_START_TOKEN + image_tokens + IMG_END_TOKEN
-            query = query.replace('<image>', image_tokens, 1)
-
-        model_inputs = tokenizer(query, return_tensors='pt')
-        input_ids = model_inputs['input_ids'].cuda()
-        attention_mask = model_inputs['attention_mask'].cuda()
-        generation_config['eos_token_id'] = eos_token_id
-        generation_output = self.generate(
-            pixel_values=pixel_values,
-            visual_features=visual_features,
-            input_ids=input_ids,
-            attention_mask=attention_mask,
-            **generation_config
-        )
-        response = tokenizer.batch_decode(generation_output, skip_special_tokens=True)[0]
-        response = response.split(template.sep)[0].strip()
-
-        query_to_print = query.replace(IMG_CONTEXT_TOKEN, '')
-        query_to_print = query_to_print.replace(f'{IMG_START_TOKEN}{IMG_END_TOKEN}', '<image>')
-        if verbose:
-            print(query_to_print, response)
-        return response
-
-    @torch.no_grad()
-    def generate(
-            self,
-            pixel_values: Optional[torch.FloatTensor] = None,
-            input_ids: Optional[torch.FloatTensor] = None,
-            attention_mask: Optional[torch.LongTensor] = None,
-            visual_features: Optional[torch.FloatTensor] = None,
-            generation_config: Optional[GenerationConfig] = None,
-            output_hidden_states: Optional[bool] = None,
-            return_dict: Optional[bool] = None,
-            **generate_kwargs,
-    ) -> torch.LongTensor:
-
-        # assert self.img_context_token_id is not None
-        if pixel_values is not None:
-            if visual_features is not None:
-                vit_embeds = visual_features.cuda()
-                vit_embeds = self.mlp1(vit_embeds)
-            else:
-                vit_embeds = self.extract_feature(pixel_values)
-
-            input_embeds = self.language_model.get_input_embeddings()(input_ids)
-            B, N, C = input_embeds.shape
-            input_embeds = input_embeds.reshape(B * N, C)
-
-            input_ids = input_ids.reshape(B * N)
-            selected = (input_ids == self.img_context_token_id)
-            assert selected.sum() != 0
-            input_embeds[selected] = vit_embeds.reshape(-1, C).to(input_embeds.device)
-
-            input_embeds = input_embeds.reshape(B, N, C)
-        else:
-            input_embeds = self.language_model.get_input_embeddings()(input_ids)
-
-        outputs = self.language_model.generate(
-            inputs_embeds=input_embeds,
-            attention_mask=attention_mask,
-            generation_config=generation_config,
-            output_hidden_states=output_hidden_states,
-            return_dict=return_dict,
-            use_cache=True,
-            **generate_kwargs,
-        )
-
-        return outputs
-
-    def chat_batch(
-            self,
-            tokenizer,
-            pixel_values_list,
-            questions,
-            generation_config,
-            histories=None,
-            return_histories=False,
-            num_patches_lists=None,
-            IMG_START_TOKEN='<|vision_start|>',
-            IMG_END_TOKEN='<|vision_end|>',
-            IMG_CONTEXT_TOKEN='<|vision_pad|>',
-            verbose=False,
-            visual_features_list=None
-    ):
-
-        if histories is None:
-            histories = [[] for _ in questions]
-
-        img_context_token_id = tokenizer.convert_tokens_to_ids(IMG_CONTEXT_TOKEN)
-        self.img_context_token_id = img_context_token_id
-        # Get eos_token_id from the template
-        template = get_conv_template(self.template)
-        template.system_message = self.system_message
-        eos_token_id = tokenizer.convert_tokens_to_ids(template.sep)
-        generation_config['eos_token_id'] = eos_token_id
-
-        queries = []
-        input_ids_list = []
-        attention_mask_list = []
-
-        for idx in range(len(questions)):
-            question = questions[idx]
-            history = histories[idx]
-            pixel_values = pixel_values_list[idx] if pixel_values_list[idx] is not None else None
-            num_patches_list = [pixel_values.shape[0]] if pixel_values is not None else []
-
-            if not history and pixel_values is not None and '<image>' not in question:
-                question = '<image>\n' + question
-
-            template_i = get_conv_template(self.template)
-            template_i.system_message = self.system_message
-            for (old_question, old_answer) in history:
-                template_i.append_message(template_i.roles[0], old_question)
-                template_i.append_message(template_i.roles[1], old_answer)
-            template_i.append_message(template_i.roles[0], question)
-            template_i.append_message(template_i.roles[1], None)
-            query = template_i.get_prompt()
-            # Handle image tokens
-            if pixel_values is not None:
-                for num_patches in num_patches_list:
-                    tile_pos_identifiers = [f"<tile_{i}>" for i in range(1, num_patches)] + ["<tile_global_thumbnail>"]
-                    image_tokens = ''
-                    for tile_pos_identifier in tile_pos_identifiers:
-                        image_tokens += tile_pos_identifier + IMG_CONTEXT_TOKEN * self.num_image_token
-                    image_tokens = IMG_START_TOKEN + image_tokens + IMG_END_TOKEN
-                    query = query.replace('<image>', image_tokens, 1)
-
-            model_inputs = tokenizer(
-                query,
-                return_tensors='pt',
-                padding=True,
-                truncation=True
-            )
-            input_ids = model_inputs['input_ids'].cuda()
-            attention_mask = model_inputs['attention_mask'].cuda()
-            input_ids_list.append(input_ids)
-            attention_mask_list.append(attention_mask)
-
-        # Call the generate function
-        generation_output = self.generate_batch(
-            pixel_values_list=pixel_values_list,
-            input_ids_list=input_ids_list,
-            attention_mask_list=attention_mask_list,
-            **generation_config
-        )
-        responses = tokenizer.batch_decode(generation_output, skip_special_tokens=True)
-
-        outputs = []
-        for idx, response in enumerate(responses):
-            response = response.split(template.sep)[0].strip()
-            histories[idx].append((questions[idx], response))
-            outputs.append(response)
-
-        if return_histories:
-            return outputs, histories
-        else:
-            if verbose:
-                for idx, query in enumerate(queries):
-                    query_to_print = query.replace(IMG_CONTEXT_TOKEN, '')
-                    query_to_print = query_to_print.replace(f'{IMG_START_TOKEN}{IMG_END_TOKEN}', '<image>')
-                    print(query_to_print, outputs[idx])
-            return outputs
-
-    @torch.no_grad()
-    def generate_batch(
-            self,
-            pixel_values_list: Optional[List[torch.FloatTensor]] = None,
-            input_ids_list: Optional[List[torch.FloatTensor]] = None,
-            attention_mask_list: Optional[List[torch.LongTensor]] = None,
-            visual_features: Optional[torch.FloatTensor] = None,
-            generation_config: Optional[GenerationConfig] = None,
-            output_hidden_states: Optional[bool] = None,
-            return_dict: Optional[bool] = None,
-            **generate_kwargs,
-    ) -> torch.LongTensor:
-        input_embeds_list = []
-        attention_mask_padded_list = []
-
-        max_seq_length = max(input_ids.shape[1] for input_ids in input_ids_list)
-
-        for pixel_values, input_ids, attention_mask in zip(pixel_values_list, input_ids_list, attention_mask_list):
-            if pixel_values is not None:
-                if visual_features is not None:
-                    vit_embeds = visual_features.cuda()
-                    vit_embeds = self.mlp1(vit_embeds)
-                else:
-                    vit_embeds = self.extract_feature(pixel_values)
-
-                input_embeds = self.language_model.get_input_embeddings()(input_ids)
-                B, N, C = input_embeds.shape
-                input_embeds = input_embeds.reshape(B * N, C)
-
-                input_ids = input_ids.reshape(B * N)
-                selected = (input_ids == self.img_context_token_id)
-                assert selected.sum() != 0, "No valid image context token IDs found."
-                input_embeds[selected] = vit_embeds.reshape(-1, C).to(input_embeds.device)
-
-                input_embeds = input_embeds.reshape(B, N, C)
-            else:
-                input_embeds = self.language_model.get_input_embeddings()(input_ids)
-
-            seq_length = input_embeds.shape[1]
-            if seq_length < max_seq_length:
-                pad_size = max_seq_length - seq_length
-                input_embeds = F.pad(input_embeds, (0, 0, 0, pad_size))
-                attention_mask = F.pad(attention_mask, (0, pad_size))
-
-            input_embeds_list.append(input_embeds)
-            attention_mask_padded_list.append(attention_mask)
-
-        input_embeds = torch.cat(input_embeds_list, dim=0)
-        attention_mask = torch.cat(attention_mask_padded_list, dim=0)
-
-        outputs = self.language_model.generate(
-            inputs_embeds=input_embeds,
-            attention_mask=attention_mask,
-            generation_config=generation_config,
-            output_hidden_states=output_hidden_states,
-            return_dict=return_dict,
-            use_cache=True,
-            **generate_kwargs,
-        )
-
-        return outputs

model/special_tokens_map.json

@@ -1,31 +0,0 @@
-{
-  "additional_special_tokens": [
-    "<|im_start|>",
-    "<|im_end|>",
-    "<|object_ref_start|>",
-    "<|object_ref_end|>",
-    "<|box_start|>",
-    "<|box_end|>",
-    "<|quad_start|>",
-    "<|quad_end|>",
-    "<|vision_start|>",
-    "<|vision_end|>",
-    "<|vision_pad|>",
-    "<|image_pad|>",
-    "<|video_pad|>"
-  ],
-  "eos_token": {
-    "content": "<|im_end|>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  },
-  "pad_token": {
-    "content": "<|endoftext|>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  }
-}

model/tokenizer_config.json

@@ -1,207 +0,0 @@
-{
-  "add_bos_token": false,
-  "add_prefix_space": false,
-  "added_tokens_decoder": {
-    "151643": {
-      "content": "<|endoftext|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "151644": {
-      "content": "<|im_start|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "151645": {
-      "content": "<|im_end|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "151646": {
-      "content": "<|object_ref_start|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "151647": {
-      "content": "<|object_ref_end|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "151648": {
-      "content": "<|box_start|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "151649": {
-      "content": "<|box_end|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "151650": {
-      "content": "<|quad_start|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "151651": {
-      "content": "<|quad_end|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "151652": {
-      "content": "<|vision_start|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "151653": {
-      "content": "<|vision_end|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "151654": {
-      "content": "<|vision_pad|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "151655": {
-      "content": "<|image_pad|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "151656": {
-      "content": "<|video_pad|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": true
-    },
-    "151657": {
-      "content": "<tool_call>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "151658": {
-      "content": "</tool_call>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "151659": {
-      "content": "<|fim_prefix|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "151660": {
-      "content": "<|fim_middle|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "151661": {
-      "content": "<|fim_suffix|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "151662": {
-      "content": "<|fim_pad|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "151663": {
-      "content": "<|repo_name|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "151664": {
-      "content": "<|file_sep|>",
-      "lstrip": false,
-      "normalized": false,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    }
-  },
-  "additional_special_tokens": [
-    "<|im_start|>",
-    "<|im_end|>",
-    "<|object_ref_start|>",
-    "<|object_ref_end|>",
-    "<|box_start|>",
-    "<|box_end|>",
-    "<|quad_start|>",
-    "<|quad_end|>",
-    "<|vision_start|>",
-    "<|vision_end|>",
-    "<|vision_pad|>",
-    "<|image_pad|>",
-    "<|video_pad|>"
-  ],
-  "bos_token": null,
-  "chat_template": "{%- if tools %}\n    {{- '<|im_start|>system\\n' }}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- messages[0]['content'] }}\n    {%- else %}\n        {{- 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.' }}\n    {%- endif %}\n    {{- \"\\n\\n# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within <tools></tools> XML tags:\\n<tools>\" }}\n    {%- for tool in tools %}\n        {{- \"\\n\" }}\n        {{- tool | tojson }}\n    {%- endfor %}\n    {{- \"\\n</tools>\\n\\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\\n<tool_call>\\n{\\\"name\\\": <function-name>, \\\"arguments\\\": <args-json-object>}\\n</tool_call><|im_end|>\\n\" }}\n{%- else %}\n    {%- if messages[0]['role'] == 'system' %}\n        {{- '<|im_start|>system\\n' + messages[0]['content'] + '<|im_end|>\\n' }}\n    {%- else %}\n        {{- '<|im_start|>system\\nYou are Qwen, created by Alibaba Cloud. You are a helpful assistant.<|im_end|>\\n' }}\n    {%- endif %}\n{%- endif %}\n{%- for message in messages %}\n    {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) or (message.role == \"assistant\" and not message.tool_calls) %}\n        {{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>' + '\\n' }}\n    {%- elif message.role == \"assistant\" %}\n        {{- '<|im_start|>' + message.role }}\n        {%- if message.content %}\n            {{- '\\n' + message.content }}\n        {%- endif %}\n        {%- for tool_call in message.tool_calls %}\n            {%- if tool_call.function is defined %}\n                {%- set tool_call = tool_call.function %}\n            {%- endif %}\n            {{- '\\n<tool_call>\\n{\"name\": \"' }}\n            {{- tool_call.name }}\n            {{- '\", \"arguments\": ' }}\n            {{- tool_call.arguments | tojson }}\n            {{- '}\\n</tool_call>' }}\n        {%- endfor %}\n        {{- '<|im_end|>\\n' }}\n    {%- elif message.role == \"tool\" %}\n        {%- if (loop.index0 == 0) or (messages[loop.index0 - 1].role != \"tool\") %}\n            {{- '<|im_start|>user' }}\n        {%- endif %}\n        {{- '\\n<tool_response>\\n' }}\n        {{- message.content }}\n        {{- '\\n</tool_response>' }}\n        {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n            {{- '<|im_end|>\\n' }}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n    {{- '<|im_start|>assistant\\n' }}\n{%- endif %}\n",
-  "clean_up_tokenization_spaces": false,
-  "eos_token": "<|im_end|>",
-  "errors": "replace",
-  "model_max_length": 131072,
-  "pad_token": "<|endoftext|>",
-  "split_special_tokens": false,
-  "tokenizer_class": "Qwen2Tokenizer",
-  "unk_token": null
-}