Upload folder using huggingface_hub

README.md

@@ -1,3 +1,84 @@
----
-license: cc-by-nc-nd-4.0
----
+---
+library_name: transformers
+license: cc-by-nc-4.0
+---
+
+```
+python=3.10
+torch==2.7.1 # may work with more recent version
+torchvision==0.22.1
+transformers==4.40.0
+opencv-python
+albumentations
+accelerate
+Pillow
+matplotlib
+einops
+pyarrow
+sentencepiece
+protobuf
+```
+
+<!-- markdownlint-disable first-line-h1 -->
+<!-- markdownlint-disable html -->
+
+<div align="center">
+<h1>
+  CheXagent
+</h1>
+</div>
+
+<p align="center">
+📝 <a href="https://arxiv.org/abs/2401.12208" target="_blank">Paper</a> • 🤗 <a href="https://huggingface.co/StanfordAIMI/CheXagent-2-3b/" target="_blank">Hugging Face</a> • 🧩 <a href="https://github.com/Stanford-AIMI/CheXagent" target="_blank">Github</a> • 🪄 <a href="https://stanford-aimi.github.io/chexagent.html" target="_blank">Project</a>
+</p>
+
+<div align="center">
+</div>
+
+## ✨ Latest News
+
+- [04/29/2024]: Model released in [Hugging Face](https://huggingface.co/StanfordAIMI/CheXagent-2-3b/).
+
+## 🎬 Get Started
+
+```python
+import io
+
+import requests
+import torch
+from PIL import Image
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+# step 1: Setup constant
+model_name = "StanfordAIMI/CheXagent-2-3b"
+dtype = torch.bfloat16
+device = "cuda"
+
+# step 2: Load Processor and Model
+tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
+model = AutoModelForCausalLM.from_pretrained(model_name, device_map="auto", trust_remote_code=True)
+model = model.to(dtype)
+model.eval()
+
+# step 3: Inference
+query = tokenizer.from_list_format([*[{'image': path} for path in paths], {'text': prompt}])
+conv = [{"from": "system", "value": "You are a helpful assistant."}, {"from": "human", "value": query}]
+input_ids = tokenizer.apply_chat_template(conv, add_generation_prompt=True, return_tensors="pt")
+output = model.generate(
+    input_ids.to(device), do_sample=False, num_beams=1, temperature=1., top_p=1., use_cache=True,
+    max_new_tokens=512
+)[0]
+response = tokenizer.decode(output[input_ids.size(1):-1])
+```
+
+## ✏️ Citation
+
+```
+@article{chexagent-2024,
+  title={CheXagent: Towards a Foundation Model for Chest X-Ray Interpretation},
+  author={Chen, Zhihong and Varma, Maya and Delbrouck, Jean-Benoit and Paschali, Magdalini and Blankemeier, Louis and Veen, Dave Van and Valanarasu, Jeya Maria Jose and Youssef, Alaa and Cohen, Joseph Paul and Reis, Eduardo Pontes and Tsai, Emily B. and Johnston, Andrew and Olsen, Cameron and Abraham, Tanishq Mathew and Gatidis, Sergios and Chaudhari, Akshay S and Langlotz, Curtis},
+  journal={arXiv preprint arXiv:2401.12208},
+  url={https://arxiv.org/abs/2401.12208},
+  year={2024}
+}
+```

added_tokens.json

@@ -0,0 +1,113 @@
+{
+  "\t\t": 50294,
+  "\t\t\t": 50293,
+  "\t\t\t\t": 50292,
+  "\t\t\t\t\t": 50291,
+  "\t\t\t\t\t\t": 50290,
+  "\t\t\t\t\t\t\t": 50289,
+  "\t\t\t\t\t\t\t\t": 50288,
+  "\t\t\t\t\t\t\t\t\t": 50287,
+  "  ": 50286,
+  "   ": 50285,
+  "    ": 50284,
+  "     ": 50283,
+  "      ": 50282,
+  "       ": 50281,
+  "        ": 50280,
+  "         ": 50279,
+  "          ": 50278,
+  "           ": 50277,
+  "            ": 50276,
+  "             ": 50275,
+  "              ": 50274,
+  "               ": 50273,
+  "                ": 50272,
+  "                 ": 50271,
+  "                  ": 50270,
+  "                   ": 50269,
+  "                    ": 50268,
+  "                     ": 50267,
+  "                      ": 50266,
+  "                       ": 50265,
+  "                        ": 50264,
+  "                         ": 50263,
+  "                          ": 50262,
+  "                           ": 50261,
+  "                            ": 50260,
+  "                             ": 50259,
+  "                              ": 50258,
+  "                               ": 50257,
+  "<|/box|>": 50301,
+  "<|/img|>": 50296,
+  "<|/quad|>": 50303,
+  "<|/ref|>": 50299,
+  "<|box|>": 50300,
+  "<|coord_0|>": 50304,
+  "<|coord_1|>": 50305,
+  "<|coord_2|>": 50306,
+  "<|coord_3|>": 50307,
+  "<|coord_4|>": 50308,
+  "<|coord_5|>": 50309,
+  "<|coord_6|>": 50310,
+  "<|coord_7|>": 50311,
+  "<|coord_8|>": 50312,
+  "<|coord_9|>": 50313,
+  "<|extra_0|>": 50314,
+  "<|extra_10|>": 50324,
+  "<|extra_11|>": 50325,
+  "<|extra_12|>": 50326,
+  "<|extra_13|>": 50327,
+  "<|extra_14|>": 50328,
+  "<|extra_15|>": 50329,
+  "<|extra_16|>": 50330,
+  "<|extra_17|>": 50331,
+  "<|extra_18|>": 50332,
+  "<|extra_19|>": 50333,
+  "<|extra_1|>": 50315,
+  "<|extra_20|>": 50334,
+  "<|extra_21|>": 50335,
+  "<|extra_22|>": 50336,
+  "<|extra_23|>": 50337,
+  "<|extra_24|>": 50338,
+  "<|extra_25|>": 50339,
+  "<|extra_26|>": 50340,
+  "<|extra_27|>": 50341,
+  "<|extra_28|>": 50342,
+  "<|extra_29|>": 50343,
+  "<|extra_2|>": 50316,
+  "<|extra_30|>": 50344,
+  "<|extra_31|>": 50345,
+  "<|extra_32|>": 50346,
+  "<|extra_33|>": 50347,
+  "<|extra_34|>": 50348,
+  "<|extra_35|>": 50349,
+  "<|extra_36|>": 50350,
+  "<|extra_37|>": 50351,
+  "<|extra_38|>": 50352,
+  "<|extra_39|>": 50353,
+  "<|extra_3|>": 50317,
+  "<|extra_40|>": 50354,
+  "<|extra_41|>": 50355,
+  "<|extra_42|>": 50356,
+  "<|extra_43|>": 50357,
+  "<|extra_44|>": 50358,
+  "<|extra_45|>": 50359,
+  "<|extra_46|>": 50360,
+  "<|extra_47|>": 50361,
+  "<|extra_48|>": 50362,
+  "<|extra_49|>": 50363,
+  "<|extra_4|>": 50318,
+  "<|extra_50|>": 50364,
+  "<|extra_51|>": 50365,
+  "<|extra_52|>": 50366,
+  "<|extra_53|>": 50367,
+  "<|extra_5|>": 50319,
+  "<|extra_6|>": 50320,
+  "<|extra_7|>": 50321,
+  "<|extra_8|>": 50322,
+  "<|extra_9|>": 50323,
+  "<|imgpad|>": 50297,
+  "<|img|>": 50295,
+  "<|quad|>": 50302,
+  "<|ref|>": 50298
+}

config.json

@@ -0,0 +1,39 @@
+{
+  "_name_or_path": "StanfordAIMI/CheXagent-2-3b",
+  "architectures": [
+    "CheXagentForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 50256,
+  "embd_pdrop": 0.0,
+  "eos_token_id": 50256,
+  "hidden_act": "gelu_new",
+  "hidden_size": 2560,
+  "initializer_range": 0.02,
+  "intermediate_size": 10240,
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 2048,
+  "model_type": "phi",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 32,
+  "partial_rotary_factor": 0.4,
+  "qk_layernorm": false,
+  "resid_pdrop": 0.1,
+  "rope_scaling": null,
+  "rope_theta": 10000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float32",
+  "transformers_version": "4.40.0",
+  "use_cache": false,
+  "visual": {
+    "image_size": 512,
+    "output_dim": 2560,
+    "vision_model_name_or_path": "StanfordAIMI/XraySigLIP__vit-l-16-siglip-384__webli"
+  },
+  "vocab_size": 51200,
+  "auto_map": {
+    "AutoModelForCausalLM": "modeling_chexagent.CheXagentForCausalLM",
+    "AutoConfig": "configuration_chexagent.CheXagentConfig"
+  }
+}

configuration_chexagent.py

@@ -0,0 +1,184 @@
+# coding=utf-8
+# Copyright 2023 Microsoft and the HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+""" Phi model configuration"""
+
+from transformers.configuration_utils import PretrainedConfig
+from transformers.utils import logging
+
+logger = logging.get_logger(__name__)
+
+PHI_PRETRAINED_CONFIG_ARCHIVE_MAP = {
+    "microsoft/phi-2": "https://huggingface.co/microsoft/phi-2/resolve/main/config.json",
+}
+
+
+class CheXagentConfig(PretrainedConfig):
+    r"""
+    This is the configuration class to store the configuration of a [`PhiModel`]. It is used to instantiate an Phi
+    model according to the specified arguments, defining the model architecture. Instantiating a configuration with the
+    defaults will yield a similar configuration to that of the Phi
+    [microsoft/phi-1](https://huggingface.co/microsoft/phi-1).
+    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+    documentation from [`PretrainedConfig`] for more information.
+    Args:
+        vocab_size (`int`, *optional*, defaults to 51200):
+            Vocabulary size of the Phi model. Defines the number of different tokens that can be represented by the
+            `inputs_ids` passed when calling [`PhiModel`].
+        hidden_size (`int`, *optional*, defaults to 2048):
+            Dimension of the hidden representations.
+        intermediate_size (`int`, *optional*, defaults to 8192):
+            Dimension of the MLP representations.
+        num_hidden_layers (`int`, *optional*, defaults to 24):
+            Number of hidden layers in the Transformer decoder.
+        num_attention_heads (`int`, *optional*, defaults to 32):
+            Number of attention heads for each attention layer in the Transformer decoder.
+        num_key_value_heads (`int`, *optional*):
+            This is the number of key_value heads that should be used to implement Grouped Query Attention. If
+            `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
+            `num_key_value_heads=1 the model will use Multi Query Attention (MQA) otherwise GQA is used. When
+            converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
+            by meanpooling all the original heads within that group. For more details checkout [this
+            paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to
+            `num_attention_heads`.
+        resid_pdrop (`float`, *optional*, defaults to 0.0):
+            Dropout probability for mlp outputs.
+        embd_pdrop (`int`, *optional*, defaults to 0.0):
+            The dropout ratio for the embeddings.
+        attention_dropout (`float`, *optional*, defaults to 0.0):
+            The dropout ratio after computing the attention scores.
+        hidden_act (`str` or `function`, *optional*, defaults to `"gelu_new"`):
+            The non-linear activation function (function or string) in the decoder.
+        max_position_embeddings (`int`, *optional*, defaults to 2048):
+            The maximum sequence length that this model might ever be used with. Phi-1 and Phi-1.5 supports up to 2048
+            tokens.
+        initializer_range (`float`, *optional*, defaults to 0.02):
+            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
+        layer_norm_eps (`float`, *optional*, defaults to 1e-05):
+            The epsilon used by the rms normalization layers.
+        use_cache (`bool`, *optional*, defaults to `True`):
+            Whether or not the model should return the last key/values attentions (not used by all models). Only
+            relevant if `config.is_decoder=True`. Whether to tie weight embeddings or not.
+        tie_word_embeddings (`bool`, *optional*, defaults to `False`):
+            Whether to tie weight embeddings
+        rope_theta (`float`, *optional*, defaults to 10000.0):
+            The base period of the RoPE embeddings.
+        rope_scaling (`Dict`, *optional*):
+            Dictionary containing the scaling configuration for the RoPE embeddings. Currently supports two scaling
+            strategies: linear and dynamic. Their scaling factor must be an float greater than 1. The expected format
+            is `{"type": strategy name, "factor": scaling factor}`. When using this flag, don't update
+            `max_position_embeddings` to the expected new maximum. See the following thread for more information on how
+            these scaling strategies behave:
+            https://www.reddit.com/r/LocalPersimmon/comments/14mrgpr/dynamically_scaled_rope_further_increases/. This
+            is an experimental feature, subject to breaking API changes in future versions.
+        partial_rotary_factor (`float`, *optional*, defaults to 0.5):
+            Percentage of the query and keys which will have rotary embedding.
+        qk_layernorm (`bool`, *optional*, defaults to `False`):
+            Whether or not to normalize the Queries and Keys after projecting the hidden states.
+        bos_token_id (`int`, *optional*, defaults to 1):
+            Denotes beginning of sequences token id.
+        eos_token_id (`int`, *optional*, defaults to 2):
+            Denotes end of sequences token id.
+    Example:
+    ```python
+    >>> from transformers import PhiModel, PhiConfig
+    >>> # Initializing a Phi-1 style configuration
+    >>> configuration = PhiConfig.from_pretrained("microsoft/phi-1")
+    >>> # Initializing a model from the configuration
+    >>> model = PhiModel(configuration)
+    >>> # Accessing the model configuration
+    >>> configuration = model.config
+    ```"""
+
+    model_type = "phi"
+    keys_to_ignore_at_inference = ["past_key_values"]
+
+    def __init__(
+            self,
+            vocab_size=51200,
+            hidden_size=2048,
+            intermediate_size=8192,
+            num_hidden_layers=24,
+            num_attention_heads=32,
+            num_key_value_heads=None,
+            resid_pdrop=0.0,
+            embd_pdrop=0.0,
+            attention_dropout=0.0,
+            hidden_act="gelu_new",
+            max_position_embeddings=2048,
+            initializer_range=0.02,
+            layer_norm_eps=1e-5,
+            use_cache=True,
+            tie_word_embeddings=False,
+            rope_theta=10000.0,
+            rope_scaling=None,
+            partial_rotary_factor=0.5,
+            qk_layernorm=False,
+            bos_token_id=1,
+            eos_token_id=2,
+            **kwargs,
+    ):
+        self.vocab_size = vocab_size
+        self.hidden_size = hidden_size
+        self.intermediate_size = intermediate_size
+        self.num_hidden_layers = num_hidden_layers
+        self.num_attention_heads = num_attention_heads
+
+        if num_key_value_heads is None:
+            num_key_value_heads = num_attention_heads
+
+        self.num_key_value_heads = num_key_value_heads
+        self.resid_pdrop = resid_pdrop
+        self.embd_pdrop = embd_pdrop
+        self.attention_dropout = attention_dropout
+        self.hidden_act = hidden_act
+        self.max_position_embeddings = max_position_embeddings
+        self.initializer_range = initializer_range
+        self.layer_norm_eps = layer_norm_eps
+        self.use_cache = use_cache
+        self.rope_theta = rope_theta
+        self.rope_scaling = rope_scaling
+        self.partial_rotary_factor = partial_rotary_factor
+        self.qk_layernorm = qk_layernorm
+        self._rope_scaling_validation()
+
+        super().__init__(
+            bos_token_id=bos_token_id,
+            eos_token_id=eos_token_id,
+            tie_word_embeddings=tie_word_embeddings,
+            **kwargs,
+        )
+
+    # Copied from transformers.models.llama.configuration_llama.LlamaConfig._rope_scaling_validation
+    def _rope_scaling_validation(self):
+        """
+        Validate the `rope_scaling` configuration.
+        """
+        if self.rope_scaling is None:
+            return
+
+        if not isinstance(self.rope_scaling, dict) or len(self.rope_scaling) != 2:
+            raise ValueError(
+                "`rope_scaling` must be a dictionary with with two fields, `type` and `factor`, "
+                f"got {self.rope_scaling}"
+            )
+        rope_scaling_type = self.rope_scaling.get("type", None)
+        rope_scaling_factor = self.rope_scaling.get("factor", None)
+        if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic"]:
+            raise ValueError(
+                f"`rope_scaling`'s type field must be one of ['linear', 'dynamic'], got {rope_scaling_type}"
+            )
+        if rope_scaling_factor is None or not isinstance(rope_scaling_factor, float) or rope_scaling_factor <= 1.0:
+            raise ValueError(f"`rope_scaling`'s factor field must be a float > 1, got {rope_scaling_factor}")

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 50256,
+  "eos_token_id": 50256,
+  "transformers_version": "4.40.0"
+}

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6260448fbee0a4278ebfd7943f52096e733a105f730a28aace9732509cde3499
+size 135

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1628f60b3bc0cbab16b15a7019f7bafd9befce39d03aedad6f3d373a1999f5a3
+size 135

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a48253caadf1548dbfb0c328c0d3988a63745d171c428252b0392bd1759c1faf
+size 135

model.safetensors.index.json

@@ -0,0 +1,868 @@
+{
+  "metadata": {
+    "total_size": 12562987008
+  },
+  "weight_map": {
+    "lm_head.bias": "model-00003-of-00003.safetensors",
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.final_layernorm.bias": "model-00003-of-00003.safetensors",
+    "model.final_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.0.input_layernorm.bias": "model-00001-of-00003.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.dense.bias": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.dense.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.input_layernorm.bias": "model-00001-of-00003.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.dense.bias": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.dense.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.input_layernorm.bias": "model-00001-of-00003.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.dense.bias": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.dense.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.input_layernorm.bias": "model-00001-of-00003.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "model.layers.11.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "model.layers.11.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.self_attn.dense.bias": "model-00001-of-00003.safetensors",
+    "model.layers.11.self_attn.dense.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.12.input_layernorm.bias": "model-00001-of-00003.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.12.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "model.layers.12.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "model.layers.12.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "model.layers.12.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "model.layers.12.self_attn.dense.bias": "model-00001-of-00003.safetensors",
+    "model.layers.12.self_attn.dense.weight": "model-00001-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.13.input_layernorm.bias": "model-00001-of-00003.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.13.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "model.layers.13.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "model.layers.13.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "model.layers.13.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "model.layers.13.self_attn.dense.bias": "model-00001-of-00003.safetensors",
+    "model.layers.13.self_attn.dense.weight": "model-00001-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.14.input_layernorm.bias": "model-00002-of-00003.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.fc1.bias": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.fc1.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.fc2.bias": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.fc2.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.dense.bias": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.dense.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.input_layernorm.bias": "model-00002-of-00003.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.fc1.bias": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.fc1.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.fc2.bias": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.fc2.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.dense.bias": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.dense.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.input_layernorm.bias": "model-00002-of-00003.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.fc1.bias": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.fc1.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.fc2.bias": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.fc2.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.dense.bias": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.dense.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.input_layernorm.bias": "model-00002-of-00003.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.fc1.bias": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.fc1.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.fc2.bias": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.fc2.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.dense.bias": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.dense.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.input_layernorm.bias": "model-00002-of-00003.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.fc1.bias": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.fc1.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.fc2.bias": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.fc2.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.dense.bias": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.dense.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.input_layernorm.bias": "model-00002-of-00003.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.fc1.bias": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.fc1.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.fc2.bias": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.fc2.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.dense.bias": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.dense.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.input_layernorm.bias": "model-00001-of-00003.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.dense.bias": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.dense.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.input_layernorm.bias": "model-00002-of-00003.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.fc1.bias": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.fc1.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.fc2.bias": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.fc2.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.dense.bias": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.dense.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.input_layernorm.bias": "model-00002-of-00003.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.fc1.bias": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.fc1.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.fc2.bias": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.fc2.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.dense.bias": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.dense.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.input_layernorm.bias": "model-00002-of-00003.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.mlp.fc1.bias": "model-00002-of-00003.safetensors",
+    "model.layers.22.mlp.fc1.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.mlp.fc2.bias": "model-00002-of-00003.safetensors",
+    "model.layers.22.mlp.fc2.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.dense.bias": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.dense.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.input_layernorm.bias": "model-00002-of-00003.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.mlp.fc1.bias": "model-00002-of-00003.safetensors",
+    "model.layers.23.mlp.fc1.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.mlp.fc2.bias": "model-00002-of-00003.safetensors",
+    "model.layers.23.mlp.fc2.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.self_attn.dense.bias": "model-00002-of-00003.safetensors",
+    "model.layers.23.self_attn.dense.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.24.input_layernorm.bias": "model-00002-of-00003.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.24.mlp.fc1.bias": "model-00002-of-00003.safetensors",
+    "model.layers.24.mlp.fc1.weight": "model-00002-of-00003.safetensors",
+    "model.layers.24.mlp.fc2.bias": "model-00002-of-00003.safetensors",
+    "model.layers.24.mlp.fc2.weight": "model-00002-of-00003.safetensors",
+    "model.layers.24.self_attn.dense.bias": "model-00002-of-00003.safetensors",
+    "model.layers.24.self_attn.dense.weight": "model-00002-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.25.input_layernorm.bias": "model-00002-of-00003.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.25.mlp.fc1.bias": "model-00002-of-00003.safetensors",
+    "model.layers.25.mlp.fc1.weight": "model-00002-of-00003.safetensors",
+    "model.layers.25.mlp.fc2.bias": "model-00002-of-00003.safetensors",
+    "model.layers.25.mlp.fc2.weight": "model-00002-of-00003.safetensors",
+    "model.layers.25.self_attn.dense.bias": "model-00002-of-00003.safetensors",
+    "model.layers.25.self_attn.dense.weight": "model-00002-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.26.input_layernorm.bias": "model-00002-of-00003.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.26.mlp.fc1.bias": "model-00002-of-00003.safetensors",
+    "model.layers.26.mlp.fc1.weight": "model-00002-of-00003.safetensors",
+    "model.layers.26.mlp.fc2.bias": "model-00002-of-00003.safetensors",
+    "model.layers.26.mlp.fc2.weight": "model-00002-of-00003.safetensors",
+    "model.layers.26.self_attn.dense.bias": "model-00002-of-00003.safetensors",
+    "model.layers.26.self_attn.dense.weight": "model-00002-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.27.input_layernorm.bias": "model-00002-of-00003.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.27.mlp.fc1.bias": "model-00002-of-00003.safetensors",
+    "model.layers.27.mlp.fc1.weight": "model-00002-of-00003.safetensors",
+    "model.layers.27.mlp.fc2.bias": "model-00002-of-00003.safetensors",
+    "model.layers.27.mlp.fc2.weight": "model-00002-of-00003.safetensors",
+    "model.layers.27.self_attn.dense.bias": "model-00002-of-00003.safetensors",
+    "model.layers.27.self_attn.dense.weight": "model-00002-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.28.input_layernorm.bias": "model-00002-of-00003.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.28.mlp.fc1.bias": "model-00002-of-00003.safetensors",
+    "model.layers.28.mlp.fc1.weight": "model-00002-of-00003.safetensors",
+    "model.layers.28.mlp.fc2.bias": "model-00002-of-00003.safetensors",
+    "model.layers.28.mlp.fc2.weight": "model-00002-of-00003.safetensors",
+    "model.layers.28.self_attn.dense.bias": "model-00002-of-00003.safetensors",
+    "model.layers.28.self_attn.dense.weight": "model-00002-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.29.input_layernorm.bias": "model-00002-of-00003.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.29.mlp.fc1.bias": "model-00002-of-00003.safetensors",
+    "model.layers.29.mlp.fc1.weight": "model-00002-of-00003.safetensors",
+    "model.layers.29.mlp.fc2.bias": "model-00002-of-00003.safetensors",
+    "model.layers.29.mlp.fc2.weight": "model-00002-of-00003.safetensors",
+    "model.layers.29.self_attn.dense.bias": "model-00002-of-00003.safetensors",
+    "model.layers.29.self_attn.dense.weight": "model-00002-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.bias": "model-00002-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.3.input_layernorm.bias": "model-00001-of-00003.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.dense.bias": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.dense.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.input_layernorm.bias": "model-00003-of-00003.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.fc1.bias": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.fc1.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.fc2.bias": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.fc2.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.dense.bias": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.dense.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.input_layernorm.bias": "model-00003-of-00003.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.fc1.bias": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.fc1.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.fc2.bias": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.fc2.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.dense.bias": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.dense.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.bias": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.input_layernorm.bias": "model-00001-of-00003.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.dense.bias": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.dense.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.input_layernorm.bias": "model-00001-of-00003.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.dense.bias": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.dense.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.input_layernorm.bias": "model-00001-of-00003.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.dense.bias": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.dense.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.input_layernorm.bias": "model-00001-of-00003.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.dense.bias": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.dense.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.input_layernorm.bias": "model-00001-of-00003.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.dense.bias": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.dense.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.input_layernorm.bias": "model-00001-of-00003.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.fc1.bias": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.fc1.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.fc2.bias": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.fc2.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.dense.bias": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.dense.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.bias": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.visual.attn_pool.0.bias": "model-00003-of-00003.safetensors",
+    "model.visual.attn_pool.0.weight": "model-00003-of-00003.safetensors",
+    "model.visual.attn_pool.2.bias": "model-00003-of-00003.safetensors",
+    "model.visual.attn_pool.2.weight": "model-00003-of-00003.safetensors",
+    "model.visual.ln_post.bias": "model-00003-of-00003.safetensors",
+    "model.visual.ln_post.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.embeddings.patch_embedding.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.embeddings.patch_embedding.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.embeddings.position_embedding.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.0.layer_norm1.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.0.layer_norm1.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.0.layer_norm2.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.0.layer_norm2.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.0.mlp.fc1.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.0.mlp.fc1.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.0.mlp.fc2.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.0.mlp.fc2.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.0.self_attn.k_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.0.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.0.self_attn.out_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.0.self_attn.out_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.0.self_attn.q_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.0.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.0.self_attn.v_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.0.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.1.layer_norm1.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.1.layer_norm1.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.1.layer_norm2.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.1.layer_norm2.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.1.mlp.fc1.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.1.mlp.fc1.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.1.mlp.fc2.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.1.mlp.fc2.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.1.self_attn.k_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.1.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.1.self_attn.out_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.1.self_attn.out_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.1.self_attn.q_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.1.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.1.self_attn.v_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.1.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.10.layer_norm1.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.10.layer_norm1.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.10.layer_norm2.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.10.layer_norm2.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.10.mlp.fc1.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.10.mlp.fc1.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.10.mlp.fc2.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.10.mlp.fc2.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.10.self_attn.k_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.10.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.10.self_attn.out_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.10.self_attn.out_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.10.self_attn.q_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.10.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.10.self_attn.v_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.10.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.11.layer_norm1.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.11.layer_norm1.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.11.layer_norm2.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.11.layer_norm2.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.11.mlp.fc1.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.11.mlp.fc1.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.11.mlp.fc2.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.11.mlp.fc2.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.11.self_attn.k_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.11.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.11.self_attn.out_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.11.self_attn.out_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.11.self_attn.q_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.11.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.11.self_attn.v_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.11.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.12.layer_norm1.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.12.layer_norm1.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.12.layer_norm2.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.12.layer_norm2.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.12.mlp.fc1.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.12.mlp.fc1.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.12.mlp.fc2.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.12.mlp.fc2.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.12.self_attn.k_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.12.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.12.self_attn.out_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.12.self_attn.out_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.12.self_attn.q_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.12.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.12.self_attn.v_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.12.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.13.layer_norm1.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.13.layer_norm1.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.13.layer_norm2.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.13.layer_norm2.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.13.mlp.fc1.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.13.mlp.fc1.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.13.mlp.fc2.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.13.mlp.fc2.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.13.self_attn.k_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.13.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.13.self_attn.out_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.13.self_attn.out_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.13.self_attn.q_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.13.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.13.self_attn.v_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.13.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.14.layer_norm1.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.14.layer_norm1.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.14.layer_norm2.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.14.layer_norm2.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.14.mlp.fc1.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.14.mlp.fc1.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.14.mlp.fc2.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.14.mlp.fc2.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.14.self_attn.k_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.14.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.14.self_attn.out_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.14.self_attn.out_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.14.self_attn.q_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.14.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.14.self_attn.v_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.14.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.15.layer_norm1.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.15.layer_norm1.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.15.layer_norm2.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.15.layer_norm2.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.15.mlp.fc1.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.15.mlp.fc1.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.15.mlp.fc2.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.15.mlp.fc2.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.15.self_attn.k_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.15.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.15.self_attn.out_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.15.self_attn.out_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.15.self_attn.q_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.15.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.15.self_attn.v_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.15.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.16.layer_norm1.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.16.layer_norm1.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.16.layer_norm2.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.16.layer_norm2.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.16.mlp.fc1.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.16.mlp.fc1.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.16.mlp.fc2.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.16.mlp.fc2.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.16.self_attn.k_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.16.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.16.self_attn.out_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.16.self_attn.out_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.16.self_attn.q_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.16.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.16.self_attn.v_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.16.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.17.layer_norm1.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.17.layer_norm1.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.17.layer_norm2.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.17.layer_norm2.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.17.mlp.fc1.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.17.mlp.fc1.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.17.mlp.fc2.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.17.mlp.fc2.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.17.self_attn.k_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.17.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.17.self_attn.out_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.17.self_attn.out_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.17.self_attn.q_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.17.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.17.self_attn.v_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.17.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.18.layer_norm1.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.18.layer_norm1.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.18.layer_norm2.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.18.layer_norm2.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.18.mlp.fc1.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.18.mlp.fc1.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.18.mlp.fc2.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.18.mlp.fc2.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.18.self_attn.k_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.18.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.18.self_attn.out_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.18.self_attn.out_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.18.self_attn.q_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.18.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.18.self_attn.v_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.18.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.19.layer_norm1.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.19.layer_norm1.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.19.layer_norm2.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.19.layer_norm2.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.19.mlp.fc1.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.19.mlp.fc1.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.19.mlp.fc2.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.19.mlp.fc2.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.19.self_attn.k_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.19.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.19.self_attn.out_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.19.self_attn.out_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.19.self_attn.q_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.19.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.19.self_attn.v_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.19.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.2.layer_norm1.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.2.layer_norm1.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.2.layer_norm2.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.2.layer_norm2.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.2.mlp.fc1.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.2.mlp.fc1.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.2.mlp.fc2.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.2.mlp.fc2.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.2.self_attn.k_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.2.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.2.self_attn.out_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.2.self_attn.out_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.2.self_attn.q_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.2.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.2.self_attn.v_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.2.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.20.layer_norm1.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.20.layer_norm1.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.20.layer_norm2.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.20.layer_norm2.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.20.mlp.fc1.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.20.mlp.fc1.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.20.mlp.fc2.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.20.mlp.fc2.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.20.self_attn.k_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.20.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.20.self_attn.out_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.20.self_attn.out_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.20.self_attn.q_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.20.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.20.self_attn.v_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.20.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.21.layer_norm1.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.21.layer_norm1.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.21.layer_norm2.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.21.layer_norm2.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.21.mlp.fc1.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.21.mlp.fc1.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.21.mlp.fc2.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.21.mlp.fc2.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.21.self_attn.k_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.21.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.21.self_attn.out_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.21.self_attn.out_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.21.self_attn.q_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.21.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.21.self_attn.v_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.21.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.22.layer_norm1.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.22.layer_norm1.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.22.layer_norm2.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.22.layer_norm2.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.22.mlp.fc1.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.22.mlp.fc1.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.22.mlp.fc2.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.22.mlp.fc2.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.22.self_attn.k_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.22.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.22.self_attn.out_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.22.self_attn.out_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.22.self_attn.q_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.22.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.22.self_attn.v_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.22.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.23.layer_norm1.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.23.layer_norm1.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.23.layer_norm2.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.23.layer_norm2.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.23.mlp.fc1.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.23.mlp.fc1.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.23.mlp.fc2.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.23.mlp.fc2.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.23.self_attn.k_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.23.self_attn.out_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.23.self_attn.out_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.23.self_attn.q_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.23.self_attn.v_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.3.layer_norm1.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.3.layer_norm1.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.3.layer_norm2.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.3.layer_norm2.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.3.mlp.fc1.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.3.mlp.fc1.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.3.mlp.fc2.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.3.mlp.fc2.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.3.self_attn.k_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.3.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.3.self_attn.out_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.3.self_attn.out_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.3.self_attn.q_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.3.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.3.self_attn.v_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.3.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.4.layer_norm1.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.4.layer_norm1.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.4.layer_norm2.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.4.layer_norm2.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.4.mlp.fc1.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.4.mlp.fc1.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.4.mlp.fc2.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.4.mlp.fc2.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.4.self_attn.k_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.4.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.4.self_attn.out_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.4.self_attn.out_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.4.self_attn.q_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.4.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.4.self_attn.v_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.4.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.5.layer_norm1.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.5.layer_norm1.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.5.layer_norm2.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.5.layer_norm2.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.5.mlp.fc1.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.5.mlp.fc1.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.5.mlp.fc2.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.5.mlp.fc2.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.5.self_attn.k_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.5.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.5.self_attn.out_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.5.self_attn.out_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.5.self_attn.q_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.5.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.5.self_attn.v_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.5.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.6.layer_norm1.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.6.layer_norm1.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.6.layer_norm2.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.6.layer_norm2.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.6.mlp.fc1.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.6.mlp.fc1.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.6.mlp.fc2.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.6.mlp.fc2.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.6.self_attn.k_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.6.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.6.self_attn.out_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.6.self_attn.out_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.6.self_attn.q_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.6.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.6.self_attn.v_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.6.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.7.layer_norm1.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.7.layer_norm1.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.7.layer_norm2.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.7.layer_norm2.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.7.mlp.fc1.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.7.mlp.fc1.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.7.mlp.fc2.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.7.mlp.fc2.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.7.self_attn.k_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.7.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.7.self_attn.out_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.7.self_attn.out_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.7.self_attn.q_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.7.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.7.self_attn.v_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.7.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.8.layer_norm1.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.8.layer_norm1.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.8.layer_norm2.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.8.layer_norm2.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.8.mlp.fc1.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.8.mlp.fc1.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.8.mlp.fc2.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.8.mlp.fc2.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.8.self_attn.k_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.8.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.8.self_attn.out_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.8.self_attn.out_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.8.self_attn.q_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.8.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.8.self_attn.v_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.8.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.9.layer_norm1.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.9.layer_norm1.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.9.layer_norm2.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.9.layer_norm2.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.9.mlp.fc1.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.9.mlp.fc1.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.9.mlp.fc2.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.9.mlp.fc2.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.9.self_attn.k_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.9.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.9.self_attn.out_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.9.self_attn.out_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.9.self_attn.q_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.9.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.9.self_attn.v_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.encoder.layers.9.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.head.attention.in_proj_bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.head.attention.in_proj_weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.head.attention.out_proj.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.head.attention.out_proj.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.head.layernorm.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.head.layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.head.mlp.fc1.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.head.mlp.fc1.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.head.mlp.fc2.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.head.mlp.fc2.weight": "model-00003-of-00003.safetensors",
+    "model.visual.model.head.probe": "model-00003-of-00003.safetensors",
+    "model.visual.model.post_layernorm.bias": "model-00003-of-00003.safetensors",
+    "model.visual.model.post_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.visual.pos_embed": "model-00003-of-00003.safetensors",
+    "model.visual.proj": "model-00003-of-00003.safetensors"
+  }
+}

modeling_chexagent.py

@@ -0,0 +1,1141 @@
+# coding=utf-8
+# Copyright 2023 Microsoft and the HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+""" PyTorch Phi model."""
+
+import math
+from typing import List, Optional, Tuple, Union
+
+import torch
+import torch.nn.functional as F
+import torch.utils.checkpoint
+from torch import nn
+from torch.nn import CrossEntropyLoss
+import transformers
+from transformers.activations import ACT2FN
+from transformers.cache_utils import Cache, DynamicCache
+from transformers.modeling_attn_mask_utils import _prepare_4d_causal_attention_mask
+from transformers.modeling_outputs import (
+    BaseModelOutputWithPast,
+    CausalLMOutputWithPast,
+)
+from transformers.modeling_utils import PreTrainedModel
+from transformers.utils import (
+    add_start_docstrings,
+    add_start_docstrings_to_model_forward,
+    is_flash_attn_greater_or_equal_2_10,
+    logging,
+)
+
+from .configuration_chexagent import CheXagentConfig
+from .modeling_visual import CLIPModel
+from .tokenization_chexagent import CheXagentTokenizer
+
+try:
+    from flash_attn import flash_attn_func, flash_attn_varlen_func
+    from flash_attn.bert_padding import index_first_axis, pad_input, unpad_input  # noqa
+except:
+    pass
+
+assert transformers.__version__ == "4.40.0", "Please install a specific HF transformers version: pip install transformers==4.40.0"
+
+logger = logging.get_logger(__name__)
+
+_CHECKPOINT_FOR_DOC = "microsoft/phi-2"
+_CONFIG_FOR_DOC = "CheXagentConfig"
+
+PHI_PRETRAINED_MODEL_ARCHIVE_LIST = [
+    "microsoft/phi-2",
+    # See all Phi models at https://huggingface.co/models?filter=phi
+]
+
+
+# Copied from transformers.models.llama.modeling_llama._get_unpad_data
+def _get_unpad_data(attention_mask):
+    seqlens_in_batch = attention_mask.sum(dim=-1, dtype=torch.int32)
+    indices = torch.nonzero(attention_mask.flatten(), as_tuple=False).flatten()
+    max_seqlen_in_batch = seqlens_in_batch.max().item()
+    cu_seqlens = F.pad(torch.cumsum(seqlens_in_batch, dim=0, dtype=torch.torch.int32), (1, 0))
+    return (
+        indices,
+        cu_seqlens,
+        max_seqlen_in_batch,
+    )
+
+
+# Copied from transformers.models.llama.modeling_llama.LlamaRotaryEmbedding with Llama->Phi
+class PhiRotaryEmbedding(nn.Module):
+    def __init__(self, dim, max_position_embeddings=2048, base=10000, device=None):
+        super().__init__()
+
+        self.dim = dim
+        self.max_position_embeddings = max_position_embeddings
+        self.base = base
+        inv_freq = 1.0 / (self.base ** (torch.arange(0, self.dim, 2).float().to(device) / self.dim))
+        self.register_buffer("inv_freq", inv_freq, persistent=False)
+
+        # Build here to make `torch.jit.trace` work.
+        self._set_cos_sin_cache(
+            seq_len=max_position_embeddings, device=self.inv_freq.device, dtype=torch.get_default_dtype()
+        )
+
+    def _set_cos_sin_cache(self, seq_len, device, dtype):
+        self.max_seq_len_cached = seq_len
+        t = torch.arange(self.max_seq_len_cached, device=device, dtype=self.inv_freq.dtype)
+
+        freqs = torch.outer(t, self.inv_freq)
+        # Different from paper, but it uses a different permutation in order to obtain the same calculation
+        emb = torch.cat((freqs, freqs), dim=-1)
+        self.register_buffer("cos_cached", emb.cos().to(dtype), persistent=False)
+        self.register_buffer("sin_cached", emb.sin().to(dtype), persistent=False)
+
+    def forward(self, x, seq_len=None):
+        # x: [bs, num_attention_heads, seq_len, head_size]
+        if seq_len > self.max_seq_len_cached:
+            self._set_cos_sin_cache(seq_len=seq_len, device=x.device, dtype=x.dtype)
+
+        return (
+            self.cos_cached[:seq_len].to(dtype=x.dtype),
+            self.sin_cached[:seq_len].to(dtype=x.dtype),
+        )
+
+
+# Copied from transformers.models.llama.modeling_llama.LlamaLinearScalingRotaryEmbedding with Llama->Phi
+class PhiLinearScalingRotaryEmbedding(PhiRotaryEmbedding):
+    """PhiRotaryEmbedding extended with linear scaling. Credits to the Reddit user /u/kaiokendev"""
+
+    def __init__(self, dim, max_position_embeddings=2048, base=10000, device=None, scaling_factor=1.0):
+        self.scaling_factor = scaling_factor
+        super().__init__(dim, max_position_embeddings, base, device)
+
+    def _set_cos_sin_cache(self, seq_len, device, dtype):
+        self.max_seq_len_cached = seq_len
+        t = torch.arange(self.max_seq_len_cached, device=device, dtype=self.inv_freq.dtype)
+        t = t / self.scaling_factor
+
+        freqs = torch.outer(t, self.inv_freq)
+        # Different from paper, but it uses a different permutation in order to obtain the same calculation
+        emb = torch.cat((freqs, freqs), dim=-1)
+        self.register_buffer("cos_cached", emb.cos().to(dtype), persistent=False)
+        self.register_buffer("sin_cached", emb.sin().to(dtype), persistent=False)
+
+
+# Copied from transformers.models.llama.modeling_llama.LlamaDynamicNTKScalingRotaryEmbedding with Llama->Phi
+class PhiDynamicNTKScalingRotaryEmbedding(PhiRotaryEmbedding):
+    """PhiRotaryEmbedding extended with Dynamic NTK scaling. Credits to the Reddit users /u/bloc97 and /u/emozilla"""
+
+    def __init__(self, dim, max_position_embeddings=2048, base=10000, device=None, scaling_factor=1.0):
+        self.scaling_factor = scaling_factor
+        super().__init__(dim, max_position_embeddings, base, device)
+
+    def _set_cos_sin_cache(self, seq_len, device, dtype):
+        self.max_seq_len_cached = seq_len
+
+        if seq_len > self.max_position_embeddings:
+            base = self.base * (
+                    (self.scaling_factor * seq_len / self.max_position_embeddings) - (self.scaling_factor - 1)
+            ) ** (self.dim / (self.dim - 2))
+            inv_freq = 1.0 / (base ** (torch.arange(0, self.dim, 2).float().to(device) / self.dim))
+            self.register_buffer("inv_freq", inv_freq, persistent=False)
+
+        t = torch.arange(self.max_seq_len_cached, device=device, dtype=self.inv_freq.dtype)
+
+        freqs = torch.outer(t, self.inv_freq)
+        # Different from paper, but it uses a different permutation in order to obtain the same calculation
+        emb = torch.cat((freqs, freqs), dim=-1)
+        self.register_buffer("cos_cached", emb.cos().to(dtype), persistent=False)
+        self.register_buffer("sin_cached", emb.sin().to(dtype), persistent=False)
+
+
+# Copied from transformers.models.llama.modeling_llama.rotate_half
+def rotate_half(x):
+    """Rotates half the hidden dims of the input."""
+    x1 = x[..., : x.shape[-1] // 2]
+    x2 = x[..., x.shape[-1] // 2:]
+    return torch.cat((-x2, x1), dim=-1)
+
+
+# Copied from transformers.models.llama.modeling_llama.apply_rotary_pos_emb
+def apply_rotary_pos_emb(q, k, cos, sin, position_ids, unsqueeze_dim=1):
+    """Applies Rotary Position Embedding to the query and key tensors.
+    Args:
+        q (`torch.Tensor`): The query tensor.
+        k (`torch.Tensor`): The key tensor.
+        cos (`torch.Tensor`): The cosine part of the rotary embedding.
+        sin (`torch.Tensor`): The sine part of the rotary embedding.
+        position_ids (`torch.Tensor`):
+            The position indices of the tokens corresponding to the query and key tensors. For example, this can be
+            used to pass offsetted position ids when working with a KV-cache.
+        unsqueeze_dim (`int`, *optional*, defaults to 1):
+            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
+            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
+            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
+            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
+            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
+            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
+    Returns:
+        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
+    """
+    cos = cos[position_ids].unsqueeze(unsqueeze_dim)
+    sin = sin[position_ids].unsqueeze(unsqueeze_dim)
+    q_embed = (q * cos) + (rotate_half(q) * sin)
+    k_embed = (k * cos) + (rotate_half(k) * sin)
+    return q_embed, k_embed
+
+
+# Copied from transformers.models.clip.modeling_clip.CLIPMLP with CLIP->Phi
+class PhiMLP(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.config = config
+        self.activation_fn = ACT2FN[config.hidden_act]
+        self.fc1 = nn.Linear(config.hidden_size, config.intermediate_size)
+        self.fc2 = nn.Linear(config.intermediate_size, config.hidden_size)
+
+    def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+        hidden_states = self.fc1(hidden_states)
+        hidden_states = self.activation_fn(hidden_states)
+        hidden_states = self.fc2(hidden_states)
+        return hidden_states
+
+
+# Copied from transformers.models.llama.modeling_llama.repeat_kv with llama->phi
+def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor:
+    """
+    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
+    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
+    """
+    batch, num_key_value_heads, slen, head_dim = hidden_states.shape
+    if n_rep == 1:
+        return hidden_states
+    hidden_states = hidden_states[:, :, None, :, :].expand(batch, num_key_value_heads, n_rep, slen, head_dim)
+    return hidden_states.reshape(batch, num_key_value_heads * n_rep, slen, head_dim)
+
+
+class PhiAttention(nn.Module):
+    """Multi-headed attention from 'Attention Is All You Need' paper"""
+
+    def __init__(self, config: CheXagentConfig, layer_idx: Optional[int] = None):
+        super().__init__()
+        self.config = config
+        self.layer_idx = layer_idx
+        if layer_idx is None:
+            logger.warning_once(
+                f"Instantiating {self.__class__.__name__} without passing `layer_idx` is not recommended and will "
+                "to errors during the forward call, if caching is used. Please make sure to provide a `layer_idx` "
+                "when creating this class."
+            )
+
+        self.attention_dropout = config.attention_dropout
+        self.hidden_size = config.hidden_size
+        self.num_heads = config.num_attention_heads
+        self.head_dim = self.hidden_size // self.num_heads
+        self.num_key_value_heads = config.num_key_value_heads
+        self.num_key_value_groups = self.num_heads // self.num_key_value_heads
+        self.max_position_embeddings = config.max_position_embeddings
+        self.rope_theta = config.rope_theta
+        self.partial_rotary_factor = config.partial_rotary_factor
+        self.is_causal = True
+
+        if (self.head_dim * self.num_heads) != self.hidden_size:
+            raise ValueError(
+                f"hidden_size must be divisible by num_heads (got `hidden_size`: {self.hidden_size}"
+                f" and `num_heads`: {self.num_heads})."
+            )
+
+        self.q_proj = nn.Linear(self.hidden_size, self.num_heads * self.head_dim, bias=True)
+        self.k_proj = nn.Linear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=True)
+        self.v_proj = nn.Linear(self.hidden_size, self.num_key_value_heads * self.head_dim, bias=True)
+        self.dense = nn.Linear(self.num_heads * self.head_dim, self.hidden_size, bias=True)
+
+        self.qk_layernorm = config.qk_layernorm
+        if self.qk_layernorm:
+            self.q_layernorm = nn.LayerNorm(
+                config.hidden_size // self.num_heads, eps=config.layer_norm_eps, elementwise_affine=True
+            )
+            self.k_layernorm = nn.LayerNorm(
+                config.hidden_size // self.num_heads, eps=config.layer_norm_eps, elementwise_affine=True
+            )
+
+        self._init_rope()
+
+    def _init_rope(self):
+        if self.config.rope_scaling is None:
+            self.rotary_emb = PhiRotaryEmbedding(
+                int(self.partial_rotary_factor * self.head_dim),
+                max_position_embeddings=self.max_position_embeddings,
+                base=self.rope_theta,
+            )
+        else:
+            scaling_type = self.config.rope_scaling["type"]
+            scaling_factor = self.config.rope_scaling["factor"]
+            if scaling_type == "linear":
+                self.rotary_emb = PhiLinearScalingRotaryEmbedding(
+                    int(self.partial_rotary_factor * self.head_dim),
+                    max_position_embeddings=self.max_position_embeddings,
+                    scaling_factor=scaling_factor,
+                    base=self.rope_theta,
+                )
+            elif scaling_type == "dynamic":
+                self.rotary_emb = PhiDynamicNTKScalingRotaryEmbedding(
+                    int(self.partial_rotary_factor * self.head_dim),
+                    max_position_embeddings=self.max_position_embeddings,
+                    scaling_factor=scaling_factor,
+                    base=self.rope_theta,
+                )
+            else:
+                raise ValueError(f"Unknown RoPE scaling type {scaling_type}")
+
+    # Phi-2 has an attention overflow issue (with FP16) and requires autocast to be disabled
+    @torch.autocast("cpu", enabled=False)
+    @torch.autocast("cuda", enabled=False)
+    def forward(
+            self,
+            hidden_states: torch.Tensor,
+            attention_mask: Optional[torch.Tensor] = None,
+            position_ids: Optional[torch.LongTensor] = None,
+            past_key_value: Optional[Cache] = None,
+            output_attentions: bool = False,
+            use_cache: bool = False,
+    ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
+        bsz, q_len, _ = hidden_states.size()
+
+        query_states = self.q_proj(hidden_states)
+        key_states = self.k_proj(hidden_states)
+        value_states = self.v_proj(hidden_states)
+
+        if self.qk_layernorm:
+            query_states = self.q_layernorm(query_states)
+            key_states = self.k_layernorm(key_states)
+
+        query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)
+        key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
+        value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
+
+        kv_seq_len = key_states.shape[-2]
+        if past_key_value is not None:
+            if self.layer_idx is None:
+                raise ValueError(
+                    f"The cache structure has changed since version v4.36. If you are using {self.__class__.__name__} "
+                    "for auto-regressive decoding with k/v caching, please make sure to initialize the attention class "
+                    "with a layer index."
+                )
+            kv_seq_len += past_key_value.get_usable_length(kv_seq_len, self.layer_idx)
+        cos, sin = self.rotary_emb(value_states, seq_len=kv_seq_len)
+
+        # Partial rotary embedding
+        query_rot, query_pass = (
+            query_states[..., : self.rotary_emb.dim],
+            query_states[..., self.rotary_emb.dim:],
+        )
+        key_rot, key_pass = (
+            key_states[..., : self.rotary_emb.dim],
+            key_states[..., self.rotary_emb.dim:],
+        )
+        # [batch_size, seq_length, num_heads, head_dim // config.partial_rotary_factor]
+        query_rot, key_rot = apply_rotary_pos_emb(query_rot, key_rot, cos, sin, position_ids)
+
+        # [batch_size, seq_length, num_heads, head_dim]
+        query_states = torch.cat((query_rot, query_pass), dim=-1)
+        key_states = torch.cat((key_rot, key_pass), dim=-1)
+
+        if past_key_value is not None:
+            cache_kwargs = {"sin": sin, "cos": cos, "partial_rotation_size": self.rotary_emb.dim}
+            key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs)
+
+        key_states = repeat_kv(key_states, self.num_key_value_groups)
+        value_states = repeat_kv(value_states, self.num_key_value_groups)
+
+        # Queries and keys upcast to fp32 is required by Phi-2 to avoid overflow
+        attn_weights = torch.matmul(
+            query_states.to(torch.float32), key_states.to(torch.float32).transpose(2, 3)
+        ) / math.sqrt(self.head_dim)
+
+        if attn_weights.size() != (bsz, self.num_heads, q_len, kv_seq_len):
+            raise ValueError(
+                f"Attention weights should be of size {(bsz, self.num_heads, q_len, kv_seq_len)}, but is"
+                f" {attn_weights.size()}"
+            )
+
+        if attention_mask is not None:
+            if attention_mask.size() != (bsz, 1, q_len, kv_seq_len):
+                raise ValueError(
+                    f"Attention mask should be of size {(bsz, 1, q_len, kv_seq_len)}, but is {attention_mask.size()}"
+                )
+            attn_weights = attn_weights + attention_mask
+
+        # upcast attention to fp32
+        attn_weights = nn.functional.softmax(attn_weights, dim=-1, dtype=torch.float32).to(value_states.dtype)
+        attn_weights = nn.functional.dropout(attn_weights, p=self.attention_dropout, training=self.training)
+
+        attn_output = torch.matmul(attn_weights, value_states)
+
+        if attn_output.size() != (bsz, self.num_heads, q_len, self.head_dim):
+            raise ValueError(
+                f"`attn_output` should be of size {(bsz, self.num_heads, q_len, self.head_dim)}, but is"
+                f" {attn_output.size()}"
+            )
+
+        attn_output = attn_output.transpose(1, 2).contiguous()
+        attn_output = attn_output.reshape(bsz, q_len, self.hidden_size)
+
+        attn_output = self.dense(attn_output)
+
+        if not output_attentions:
+            attn_weights = None
+
+        return attn_output, attn_weights, past_key_value
+
+
+class PhiFlashAttention2(PhiAttention):
+    """
+    Phi flash attention module. This module inherits from `PhiAttention` as the weights of the module stays
+    untouched. The only required change would be on the forward pass where it needs to correctly call the public API of
+    flash attention and deal with padding tokens in case the input contains any of them.
+    """
+
+    # Copied from transformers.models.llama.modeling_llama.LlamaFlashAttention2.__init__
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        # TODO: Should be removed once Flash Attention for RoCm is bumped to 2.1.
+        # flash_attn<2.1 generates top-left aligned causal mask, while what is needed here is bottom-right alignement, that was made default for flash_attn>=2.1. This attribute is used to handle this difference. Reference: https://github.com/Dao-AILab/flash-attention/releases/tag/v2.1.0.
+        # Beware that with flash_attn<2.1, using q_seqlen != k_seqlen (except for the case q_seqlen == 1) produces a wrong mask (top-left).
+        self._flash_attn_uses_top_left_mask = not is_flash_attn_greater_or_equal_2_10()
+
+    def forward(
+            self,
+            hidden_states: torch.Tensor,
+            attention_mask: Optional[torch.LongTensor] = None,
+            position_ids: Optional[torch.LongTensor] = None,
+            past_key_value: Optional[Cache] = None,
+            output_attentions: bool = False,
+            use_cache: bool = False,
+            **kwargs,
+    ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
+        # PhiFlashAttention2 attention does not support output_attentions
+
+        output_attentions = False
+
+        bsz, q_len, _ = hidden_states.size()
+
+        query_states = self.q_proj(hidden_states)
+        key_states = self.k_proj(hidden_states)
+        value_states = self.v_proj(hidden_states)
+
+        if self.qk_layernorm:
+            query_states = self.q_layernorm(query_states)
+            key_states = self.k_layernorm(key_states)
+
+        # Flash attention requires the input to have the shape
+        # batch_size x seq_length x head_dim x hidden_dim
+        # therefore we just need to keep the original shape
+        query_states = query_states.view(bsz, q_len, self.num_heads, self.head_dim).transpose(1, 2)
+        key_states = key_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
+        value_states = value_states.view(bsz, q_len, self.num_key_value_heads, self.head_dim).transpose(1, 2)
+
+        kv_seq_len = key_states.shape[-2]
+        if past_key_value is not None:
+            kv_seq_len += past_key_value.get_usable_length(kv_seq_len, self.layer_idx)
+        cos, sin = self.rotary_emb(value_states, seq_len=kv_seq_len)
+
+        # Partial rotary embedding
+        query_rot, query_pass = (
+            query_states[..., : self.rotary_emb.dim],
+            query_states[..., self.rotary_emb.dim:],
+        )
+        key_rot, key_pass = (
+            key_states[..., : self.rotary_emb.dim],
+            key_states[..., self.rotary_emb.dim:],
+        )
+        # [batch_size, seq_length, num_heads, head_dim // config.partial_rotary_factor]
+        query_rot, key_rot = apply_rotary_pos_emb(query_rot, key_rot, cos, sin, position_ids)
+
+        # [batch_size, seq_length, num_heads, head_dim]
+        query_states = torch.cat((query_rot, query_pass), dim=-1)
+        key_states = torch.cat((key_rot, key_pass), dim=-1)
+
+        if past_key_value is not None:
+            cache_kwargs = {"sin": sin, "cos": cos, "partial_rotation_size": self.rotary_emb.dim}
+            key_states, value_states = past_key_value.update(key_states, value_states, self.layer_idx, cache_kwargs)
+
+        # TODO: These transpose are quite inefficient but Flash Attention requires the layout [batch_size, sequence_length, num_heads, head_dim]. We would need to refactor the KV cache
+        # to be able to avoid many of these transpose/reshape/view.
+        query_states = query_states.transpose(1, 2)
+        key_states = key_states.transpose(1, 2)
+        value_states = value_states.transpose(1, 2)
+
+        attn_dropout = self.attention_dropout if self.training else 0.0
+
+        # In PEFT, usually we cast the layer norms in float32 for training stability reasons
+        # therefore the input hidden states gets silently casted in float32. Hence, we need
+        # cast them back in the correct dtype just to be sure everything works as expected.
+        # This might slowdown training & inference so it is recommended to not cast the LayerNorms
+        # in fp32.
+
+        if query_states.dtype == torch.float32:
+            if torch.is_autocast_enabled():
+                target_dtype = torch.get_autocast_gpu_dtype()
+            # Handle the case where the model is quantized
+            elif hasattr(self.config, "_pre_quantization_dtype"):
+                target_dtype = self.config._pre_quantization_dtype
+            else:
+                target_dtype = self.q_proj.weight.dtype
+
+            logger.warning_once(
+                f"The input hidden states seems to be silently casted in float32, this might be related to"
+                f" the fact you have upcasted embedding or layer norm layers in float32. We will cast back the input in"
+                f" {target_dtype}."
+            )
+
+            query_states = query_states.to(target_dtype)
+            key_states = key_states.to(target_dtype)
+            value_states = value_states.to(target_dtype)
+
+        attn_output = self._flash_attention_forward(
+            query_states, key_states, value_states, attention_mask, q_len, dropout=attn_dropout, softmax_scale=None
+        )
+
+        attn_output = attn_output.reshape(bsz, q_len, self.hidden_size).contiguous()
+        attn_output = self.dense(attn_output)
+
+        if not output_attentions:
+            attn_weights = None
+
+        return attn_output, attn_weights, past_key_value
+
+    # Copied from transformers.models.llama.modeling_llama.LlamaFlashAttention2._flash_attention_forward
+    def _flash_attention_forward(
+            self, query_states, key_states, value_states, attention_mask, query_length, dropout=0.0, softmax_scale=None
+    ):
+        """
+        Calls the forward method of Flash Attention - if the input hidden states contain at least one padding token
+        first unpad the input, then computes the attention scores and pad the final attention scores.
+        Args:
+            query_states (`torch.Tensor`):
+                Input query states to be passed to Flash Attention API
+            key_states (`torch.Tensor`):
+                Input key states to be passed to Flash Attention API
+            value_states (`torch.Tensor`):
+                Input value states to be passed to Flash Attention API
+            attention_mask (`torch.Tensor`):
+                The padding mask - corresponds to a tensor of size `(batch_size, seq_len)` where 0 stands for the
+                position of padding tokens and 1 for the position of non-padding tokens.
+            dropout (`int`, *optional*):
+                Attention dropout
+            softmax_scale (`float`, *optional*):
+                The scaling of QK^T before applying softmax. Default to 1 / sqrt(head_dim)
+        """
+        if not self._flash_attn_uses_top_left_mask:
+            causal = self.is_causal
+        else:
+            # TODO: Remove the `query_length != 1` check once Flash Attention for RoCm is bumped to 2.1. For details, please see the comment in LlamaFlashAttention2 __init__.
+            causal = self.is_causal and query_length != 1
+
+        # Contains at least one padding token in the sequence
+        if attention_mask is not None:
+            batch_size = query_states.shape[0]
+            query_states, key_states, value_states, indices_q, cu_seq_lens, max_seq_lens = self._upad_input(
+                query_states, key_states, value_states, attention_mask, query_length
+            )
+
+            cu_seqlens_q, cu_seqlens_k = cu_seq_lens
+            max_seqlen_in_batch_q, max_seqlen_in_batch_k = max_seq_lens
+
+            attn_output_unpad = flash_attn_varlen_func(
+                query_states,
+                key_states,
+                value_states,
+                cu_seqlens_q=cu_seqlens_q,
+                cu_seqlens_k=cu_seqlens_k,
+                max_seqlen_q=max_seqlen_in_batch_q,
+                max_seqlen_k=max_seqlen_in_batch_k,
+                dropout_p=dropout,
+                softmax_scale=softmax_scale,
+                causal=causal,
+            )
+
+            attn_output = pad_input(attn_output_unpad, indices_q, batch_size, query_length)
+        else:
+            attn_output = flash_attn_func(
+                query_states, key_states, value_states, dropout, softmax_scale=softmax_scale, causal=causal
+            )
+
+        return attn_output
+
+    # Copied from transformers.models.llama.modeling_llama.LlamaFlashAttention2._upad_input
+    def _upad_input(self, query_layer, key_layer, value_layer, attention_mask, query_length):
+        indices_k, cu_seqlens_k, max_seqlen_in_batch_k = _get_unpad_data(attention_mask)
+        batch_size, kv_seq_len, num_key_value_heads, head_dim = key_layer.shape
+
+        key_layer = index_first_axis(
+            key_layer.reshape(batch_size * kv_seq_len, num_key_value_heads, head_dim), indices_k
+        )
+        value_layer = index_first_axis(
+            value_layer.reshape(batch_size * kv_seq_len, num_key_value_heads, head_dim), indices_k
+        )
+        if query_length == kv_seq_len:
+            query_layer = index_first_axis(
+                query_layer.reshape(batch_size * kv_seq_len, self.num_heads, head_dim), indices_k
+            )
+            cu_seqlens_q = cu_seqlens_k
+            max_seqlen_in_batch_q = max_seqlen_in_batch_k
+            indices_q = indices_k
+        elif query_length == 1:
+            max_seqlen_in_batch_q = 1
+            cu_seqlens_q = torch.arange(
+                batch_size + 1, dtype=torch.int32, device=query_layer.device
+            )  # There is a memcpy here, that is very bad.
+            indices_q = cu_seqlens_q[:-1]
+            query_layer = query_layer.squeeze(1)
+        else:
+            # The -q_len: slice assumes left padding.
+            attention_mask = attention_mask[:, -query_length:]
+            query_layer, indices_q, cu_seqlens_q, max_seqlen_in_batch_q = unpad_input(query_layer, attention_mask)
+
+        return (
+            query_layer,
+            key_layer,
+            value_layer,
+            indices_q,
+            (cu_seqlens_q, cu_seqlens_k),
+            (max_seqlen_in_batch_q, max_seqlen_in_batch_k),
+        )
+
+
+PHI_ATTENTION_CLASSES = {
+    "eager": PhiAttention,
+    "flash_attention_2": PhiFlashAttention2,
+}
+
+
+class PhiDecoderLayer(nn.Module):
+    def __init__(self, config: CheXagentConfig, layer_idx: int):
+        super().__init__()
+        self.self_attn = PHI_ATTENTION_CLASSES[config._attn_implementation](config, layer_idx=layer_idx)
+        self.mlp = PhiMLP(config)
+        self.input_layernorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
+        self.resid_dropout = nn.Dropout(config.resid_pdrop)
+
+    def forward(
+            self,
+            hidden_states: torch.Tensor,
+            attention_mask: Optional[torch.Tensor] = None,
+            position_ids: Optional[torch.LongTensor] = None,
+            output_attentions: Optional[bool] = False,
+            use_cache: Optional[bool] = False,
+            past_key_value: Optional[Tuple[torch.Tensor]] = None,
+    ) -> Tuple[torch.FloatTensor, Optional[Tuple[torch.FloatTensor, torch.FloatTensor]]]:
+        """
+        Args:
+            hidden_states (`torch.FloatTensor`):
+                input to the layer of shape `(batch, seq_len, embed_dim)`
+            attention_mask (`torch.FloatTensor`, *optional*): attention mask of size
+                `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values.
+            position_ids (`torch.LongTensor` of shape `({0})`, *optional*):
+                Indices of positions of each input sequence tokens in the position embeddings. Selected in the range
+                `[0, config.n_positions - 1]`. [What are position IDs?](../glossary#position-ids)
+            output_attentions (`bool`, *optional*):
+                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
+                returned tensors for more detail.
+            use_cache (`bool`, *optional*):
+                If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
+                (see `past_key_values`).
+            past_key_value (`Tuple(torch.FloatTensor)`, *optional*): cached past key and value projection states
+        """
+
+        residual = hidden_states
+
+        hidden_states = self.input_layernorm(hidden_states)
+
+        # Self Attention
+        attn_outputs, self_attn_weights, present_key_value = self.self_attn(
+            hidden_states=hidden_states,
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            past_key_value=past_key_value,
+            output_attentions=output_attentions,
+            use_cache=use_cache,
+        )
+        attn_outputs = self.resid_dropout(attn_outputs)
+
+        feed_forward_hidden_states = self.resid_dropout(self.mlp(hidden_states))
+        hidden_states = attn_outputs + feed_forward_hidden_states + residual
+        outputs = (hidden_states,)
+
+        if output_attentions:
+            outputs += (self_attn_weights,)
+
+        if use_cache:
+            outputs += (present_key_value,)
+
+        return outputs
+
+
+PHI_START_DOCSTRING = r"""
+    This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
+    library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
+    etc.)
+    This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
+    Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
+    and behavior.
+    Parameters:
+        config ([`CheXagentConfig`]):
+            Model configuration class with all the parameters of the model. Initializing with a config file does not
+            load the weights associated with the model, only the configuration. Check out the
+            [`~PreTrainedModel.from_pretrained`] method to load the model weights.
+"""
+
+
+@add_start_docstrings(
+    "The bare Phi Model outputting raw hidden-states without any specific head on top.",
+    PHI_START_DOCSTRING,
+)
+class PhiPreTrainedModel(PreTrainedModel):
+    config_class = CheXagentConfig
+    base_model_prefix = "model"
+    supports_gradient_checkpointing = True
+    _no_split_modules = ["PhiDecoderLayer"]
+    _skip_keys_device_placement = "past_key_values"
+    _supports_flash_attn_2 = True
+    _supports_cache_class = True
+
+    def _init_weights(self, module):
+        std = self.config.initializer_range
+        if isinstance(module, nn.Linear):
+            module.weight.data.normal_(mean=0.0, std=std)
+            if module.bias is not None:
+                module.bias.data.zero_()
+        elif isinstance(module, nn.Embedding):
+            module.weight.data.normal_(mean=0.0, std=std)
+            if module.padding_idx is not None:
+                module.weight.data[module.padding_idx].zero_()
+
+
+PHI_INPUTS_DOCSTRING = r"""
+    Args:
+        input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
+            Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
+            it.
+            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+            [`PreTrainedTokenizer.__call__`] for details.
+            [What are input IDs?](../glossary#input-ids)
+        attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
+            Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:
+            - 1 for tokens that are **not masked**,
+            - 0 for tokens that are **masked**.
+            [What are attention masks?](../glossary#attention-mask)
+            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
+            [`PreTrainedTokenizer.__call__`] for details.
+            If `past_key_values` is used, optionally only the last `input_ids` have to be input (see
+            `past_key_values`).
+            If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`]
+            and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more
+            information on the default strategy.
+            - 1 indicates the head is **not masked**,
+            - 0 indicates the head is **masked**.
+        position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
+            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
+            config.n_positions - 1]`.
+            [What are position IDs?](../glossary#position-ids)
+        past_key_values (`Cache` or `tuple(tuple(torch.FloatTensor))`, *optional*):
+            Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention
+            blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values`
+            returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`.
+            Two formats are allowed:
+            - a [`~cache_utils.Cache`] instance;
+            - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of
+            shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`). This is also known as the legacy
+            cache format.
+            The model will output the same cache format that is fed as input. If no `past_key_values` are passed, the
+            legacy cache format will be returned.
+            If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't
+            have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids`
+            of shape `(batch_size, sequence_length)`.
+        inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
+            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
+            is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
+            model's internal embedding lookup matrix.
+        use_cache (`bool`, *optional*):
+            If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
+            `past_key_values`).
+        output_attentions (`bool`, *optional*):
+            Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
+            tensors for more detail.
+        output_hidden_states (`bool`, *optional*):
+            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
+            more detail.
+        return_dict (`bool`, *optional*):
+            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
+"""
+
+
+class CheXagentModel(PhiPreTrainedModel):
+    """
+    Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`PhiDecoderLayer`]
+    Args:
+        config: CheXagentConfig
+    """
+
+    def __init__(self, config: CheXagentConfig):
+        super().__init__(config)
+        self.padding_idx = config.pad_token_id
+        self.vocab_size = config.vocab_size
+
+        self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size, self.padding_idx)
+        self.embed_dropout = nn.Dropout(config.embd_pdrop)
+        self.layers = nn.ModuleList(
+            [PhiDecoderLayer(config, layer_idx) for layer_idx in range(config.num_hidden_layers)]
+        )
+        self.final_layernorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
+        self._use_flash_attention_2 = config._attn_implementation == "flash_attention_2"
+
+        self.gradient_checkpointing = False
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+        # IMAGE
+        self.tokenizer = CheXagentTokenizer.from_pretrained(config.name_or_path)
+        # self.visual = VisionEnsembleModel(**config.visual)
+        self.visual = CLIPModel(**config.visual)
+        # self.visual = DINOv2Model(**config.visual)
+
+    def get_input_embeddings(self):
+        return self.embed_tokens
+
+    def set_input_embeddings(self, value):
+        self.embed_tokens = value
+
+    @add_start_docstrings_to_model_forward(PHI_INPUTS_DOCSTRING)
+    def forward(
+            self,
+            input_ids: torch.LongTensor = None,
+            attention_mask: Optional[torch.Tensor] = None,
+            position_ids: Optional[torch.LongTensor] = None,
+            past_key_values: Optional[List[torch.FloatTensor]] = None,
+            inputs_embeds: Optional[torch.FloatTensor] = None,
+            use_cache: Optional[bool] = None,
+            output_attentions: Optional[bool] = None,
+            output_hidden_states: Optional[bool] = None,
+            return_dict: Optional[bool] = None,
+    ) -> Union[Tuple, BaseModelOutputWithPast]:
+        # IMAGE: encode images
+        if past_key_values is None and torch.any(input_ids == self.tokenizer.img_start_id):
+            bos_pos = torch.where(input_ids == self.tokenizer.img_start_id)
+            eos_pos = torch.where(input_ids == self.tokenizer.img_end_id)
+            assert (bos_pos[0] == eos_pos[0]).all()
+            img_pos = torch.stack((bos_pos[0], bos_pos[1], eos_pos[1]), dim=1)
+            image_paths = []
+            for i, a, b in img_pos:
+                image = input_ids[i][a + 1: b - 1].tolist()
+                image = image[: image.index(self.tokenizer.img_pad_id)]
+                image_paths.append(self.tokenizer.decode(image))
+            images = self.visual.encode(image_paths, training=self.training)
+            assert images.shape[0] == len(images)
+            fake_images = None
+        elif self.training:
+            fake_images = torch.zeros(1, 3, 512, 512).to(
+                dtype=next(self.visual.parameters()).dtype, device=next(self.visual.parameters()).device)
+            images = self.visual(fake_images)
+        else:
+            fake_images = None
+            images = None
+
+        # set constants
+        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+        output_hidden_states = (
+            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+        )
+        use_cache = use_cache if use_cache is not None else self.config.use_cache
+
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+        # IMAGE: retrieve input_ids and inputs_embeds
+        if input_ids is not None and inputs_embeds is not None:
+            raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time")
+        elif input_ids is not None:
+            input_shape = input_ids.size()
+            batch_size, seq_length = input_ids.shape[:2]
+        elif inputs_embeds is not None:
+            input_shape = inputs_embeds.size()[:-1]
+            batch_size, seq_length = inputs_embeds.shape[:2]
+        else:
+            raise ValueError("You have to specify either input_ids or inputs_embeds")
+
+        past_key_values_length = 0
+
+        if self.gradient_checkpointing and self.training:
+            if use_cache:
+                logger.warning_once(
+                    "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..."
+                )
+                use_cache = False
+
+        if use_cache:
+            use_legacy_cache = not isinstance(past_key_values, Cache)
+            if use_legacy_cache:
+                past_key_values = DynamicCache.from_legacy_cache(past_key_values)
+            past_key_values_length = past_key_values.get_usable_length(seq_length)
+
+        if position_ids is None:
+            device = input_ids.device if input_ids is not None else inputs_embeds.device
+            position_ids = torch.arange(
+                past_key_values_length, seq_length + past_key_values_length, dtype=torch.long, device=device
+            )
+            position_ids = position_ids.unsqueeze(0)
+
+        if inputs_embeds is None:
+            inputs_embeds = self.embed_tokens(input_ids)
+
+        inputs_embeds = self.embed_dropout(inputs_embeds)
+
+        # Attention mask.
+        if self._use_flash_attention_2:
+            # 2d mask is passed through the layers
+            attention_mask = attention_mask if (attention_mask is not None and 0 in attention_mask) else None
+        else:
+            # 4d mask is passed through the layers
+            attention_mask = _prepare_4d_causal_attention_mask(
+                attention_mask, (batch_size, seq_length), inputs_embeds, past_key_values_length
+            )
+
+        # IMAGE: embed positions
+        hidden_states = inputs_embeds.clone()
+        if fake_images is not None:
+            hidden_states = hidden_states + images.mean() * 0
+        elif images is not None:
+            for idx, (i, a, b) in enumerate(img_pos):
+                hidden_states[i][a + 1: b] = images[idx]
+        output_shape = input_shape + (hidden_states.size(-1),)
+
+        # decoder layers
+        all_hidden_states = () if output_hidden_states else None
+        all_self_attns = () if output_attentions else None
+        next_decoder_cache = None
+
+        for decoder_layer in self.layers:
+            if output_hidden_states:
+                all_hidden_states += (hidden_states,)
+
+            if self.gradient_checkpointing and self.training:
+                layer_outputs = self._gradient_checkpointing_func(
+                    decoder_layer.__call__,
+                    hidden_states,
+                    attention_mask,
+                    position_ids,
+                    past_key_values,
+                    output_attentions,
+                )
+            else:
+                layer_outputs = decoder_layer(
+                    hidden_states,
+                    attention_mask=attention_mask,
+                    position_ids=position_ids,
+                    past_key_value=past_key_values,
+                    output_attentions=output_attentions,
+                    use_cache=use_cache,
+                )
+
+            hidden_states = layer_outputs[0]
+
+            if use_cache:
+                next_decoder_cache = layer_outputs[2 if output_attentions else 1]
+
+            if output_attentions:
+                all_self_attns += (layer_outputs[1],)
+
+        hidden_states = self.final_layernorm(hidden_states)
+
+        # add hidden states from the last decoder layer
+        if output_hidden_states:
+            all_hidden_states += (hidden_states,)
+
+        next_cache = None
+        if use_cache:
+            next_cache = next_decoder_cache.to_legacy_cache() if use_legacy_cache else next_decoder_cache
+        if not return_dict:
+            return tuple(v for v in [hidden_states, next_cache, all_hidden_states, all_self_attns] if v is not None)
+        return BaseModelOutputWithPast(
+            last_hidden_state=hidden_states,
+            past_key_values=next_cache,
+            hidden_states=all_hidden_states,
+            attentions=all_self_attns,
+        )
+
+
+class CheXagentForCausalLM(PhiPreTrainedModel):
+    _tied_weights_keys = ["lm_head.weight"]
+
+    def __init__(self, config):
+        super().__init__(config)
+        self.model = CheXagentModel(config)
+        self.vocab_size = config.vocab_size
+        self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=True)
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    # Copied from transformers.models.llama.modeling_llama.LlamaForCausalLM.get_input_embeddings
+    def get_input_embeddings(self):
+        return self.model.embed_tokens
+
+    # Copied from transformers.models.llama.modeling_llama.LlamaForCausalLM.set_input_embeddings
+    def set_input_embeddings(self, value):
+        self.model.embed_tokens = value
+
+    # Copied from transformers.models.llama.modeling_llama.LlamaForCausalLM.get_output_embeddings
+    def get_output_embeddings(self):
+        return self.lm_head
+
+    # Copied from transformers.models.llama.modeling_llama.LlamaForCausalLM.set_output_embeddings
+    def set_output_embeddings(self, new_embeddings):
+        self.lm_head = new_embeddings
+
+    # Copied from transformers.models.llama.modeling_llama.LlamaForCausalLM.set_decoder
+    def set_decoder(self, decoder):
+        self.model = decoder
+
+    # Copied from transformers.models.llama.modeling_llama.LlamaForCausalLM.get_decoder
+    def get_decoder(self):
+        return self.model
+
+    def forward(
+            self,
+            input_ids: torch.LongTensor = None,
+            attention_mask: Optional[torch.Tensor] = None,
+            position_ids: Optional[torch.LongTensor] = None,
+            past_key_values: Optional[List[torch.FloatTensor]] = None,
+            inputs_embeds: Optional[torch.FloatTensor] = None,
+            labels: Optional[torch.LongTensor] = None,
+            use_cache: Optional[bool] = None,
+            output_attentions: Optional[bool] = None,
+            output_hidden_states: Optional[bool] = None,
+            return_dict: Optional[bool] = None,
+    ) -> Union[Tuple, CausalLMOutputWithPast]:
+        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+        output_hidden_states = (
+            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+        )
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+        # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn)
+        outputs = self.model(
+            input_ids=input_ids,
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            past_key_values=past_key_values,
+            inputs_embeds=inputs_embeds,
+            use_cache=use_cache,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+        )
+
+        hidden_states = outputs[0]
+        logits = self.lm_head(hidden_states)
+        logits = logits.float()
+
+        loss = None
+        if labels is not None:
+            # Shift so that tokens < n predict n
+            shift_logits = logits[..., :-1, :].contiguous()
+            shift_labels = labels[..., 1:].contiguous()
+            # Flatten the tokens
+            loss_fct = CrossEntropyLoss()
+            shift_logits = shift_logits.view(-1, self.config.vocab_size)
+            shift_labels = shift_labels.view(-1)
+            # Enable model parallelism
+            shift_labels = shift_labels.to(shift_logits.device)
+            loss = loss_fct(shift_logits, shift_labels)
+
+        if not return_dict:
+            output = (logits,) + outputs[1:]
+            return (loss,) + output if loss is not None else output
+
+        return CausalLMOutputWithPast(
+            loss=loss,
+            logits=logits,
+            past_key_values=outputs.past_key_values,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
+        )
+
+    # Copied from transformers.models.llama.modeling_llama.LlamaForCausalLM.prepare_inputs_for_generation
+    def prepare_inputs_for_generation(
+            self, input_ids, past_key_values=None, attention_mask=None, inputs_embeds=None, **kwargs
+    ):
+        if past_key_values is not None:
+            if isinstance(past_key_values, Cache):
+                cache_length = past_key_values.get_seq_length()
+                past_length = past_key_values.seen_tokens
+                max_cache_length = past_key_values.get_max_length()
+            else:
+                cache_length = past_length = past_key_values[0][0].shape[2]
+                max_cache_length = None
+
+            # Keep only the unprocessed tokens:
+            # 1 - If the length of the attention_mask exceeds the length of input_ids, then we are in a setting where
+            # some of the inputs are exclusively passed as part of the cache (e.g. when passing input_embeds as
+            # input)
+            if attention_mask is not None and attention_mask.shape[1] > input_ids.shape[1]:
+                input_ids = input_ids[:, -(attention_mask.shape[1] - past_length):]
+            # 2 - If the past_length is smaller than input_ids', then input_ids holds all input tokens. We can discard
+            # input_ids based on the past_length.
+            elif past_length < input_ids.shape[1]:
+                input_ids = input_ids[:, past_length:]
+            # 3 - Otherwise (past_length >= input_ids.shape[1]), let's assume input_ids only has unprocessed tokens.
+
+            # If we are about to go beyond the maximum cache length, we need to crop the input attention mask.
+            if (
+                    max_cache_length is not None
+                    and attention_mask is not None
+                    and cache_length + input_ids.shape[1] > max_cache_length
+            ):
+                attention_mask = attention_mask[:, -max_cache_length:]
+
+        position_ids = kwargs.get("position_ids", None)
+        if attention_mask is not None and position_ids is None:
+            # create position_ids on the fly for batch generation
+            position_ids = attention_mask.long().cumsum(-1) - 1
+            position_ids.masked_fill_(attention_mask == 0, 1)
+            if past_key_values:
+                position_ids = position_ids[:, -input_ids.shape[1]:]
+
+        # if `inputs_embeds` are passed, we only want to use them in the 1st generation step
+        if inputs_embeds is not None and past_key_values is None:
+            model_inputs = {"inputs_embeds": inputs_embeds}
+        else:
+            model_inputs = {"input_ids": input_ids}
+
+        model_inputs.update(
+            {
+                "position_ids": position_ids,
+                "past_key_values": past_key_values,
+                "use_cache": kwargs.get("use_cache"),
+                "attention_mask": attention_mask,
+            }
+        )
+        return model_inputs
+
+    @staticmethod
+    # Copied from transformers.models.llama.modeling_llama.LlamaForCausalLM._reorder_cache
+    def _reorder_cache(past_key_values, beam_idx):
+        reordered_past = ()
+        for layer_past in past_key_values:
+            reordered_past += (
+                tuple(past_state.index_select(0, beam_idx.to(past_state.device)) for past_state in layer_past),
+            )
+        return reordered_past

modeling_visual.py

@@ -0,0 +1,248 @@
+import io
+import math
+import re
+from functools import partial
+from typing import List
+
+import albumentations as A
+import cv2
+import numpy as np
+import pyarrow as pa
+import requests
+import torch
+import transformers
+from PIL import Image
+from albumentations.pytorch import ToTensorV2
+from einops import rearrange
+from torch import nn
+from torch.nn import functional as F
+from torch.nn.init import trunc_normal_
+from torchvision import transforms
+from torchvision.transforms import InterpolationMode
+from transformers import AutoModel, AutoProcessor
+from transformers.activations import ACT2FN
+
+assert transformers.__version__ == "4.40.0", "Please install a specific HF transformers version: pip install transformers==4.40.0"
+
+
+def get_abs_pos(abs_pos, tgt_size):
+    # abs_pos: L, C
+    # tgt_size: M
+    # return: M, C
+    src_size = int(math.sqrt(abs_pos.size(0)))
+    tgt_size = int(math.sqrt(tgt_size))
+    dtype = abs_pos.dtype
+
+    if src_size != tgt_size:
+        return F.interpolate(
+            abs_pos.float().reshape(1, src_size, src_size, -1).permute(0, 3, 1, 2),
+            size=(tgt_size, tgt_size),
+            mode="bicubic",
+            align_corners=False,
+        ).permute(0, 2, 3, 1).flatten(0, 2).to(dtype=dtype)
+    else:
+        return abs_pos
+
+
+def get_2d_sincos_pos_embed(embed_dim, grid_size, cls_token=False):
+    """
+    grid_size: int of the grid height and width
+    return:
+    pos_embed: [grid_size*grid_size, embed_dim] or [1+grid_size*grid_size, embed_dim] (w/ or w/o cls_token)
+    """
+    grid_h = np.arange(grid_size, dtype=np.float32)
+    grid_w = np.arange(grid_size, dtype=np.float32)
+    grid = np.meshgrid(grid_w, grid_h)  # here w goes first
+    grid = np.stack(grid, axis=0)
+
+    grid = grid.reshape([2, 1, grid_size, grid_size])
+    pos_embed = get_2d_sincos_pos_embed_from_grid(embed_dim, grid)
+    if cls_token:
+        pos_embed = np.concatenate([np.zeros([1, embed_dim]), pos_embed], axis=0)
+    return pos_embed
+
+
+def get_2d_sincos_pos_embed_from_grid(embed_dim, grid):
+    assert embed_dim % 2 == 0
+
+    # use half of dimensions to encode grid_h
+    emb_h = get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[0])  # (H*W, D/2)
+    emb_w = get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[1])  # (H*W, D/2)
+
+    emb = np.concatenate([emb_h, emb_w], axis=1)  # (H*W, D)
+    return emb
+
+
+def get_1d_sincos_pos_embed_from_grid(embed_dim, pos):
+    """
+    embed_dim: output dimension for each position
+    pos: a list of positions to be encoded: size (M,)
+    out: (M, D)
+    """
+    assert embed_dim % 2 == 0
+    omega = np.arange(embed_dim // 2, dtype=np.float32)
+    omega /= embed_dim / 2.
+    omega = 1. / 10000 ** omega  # (D/2,)
+
+    pos = pos.reshape(-1)  # (M,)
+    out = np.einsum('m,d->md', pos, omega)  # (M, D/2), outer product
+
+    emb_sin = np.sin(out)  # (M, D/2)
+    emb_cos = np.cos(out)  # (M, D/2)
+
+    emb = np.concatenate([emb_sin, emb_cos], axis=1)  # (M, D)
+    return emb
+
+
+class Resampler(nn.Module):
+    """
+    A 2D perceiver-resampler network with one cross attention layers by
+        (grid_size**2) learnable queries and 2d sincos pos_emb
+    Outputs:
+        A tensor with the shape of (grid_size**2, embed_dim)
+    """
+
+    def __init__(
+            self,
+            grid_size,
+            embed_dim,
+            num_heads,
+            kv_dim=None,
+            norm_layer=nn.LayerNorm
+    ):
+        super().__init__()
+        self.num_queries = grid_size ** 2
+        self.embed_dim = embed_dim
+        self.num_heads = num_heads
+
+        self.pos_embed = nn.Parameter(
+            torch.from_numpy(get_2d_sincos_pos_embed(embed_dim, grid_size)).float()
+        ).requires_grad_(False)
+
+        self.query = nn.Parameter(torch.zeros(self.num_queries, embed_dim))
+        # trunc_normal_(self.query, std=.02)
+
+        if kv_dim is not None and kv_dim != embed_dim:
+            self.kv_proj = nn.Linear(kv_dim, embed_dim, bias=False)
+        else:
+            self.kv_proj = nn.Identity()
+
+        self.attn = nn.MultiheadAttention(embed_dim, num_heads)
+        self.ln_q = norm_layer(embed_dim)
+        self.ln_kv = norm_layer(embed_dim)
+        # self.apply(self._init_weights)
+
+    def _init_weights(self, m):
+        if isinstance(m, nn.Linear):
+            trunc_normal_(m.weight, std=.02)
+            if isinstance(m, nn.Linear) and m.bias is not None:
+                nn.init.constant_(m.bias, 0)
+        elif isinstance(m, nn.LayerNorm):
+            nn.init.constant_(m.bias, 0)
+            nn.init.constant_(m.weight, 1.0)
+
+    def forward(self, x, attn_mask=None):
+
+        pos_embed = get_abs_pos(self.pos_embed, x.size(1))
+
+        x = self.kv_proj(x)
+        x = self.ln_kv(x).permute(1, 0, 2)
+
+        N = x.shape[1]
+        q = self.ln_q(self.query)
+        out = self.attn(
+            self._repeat(q, N) + self.pos_embed.unsqueeze(1),
+            x + pos_embed.unsqueeze(1),
+            x,
+            attn_mask=attn_mask
+        )[0]
+        return out.permute(1, 0, 2)
+
+    def _repeat(self, query, N: int):
+        return query.unsqueeze(1).repeat(1, N, 1)
+
+
+class CLIPModel(nn.Module):
+
+    def __init__(
+            self,
+            image_size: int,
+            n_queries: int = 256,
+            output_dim: int = 512,
+            vision_model_name_or_path: str = "StanfordAIMI/XraySigLIP__vit-l-16-siglip-384__webli",
+            **kwargs
+    ):
+        super().__init__()
+        # load model and processor
+        self.model = AutoModel.from_pretrained(vision_model_name_or_path).vision_model
+        self.processor = AutoProcessor.from_pretrained(vision_model_name_or_path).image_processor
+
+        # set constants
+        self.image_height, self.image_width = self.image_size = (image_size, image_size)
+        width = self.model.config.hidden_size
+        patch_height, patch_width = self.model.embeddings.patch_embedding.kernel_size
+        self.grid_size = (self.image_height // patch_height, self.image_width // patch_width)
+        self.output_dim = output_dim
+
+        # Transforms
+        self.mean = self.processor.image_mean
+        self.std = self.processor.image_std
+        
+        self.image_transform = transforms.Compose([
+            transforms.Resize(
+                (image_size, image_size),
+                interpolation=InterpolationMode.BICUBIC
+            ),
+            transforms.ToTensor(),
+            transforms.Normalize(mean=self.mean, std=self.std),
+        ])
+
+        # MLP
+        self.pos_embed = nn.Parameter(
+            torch.from_numpy(get_2d_sincos_pos_embed(width, self.grid_size[0])).float()
+        ).requires_grad_(False)
+        self.attn_pool = nn.Sequential(
+            nn.Linear(width, output_dim * 4, bias=True),
+            ACT2FN["gelu"],
+            nn.Linear(output_dim * 4, output_dim, bias=True)
+        )
+        norm_layer = partial(nn.LayerNorm, eps=1e-6)
+        self.ln_post = norm_layer(output_dim)
+        self.proj = nn.Parameter((output_dim ** -0.5) * torch.randn(output_dim, output_dim), requires_grad=True)
+
+    def forward_resampler(self, x):
+        pos_embed = get_abs_pos(self.pos_embed, x.size(1))
+        x = x + pos_embed.unsqueeze(0)
+        x = self.attn_pool(x)
+        x = self.ln_post(x)
+        x = x @ self.proj
+        return x
+
+    def forward(self, x: torch.Tensor):
+        # get feature
+        x = self.model(x, output_hidden_states=True).hidden_states[-1]
+
+        # resampler
+        x = self.forward_resampler(x)
+        return x
+
+    def load_image(self, image_path, training):
+        if image_path.startswith("http://") or image_path.startswith("https://"):
+            image = Image.open(requests.get(image_path, stream=True).raw)
+        else:
+            image = Image.open(image_path)
+
+        image = image.convert("RGB")
+
+        image_tensor = self.image_transform(image)
+        return image_tensor
+
+    def encode(self, image_paths: List[str], training):
+        images = []
+        for image_path in image_paths:
+            image = self.load_image(image_path, training)
+            images.append(image)
+        images = torch.stack(images, dim=0)
+        images = images.to(dtype=next(self.parameters()).dtype, device=next(self.parameters()).device)
+        outputs = self.forward(images)
+        return outputs

special_tokens_map.json

@@ -0,0 +1,39 @@
+{
+  "additional_special_tokens": [
+    {
+      "content": "<|coord_9|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    }
+  ],
+  "bos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenization_chexagent.py

@@ -0,0 +1,675 @@
+import json
+from functools import lru_cache
+from typing import TYPE_CHECKING
+
+import regex as re
+from transformers.tokenization_utils_base import TextInput
+from transformers.utils import is_tf_available, is_torch_available, to_py_obj
+
+if TYPE_CHECKING:
+    if is_torch_available():
+        import torch
+    if is_tf_available():
+        import tensorflow as tf
+
+import os
+import random
+from typing import Dict, List, Tuple, Union, Any, Callable, Optional
+
+import matplotlib as mpl
+import matplotlib.colors as mcolors
+import matplotlib.colors as mplc
+import matplotlib.figure as mplfigure
+import numpy as np
+import requests
+import torch
+from PIL import Image
+from matplotlib.backends.backend_agg import FigureCanvasAgg
+from transformers import PreTrainedTokenizer, AddedToken
+from transformers.utils import logging
+
+logger = logging.get_logger(__name__)
+
+VOCAB_FILES_NAMES = {
+    "vocab_file": "vocab.json",
+    "merges_file": "merges.txt",
+}
+
+PRETRAINED_VOCAB_FILES_MAP = {
+    "vocab_file": {
+        "Salesforce/codegen-350M-mono": "https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json",
+    },
+    "merges_file": {
+        "Salesforce/codegen-350M-mono": "https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt",
+    },
+}
+
+PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
+    "Salesforce/codegen-350M-mono": 2048,
+}
+
+IMG_TOKEN_SPAN = 1024
+
+DEFAULT_CHAT_TEMPLATE = "{% for message in messages %}\n{% if message['from'] == 'human' %}\n{{ '<|user|>\n' + message['value'] + eos_token }}\n{% elif message['from'] == 'system' %}\n{{ '<|system|>\n' + message['value'] + eos_token }}\n{% elif message['from'] == 'gpt' %}\n{{ '<|assistant|>\n'  + message['value'] + eos_token }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '<|assistant|>' }}\n{% endif %}\n{% endfor %}"
+
+
+@lru_cache()
+def bytes_to_unicode():
+    """
+    Returns list of utf-8 byte and a mapping to unicode strings. We specifically avoids mapping to whitespace/control
+    characters the bpe code barfs on.
+
+    The reversible bpe codes work on unicode strings. This means you need a large # of unicode characters in your vocab
+    if you want to avoid UNKs. When you're at something like a 10B token dataset you end up needing around 5K for
+    decent coverage. This is a significant percentage of your normal, say, 32K bpe vocab. To avoid that, we want lookup
+    tables between utf-8 bytes and unicode strings.
+    """
+    bs = (
+            list(range(ord("!"), ord("~") + 1)) + list(range(ord("¡"), ord("¬") + 1)) + list(
+        range(ord("®"), ord("ÿ") + 1))
+    )
+    cs = bs[:]
+    n = 0
+    for b in range(2 ** 8):
+        if b not in bs:
+            bs.append(b)
+            cs.append(2 ** 8 + n)
+            n += 1
+    cs = [chr(n) for n in cs]
+    return dict(zip(bs, cs))
+
+
+def get_pairs(word):
+    """
+    Return set of symbol pairs in a word.
+
+    Word is represented as tuple of symbols (symbols being variable-length strings).
+    """
+    pairs = set()
+    prev_char = word[0]
+    for char in word[1:]:
+        pairs.add((prev_char, char))
+        prev_char = char
+    return pairs
+
+
+def _list_find(
+        input_list: List[Any],
+        candidates: Tuple[Any],
+        start: int = 0,
+):
+    for i in range(start, len(input_list)):
+        if input_list[i] in candidates:
+            return i
+    return -1
+
+
+def _replace_closed_tag(
+        input_tokens: List[Any],
+        start_tags: Union[Any, Tuple[Any]],
+        end_tags: Union[Any, Tuple[Any]],
+        inclusive_replace_func: Callable,
+        exclusive_replace_func: Callable = lambda x: x,
+):
+    if isinstance(start_tags, (str, int)):
+        start_tags = (start_tags,)
+    if isinstance(end_tags, (str, int)):
+        end_tags = (end_tags,)
+    assert len(start_tags) == len(end_tags)
+
+    output_tokens = []
+    end = 0
+    while True:
+        start = _list_find(input_tokens, start_tags, end)
+        if start == -1:
+            break
+        output_tokens.extend(exclusive_replace_func(input_tokens[end: start]))
+        tag_idx = start_tags.index(input_tokens[start])
+        end = _list_find(input_tokens, (end_tags[tag_idx],), start)
+        if end == -1:
+            raise ValueError("Unclosed image token")
+        output_tokens.extend(inclusive_replace_func(input_tokens[start: end + 1]))
+        end += 1
+    output_tokens.extend(exclusive_replace_func(input_tokens[end:]))
+    return output_tokens
+
+
+class CheXagentTokenizer(PreTrainedTokenizer):
+    vocab_files_names = VOCAB_FILES_NAMES
+    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
+    max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
+    model_input_names = ["input_ids", "attention_mask"]
+
+    def __init__(
+            self,
+            vocab_file,
+            merges_file,
+            errors="replace",
+            unk_token="<|endoftext|>",
+            bos_token="<|endoftext|>",
+            eos_token="<|endoftext|>",
+            pad_token=None,
+            add_prefix_space=False,
+            add_bos_token=False,
+            image_start_tag='<|img|>',
+            image_end_tag='<|/img|>',
+            image_pad_tag='<|imgpad|>',
+            ref_start_tag='<|ref|>',
+            ref_end_tag='<|/ref|>',
+            box_start_tag='<|box|>',
+            box_end_tag='<|/box|>',
+            quad_start_tag='<|quad|>',
+            quad_end_tag='<|/quad|>',
+            **kwargs,
+    ):
+        bos_token = AddedToken(bos_token, special=True) if isinstance(bos_token, str) else bos_token
+        eos_token = AddedToken(eos_token, special=True) if isinstance(eos_token, str) else eos_token
+        unk_token = AddedToken(unk_token, special=True) if isinstance(unk_token, str) else unk_token
+        pad_token = AddedToken(pad_token, special=True) if isinstance(pad_token, str) else pad_token
+        self.add_bos_token = add_bos_token
+
+        with open(vocab_file, encoding="utf-8") as vocab_handle:
+            self.encoder = json.load(vocab_handle)
+        self.decoder = {v: k for k, v in self.encoder.items()}
+        self.errors = errors  # how to handle errors in decoding
+        self.byte_encoder = bytes_to_unicode()
+        self.byte_decoder = {v: k for k, v in self.byte_encoder.items()}
+        with open(merges_file, encoding="utf-8") as merges_handle:
+            bpe_merges = merges_handle.read().split("\n")[1:-1]
+        bpe_merges = [tuple(merge.split()) for merge in bpe_merges]
+        self.bpe_ranks = dict(zip(bpe_merges, range(len(bpe_merges))))
+        self.cache = {}
+        self.add_prefix_space = add_prefix_space
+
+        # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
+        self.pat = re.compile(r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+""")
+        super().__init__(
+            errors=errors,
+            unk_token=unk_token,
+            bos_token=bos_token,
+            eos_token=eos_token,
+            pad_token=pad_token,
+            add_prefix_space=add_prefix_space,
+            add_bos_token=add_bos_token,
+            **kwargs,
+        )
+
+        self.image_start_tag = image_start_tag
+        self.image_end_tag = image_end_tag
+        self.image_pad_tag = image_pad_tag
+        self.ref_start_tag = ref_start_tag
+        self.ref_end_tag = ref_end_tag
+        self.box_start_tag = box_start_tag
+        self.box_end_tag = box_end_tag
+        self.quad_start_tag = quad_start_tag
+        self.quad_end_tag = quad_end_tag
+        self.IMAGE_ST = (
+            image_start_tag, image_end_tag, image_pad_tag,
+            ref_start_tag, ref_end_tag, box_start_tag, box_end_tag,
+            quad_start_tag, quad_end_tag,
+        )
+        for special_token in self.IMAGE_ST:
+            if special_token not in self.get_vocab():
+                self.add_special_tokens({"additional_special_tokens": [special_token]})
+        for coordinate in range(10):
+            if f"<{coordinate}>" not in self.get_vocab():
+                self.add_special_tokens({"additional_special_tokens": [f"<|coord_{coordinate}|>"]})
+        if len(self) % 64 != 0:
+            for extra in range(((len(self) // 64) + 1) * 64 - len(self)):
+                if f"<extra_{extra}>" not in self.get_vocab():
+                    self.add_special_tokens({"additional_special_tokens": [f"<|extra_{extra}|>"]})
+        self.img_start_id = self.convert_tokens_to_ids(self.image_start_tag)
+        self.img_end_id = self.convert_tokens_to_ids(self.image_end_tag)
+        self.img_pad_id = self.convert_tokens_to_ids(self.image_pad_tag)
+        self.ref_start_id = self.convert_tokens_to_ids(self.ref_start_tag)
+        self.ref_end_id = self.convert_tokens_to_ids(self.ref_end_tag)
+        self.box_start_id = self.convert_tokens_to_ids(self.box_start_tag)
+        self.box_end_id = self.convert_tokens_to_ids(self.box_end_tag)
+        self.quad_start_id = self.convert_tokens_to_ids(self.quad_start_tag)
+        self.quad_end_id = self.convert_tokens_to_ids(self.quad_end_tag)
+        self.chat_template = DEFAULT_CHAT_TEMPLATE
+
+    @property
+    def vocab_size(self):
+        return len(self.encoder)
+
+    def get_vocab(self):
+        return dict(self.encoder, **self.added_tokens_encoder)
+
+    def bpe(self, token):
+        if token in self.cache:
+            return self.cache[token]
+        word = tuple(token)
+        pairs = get_pairs(word)
+
+        if not pairs:
+            return token
+
+        while True:
+            bigram = min(pairs, key=lambda pair: self.bpe_ranks.get(pair, float("inf")))
+            if bigram not in self.bpe_ranks:
+                break
+            first, second = bigram
+            new_word = []
+            i = 0
+            while i < len(word):
+                try:
+                    j = word.index(first, i)
+                except ValueError:
+                    new_word.extend(word[i:])
+                    break
+                else:
+                    new_word.extend(word[i:j])
+                    i = j
+
+                if word[i] == first and i < len(word) - 1 and word[i + 1] == second:
+                    new_word.append(first + second)
+                    i += 2
+                else:
+                    new_word.append(word[i])
+                    i += 1
+            new_word = tuple(new_word)
+            word = new_word
+            if len(word) == 1:
+                break
+            else:
+                pairs = get_pairs(word)
+        word = " ".join(word)
+        self.cache[token] = word
+        return word
+
+    def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None):
+        if self.add_bos_token:
+            bos_token_ids = [self.bos_token_id]
+        else:
+            bos_token_ids = []
+
+        output = bos_token_ids + token_ids_0
+
+        if token_ids_1 is None:
+            return output
+
+        return output + bos_token_ids + token_ids_1
+
+    def tokenize(self, text: TextInput, **kwargs) -> List[str]:
+        def _encode_imgurl(img_tokens):
+            assert img_tokens[0] == self.image_start_tag and img_tokens[-1] == self.image_end_tag
+            img_tokens = img_tokens[1:-1]
+            img_url = ''.join(img_tokens)
+            out_img_tokens = list(img_url)
+            if len(out_img_tokens) > IMG_TOKEN_SPAN:
+                raise ValueError("The content in {}..{} is too long".format(self.image_start_tag, self.image_end_tag))
+            out_img_tokens.extend([self.image_pad_tag] * (IMG_TOKEN_SPAN - len(out_img_tokens)))
+            out_img_tokens = [self.image_start_tag] + out_img_tokens + [self.image_end_tag]
+            return out_img_tokens
+
+        tokens = super().tokenize(text, **kwargs)
+        tokens = _replace_closed_tag(tokens, self.image_start_tag, self.image_end_tag, _encode_imgurl)
+        return tokens
+
+    def _tokenize(self, text):
+        """Tokenize a string."""
+
+        bpe_tokens = []
+        for token in re.findall(self.pat, text):
+            token = "".join(
+                self.byte_encoder[b] for b in token.encode("utf-8")
+            )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
+            bpe_tokens.extend(bpe_token for bpe_token in self.bpe(token).split(" "))
+        return bpe_tokens
+
+    def _convert_token_to_id(self, token):
+        """Converts a token (str) in an id using the vocab."""
+        return self.encoder.get(token, self.encoder.get(self.unk_token))
+
+    def _convert_id_to_token(self, index):
+        """Converts an index (integer) in a token (str) using the vocab."""
+        return self.decoder.get(index)
+
+    def convert_tokens_to_string(self, tokens):
+        """Converts a sequence of tokens (string) in a single string."""
+        text = "".join(tokens)
+        text = bytearray([self.byte_decoder[c] for c in text]).decode("utf-8", errors=self.errors)
+        return text
+
+    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
+        if not os.path.isdir(save_directory):
+            logger.error(f"Vocabulary path ({save_directory}) should be a directory")
+            return
+        vocab_file = os.path.join(
+            save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]
+        )
+        merge_file = os.path.join(
+            save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]
+        )
+
+        with open(vocab_file, "w", encoding="utf-8") as f:
+            f.write(json.dumps(self.encoder, indent=2, sort_keys=True, ensure_ascii=False) + "\n")
+
+        index = 0
+        with open(merge_file, "w", encoding="utf-8") as writer:
+            writer.write("#version: 0.2\n")
+            for bpe_tokens, token_index in sorted(self.bpe_ranks.items(), key=lambda kv: kv[1]):
+                if index != token_index:
+                    logger.warning(
+                        f"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
+                        " Please check that the tokenizer is not corrupted!"
+                    )
+                    index = token_index
+                writer.write(" ".join(bpe_tokens) + "\n")
+                index += 1
+
+        return vocab_file, merge_file
+
+    def prepare_for_tokenization(self, text, is_split_into_words=False, **kwargs):
+        add_prefix_space = kwargs.pop("add_prefix_space", self.add_prefix_space)
+        if is_split_into_words or add_prefix_space:
+            text = " " + text
+        return (text, kwargs)
+
+    def decode(
+            self,
+            token_ids: Union[int, List[int], "np.ndarray", "torch.Tensor", "tf.Tensor"],
+            skip_special_tokens: bool = False,
+            clean_up_tokenization_spaces: bool = None,
+            truncate_before_pattern: Optional[List[str]] = None,
+            **kwargs,
+    ) -> str:
+        """
+        Converts a sequence of ids in a string, using the tokenizer and vocabulary with options to remove special
+        tokens and clean up tokenization spaces.
+
+        Similar to doing `self.convert_tokens_to_string(self.convert_ids_to_tokens(token_ids))`.
+
+        Args:
+            token_ids (`Union[int, List[int], np.ndarray, torch.Tensor, tf.Tensor]`):
+                List of tokenized input ids. Can be obtained using the `__call__` method.
+            skip_special_tokens (`bool`, *optional*, defaults to `False`):
+                Whether or not to remove special tokens in the decoding.
+            clean_up_tokenization_spaces (`bool`, *optional*):
+                Whether or not to clean up the tokenization spaces. If `None`, will default to
+                `self.clean_up_tokenization_spaces` (available in the `tokenizer_config`).
+            truncate_before_pattern (`List[str]`, *optional*, defaults to `None`):
+                A list of regular expression strings that will be used to truncate the returned string. This can be
+                used to remove extra pieces of code (e.g. truncate if observing a comment symbol "#" at the beginning
+                of a new line). An example pattern could be `["^#", re.escape("<|endoftext|>"), "^'''", "\n\n\n"]`.
+            kwargs (additional keyword arguments, *optional*):
+                Will be passed to the underlying model specific decode method.
+
+        Returns:
+            `str`: The decoded sentence.
+        """
+
+        token_ids = to_py_obj(token_ids)
+
+        decoded_text = self._decode(
+            token_ids=token_ids,
+            skip_special_tokens=skip_special_tokens,
+            clean_up_tokenization_spaces=clean_up_tokenization_spaces,
+            **kwargs,
+        )
+
+        if truncate_before_pattern is not None and len(truncate_before_pattern) > 0:
+            decoded_text = self.truncate(decoded_text, truncate_before_pattern)
+
+        return decoded_text
+
+    def _decode(
+            self,
+            token_ids: List[int],
+            skip_special_tokens: bool = False,
+            clean_up_tokenization_spaces: bool = None,
+            spaces_between_special_tokens: bool = True,
+            **kwargs,
+    ) -> str:
+
+        def _decode_imgurl(img_token_ids):
+            assert img_token_ids[0] == self.img_start_id and img_token_ids[-1] == self.img_end_id
+            img_token_ids = img_token_ids[1:-1]
+            img_token_ids = img_token_ids[: img_token_ids.index(self.img_pad_id)]
+            return [self.img_start_id] + img_token_ids + [self.img_end_id]
+
+        token_ids = _replace_closed_tag(token_ids, self.img_start_id, self.img_end_id, _decode_imgurl)
+
+        return super()._decode(
+            token_ids, skip_special_tokens, clean_up_tokenization_spaces, spaces_between_special_tokens, **kwargs
+        )
+
+    def truncate(self, completion, truncate_before_pattern):
+        def find_re(string, pattern, start_pos):
+            m = pattern.search(string, start_pos)
+            return m.start() if m else -1
+
+        terminals = [re.compile(pattern, re.MULTILINE) for pattern in truncate_before_pattern]
+
+        prints = list(re.finditer("^print", completion, re.MULTILINE))
+
+        if len(prints) > 1:
+            completion = completion[: prints[1].start()]
+
+        defs = list(re.finditer("^def", completion, re.MULTILINE))
+
+        if len(defs) > 1:
+            completion = completion[: defs[1].start()]
+
+        start_pos = 0
+
+        terminals_pos = [
+            pos for pos in [find_re(completion, terminal, start_pos) for terminal in terminals] if pos != -1
+        ]
+
+        if len(terminals_pos) > 0:
+            return completion[: min(terminals_pos)]
+        else:
+            return completion
+
+    def from_list_format(self, list_format: List[Dict]):
+        text = ''
+        num_images = 0
+        for ele in list_format:
+            if 'image' in ele:
+                num_images += 1
+                text += f'Picture {num_images}:'
+                text += self.image_start_tag + ele['image'] + self.image_end_tag
+                text += '\n'
+            elif 'text' in ele:
+                text += ele['text']
+            elif 'box' in ele:
+                if 'ref' in ele:
+                    text += self.ref_start_tag + ele['ref'] + self.ref_end_tag
+                for box in ele['box']:
+                    text += self.box_start_tag + '(%d,%d),(%d,%d)' % (box[0], box[1], box[2], box[3]) + self.box_end_tag
+            else:
+                raise ValueError("Unsupport element: " + str(ele))
+        return text
+
+    def to_list_format(self, text: str):
+        token_ids = self.encode(text)
+
+        def _encode_vl_info(tokens):
+            if len(tokens) == 0:
+                return []
+            if tokens[0] == self.img_start_id and tokens[-1] == self.img_end_id:
+                key = 'image'
+            elif tokens[0] == self.ref_start_id and tokens[-1] == self.ref_end_id:
+                key = 'ref'
+            elif tokens[0] == self.box_start_id and tokens[-1] == self.box_end_id:
+                key = 'box'
+            elif tokens[0] == self.quad_start_id and tokens[-1] == self.quad_end_id:
+                key = 'quad'
+            else:
+                val = self.decode(tokens)
+                return [{'text': val}]
+            tokens = [token for token in tokens[1:-1] if token != self.img_pad_id]
+            val = self.decode(tokens, skip_special_tokens=True)
+            return [{key: val}]
+
+        return _replace_closed_tag(
+            token_ids,
+            (self.img_start_id, self.ref_start_id, self.box_start_id, self.quad_start_id),
+            (self.img_end_id, self.ref_end_id, self.box_end_id, self.quad_end_id),
+            _encode_vl_info,
+            _encode_vl_info,
+        )
+
+    def _fetch_latest_picture(self, response, history):
+        if history is None:
+            history = []
+        _history = history + [(response, None)]
+        for q, r in _history[::-1]:
+            for ele in self.to_list_format(q)[::-1]:
+                if 'image' in ele:
+                    return ele['image']
+        return None
+
+    def _fetch_all_box_with_ref(self, text):
+        list_format = self.to_list_format(text)
+        output = []
+        for i, ele in enumerate(list_format):
+            if 'box' in ele:
+                bbox = tuple(map(int, ele['box'].replace('(', '').replace(')', '').split(',')))
+                assert len(bbox) == 4
+                output.append({'box': bbox})
+                if i > 0 and 'ref' in list_format[i - 1]:
+                    output[-1]['ref'] = list_format[i - 1]['ref'].strip()
+        return output
+
+    def draw_bbox_on_latest_picture(
+            self,
+            response,
+            history=None,
+    ) -> Optional[Image.Image]:
+        image = self._fetch_latest_picture(response, history)
+        if image is None:
+            return None
+        if image.startswith("http://") or image.startswith("https://"):
+            image = Image.open(requests.get(image, stream=True).raw).convert("RGB")
+            h, w = image.height, image.width
+        else:
+            image = np.asarray(Image.open(image).convert("RGB"))
+            h, w = image.shape[0], image.shape[1]
+        visualizer = Visualizer(image)
+
+        boxes = self._fetch_all_box_with_ref(response)
+        if not boxes:
+            return None
+        color = random.choice([_ for _ in mcolors.TABLEAU_COLORS.keys()])  # init color
+        for box in boxes:
+            if 'ref' in box:  # random new color for new refexps
+                color = random.choice([_ for _ in mcolors.TABLEAU_COLORS.keys()])
+            x1, y1, x2, y2 = box['box']
+            x1, y1, x2, y2 = (int(x1 / 1000 * w), int(y1 / 1000 * h), int(x2 / 1000 * w), int(y2 / 1000 * h))
+            visualizer.draw_box((x1, y1, x2, y2), alpha=1, edge_color=color)
+            if 'ref' in box:
+                visualizer.draw_text(box['ref'], (x1, y1), color=color, horizontal_alignment="left")
+        return visualizer.output
+
+
+class VisImage:
+    def __init__(self, img, scale=1.0):
+        self.img = img
+        self.scale = scale
+        self.width, self.height = img.shape[1], img.shape[0]
+        self._setup_figure(img)
+
+    def _setup_figure(self, img):
+        fig = mplfigure.Figure(frameon=False)
+        self.dpi = fig.get_dpi()
+        # add a small 1e-2 to avoid precision lost due to matplotlib's truncation
+        # (https://github.com/matplotlib/matplotlib/issues/15363)
+        fig.set_size_inches(
+            (self.width * self.scale + 1e-2) / self.dpi,
+            (self.height * self.scale + 1e-2) / self.dpi,
+        )
+        self.canvas = FigureCanvasAgg(fig)
+        # self.canvas = mpl.backends.backend_cairo.FigureCanvasCairo(fig)
+        ax = fig.add_axes([0.0, 0.0, 1.0, 1.0])
+        ax.axis("off")
+        self.fig = fig
+        self.ax = ax
+        self.reset_image(img)
+
+    def reset_image(self, img):
+        img = img.astype("uint8")
+        self.ax.imshow(img, extent=(0, self.width, self.height, 0), interpolation="nearest")
+
+    def save(self, filepath):
+        self.fig.savefig(filepath)
+
+    def get_image(self):
+        canvas = self.canvas
+        s, (width, height) = canvas.print_to_buffer()
+
+        buffer = np.frombuffer(s, dtype="uint8")
+
+        img_rgba = buffer.reshape(height, width, 4)
+        rgb, alpha = np.split(img_rgba, [3], axis=2)
+        return rgb.astype("uint8")
+
+
+class Visualizer:
+    def __init__(self, img_rgb, metadata=None, scale=1.0):
+        self.img = np.asarray(img_rgb).clip(0, 255).astype(np.uint8)
+        self.output = VisImage(self.img, scale=scale)
+        self.cpu_device = torch.device("cpu")
+
+        # too small texts are useless, therefore clamp to 14
+        self._default_font_size = max(
+            np.sqrt(self.output.height * self.output.width) // 30, 15 // scale
+        )
+
+    def draw_text(
+            self,
+            text,
+            position,
+            *,
+            font_size=None,
+            color="g",
+            horizontal_alignment="center",
+            rotation=0,
+    ):
+        if not font_size:
+            font_size = self._default_font_size
+
+        # since the text background is dark, we don't want the text to be dark
+        color = np.maximum(list(mplc.to_rgb(color)), 0.2)
+        color[np.argmax(color)] = max(0.8, np.max(color))
+
+        x, y = position
+        self.output.ax.text(
+            x,
+            y,
+            text,
+            size=font_size * self.output.scale,
+            bbox={"facecolor": "black", "alpha": 0.8, "pad": 0.7, "edgecolor": "none"},
+            verticalalignment="top",
+            horizontalalignment=horizontal_alignment,
+            color=color,
+            zorder=10,
+            rotation=rotation,
+        )
+        return self.output
+
+    def draw_box(self, box_coord, alpha=0.5, edge_color="g", line_style="-"):
+        x0, y0, x1, y1 = box_coord
+        width = x1 - x0
+        height = y1 - y0
+
+        linewidth = max(self._default_font_size / 4, 1)
+
+        self.output.ax.add_patch(
+            mpl.patches.Rectangle(
+                (x0, y0),
+                width,
+                height,
+                fill=False,
+                edgecolor=edge_color,
+                linewidth=linewidth * self.output.scale,
+                alpha=alpha,
+                linestyle=line_style,
+            )
+        )
+        return self.output
+
+    def get_output(self):
+        return self.output

tokenizer_config.json

@@ -0,0 +1,922 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "50256": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50257": {
+      "content": "                               ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50258": {
+      "content": "                              ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50259": {
+      "content": "                             ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50260": {
+      "content": "                            ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50261": {
+      "content": "                           ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50262": {
+      "content": "                          ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50263": {
+      "content": "                         ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50264": {
+      "content": "                        ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50265": {
+      "content": "                       ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50266": {
+      "content": "                      ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50267": {
+      "content": "                     ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50268": {
+      "content": "                    ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50269": {
+      "content": "                   ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50270": {
+      "content": "                  ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50271": {
+      "content": "                 ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50272": {
+      "content": "                ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50273": {
+      "content": "               ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50274": {
+      "content": "              ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50275": {
+      "content": "             ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50276": {
+      "content": "            ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50277": {
+      "content": "           ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50278": {
+      "content": "          ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50279": {
+      "content": "         ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50280": {
+      "content": "        ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50281": {
+      "content": "       ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50282": {
+      "content": "      ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50283": {
+      "content": "     ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50284": {
+      "content": "    ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50285": {
+      "content": "   ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50286": {
+      "content": "  ",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50287": {
+      "content": "\t\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50288": {
+      "content": "\t\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50289": {
+      "content": "\t\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50290": {
+      "content": "\t\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50291": {
+      "content": "\t\t\t\t\t",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50292": {
+      "content": "\t\t\t\t",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50293": {
+      "content": "\t\t\t",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50294": {
+      "content": "\t\t",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "50295": {
+      "content": "<|img|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50296": {
+      "content": "<|/img|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50297": {
+      "content": "<|imgpad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50298": {
+      "content": "<|ref|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50299": {
+      "content": "<|/ref|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50300": {
+      "content": "<|box|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50301": {
+      "content": "<|/box|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50302": {
+      "content": "<|quad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50303": {
+      "content": "<|/quad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50304": {
+      "content": "<|coord_0|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50305": {
+      "content": "<|coord_1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50306": {
+      "content": "<|coord_2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50307": {
+      "content": "<|coord_3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50308": {
+      "content": "<|coord_4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50309": {
+      "content": "<|coord_5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50310": {
+      "content": "<|coord_6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50311": {
+      "content": "<|coord_7|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50312": {
+      "content": "<|coord_8|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50313": {
+      "content": "<|coord_9|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50314": {
+      "content": "<|extra_0|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50315": {
+      "content": "<|extra_1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50316": {
+      "content": "<|extra_2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50317": {
+      "content": "<|extra_3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50318": {
+      "content": "<|extra_4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50319": {
+      "content": "<|extra_5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50320": {
+      "content": "<|extra_6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50321": {
+      "content": "<|extra_7|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50322": {
+      "content": "<|extra_8|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50323": {
+      "content": "<|extra_9|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50324": {
+      "content": "<|extra_10|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50325": {
+      "content": "<|extra_11|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50326": {
+      "content": "<|extra_12|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50327": {
+      "content": "<|extra_13|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50328": {
+      "content": "<|extra_14|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50329": {
+      "content": "<|extra_15|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50330": {
+      "content": "<|extra_16|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50331": {
+      "content": "<|extra_17|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50332": {
+      "content": "<|extra_18|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50333": {
+      "content": "<|extra_19|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50334": {
+      "content": "<|extra_20|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50335": {
+      "content": "<|extra_21|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50336": {
+      "content": "<|extra_22|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50337": {
+      "content": "<|extra_23|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50338": {
+      "content": "<|extra_24|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50339": {
+      "content": "<|extra_25|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50340": {
+      "content": "<|extra_26|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50341": {
+      "content": "<|extra_27|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50342": {
+      "content": "<|extra_28|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50343": {
+      "content": "<|extra_29|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50344": {
+      "content": "<|extra_30|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50345": {
+      "content": "<|extra_31|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50346": {
+      "content": "<|extra_32|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50347": {
+      "content": "<|extra_33|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50348": {
+      "content": "<|extra_34|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50349": {
+      "content": "<|extra_35|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50350": {
+      "content": "<|extra_36|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50351": {
+      "content": "<|extra_37|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50352": {
+      "content": "<|extra_38|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50353": {
+      "content": "<|extra_39|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50354": {
+      "content": "<|extra_40|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50355": {
+      "content": "<|extra_41|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50356": {
+      "content": "<|extra_42|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50357": {
+      "content": "<|extra_43|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50358": {
+      "content": "<|extra_44|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50359": {
+      "content": "<|extra_45|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50360": {
+      "content": "<|extra_46|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50361": {
+      "content": "<|extra_47|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50362": {
+      "content": "<|extra_48|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50363": {
+      "content": "<|extra_49|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50364": {
+      "content": "<|extra_50|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50365": {
+      "content": "<|extra_51|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50366": {
+      "content": "<|extra_52|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "50367": {
+      "content": "<|extra_53|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|coord_9|>"
+  ],
+  "bos_token": "<|endoftext|>",
+  "chat_template": "{% for message in messages %}\n{% if message['from'] == 'human' %}\n{{ '<|user|>\n' + message['value'] + eos_token }}\n{% elif message['from'] == 'system' %}\n{{ '<|system|>\n' + message['value'] + eos_token }}\n{% elif message['from'] == 'gpt' %}\n{{ '<|assistant|>\n'  + message['value'] + eos_token }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '<|assistant|>' }}\n{% endif %}\n{% endfor %}",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "model_max_length": 3072,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "right",
+  "tokenizer_class": "CheXagentTokenizer",
+  "unk_token": "<|endoftext|>",
+  "use_fast": false,
+  "auto_map": {
+    "AutoTokenizer": [
+      "tokenization_chexagent.CheXagentTokenizer",
+      null
+      ]
+  }
+}