jiosephlee/sft_rejection_sampling_pgb_clin_herg_Intern-s1-mini-distill-dsv32-11k-samples_lr1e-05

README.md

@@ -0,0 +1,57 @@
+---
+base_model: Kiria-Nozan/Intern-s1-mini-distill-dsv32-11k-samples
+library_name: transformers
+model_name: 2026-01-22_06-56
+tags:
+- generated_from_trainer
+- sft
+- trl
+licence: license
+---
+
+# Model Card for 2026-01-22_06-56
+
+This model is a fine-tuned version of [Kiria-Nozan/Intern-s1-mini-distill-dsv32-11k-samples](https://huggingface.co/Kiria-Nozan/Intern-s1-mini-distill-dsv32-11k-samples).
+It has been trained using [TRL](https://github.com/huggingface/trl).
+
+## Quick start
+
+```python
+from transformers import pipeline
+
+question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?"
+generator = pipeline("text-generation", model="jiosephlee/2026-01-22_06-56", device="cuda")
+output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0]
+print(output["generated_text"])
+```
+
+## Training procedure
+
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/upenn-ml/therapeutic-sft/runs/jqbteigz) 
+
+
+This model was trained with SFT.
+
+### Framework versions
+
+- TRL: 0.28.0.dev0
+- Transformers: 4.57.6
+- Pytorch: 2.9.0
+- Datasets: 4.5.0
+- Tokenizers: 0.22.1
+
+## Citations
+
+
+
+Cite TRL as:
+    
+```bibtex
+@software{vonwerra2020trl,
+  title   = {{TRL: Transformers Reinforcement Learning}},
+  author  = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin},
+  license = {Apache-2.0},
+  url     = {https://github.com/huggingface/trl},
+  year    = {2020}
+}
+```

chat_template.jinja

@@ -0,0 +1,120 @@
+{% set default_thinking_sys %}You are an expert reasoner with extensive experience in all areas. You approach problems through systematic thinking and rigorous reasoning. Your response should reflect deep understanding and precise logical thinking, making your solution path and reasoning clear to others. Please put your thinking process within <think>...</think> tags.{% endset %}
+{%- set tool_instruction %}Your response should consist of a reasoning step (**thought**) followed immediately by a function call in valid JSON format. Wrap each function call using the `<|action_start|><|plugin|>` and `<|action_end|>` tags.
+
+**Format example:**
+
+```
+(Your thought goes here...)
+
+<|action_start|><|plugin|>
+{
+    "name": "tool_name",
+    "parameters": {
+        "parameter1": "value1",
+        "parameter2": "value2"
+    }
+}
+<|action_end|>
+```
+
+# External Tools
+You have access to these tools:
+{% if tools %}{{ tools | tojson(indent=2) }}{% else %}[]{% endif %}{% endset %}
+{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
+{%- for message in messages[::-1] %}
+    {%- set index = (messages|length - 1) - loop.index0 %}
+    {%- if ns.multi_step_tool and message.role == "user" %}
+        {%- set ns.multi_step_tool = false %}
+        {%- set ns.last_query_index = index %}
+    {%- endif %}
+{%- endfor %}
+{%- for message in messages %}
+  {%- set role = message.role if message.role != 'tool' else 'environment' %}
+  {%- set reasoning_content = '' %}
+  {%- set content = message.content %}
+  {%- set ns.tool_calls = '' %}
+  {%- if role == 'assistant' %}
+    {%- if message.reasoning_content is string %}
+      {%- set reasoning_content = message.reasoning_content %}
+    {%- elif '</think>' in content %}
+      {%- set reasoning_content = content.split('</think>')[0].strip().split('<think>')[-1].strip() %}
+      {%- set content = content.split('</think>')[-1].lstrip('
+') %}
+    {%- endif %}
+    {%- if message.tool_calls %}
+      {%- for tool_call in message.tool_calls %}
+        {%- if not loop.first %}
+            {%- set ns.tool_calls = ns.tool_calls + '
+' %}
+        {%- endif %}
+        {%- if tool_call.function %}
+            {%- set tool_call = tool_call.function %}
+        {%- endif %}
+        {%- set ns.tool_calls = ns.tool_calls + '<|action_start|><|plugin|>
+{"name": "' + tool_call.name + '", "parameters": ' %}
+        {%- if tool_call.arguments is string %}
+            {%- set ns.tool_calls = ns.tool_calls + tool_call.arguments %}
+        {%- else %}
+            {%- set ns.tool_calls = ns.tool_calls + tool_call.arguments | tojson %}
+        {%- endif %}
+        {%- set ns.tool_calls = ns.tool_calls + '}
+<|action_end|>' %}
+        {%- endfor %}
+    {%- endif %}
+  {%- set reasoning_content = '<think>
+' + reasoning_content.strip('
+') + '
+</think>
+' %}
+  {%- endif %}
+  {%- if not content is string %}
+    {%- set ns.content = '' %}
+    {%- for _content in message.content %}
+      {%- if _content.type == 'image' %}
+        {%- set ns.content = ns.content ~ '
+<IMG_CONTEXT>' %}
+      {%- elif _content.type == 'video' %}
+        {%- set ns.content = ns.content ~ '
+<video>' %}
+      {%- elif _content.type == 'text' %}
+        {%- set ns.content = ns.content ~ '
+' ~ _content.text %}
+      {%- endif %}
+    {%- endfor %}
+    {%- set content = ns.content %}
+  {%- endif %}
+  {%- set content = content.lstrip('
+') %}
+  {%- if ns.tool_calls %}
+    {%- set content = content + ns.tool_calls %}
+  {%- endif %}
+  {%- if loop.index0 == 0 %}
+    {%- set system_prompt = '' %}
+    {%- if role == 'system' %}
+      {%- set system_prompt = system_prompt + content %}
+    {%- elif enable_thinking is not defined or enable_thinking %}
+      {%- set system_prompt = system_prompt + default_thinking_sys %}
+    {%- endif %}
+    {%- if tools %}
+      {%- set system_prompt = system_prompt.rstrip('
+') + '
+
+' + tool_instruction %}
+    {%- endif %}
+    {%- set system_prompt = system_prompt.strip('
+') %}
+  {%- endif %}
+{%- if loop.index0 == 0 and system_prompt %}<|im_start|>system{% if tools %} name=<|plugin|>{% endif %}
+
+{{ system_prompt }}<|im_end|>
+{% endif %}
+{%- if role != 'system' %}<|im_start|>{{ role }}{% if role == 'environment' or role == 'tool' %} name=<|plugin|>{% endif %}
+
+{% if loop.index0 > ns.last_query_index and (loop.last or (not loop.last and reasoning_content)) %}{{ reasoning_content }}
+{%- endif %}{{ content }}<|im_end|>
+{% endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}<|im_start|>assistant
+{% if enable_thinking is not defined or enable_thinking %}
+<think>{% endif %}
+{% endif %}

config.json

@@ -0,0 +1,131 @@
+{
+  "architectures": [
+    "InternS1ForConditionalGeneration"
+  ],
+  "auto_map": {
+    "AutoConfig": "configuration_interns1.InternS1Config",
+    "AutoModel": "modeling_interns1.InternS1ForConditionalGeneration",
+    "AutoModelForCausalLM": "modeling_interns1.InternS1ForConditionalGeneration"
+  },
+  "downsample_ratio": 0.5,
+  "dtype": "bfloat16",
+  "eos_token_id": 151645,
+  "hidden_size": 4096,
+  "image_seq_length": 256,
+  "image_token_id": 152957,
+  "model_type": "interns1",
+  "pad_token_id": 151643,
+  "projector_hidden_act": "gelu",
+  "text_config": {
+    "_attn_implementation_autoset": true,
+    "architectures": [
+      "Qwen3ForCausalLM"
+    ],
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "bos_token_id": 151643,
+    "dtype": "bfloat16",
+    "eos_token_id": 151645,
+    "head_dim": 128,
+    "hidden_act": "silu",
+    "hidden_size": 4096,
+    "initializer_range": 0.02,
+    "intermediate_size": 12288,
+    "layer_types": [
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention"
+    ],
+    "max_position_embeddings": 65536,
+    "max_window_layers": 36,
+    "model_type": "qwen3",
+    "num_attention_heads": 32,
+    "num_hidden_layers": 36,
+    "num_key_value_heads": 8,
+    "rms_norm_eps": 1e-06,
+    "rope_scaling": null,
+    "rope_theta": 1000000,
+    "sliding_window": null,
+    "use_cache": false,
+    "use_sliding_window": false,
+    "vocab_size": 153216
+  },
+  "transformers_version": "4.57.6",
+  "use_cache": false,
+  "vision_config": {
+    "_attn_implementation_autoset": true,
+    "architectures": [
+      "InternVisionModel"
+    ],
+    "attention_bias": true,
+    "attention_dropout": 0.0,
+    "auto_map": {
+      "AutoConfig": "configuration_interns1.InternS1VisionConfig",
+      "AutoModel": "modeling_interns1.InternS1VisionModel"
+    },
+    "drop_path_rate": 0.0,
+    "dropout": 0.0,
+    "dtype": "bfloat16",
+    "hidden_act": "gelu",
+    "hidden_dropout_prob": 0.0,
+    "hidden_size": 1024,
+    "image_size": [
+      448,
+      448
+    ],
+    "initializer_factor": 1.0,
+    "initializer_range": 0.02,
+    "intermediate_size": 4096,
+    "layer_norm_eps": 1e-06,
+    "layer_scale_init_value": 0.1,
+    "model_type": "interns1_vision",
+    "norm_type": "layer_norm",
+    "num_attention_heads": 16,
+    "num_channels": 3,
+    "num_hidden_layers": 24,
+    "patch_size": [
+      14,
+      14
+    ],
+    "projection_dropout": 0.0,
+    "use_absolute_position_embeddings": true,
+    "use_mask_token": false,
+    "use_mean_pooling": true,
+    "use_qk_norm": false
+  },
+  "vision_feature_layer": -1,
+  "vision_feature_select_strategy": "default"
+}

configuration_interns1.py

@@ -0,0 +1,225 @@
+# coding=utf-8
+# Copyright 2025 HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from transformers.configuration_utils import PretrainedConfig
+from transformers import AutoConfig
+
+
+class InternS1VisionConfig(PretrainedConfig):
+    r"""
+    This is the configuration class to store the configuration of a [`InternS1VisionModel`]. It is used to instantiate an InternS1VisionModel
+    model according to the specified arguments, defining the model architecture.
+
+    Args:
+        hidden_size (`int`, *optional*, defaults to 1024):
+            Dimensionality of the encoder layers and the pooler layer.
+        num_hidden_layers (`int`, *optional*, defaults to 24):
+            Number of hidden layers in the Transformer encoder.
+        num_attention_heads (`int`, *optional*, defaults to 16):
+            Number of attention heads for each attention layer in the Transformer encoder.
+        attention_bias (`bool`, *optional*, defaults to `False`):
+            Whether to add a bias to the queries, keys and values.
+        use_qk_norm (`bool`, *optional*, defaults to `False`):
+            Whether to apply normalization to the queries and keys before the attention operation.
+        intermediate_size (`int`, *optional*, defaults to 4096):
+            Dimensionality of the "intermediate" (i.e., feed-forward) layer in the Transformer encoder.
+        hidden_act (`str` or `function`, *optional*, defaults to `"gelu"`):
+            The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
+            `"relu"`, `"selu"` and `"gelu_new"` are supported.
+        hidden_dropout_prob (`float`, *optional*, defaults to 0.0):
+            The dropout probability for all fully connected layers in the embeddings, encoder, and pooler.
+        attention_dropout (`float`, *optional*, defaults to 0.0):
+            Dropout probability for attention weights.
+        projection_dropout (`float`, *optional*, defaults to 0.0):
+            Dropout probability for the projection layer.
+        initializer_range (`float`, *optional*, defaults to 0.02):
+            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
+        norm_type (`str`, *optional*, defaults to `"layer_norm"`):
+            The type of normalization to use in the encoder. Can be `"layer_norm"` or `"rms_norm"`.
+        layer_norm_eps (`float`, *optional*, defaults to 1e-06):
+            The epsilon used by the layer normalization layers.
+        image_size (`int` or `list[int]`, *optional*, defaults to `[448, 448]`):
+            The size (resolution) of each image.
+        patch_size (`int` or `list[int]`, *optional*, defaults to `[14, 14]`):
+            The size (resolution) of each patch.
+        num_channels (`int`, *optional*, defaults to 3):
+            The number of input channels.
+        use_mask_token (`bool`, *optional*, defaults to `False`):
+            Whether to use a mask token for masked image modeling.
+        use_absolute_position_embeddings (`bool`, *optional*, defaults to `True`):
+            Whether to use BERT-style absolute position embeddings.
+        layer_scale_init_value (`float`, *optional*, defaults to 0.1):
+            Scale to use in the self-attention layers. 0.1 for base, 1e-5 for large. Set 0 to disable layer scale.
+        use_mean_pooling (`bool`, *optional*, defaults to `True`):
+            Whether to mean pool the final hidden states of the patches instead of using the final hidden state of the
+            CLS token, before applying the classification head.
+
+    Example:
+
+    ```python
+    >>> from transformers import InternS1VisionConfig, InternS1VisionModel
+
+    >>> # Initializing a InternS1VisionModel
+    >>> configuration = InternS1VisionConfig()
+
+    >>> # Initializing a model (with random weights) from configuration
+    >>> model = InternS1VisionModel(configuration)
+
+    >>> # Accessing the model configuration
+    >>> configuration = model.config
+    ```"""
+
+    model_type = "interns1_vision"
+    base_config_key = "vision_config"
+
+    def __init__(
+        self,
+        hidden_size=1024,
+        num_hidden_layers=24,
+        num_attention_heads=16,
+        attention_bias=False,
+        use_qk_norm=False,
+        intermediate_size=4096,
+        hidden_act="gelu",
+        hidden_dropout_prob=0.0,
+        attention_dropout=0.0,
+        projection_dropout=0.0,
+        drop_path_rate=0.0,
+        initializer_range=0.02,
+        norm_type="layer_norm",
+        layer_norm_eps=1e-06,
+        image_size=[448, 448],
+        patch_size=[14, 14],
+        num_channels=3,
+        use_mask_token=False,
+        use_absolute_position_embeddings=True,
+        layer_scale_init_value=0.1,
+        use_mean_pooling=True,
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+
+        self.hidden_size = hidden_size
+        self.num_hidden_layers = num_hidden_layers
+        self.num_attention_heads = num_attention_heads
+        self.attention_bias = attention_bias
+        self.use_qk_norm = use_qk_norm
+        self.intermediate_size = intermediate_size
+        self.hidden_act = hidden_act
+        self.hidden_dropout_prob = hidden_dropout_prob
+        self.attention_dropout = attention_dropout
+        self.projection_dropout = projection_dropout
+        self.initializer_range = initializer_range
+        self.norm_type = norm_type
+        self.layer_norm_eps = layer_norm_eps
+        self.drop_path_rate = drop_path_rate
+
+        image_size = image_size if isinstance(image_size, (list, tuple)) else (image_size, image_size)
+        patch_size = patch_size if isinstance(patch_size, (list, tuple)) else (patch_size, patch_size)
+        self.image_size = image_size
+        self.patch_size = patch_size
+
+        self.num_channels = num_channels
+        self.use_mask_token = use_mask_token
+        self.use_absolute_position_embeddings = use_absolute_position_embeddings
+        self.layer_scale_init_value = layer_scale_init_value
+        self.use_mean_pooling = use_mean_pooling
+
+
+class InternS1Config(PretrainedConfig):
+    r"""
+    This is the configuration class to store the configuration of a [`InternS1ForConditionalGeneration`]. It is used to instantiate a
+    InternS1 model according to the specified arguments, defining the model architecture.
+
+    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+    documentation from [`PretrainedConfig`] for more information.
+
+
+    Args:
+        vision_config (`Union[AutoConfig, dict]`,  *optional*, defaults to `InternVisonConfig`):
+            The config object or dictionary of the vision backbone.
+        text_config (`Union[AutoConfig, dict]`, *optional*, defaults to `Qwen2Config`):
+            The config object or dictionary of the text backbone.
+        image_token_id (`int`, *optional*, defaults to 151667):
+            The image token index to encode the image prompt.
+        image_seq_length (`int`, *optional*, defaults to 256):
+            Number of image tokens to use per image patch.
+        downsample_ratio (`float`, *optional*, defaults to 0.5):
+            Factor by which to downsample the image.
+        projector_hidden_act (`str` or `function`, *optional*, defaults to `"gelu"`):
+            The non-linear activation function (function or string) in the projector.
+        vision_feature_layer (`int`, *optional*, defaults to -1):
+            The index of the layer to use as the image features.
+        vision_feature_select_strategy (`str`, *optional*, defaults to `"default"`):
+            The feature selection strategy used to select the vision feature from the vision backbone.
+            Can be one of `"default"` or `"full"`.
+
+    ```python
+    >>> from transformers import InternS1ForConditionalGeneration, InternS1Config
+
+    >>> # Initializing a InternS1 style configuration
+    >>> configuration = InternS1Config()
+
+    >>> # Initializing a model (with random weights) from configuration
+    >>> model = InternS1ForConditionalGeneration(configuration)
+
+    >>> # Accessing the model configuration
+    >>> configuration = model.config
+    ```"""
+
+    model_type = "interns1"
+    sub_configs = {"text_config": AutoConfig, "vision_config": InternS1VisionConfig}
+
+    def __init__(
+        self,
+        vision_config=None,
+        text_config=None,
+        image_token_id=151667,
+        image_seq_length=256,
+        downsample_ratio=0.5,
+        projector_hidden_act="gelu",
+        vision_feature_layer=-1,
+        vision_feature_select_strategy="default",
+        **kwargs,
+    ):  
+        from transformers import CONFIG_MAPPING
+
+        self.image_token_id = image_token_id
+        self.image_seq_length = image_seq_length
+        self.downsample_ratio = downsample_ratio
+        self.projector_hidden_act = projector_hidden_act
+        self.vision_feature_layer = vision_feature_layer
+        self.vision_feature_select_strategy = vision_feature_select_strategy
+
+        if isinstance(vision_config, dict):
+            self.vision_config = InternS1VisionConfig(**vision_config)
+        elif isinstance(vision_config, InternS1VisionConfig):
+            self.vision_config = vision_config
+        elif vision_config is None:
+            self.vision_config = InternS1VisionConfig()
+
+        if isinstance(text_config, dict):
+            text_config["model_type"] = text_config["model_type"] if "model_type" in text_config else "qwen3"
+            text_config = CONFIG_MAPPING[text_config["model_type"]](**text_config)
+        elif text_config is None:
+            text_config = CONFIG_MAPPING["qwen3"]()
+
+        self.text_config = text_config
+
+        super().__init__(**kwargs)
+
+
+__all__ = ["InternS1VisionConfig", "InternS1Config"]

generation_config.json

@@ -0,0 +1,9 @@
+{
+  "_from_model_config": true,
+  "eos_token_id": [
+    151645,
+    151645
+  ],
+  "pad_token_id": 151643,
+  "transformers_version": "4.57.6"
+}

model-00001-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:82e290d27eb6636506bb43b9e3d755ed4abd7302f68085309a49fbdc48a8c656
+size 4916843808

model-00002-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7ae96f4043458627025cd648f25e683628acf997fc34ba74984bf538d8f32292
+size 4915962480

model-00003-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d8cbd183879e82819644d01d48936704c24054ac737db460d27ee16350157d6d
+size 4915962496

model-00004-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0411daeaf11fa2a77f426b949a7acb129d9d069b79cba4e5ea8c0a16caf9c9c4
+size 2328949432

model.safetensors.index.json

@@ -0,0 +1,849 @@
+{
+  "metadata": {
+    "total_parameters": 8538804224,
+    "total_size": 17077608448
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00004-of-00004.safetensors",
+    "model.language_model.embed_tokens.weight": "model-00001-of-00004.safetensors",
+    "model.language_model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.language_model.layers.0.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.language_model.layers.0.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.language_model.layers.0.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.language_model.layers.0.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.language_model.layers.0.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
+    "model.language_model.layers.0.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.language_model.layers.0.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.language_model.layers.0.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
+    "model.language_model.layers.0.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.language_model.layers.0.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.language_model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.language_model.layers.1.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.language_model.layers.1.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.language_model.layers.1.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.language_model.layers.1.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.language_model.layers.1.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
+    "model.language_model.layers.1.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.language_model.layers.1.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.language_model.layers.1.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
+    "model.language_model.layers.1.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.language_model.layers.1.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.language_model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.10.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.10.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.10.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.10.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.10.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.10.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.10.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.10.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.10.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.10.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.11.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.11.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.11.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.11.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.11.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.11.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.11.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.11.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.11.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.11.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.12.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.12.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.12.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.12.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.12.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.12.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.12.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.12.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.12.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.12.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.13.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.13.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.13.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.13.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.13.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.13.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.13.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.13.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.13.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.13.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.14.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.14.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.14.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.14.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.14.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.14.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.14.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.14.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.14.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.14.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.14.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.15.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.15.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.15.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.15.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.15.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.15.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.15.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.15.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.15.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.15.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.15.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.16.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.16.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.16.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.16.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.16.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.16.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.16.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.16.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.16.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.16.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.16.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.17.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.17.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.17.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.17.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.17.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.17.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.17.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.17.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.17.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.17.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.17.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.18.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.18.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.18.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.18.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.18.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.18.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.18.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.18.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.18.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.18.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.18.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.19.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.19.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.19.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.19.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.19.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.19.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.19.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.19.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.19.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.19.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.19.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.language_model.layers.2.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.language_model.layers.2.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.language_model.layers.2.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.language_model.layers.2.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.language_model.layers.2.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
+    "model.language_model.layers.2.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.language_model.layers.2.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.language_model.layers.2.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
+    "model.language_model.layers.2.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.language_model.layers.2.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.language_model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.20.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.20.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.20.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.20.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.20.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.20.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.20.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.20.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.20.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.20.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.21.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.21.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.21.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.21.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.21.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.21.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.21.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.21.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.21.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.21.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.21.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.22.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.22.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.22.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.22.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.22.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.22.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.22.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.22.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.22.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.22.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.22.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.23.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.23.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.23.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.23.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.23.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.23.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.23.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.23.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.23.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.23.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.23.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.24.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.24.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.24.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.24.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.24.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.24.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.24.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.24.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.24.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.24.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.24.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.25.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.25.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.25.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.25.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.25.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.25.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.25.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.25.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.25.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.25.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.25.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.26.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.26.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.26.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.26.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.26.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.26.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.26.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.26.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.26.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.26.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.26.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.27.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.27.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.27.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.27.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.27.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.27.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.27.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.27.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.27.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.27.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.27.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.28.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.28.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.28.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.28.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.28.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.28.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.28.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.28.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.28.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.28.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.28.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.29.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.29.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.29.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.29.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.29.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.29.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.29.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.29.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.29.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.29.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.29.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.language_model.layers.3.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.language_model.layers.3.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.language_model.layers.3.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.language_model.layers.3.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.language_model.layers.3.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
+    "model.language_model.layers.3.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.language_model.layers.3.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.language_model.layers.3.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
+    "model.language_model.layers.3.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.language_model.layers.3.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.language_model.layers.30.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.30.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.30.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.30.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.30.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.30.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.30.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.30.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.30.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.30.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.30.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.31.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.31.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.31.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.31.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.31.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.31.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.31.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.31.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.31.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.31.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.31.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.32.input_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.32.mlp.down_proj.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.32.mlp.gate_proj.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.32.mlp.up_proj.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.32.post_attention_layernorm.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.32.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.32.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.32.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.32.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.32.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.32.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.33.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.language_model.layers.33.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.language_model.layers.33.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.language_model.layers.33.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.language_model.layers.33.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.language_model.layers.33.self_attn.k_norm.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.33.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.33.self_attn.o_proj.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.33.self_attn.q_norm.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.33.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.33.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.language_model.layers.34.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.language_model.layers.34.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.language_model.layers.34.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.language_model.layers.34.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.language_model.layers.34.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.language_model.layers.34.self_attn.k_norm.weight": "model-00004-of-00004.safetensors",
+    "model.language_model.layers.34.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.language_model.layers.34.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.language_model.layers.34.self_attn.q_norm.weight": "model-00004-of-00004.safetensors",
+    "model.language_model.layers.34.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.language_model.layers.34.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.language_model.layers.35.input_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.language_model.layers.35.mlp.down_proj.weight": "model-00004-of-00004.safetensors",
+    "model.language_model.layers.35.mlp.gate_proj.weight": "model-00004-of-00004.safetensors",
+    "model.language_model.layers.35.mlp.up_proj.weight": "model-00004-of-00004.safetensors",
+    "model.language_model.layers.35.post_attention_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.language_model.layers.35.self_attn.k_norm.weight": "model-00004-of-00004.safetensors",
+    "model.language_model.layers.35.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.language_model.layers.35.self_attn.o_proj.weight": "model-00004-of-00004.safetensors",
+    "model.language_model.layers.35.self_attn.q_norm.weight": "model-00004-of-00004.safetensors",
+    "model.language_model.layers.35.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.language_model.layers.35.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.language_model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.language_model.layers.4.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.language_model.layers.4.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.language_model.layers.4.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.language_model.layers.4.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.language_model.layers.4.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
+    "model.language_model.layers.4.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.language_model.layers.4.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.language_model.layers.4.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
+    "model.language_model.layers.4.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.language_model.layers.4.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.language_model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.language_model.layers.5.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.language_model.layers.5.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.language_model.layers.5.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.language_model.layers.5.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.language_model.layers.5.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
+    "model.language_model.layers.5.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.language_model.layers.5.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.language_model.layers.5.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
+    "model.language_model.layers.5.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.language_model.layers.5.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.language_model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.language_model.layers.6.mlp.down_proj.weight": "model-00001-of-00004.safetensors",
+    "model.language_model.layers.6.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.language_model.layers.6.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.language_model.layers.6.post_attention_layernorm.weight": "model-00001-of-00004.safetensors",
+    "model.language_model.layers.6.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
+    "model.language_model.layers.6.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.language_model.layers.6.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.language_model.layers.6.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
+    "model.language_model.layers.6.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.language_model.layers.6.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.language_model.layers.7.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.7.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.7.mlp.gate_proj.weight": "model-00001-of-00004.safetensors",
+    "model.language_model.layers.7.mlp.up_proj.weight": "model-00001-of-00004.safetensors",
+    "model.language_model.layers.7.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.7.self_attn.k_norm.weight": "model-00001-of-00004.safetensors",
+    "model.language_model.layers.7.self_attn.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.language_model.layers.7.self_attn.o_proj.weight": "model-00001-of-00004.safetensors",
+    "model.language_model.layers.7.self_attn.q_norm.weight": "model-00001-of-00004.safetensors",
+    "model.language_model.layers.7.self_attn.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.language_model.layers.7.self_attn.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.language_model.layers.8.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.8.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.8.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.8.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.8.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.8.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.8.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.8.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.8.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.8.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.8.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.9.mlp.down_proj.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.9.mlp.gate_proj.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.9.mlp.up_proj.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.9.post_attention_layernorm.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.9.self_attn.k_norm.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.9.self_attn.k_proj.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.9.self_attn.o_proj.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.9.self_attn.q_norm.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.9.self_attn.q_proj.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.layers.9.self_attn.v_proj.weight": "model-00002-of-00004.safetensors",
+    "model.language_model.norm.weight": "model-00004-of-00004.safetensors",
+    "model.multi_modal_projector.layer_norm.bias": "model-00001-of-00004.safetensors",
+    "model.multi_modal_projector.layer_norm.weight": "model-00001-of-00004.safetensors",
+    "model.multi_modal_projector.linear_1.bias": "model-00001-of-00004.safetensors",
+    "model.multi_modal_projector.linear_1.weight": "model-00001-of-00004.safetensors",
+    "model.multi_modal_projector.linear_2.bias": "model-00001-of-00004.safetensors",
+    "model.multi_modal_projector.linear_2.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.embeddings.cls_token": "model-00001-of-00004.safetensors",
+    "model.vision_tower.embeddings.patch_embeddings.projection.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.embeddings.patch_embeddings.projection.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.embeddings.position_embeddings": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.0.attention.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.0.attention.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.0.attention.projection_layer.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.0.attention.projection_layer.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.0.attention.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.0.attention.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.0.attention.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.0.attention.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.0.lambda_1": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.0.lambda_2": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.0.layernorm_after.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.0.layernorm_after.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.0.layernorm_before.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.0.layernorm_before.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.0.mlp.fc1.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.0.mlp.fc1.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.0.mlp.fc2.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.0.mlp.fc2.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.1.attention.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.1.attention.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.1.attention.projection_layer.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.1.attention.projection_layer.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.1.attention.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.1.attention.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.1.attention.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.1.attention.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.1.lambda_1": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.1.lambda_2": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.1.layernorm_after.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.1.layernorm_after.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.1.layernorm_before.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.1.layernorm_before.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.1.mlp.fc1.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.1.mlp.fc1.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.1.mlp.fc2.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.1.mlp.fc2.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.10.attention.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.10.attention.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.10.attention.projection_layer.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.10.attention.projection_layer.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.10.attention.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.10.attention.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.10.attention.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.10.attention.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.10.lambda_1": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.10.lambda_2": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.10.layernorm_after.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.10.layernorm_after.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.10.layernorm_before.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.10.layernorm_before.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.10.mlp.fc1.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.10.mlp.fc1.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.10.mlp.fc2.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.10.mlp.fc2.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.11.attention.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.11.attention.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.11.attention.projection_layer.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.11.attention.projection_layer.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.11.attention.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.11.attention.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.11.attention.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.11.attention.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.11.lambda_1": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.11.lambda_2": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.11.layernorm_after.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.11.layernorm_after.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.11.layernorm_before.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.11.layernorm_before.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.11.mlp.fc1.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.11.mlp.fc1.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.11.mlp.fc2.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.11.mlp.fc2.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.12.attention.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.12.attention.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.12.attention.projection_layer.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.12.attention.projection_layer.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.12.attention.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.12.attention.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.12.attention.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.12.attention.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.12.lambda_1": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.12.lambda_2": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.12.layernorm_after.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.12.layernorm_after.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.12.layernorm_before.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.12.layernorm_before.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.12.mlp.fc1.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.12.mlp.fc1.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.12.mlp.fc2.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.12.mlp.fc2.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.13.attention.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.13.attention.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.13.attention.projection_layer.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.13.attention.projection_layer.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.13.attention.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.13.attention.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.13.attention.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.13.attention.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.13.lambda_1": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.13.lambda_2": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.13.layernorm_after.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.13.layernorm_after.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.13.layernorm_before.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.13.layernorm_before.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.13.mlp.fc1.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.13.mlp.fc1.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.13.mlp.fc2.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.13.mlp.fc2.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.14.attention.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.14.attention.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.14.attention.projection_layer.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.14.attention.projection_layer.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.14.attention.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.14.attention.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.14.attention.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.14.attention.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.14.lambda_1": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.14.lambda_2": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.14.layernorm_after.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.14.layernorm_after.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.14.layernorm_before.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.14.layernorm_before.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.14.mlp.fc1.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.14.mlp.fc1.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.14.mlp.fc2.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.14.mlp.fc2.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.15.attention.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.15.attention.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.15.attention.projection_layer.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.15.attention.projection_layer.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.15.attention.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.15.attention.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.15.attention.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.15.attention.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.15.lambda_1": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.15.lambda_2": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.15.layernorm_after.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.15.layernorm_after.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.15.layernorm_before.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.15.layernorm_before.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.15.mlp.fc1.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.15.mlp.fc1.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.15.mlp.fc2.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.15.mlp.fc2.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.16.attention.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.16.attention.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.16.attention.projection_layer.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.16.attention.projection_layer.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.16.attention.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.16.attention.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.16.attention.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.16.attention.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.16.lambda_1": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.16.lambda_2": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.16.layernorm_after.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.16.layernorm_after.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.16.layernorm_before.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.16.layernorm_before.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.16.mlp.fc1.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.16.mlp.fc1.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.16.mlp.fc2.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.16.mlp.fc2.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.17.attention.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.17.attention.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.17.attention.projection_layer.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.17.attention.projection_layer.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.17.attention.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.17.attention.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.17.attention.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.17.attention.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.17.lambda_1": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.17.lambda_2": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.17.layernorm_after.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.17.layernorm_after.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.17.layernorm_before.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.17.layernorm_before.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.17.mlp.fc1.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.17.mlp.fc1.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.17.mlp.fc2.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.17.mlp.fc2.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.18.attention.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.18.attention.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.18.attention.projection_layer.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.18.attention.projection_layer.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.18.attention.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.18.attention.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.18.attention.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.18.attention.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.18.lambda_1": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.18.lambda_2": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.18.layernorm_after.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.18.layernorm_after.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.18.layernorm_before.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.18.layernorm_before.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.18.mlp.fc1.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.18.mlp.fc1.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.18.mlp.fc2.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.18.mlp.fc2.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.19.attention.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.19.attention.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.19.attention.projection_layer.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.19.attention.projection_layer.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.19.attention.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.19.attention.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.19.attention.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.19.attention.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.19.lambda_1": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.19.lambda_2": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.19.layernorm_after.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.19.layernorm_after.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.19.layernorm_before.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.19.layernorm_before.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.19.mlp.fc1.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.19.mlp.fc1.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.19.mlp.fc2.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.19.mlp.fc2.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.2.attention.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.2.attention.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.2.attention.projection_layer.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.2.attention.projection_layer.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.2.attention.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.2.attention.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.2.attention.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.2.attention.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.2.lambda_1": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.2.lambda_2": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.2.layernorm_after.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.2.layernorm_after.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.2.layernorm_before.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.2.layernorm_before.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.2.mlp.fc1.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.2.mlp.fc1.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.2.mlp.fc2.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.2.mlp.fc2.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.20.attention.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.20.attention.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.20.attention.projection_layer.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.20.attention.projection_layer.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.20.attention.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.20.attention.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.20.attention.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.20.attention.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.20.lambda_1": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.20.lambda_2": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.20.layernorm_after.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.20.layernorm_after.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.20.layernorm_before.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.20.layernorm_before.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.20.mlp.fc1.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.20.mlp.fc1.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.20.mlp.fc2.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.20.mlp.fc2.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.21.attention.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.21.attention.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.21.attention.projection_layer.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.21.attention.projection_layer.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.21.attention.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.21.attention.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.21.attention.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.21.attention.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.21.lambda_1": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.21.lambda_2": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.21.layernorm_after.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.21.layernorm_after.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.21.layernorm_before.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.21.layernorm_before.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.21.mlp.fc1.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.21.mlp.fc1.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.21.mlp.fc2.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.21.mlp.fc2.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.22.attention.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.22.attention.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.22.attention.projection_layer.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.22.attention.projection_layer.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.22.attention.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.22.attention.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.22.attention.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.22.attention.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.22.lambda_1": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.22.lambda_2": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.22.layernorm_after.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.22.layernorm_after.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.22.layernorm_before.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.22.layernorm_before.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.22.mlp.fc1.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.22.mlp.fc1.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.22.mlp.fc2.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.22.mlp.fc2.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.23.attention.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.23.attention.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.23.attention.projection_layer.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.23.attention.projection_layer.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.23.attention.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.23.attention.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.23.attention.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.23.attention.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.23.lambda_1": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.23.lambda_2": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.23.layernorm_after.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.23.layernorm_after.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.23.layernorm_before.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.23.layernorm_before.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.23.mlp.fc1.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.23.mlp.fc1.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.23.mlp.fc2.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.23.mlp.fc2.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.3.attention.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.3.attention.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.3.attention.projection_layer.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.3.attention.projection_layer.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.3.attention.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.3.attention.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.3.attention.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.3.attention.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.3.lambda_1": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.3.lambda_2": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.3.layernorm_after.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.3.layernorm_after.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.3.layernorm_before.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.3.layernorm_before.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.3.mlp.fc1.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.3.mlp.fc1.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.3.mlp.fc2.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.3.mlp.fc2.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.4.attention.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.4.attention.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.4.attention.projection_layer.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.4.attention.projection_layer.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.4.attention.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.4.attention.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.4.attention.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.4.attention.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.4.lambda_1": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.4.lambda_2": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.4.layernorm_after.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.4.layernorm_after.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.4.layernorm_before.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.4.layernorm_before.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.4.mlp.fc1.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.4.mlp.fc1.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.4.mlp.fc2.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.4.mlp.fc2.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.5.attention.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.5.attention.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.5.attention.projection_layer.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.5.attention.projection_layer.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.5.attention.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.5.attention.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.5.attention.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.5.attention.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.5.lambda_1": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.5.lambda_2": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.5.layernorm_after.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.5.layernorm_after.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.5.layernorm_before.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.5.layernorm_before.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.5.mlp.fc1.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.5.mlp.fc1.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.5.mlp.fc2.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.5.mlp.fc2.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.6.attention.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.6.attention.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.6.attention.projection_layer.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.6.attention.projection_layer.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.6.attention.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.6.attention.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.6.attention.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.6.attention.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.6.lambda_1": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.6.lambda_2": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.6.layernorm_after.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.6.layernorm_after.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.6.layernorm_before.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.6.layernorm_before.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.6.mlp.fc1.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.6.mlp.fc1.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.6.mlp.fc2.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.6.mlp.fc2.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.7.attention.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.7.attention.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.7.attention.projection_layer.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.7.attention.projection_layer.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.7.attention.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.7.attention.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.7.attention.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.7.attention.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.7.lambda_1": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.7.lambda_2": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.7.layernorm_after.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.7.layernorm_after.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.7.layernorm_before.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.7.layernorm_before.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.7.mlp.fc1.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.7.mlp.fc1.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.7.mlp.fc2.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.7.mlp.fc2.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.8.attention.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.8.attention.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.8.attention.projection_layer.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.8.attention.projection_layer.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.8.attention.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.8.attention.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.8.attention.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.8.attention.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.8.lambda_1": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.8.lambda_2": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.8.layernorm_after.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.8.layernorm_after.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.8.layernorm_before.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.8.layernorm_before.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.8.mlp.fc1.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.8.mlp.fc1.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.8.mlp.fc2.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.8.mlp.fc2.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.9.attention.k_proj.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.9.attention.k_proj.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.9.attention.projection_layer.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.9.attention.projection_layer.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.9.attention.q_proj.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.9.attention.q_proj.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.9.attention.v_proj.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.9.attention.v_proj.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.9.lambda_1": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.9.lambda_2": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.9.layernorm_after.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.9.layernorm_after.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.9.layernorm_before.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.9.layernorm_before.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.9.mlp.fc1.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.9.mlp.fc1.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.9.mlp.fc2.bias": "model-00001-of-00004.safetensors",
+    "model.vision_tower.encoder.layer.9.mlp.fc2.weight": "model-00001-of-00004.safetensors"
+  }
+}

modeling_interns1.py

@@ -0,0 +1,1200 @@
+#                🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
+#           This file was automatically generated from src/transformers/models/interns1/modular_interns1.py.
+#               Do NOT edit this file manually as any edits will be overwritten by the generation of
+#             the file from the modular. If any change should be done, please apply the change to the
+#                          modular_interns1.py file directly. One of our CI enforces this.
+#                🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨🚨
+# coding=utf-8
+# Copyright 2025 HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import collections.abc
+from dataclasses import dataclass
+from typing import Callable, Optional, Union
+import numpy as np
+import torch
+import torch.nn as nn
+
+from transformers.activations import ACT2FN
+from transformers.cache_utils import Cache
+from transformers.generation import GenerationMixin
+from transformers.integrations import use_kernel_forward_from_hub
+from transformers.modeling_flash_attention_utils import FlashAttentionKwargs
+from transformers.modeling_layers import GradientCheckpointingLayer
+from transformers.modeling_outputs import BaseModelOutput, BaseModelOutputWithPooling
+from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS, PreTrainedModel
+from transformers.processing_utils import Unpack
+from transformers.utils import (
+    ModelOutput,
+    auto_docstring,
+    can_return_tuple,
+    is_torchdynamo_compiling,
+    torch_int,
+)
+from transformers import AutoModel
+from .configuration_interns1 import InternS1Config, InternS1VisionConfig
+
+
+@use_kernel_forward_from_hub("RMSNorm")
+class InternS1VisionRMSNorm(nn.Module):
+    def __init__(self, hidden_size, eps=1e-6):
+        """
+        InternS1VisionRMSNorm is equivalent to T5LayerNorm
+        """
+        super().__init__()
+        self.weight = nn.Parameter(torch.ones(hidden_size))
+        self.variance_epsilon = eps
+
+    def forward(self, hidden_states):
+        input_dtype = hidden_states.dtype
+        hidden_states = hidden_states.to(torch.float32)
+        variance = hidden_states.pow(2).mean(-1, keepdim=True)
+        hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon)
+        return self.weight * hidden_states.to(input_dtype)
+
+    def extra_repr(self):
+        return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}"
+
+
+def eager_attention_forward(
+        module: nn.Module,
+        query: torch.Tensor,
+        key: torch.Tensor,
+        value: torch.Tensor,
+        attention_mask: Optional[torch.Tensor],
+        scaling: float,
+        dropout: float = 0.0,
+        **kwargs,
+):
+    key_states = key
+    value_states = value
+
+    attn_weights = torch.matmul(query, key_states.transpose(2, 3)) * scaling
+    if attention_mask is not None:
+        causal_mask = attention_mask[:, :, :, : key_states.shape[-2]]
+        attn_weights = attn_weights + causal_mask
+
+    # No upcasting of the attention weights to float32 in this implementation
+    attn_weights = nn.functional.softmax(attn_weights, dim=-1)
+    attn_weights = nn.functional.dropout(attn_weights, p=dropout, training=module.training)
+    attn_output = torch.matmul(attn_weights, value_states)
+    attn_output = attn_output.transpose(1, 2).contiguous()
+
+    return attn_output, attn_weights
+
+
+class InternS1VisionAttention(nn.Module):
+    """Attention Class for InternS1 Vision Encoder"""
+
+    def __init__(self, config: InternS1VisionConfig):
+        super().__init__()
+        self.config = config
+        self.embed_dim = config.hidden_size
+        self.num_heads = config.num_attention_heads
+        self.head_dim = self.embed_dim // self.num_heads
+        if self.head_dim * self.num_heads != self.embed_dim:
+            raise ValueError(
+                f"embed_dim must be divisible by num_heads (got `embed_dim`: {self.embed_dim} and `num_heads`:"
+                f" {self.num_heads})."
+            )
+        self.scale = self.head_dim ** -0.5
+        self.attention_dropout = config.attention_dropout
+        proj_dropout = config.projection_dropout
+        qk_norm = config.use_qk_norm
+
+        # Needed for flash attention
+        self.is_causal = False
+
+        self.q_proj = nn.Linear(self.embed_dim, self.num_heads * self.head_dim, bias=config.attention_bias)
+        self.k_proj = nn.Linear(self.embed_dim, self.num_heads * self.head_dim, bias=config.attention_bias)
+        self.v_proj = nn.Linear(self.embed_dim, self.num_heads * self.head_dim, bias=config.attention_bias)
+        self.projection_layer = nn.Linear(self.embed_dim, self.embed_dim)
+        self.projection_dropout = nn.Dropout(proj_dropout) if proj_dropout > 0 else nn.Identity()
+
+        self.q_norm = InternS1VisionRMSNorm(self.embed_dim) if qk_norm else nn.Identity()
+        self.k_norm = InternS1VisionRMSNorm(self.embed_dim) if qk_norm else nn.Identity()
+
+    def forward(
+            self,
+            hidden_states: torch.Tensor,
+            attention_mask: Optional[torch.Tensor] = None,
+            output_attentions: Optional[torch.Tensor] = None,
+            **kwargs: Unpack[FlashAttentionKwargs],
+    ):
+        batch_size, seq_len, _ = hidden_states.size()
+
+        query_states = self.q_proj(hidden_states)
+        key_states = self.k_proj(hidden_states)
+        value_states = self.v_proj(hidden_states)
+
+        query_states = self.q_norm(query_states)
+        key_states = self.k_norm(key_states)
+
+        query_states = query_states.reshape(batch_size, seq_len, self.num_heads, self.head_dim).transpose(1, 2)
+        key_states = key_states.reshape(batch_size, seq_len, self.num_heads, self.head_dim).transpose(1, 2)
+        value_states = value_states.view(batch_size, seq_len, self.num_heads, self.head_dim).transpose(1, 2)
+
+        attention_interface: Callable = eager_attention_forward
+        if self.config._attn_implementation != "eager":
+            attention_interface = ALL_ATTENTION_FUNCTIONS[self.config._attn_implementation]
+
+        attn_output, attn_weights = attention_interface(
+            self,
+            query_states,
+            key_states,
+            value_states,
+            attention_mask,
+            dropout=0.0 if not self.training else self.attention_dropout,
+            scaling=self.scale,
+            is_causal=False,
+            **kwargs,
+        )
+        attn_output = attn_output.reshape(batch_size, seq_len, self.embed_dim)
+
+        output = self.projection_layer(attn_output)
+        output = self.projection_dropout(output)
+
+        outputs = (output, attn_weights) if output_attentions else (output, None)
+        return outputs
+
+
+@auto_docstring
+class InternS1VisionPreTrainedModel(PreTrainedModel):
+    config_class = InternS1VisionConfig
+    base_model_prefix = "interns1_vision"
+    main_input_name = "pixel_values"
+    supports_gradient_checkpointing = True
+    _no_split_modules = ["InternS1VisionLayer"]
+    _supports_sdpa = True
+    _supports_flash_attn = True
+    _supports_flex_attn = True
+    _supports_attention_backend = True
+
+    def _init_weights(self, module):
+        """Initialize the weights"""
+        if isinstance(module, (nn.Linear, nn.Conv2d, nn.ConvTranspose2d)):
+            # Slightly different from the TF version which uses truncated_normal for initialization
+            # cf https://github.com/pytorch/pytorch/pull/5617
+            module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)
+            if module.bias is not None:
+                module.bias.data.zero_()
+        elif isinstance(module, nn.Embedding):
+            module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)
+            if module.padding_idx is not None:
+                module.weight.data[module.padding_idx].zero_()
+        elif isinstance(module, nn.LayerNorm):
+            module.bias.data.zero_()
+            module.weight.data.fill_(1.0)
+        elif isinstance(module, InternS1VisionEmbeddings):
+            module.cls_token.data.zero_()
+            if module.mask_token is not None:
+                module.mask_token.data.zero_()
+            if module.position_embeddings is not None:
+                module.position_embeddings.data.zero_()
+        elif isinstance(module, InternS1VisionLayer):
+            module.lambda_1.data.fill_(self.config.layer_scale_init_value)
+            module.lambda_2.data.fill_(self.config.layer_scale_init_value)
+
+
+@dataclass
+@auto_docstring(
+    custom_intro="""
+    Class for outputs of [`InternS1VisionModel`].
+    """
+)
+class InternS1VisionModelOutputWithPooling(BaseModelOutputWithPooling):
+    r"""
+    pooler_output (`torch.FloatTensor` of shape `(batch_size, hidden_size)`):
+        Average of the last layer hidden states of the patch tokens (excluding the *[CLS]* token) if
+        *config.use_mean_pooling* is set to True. If set to False, then the final hidden state of the *[CLS]* token
+        will be returned.
+    """
+
+
+class InternS1VisionPatchEmbeddings(nn.Module):
+    """
+    This class turns `pixel_values` of shape `(batch_size, num_channels, height, width)` into the initial
+    `hidden_states` (patch embeddings) of shape `(batch_size, seq_length, hidden_size)` to be consumed by a
+    Transformer.
+    """
+
+    def __init__(self, config):
+        super().__init__()
+        image_size, patch_size = config.image_size, config.patch_size
+        num_channels, hidden_size = config.num_channels, config.hidden_size
+
+        num_patches = (image_size[1] // patch_size[1]) * (image_size[0] // patch_size[0])
+        patch_shape = (image_size[0] // patch_size[0], image_size[1] // patch_size[1])
+        self.image_size = image_size
+        self.patch_size = patch_size
+        self.num_channels = num_channels
+        self.num_patches = num_patches
+        self.patch_shape = patch_shape
+
+        self.projection = nn.Conv2d(num_channels, hidden_size, kernel_size=patch_size, stride=patch_size)
+
+    def forward(self, pixel_values: torch.Tensor) -> torch.Tensor:
+        batch_size, num_channels, height, width = pixel_values.shape
+        if num_channels != self.num_channels:
+            raise ValueError(
+                "Make sure that the channel dimension of the pixel values match with the one set in the configuration."
+            )
+
+        embeddings = self.projection(pixel_values.to(self.projection.weight.dtype))
+        patch_height, patch_width = embeddings.shape[2], embeddings.shape[3]
+        embeddings = embeddings.flatten(2).transpose(1, 2)
+
+        return embeddings, (patch_height, patch_width)
+
+
+# Based on timm implementation, which can be found here:
+# https://github.com/rwightman/pytorch-image-models/blob/master/timm/models/vision_transformer.py
+class InternS1VisionEmbeddings(nn.Module):
+    """
+    Construct the CLS token, position and patch embeddings. Optionally, also the mask token.
+
+    """
+
+    def __init__(self, config: InternS1VisionConfig) -> None:
+        super().__init__()
+
+        self.cls_token = nn.Parameter(torch.zeros(1, 1, config.hidden_size))
+        if config.use_mask_token:
+            self.mask_token = nn.Parameter(torch.zeros(1, 1, config.hidden_size))
+        else:
+            self.mask_token = None
+        self.patch_embeddings = InternS1VisionPatchEmbeddings(config)
+        self.patch_size = config.patch_size
+        self.image_size = (
+            config.image_size
+            if isinstance(config.image_size, collections.abc.Iterable)
+            else (config.image_size, config.image_size)
+        )
+        num_patches = self.patch_embeddings.num_patches
+        if config.use_absolute_position_embeddings:
+            self.position_embeddings = nn.Parameter(torch.zeros(1, num_patches + 1, config.hidden_size))
+        else:
+            self.position_embeddings = None
+        self.dropout = nn.Dropout(config.hidden_dropout_prob)
+
+    def interpolate_pos_encoding(self, embeddings: torch.Tensor, height: int, width: int) -> torch.Tensor:
+        """
+        This method allows to interpolate the pre-trained position encodings, to be able to use the model on higher resolution
+        images. This method is also adapted to support torch.jit tracing.
+
+        Adapted from:
+        - https://github.com/facebookresearch/dino/blob/de9ee3df6cf39fac952ab558447af1fa1365362a/vision_transformer.py#L174-L194, and
+        - https://github.com/facebookresearch/dinov2/blob/e1277af2ba9496fbadf7aec6eba56e8d882d1e35/dinov2/models/vision_transformer.py#L179-L211
+        """
+
+        num_patches = embeddings.shape[1] - 1
+        num_positions = self.position_embeddings.shape[1] - 1
+
+        # always interpolate when tracing to ensure the exported model works for dynamic input shapes
+        if not torch.jit.is_tracing() and num_patches == num_positions and height == width:
+            return self.position_embeddings
+
+        class_pos_embed = self.position_embeddings[:, :1]
+        patch_pos_embed = self.position_embeddings[:, 1:]
+
+        dim = embeddings.shape[-1]
+
+        new_height = height // self.patch_size[0]
+        new_width = width // self.patch_size[1]
+
+        sqrt_num_positions = torch_int(num_positions ** 0.5)
+        patch_pos_embed = patch_pos_embed.reshape(1, sqrt_num_positions, sqrt_num_positions, dim)
+        patch_pos_embed = patch_pos_embed.permute(0, 3, 1, 2)
+
+        patch_pos_embed = nn.functional.interpolate(
+            patch_pos_embed,
+            size=(new_height, new_width),
+            mode="bicubic",
+            align_corners=False,
+        )
+
+        patch_pos_embed = patch_pos_embed.permute(0, 2, 3, 1).view(1, -1, dim)
+
+        return torch.cat((class_pos_embed, patch_pos_embed), dim=1)
+
+    def forward(
+            self,
+            pixel_values: torch.Tensor,
+            bool_masked_pos: Optional[torch.BoolTensor] = None,
+    ) -> torch.Tensor:
+        _, _, height, width = pixel_values.shape
+        embeddings, (patch_height, patch_width) = self.patch_embeddings(pixel_values)
+        batch_size, seq_len, _ = embeddings.size()
+
+        if bool_masked_pos is not None:
+            mask_tokens = self.mask_token.expand(batch_size, seq_len, -1)
+            # replace the masked visual tokens by mask_tokens
+            w = bool_masked_pos.unsqueeze(-1).type_as(mask_tokens)
+            embeddings = embeddings * (1 - w) + mask_tokens * w
+
+        cls_tokens = self.cls_token.expand(batch_size, -1, -1)
+        embeddings = torch.cat((cls_tokens, embeddings), dim=1)
+
+        if self.position_embeddings is not None:
+            embeddings = embeddings + self.interpolate_pos_encoding(embeddings, height, width)
+
+        embeddings = self.dropout(embeddings)
+
+        return embeddings, (patch_height, patch_width)
+
+
+class InternS1VisionMLP(nn.Module):
+    def __init__(self, config):
+        super().__init__()
+        self.config = config
+        self.activation_fn = ACT2FN[config.hidden_act]
+        self.fc1 = nn.Linear(config.hidden_size, config.intermediate_size)
+        self.fc2 = nn.Linear(config.intermediate_size, config.hidden_size)
+
+    def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+        hidden_states = self.fc1(hidden_states)
+        hidden_states = self.activation_fn(hidden_states)
+        hidden_states = self.fc2(hidden_states)
+        return hidden_states
+
+
+NORM2FN = {"layer_norm": nn.LayerNorm, "rms_norm": InternS1VisionRMSNorm}
+
+
+class InternS1VisionLayer(GradientCheckpointingLayer):
+    """This corresponds to the Block class in the timm implementation."""
+
+    def __init__(self, config: InternS1VisionConfig, drop_path_rate=0.0) -> None:
+        super().__init__()
+        self.chunk_size_feed_forward = config.chunk_size_feed_forward
+        self.seq_len_dim = 1
+        self.attention = InternS1VisionAttention(config)
+        self.mlp = InternS1VisionMLP(config)
+        # InternS1 uses different layernorm implementations for different models
+        self.layernorm_before = NORM2FN[config.norm_type](config.hidden_size, eps=config.layer_norm_eps)
+        self.layernorm_after = NORM2FN[config.norm_type](config.hidden_size, eps=config.layer_norm_eps)
+
+        init_values = config.layer_scale_init_value
+        self.lambda_1 = nn.Parameter(init_values * torch.ones(config.hidden_size), requires_grad=True)
+        self.lambda_2 = nn.Parameter(init_values * torch.ones(config.hidden_size), requires_grad=True)
+        self.dropout = nn.Dropout(config.hidden_dropout_prob)
+
+        if drop_path_rate > 0.0:
+            try:
+                from timm.layers import DropPath
+            except ImportError:
+                raise ImportError("timm is not installed, please install it to use DropPath by 'pip install timm'. ")
+            self.drop_path1 = DropPath(drop_path_rate)
+            self.drop_path2 = DropPath(drop_path_rate)
+        else:
+            self.drop_path1 = nn.Identity()
+            self.drop_path2 = nn.Identity()
+
+    def forward(
+            self,
+            hidden_states: torch.Tensor,
+            output_attentions: bool = False,
+    ) -> Union[tuple[torch.Tensor], tuple[torch.Tensor, torch.Tensor]]:
+        attention_output, attention_weights = self.attention(
+            self.layernorm_before(hidden_states),  # in InternS1Vision, layernorm is applied before self-attention
+            output_attentions=output_attentions,
+        )
+
+        attention_output = self.lambda_1 * attention_output
+
+        # first residual connection
+        hidden_states = self.drop_path1(attention_output) + hidden_states
+
+        # in InternS1Vision, layernorm is also applied after self-attention
+        layer_output = self.layernorm_after(hidden_states)
+
+        layer_output = self.mlp(layer_output)
+        layer_output = self.dropout(layer_output)
+
+        if self.lambda_2 is not None:
+            layer_output = self.lambda_2 * layer_output
+
+        # second residual connection
+        layer_output = self.drop_path2(layer_output) + hidden_states
+
+        return layer_output, attention_weights
+
+
+class InternS1VisionEncoder(nn.Module):
+    def __init__(self, config: InternS1VisionConfig) -> None:
+        super().__init__()
+        self.config = config
+        dpr = np.linspace(0.0, float(config.drop_path_rate), int(config.num_hidden_layers))
+        self.layer = nn.ModuleList([InternS1VisionLayer(config, dpr[idx]) for idx in range(config.num_hidden_layers)])
+
+    @can_return_tuple
+    def forward(
+            self,
+            hidden_states: torch.Tensor,
+            output_attentions: bool = False,
+            output_hidden_states: bool = False,
+    ) -> Union[tuple, BaseModelOutput]:
+        all_hidden_states = () if output_hidden_states else None
+        all_self_attentions = () if output_attentions else None
+
+        for i, layer_module in enumerate(self.layer):
+            if output_hidden_states:
+                all_hidden_states = all_hidden_states + (hidden_states,)
+
+            layer_outputs = layer_module(hidden_states, output_attentions)
+
+            hidden_states = layer_outputs[0]
+
+            if output_attentions:
+                all_self_attentions = all_self_attentions + (layer_outputs[1],)
+
+        if output_hidden_states:
+            all_hidden_states = all_hidden_states + (hidden_states,)
+
+        return BaseModelOutput(
+            last_hidden_state=hidden_states,
+            hidden_states=all_hidden_states,
+            attentions=all_self_attentions,
+        )
+
+
+@auto_docstring
+class InternS1VisionModel(InternS1VisionPreTrainedModel):
+    def __init__(self, config: InternS1VisionConfig) -> None:
+        super().__init__(config)
+        self.config = config
+
+        self.embeddings = InternS1VisionEmbeddings(config)
+        self.encoder = InternS1VisionEncoder(config)
+
+        self.layernorm = (
+            nn.Identity() if config.use_mean_pooling else nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
+        )
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    def get_input_embeddings(self):
+        return self.embeddings.patch_embeddings
+
+    @can_return_tuple
+    @auto_docstring
+    def forward(
+            self,
+            pixel_values: torch.Tensor,
+            bool_masked_pos: Optional[torch.BoolTensor] = None,
+            output_attentions: Optional[bool] = None,
+            output_hidden_states: Optional[bool] = None,
+    ) -> Union[tuple, InternS1VisionModelOutputWithPooling]:
+        r"""
+        bool_masked_pos (`torch.BoolTensor` of shape `(batch_size, num_patches)`, *optional*):
+            Boolean masked positions. Indicates which patches are masked (1) and which aren't (0).
+        """
+        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+        output_hidden_states = (
+            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+        )
+
+        embedding_output, _ = self.embeddings(pixel_values, bool_masked_pos=bool_masked_pos)
+
+        encoder_outputs = self.encoder(
+            embedding_output,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+        )
+        sequence_output = encoder_outputs[0]
+        sequence_output = self.layernorm(sequence_output)
+
+        return InternS1VisionModelOutputWithPooling(
+            last_hidden_state=sequence_output,
+            hidden_states=encoder_outputs.hidden_states,
+            attentions=encoder_outputs.attentions,
+        )
+
+
+@auto_docstring
+class InternS1PreTrainedModel(PreTrainedModel):
+    config_class = InternS1Config
+    base_model_prefix = ""
+    supports_gradient_checkpointing = True
+    _skip_keys_device_placement = "past_key_values"
+
+    _supports_flash_attn = True
+    _supports_sdpa = True
+
+    _supports_static_cache = True
+    _supports_flex_attn = True
+    _supports_attention_backend = True
+
+    def _init_weights(self, module):
+        std = getattr(self.config, "initializer_range", self.config.get_text_config().initializer_range)
+
+        if isinstance(module, nn.Linear):
+            module.weight.data.normal_(mean=0.0, std=std)
+            if module.bias is not None:
+                module.bias.data.zero_()
+        elif isinstance(module, nn.LayerNorm):
+            module.bias.data.zero_()
+            module.weight.data.fill_(1.0)
+
+
+class InternS1MultiModalProjector(nn.Module):
+    def __init__(self, config: InternS1Config):
+        super().__init__()
+        self.layer_norm = nn.LayerNorm(config.vision_config.hidden_size * int(1 / config.downsample_ratio) ** 2)
+        self.linear_1 = nn.Linear(
+            config.vision_config.hidden_size * int(1 / config.downsample_ratio) ** 2, config.text_config.hidden_size
+        )
+        self.act = ACT2FN[config.projector_hidden_act]
+        self.linear_2 = nn.Linear(config.text_config.hidden_size, config.text_config.hidden_size)
+
+    def forward(self, image_features):
+        hidden_states = self.layer_norm(image_features)
+        hidden_states = self.linear_1(hidden_states)
+        hidden_states = self.act(hidden_states)
+        hidden_states = self.linear_2(hidden_states)
+        return hidden_states
+
+
+@dataclass
+@auto_docstring(
+    custom_intro="""
+    Base class for InternS1 outputs, with hidden states and attentions.
+    """
+)
+class InternS1ModelOutputWithPast(ModelOutput):
+    """
+    Base class for model's outputs, with potential hidden states and attentions.
+
+    Args:
+        last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
+            Sequence of hidden-states at the output of the last layer of the model.
+        past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
+            It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).
+
+            Contains pre-computed hidden-states (key and values in the self-attention blocks and optionally if
+            `config.is_encoder_decoder=True` in the cross-attention blocks) that can be used (see `past_key_values`
+            input) to speed up sequential decoding.
+        hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
+            Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, +
+            one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`.
+
+            Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.
+        attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
+            Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+            sequence_length)`.
+
+            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
+            heads.
+        router_logits (`tuple(torch.FloatTensor)`, *optional*, returned when `output_router_probs=True` and `config.add_router_probs=True` is passed or when `config.output_router_probs=True`):
+            Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, sequence_length, num_experts)`.
+
+            Raw router logtis (post-softmax) that are computed by MoE routers, these terms are used to compute the auxiliary
+            loss for Mixture of Experts models.
+        image_hidden_states (`torch.FloatTensor`, *optional*):
+            A `torch.FloatTensor` of size `(batch_size, num_images, sequence_length, hidden_size)`.
+            image_hidden_states of the model produced by the vision encoder and after projecting the last hidden state.
+    """
+
+    last_hidden_state: Optional[torch.FloatTensor] = None
+    past_key_values: Optional[Cache] = None
+    hidden_states: Optional[tuple[torch.FloatTensor, ...]] = None
+    attentions: Optional[tuple[torch.FloatTensor, ...]] = None
+    router_logits: Optional[tuple[torch.FloatTensor]] = None
+    image_hidden_states: Optional[torch.FloatTensor] = None
+
+
+@auto_docstring(
+    custom_intro="""
+    The InternS1 model which consists of a vision backbone and a language model, without a language modeling head.
+    """
+)
+class InternS1Model(InternS1PreTrainedModel):
+    config_class = InternS1Config
+
+    def __init__(self, config: InternS1Config):
+        super().__init__(config)
+        self.vision_tower = InternS1VisionModel._from_config(config.vision_config)
+
+        self.multi_modal_projector = InternS1MultiModalProjector(config)
+        self.language_model = AutoModel.from_config(config.text_config)
+
+        self.is_moe_model = False
+        if hasattr(config.text_config, 'output_router_logits'):
+            self.is_moe_model = True
+
+        self.post_init()
+
+    def get_input_embeddings(self):
+        return self.language_model.get_input_embeddings()
+
+    def set_input_embeddings(self, value):
+        self.language_model.set_input_embeddings(value)
+
+    def set_decoder(self, decoder):
+        self.language_model = decoder
+
+    def get_decoder(self):
+        return self.language_model
+
+    def get_image_features(
+            self,
+            pixel_values: torch.FloatTensor,
+            vision_feature_layer: Optional[Union[int, list[int]]] = None,
+            vision_feature_select_strategy: Optional[str] = None,
+            **kwargs,
+    ):
+        """
+        Obtains image last hidden states from the vision tower and apply multimodal projection.
+
+        Args:
+            pixel_values (`torch.FloatTensor]` of shape `(batch_size, channels, height, width)`)
+               The tensors corresponding to the input images.
+            vision_feature_layer (`int` or `list[int]`):
+                Layer index or list of layer indices to extract features from.
+        Returns:
+            vision_features (`torch.Tensor`): Image feature tensor of shape `(num_images, image_length, embed_dim)`.
+        """
+        vision_feature_layer = (
+            vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer
+        )
+        vision_feature_select_strategy = (
+            vision_feature_select_strategy
+            if vision_feature_select_strategy is not None
+            else self.config.vision_feature_select_strategy
+        )
+
+        downsample_ratio = self.config.downsample_ratio
+        if vision_feature_layer == -1:
+            vision_features = self.vision_tower(pixel_values=pixel_values).last_hidden_state
+        else:
+            vision_features = self.vision_model(pixel_values=pixel_values).hidden_states[vision_feature_layer]
+        if vision_feature_select_strategy == "default":
+            vision_features = vision_features[:, 1:, :]
+
+        # Calculate dimensions based on vision features
+        channels = vision_features.shape[1]
+        feature_size = int(channels ** 0.5)
+        batch_size = vision_features.shape[0]
+
+        # Reshape tensor to spatial dimensions
+        vision_features = vision_features.reshape(batch_size, feature_size, feature_size, -1)
+
+        # Apply downsampling using pixel shuffle
+        vision_features = self.pixel_shuffle(vision_features, scale_factor=downsample_ratio)
+
+        # Reshape tensor to prepare for projection
+        vision_features = vision_features.reshape(batch_size, -1, vision_features.shape[-1])
+
+        # Project features through multi-modal projector
+        vision_features = self.multi_modal_projector(vision_features)
+        return vision_features
+
+    @can_return_tuple
+    @auto_docstring
+    def forward(
+            self,
+            input_ids: torch.LongTensor = None,
+            pixel_values: torch.FloatTensor = None,
+            attention_mask: Optional[torch.Tensor] = None,
+            position_ids: Optional[torch.LongTensor] = None,
+            past_key_values: Optional[Cache] = None,
+            inputs_embeds: Optional[torch.FloatTensor] = None,
+            vision_feature_layer: Optional[Union[int, list[int]]] = None,
+            vision_feature_select_strategy: Optional[str] = None,
+            use_cache: Optional[bool] = None,
+            output_attentions: Optional[bool] = None,
+            output_hidden_states: Optional[bool] = None,
+            output_router_logits: Optional[bool] = None,
+            return_dict: Optional[bool] = None,
+            cache_position: Optional[torch.LongTensor] = None,
+            **kwargs: Unpack[FlashAttentionKwargs],
+    ) -> InternS1ModelOutputWithPast:
+        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+        output_hidden_states = (
+            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+        )
+        if self.is_moe_model:
+            output_router_logits = (
+                output_router_logits if output_router_logits is not None else self.config.text_config.output_router_logits
+            )
+            kwargs['output_router_logits'] = output_router_logits
+
+        vision_feature_layer = (
+            vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer
+        )
+        vision_feature_select_strategy = (
+            vision_feature_select_strategy
+            if vision_feature_select_strategy is not None
+            else self.config.vision_feature_select_strategy
+        )
+
+        if (input_ids is None) ^ (inputs_embeds is not None):
+            raise ValueError("You must specify exactly one of input_ids or inputs_embeds")
+
+        if inputs_embeds is None:
+            inputs_embeds = self.get_input_embeddings()(input_ids)
+
+        if pixel_values is not None:
+            image_features = self.get_image_features(
+                pixel_values=pixel_values,
+                vision_feature_layer=vision_feature_layer,
+                vision_feature_select_strategy=vision_feature_select_strategy,
+            )
+
+            if input_ids is None:
+                special_image_mask = inputs_embeds == self.get_input_embeddings()(
+                    torch.tensor(self.config.image_token_id, dtype=torch.long, device=inputs_embeds.device)
+                )
+                special_image_mask = special_image_mask.all(-1)
+            else:
+                special_image_mask = input_ids == self.config.image_token_id
+
+            n_image_tokens = (special_image_mask).sum()
+            special_image_mask = special_image_mask.unsqueeze(-1).expand_as(inputs_embeds).to(inputs_embeds.device)
+
+            if not is_torchdynamo_compiling() and inputs_embeds[special_image_mask].numel() != image_features.numel():
+                n_image_features = image_features.shape[0] * image_features.shape[1]
+                raise ValueError(
+                    f"Image features and image tokens do not match: tokens: {n_image_tokens}, features {n_image_features}"
+                )
+            image_features = image_features.to(inputs_embeds.device, inputs_embeds.dtype)
+            inputs_embeds = inputs_embeds.masked_scatter(special_image_mask, image_features)
+
+        outputs = self.language_model(
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            past_key_values=past_key_values,
+            inputs_embeds=inputs_embeds,
+            use_cache=use_cache,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            cache_position=cache_position,
+            **kwargs,
+        )
+
+        return InternS1ModelOutputWithPast(
+            last_hidden_state=outputs.last_hidden_state,
+            past_key_values=outputs.past_key_values,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
+            router_logits=outputs.router_logits if self.is_moe_model else None,
+            image_hidden_states=image_features if pixel_values is not None else None,
+        )
+
+    def pixel_shuffle(self, vision_features: torch.Tensor, scale_factor: float = 0.5):
+        """Perform pixel shuffle downsampling on vision features.
+
+        Args:
+            vision_features (`torch.Tensor`):
+                Input tensor of shape (batch_size, width, height, channels).
+            scale_factor (`float`, *optional*, defaults to `0.5`):
+                Factor by which to downsample. Default is 0.5, which halves the dimensions.
+
+        Returns:
+            vision_features (`torch.Tensor`):
+                Downsampled tensor of shape (batch_size, height*scale_factor, width*scale_factor, channels/(scale_factor^2)).
+        """
+        batch_size, width, height, channels = vision_features.size()
+
+        if height % scale_factor != 0 or width % scale_factor != 0:
+            raise ValueError("Height and width must be divisible by scale_factor for proper downsampling.")
+
+        # Reshape to allow downsampling
+        vision_features = vision_features.view(
+            batch_size, width, int(height * scale_factor), int(channels / scale_factor)
+        )
+        # Permute dimensions to align downsampled axis correctly
+        vision_features = vision_features.permute(0, 2, 1, 3).contiguous()
+
+        # Reshape to achieve final downsampled dimensions
+        vision_features = vision_features.view(
+            batch_size, int(height * scale_factor), int(width * scale_factor), int(channels / (scale_factor ** 2))
+        )
+
+        # Swap height and width back for proper orientation
+        vision_features = vision_features.permute(0, 2, 1, 3).contiguous()
+
+        return vision_features
+
+
+@dataclass
+@auto_docstring(
+    custom_intro="""
+    Base class for InternS1 causal language model (or autoregressive) outputs.
+    """
+)
+class InternS1CausalLMOutputWithPast(ModelOutput):
+    """
+    Base class for causal language model (or autoregressive) with mixture of experts outputs.
+
+    Args:
+        loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
+            Language modeling loss (for next-token prediction).
+
+        logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
+            Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
+
+        aux_loss (`torch.FloatTensor`, *optional*, returned when `labels` is provided):
+            aux_loss for the sparse modules.
+
+        router_logits (`tuple(torch.FloatTensor)`, *optional*, returned when `output_router_probs=True` and `config.add_router_probs=True` is passed or when `config.output_router_probs=True`):
+            Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, sequence_length, num_experts)`.
+
+            Raw router logtis (post-softmax) that are computed by MoE routers, these terms are used to compute the auxiliary
+            loss for Mixture of Experts models.
+
+        past_key_values (`Cache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
+            It is a [`~cache_utils.Cache`] instance. For more details, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache).
+
+            Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
+            `past_key_values` input) to speed up sequential decoding.
+        hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
+            Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, +
+            one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`.
+
+            Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.
+        attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
+            Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
+            sequence_length)`.
+
+            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
+            heads.
+        image_hidden_states (`torch.FloatTensor`, *optional*):
+            A `torch.FloatTensor` of size `(batch_size, num_images, sequence_length, hidden_size)`.
+            image_hidden_states of the model produced by the vision encoder and after projecting the last hidden state.
+    """
+
+    loss: Optional[torch.FloatTensor] = None
+    aux_loss: Optional[torch.FloatTensor] = None
+    logits: Optional[torch.FloatTensor] = None
+    past_key_values: Optional[Cache] = None
+    hidden_states: Optional[tuple[torch.FloatTensor, ...]] = None
+    attentions: Optional[tuple[torch.FloatTensor, ...]] = None
+    router_logits: Optional[tuple[torch.FloatTensor]] = None
+    image_hidden_states: Optional[torch.FloatTensor] = None
+
+
+def load_balancing_loss_func(
+        gate_logits: Union[torch.Tensor, tuple[torch.Tensor], None],
+        num_experts: Optional[int] = None,
+        top_k=2,
+        attention_mask: Optional[torch.Tensor] = None,
+) -> Union[torch.Tensor, int]:
+    r"""
+    Computes auxiliary load balancing loss as in Switch Transformer - implemented in Pytorch.
+
+    See Switch Transformer (https://huggingface.co/papers/2101.03961) for more details. This function implements the loss
+    function presented in equations (4) - (6) of the paper. It aims at penalizing cases where the routing between
+    experts is too unbalanced.
+
+    Args:
+        gate_logits:
+            Logits from the `gate`, should be a tuple of model.config.num_hidden_layers tensors of
+            shape [batch_size X sequence_length, num_experts].
+        num_experts:
+            Number of experts
+        top_k:
+            The number of experts to route per-token, can be also interpreted as the `top-k` routing
+            parameter.
+        attention_mask (`torch.Tensor`, *optional*):
+            The attention_mask used in forward function
+            shape [batch_size X sequence_length] if not None.
+
+    Returns:
+        The auxiliary loss.
+    """
+    if gate_logits is None or not isinstance(gate_logits, tuple):
+        return 0
+
+    if isinstance(gate_logits, tuple):
+        compute_device = gate_logits[0].device
+        concatenated_gate_logits = torch.cat([layer_gate.to(compute_device) for layer_gate in gate_logits], dim=0)
+
+    routing_weights = torch.nn.functional.softmax(concatenated_gate_logits, dim=-1)
+
+    _, selected_experts = torch.topk(routing_weights, top_k, dim=-1)
+
+    expert_mask = torch.nn.functional.one_hot(selected_experts, num_experts)
+
+    if attention_mask is None:
+        # Compute the percentage of tokens routed to each experts
+        tokens_per_expert = torch.mean(expert_mask.float(), dim=0)
+
+        # Compute the average probability of routing to these experts
+        router_prob_per_expert = torch.mean(routing_weights, dim=0)
+    else:
+        batch_size, sequence_length = attention_mask.shape
+        num_hidden_layers = concatenated_gate_logits.shape[0] // (batch_size * sequence_length)
+
+        # Compute the mask that masks all padding tokens as 0 with the same shape of expert_mask
+        expert_attention_mask = (
+            attention_mask[None, :, :, None, None]
+            .expand((num_hidden_layers, batch_size, sequence_length, top_k, num_experts))
+            .reshape(-1, top_k, num_experts)
+            .to(compute_device)
+        )
+
+        # Compute the percentage of tokens routed to each experts
+        tokens_per_expert = torch.sum(expert_mask.float() * expert_attention_mask, dim=0) / torch.sum(
+            expert_attention_mask, dim=0
+        )
+
+        # Compute the mask that masks all padding tokens as 0 with the same shape of tokens_per_expert
+        router_per_expert_attention_mask = (
+            attention_mask[None, :, :, None]
+            .expand((num_hidden_layers, batch_size, sequence_length, num_experts))
+            .reshape(-1, num_experts)
+            .to(compute_device)
+        )
+
+        # Compute the average probability of routing to these experts
+        router_prob_per_expert = torch.sum(routing_weights * router_per_expert_attention_mask, dim=0) / torch.sum(
+            router_per_expert_attention_mask, dim=0
+        )
+
+    overall_loss = torch.sum(tokens_per_expert * router_prob_per_expert.unsqueeze(0))
+    return overall_loss * num_experts
+
+
+@auto_docstring(
+    custom_intro="""
+    The INTERNS1 model which consists of a vision backbone and a language model.
+    """
+)
+class InternS1ForConditionalGeneration(InternS1PreTrainedModel, GenerationMixin):
+    config_class = InternS1Config
+    _tied_weights_keys = ["lm_head.weight"]
+
+    def __init__(self, config: InternS1Config):
+        super().__init__(config)
+        self.model = InternS1Model(config)
+        self.lm_head = nn.Linear(config.text_config.hidden_size, config.text_config.vocab_size, bias=False)
+
+        self.is_moe_model = False
+        if hasattr(config.text_config, 'output_router_logits'):
+            self.is_moe_model = True
+        self.post_init()
+
+    def get_input_embeddings(self):
+        return self.model.get_input_embeddings()
+
+    def set_input_embeddings(self, value):
+        self.model.set_input_embeddings(value)
+
+    def get_output_embeddings(self) -> nn.Module:
+        return self.lm_head
+
+    def set_output_embeddings(self, new_embeddings):
+        self.lm_head = new_embeddings
+
+    def set_decoder(self, decoder):
+        self.model.set_decoder(decoder)
+
+    def get_decoder(self):
+        return self.model.get_decoder
+
+    def get_image_features(
+            self,
+            pixel_values: torch.FloatTensor,
+            vision_feature_layer: Optional[Union[int, list[int]]] = None,
+            vision_feature_select_strategy: Optional[str] = None,
+            **kwargs,
+    ):
+        return self.model.get_image_features(
+            pixel_values=pixel_values,
+            vision_feature_layer=vision_feature_layer,
+            vision_feature_select_strategy=vision_feature_select_strategy,
+            **kwargs,
+        )
+
+    # Make modules available throught conditional class for BC
+    @property
+    def language_model(self):
+        return self.model.language_model
+
+    @property
+    def vision_tower(self):
+        return self.model.vision_tower
+
+    @property
+    def multi_modal_projector(self):
+        return self.model.multi_modal_projector
+
+    @can_return_tuple
+    @auto_docstring
+    def forward(
+            self,
+            input_ids: torch.LongTensor = None,
+            pixel_values: torch.FloatTensor = None,
+            attention_mask: Optional[torch.Tensor] = None,
+            position_ids: Optional[torch.LongTensor] = None,
+            past_key_values: Optional[Cache] = None,
+            inputs_embeds: Optional[torch.FloatTensor] = None,
+            vision_feature_layer: Optional[Union[int, list[int]]] = None,
+            vision_feature_select_strategy: Optional[str] = None,
+            labels: Optional[torch.LongTensor] = None,
+            use_cache: Optional[bool] = None,
+            output_attentions: Optional[bool] = None,
+            output_hidden_states: Optional[bool] = None,
+            output_router_logits: Optional[bool] = None,
+            return_dict: Optional[bool] = None,
+            cache_position: Optional[torch.LongTensor] = None,
+            logits_to_keep: Union[int, torch.Tensor] = 0,
+            image_sizes: Optional[torch.Tensor] = None,
+            **kwargs,
+    ) -> Union[tuple, InternS1CausalLMOutputWithPast]:
+        r"""
+        Example:
+
+        ```python
+        >>> import torch
+        >>> from transformers import AutoProcessor, AutoModelForImageTextToText
+
+        >>> torch_device = "cuda"
+        >>> processor = AutoProcessor.from_pretrained("InternLM/InternS1") # todo
+        >>> model = AutoModelForImageTextToText.from_pretrained(
+        ...     "InternLM/InternS1", torch_dtype=torch.bfloat16, device_map=torch_device
+        ... )
+
+        >>> messages = [
+        ...     {
+        ...         "role": "user",
+        ...         "content": [
+        ...             {
+        ...                 "type": "image",
+        ...                 "url": "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg",
+        ...             },
+        ...             {
+        ...                 "type": "image",
+        ...                 "url": "https://thumbs.dreamstime.com/b/golden-gate-bridge-san-francisco-purple-flowers-california-echium-candicans-36805947.jpg",
+        ...             },
+        ...             {"type": "text", "text": "These images depict two different landmarks. Can you identify them?"},
+        ...         ],
+        ...     },
+        ... ]
+
+        >>> inputs = processor.apply_chat_template(messages, add_generation_prompt=True, tokenize=True, return_dict=True, return_tensors="pt").to(torch_device)
+        >>> generate_ids = model.generate(**inputs, max_new_tokens=200)
+        >>> print(processor.decode(generate_ids[0, inputs["input_ids"].shape[1] :], skip_special_tokens=True))
+        The images depict the Statue of Liberty and the Golden Gate Bridge.
+        ```"""
+        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+        output_hidden_states = (
+            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+        )
+
+        if self.is_moe_model:
+            output_router_logits = (
+                output_router_logits if output_router_logits is not None else self.config.text_config.output_router_logits
+            )
+            kwargs['output_router_logits'] = output_router_logits
+
+        vision_feature_layer = (
+            vision_feature_layer if vision_feature_layer is not None else self.config.vision_feature_layer
+        )
+        vision_feature_select_strategy = (
+            vision_feature_select_strategy
+            if vision_feature_select_strategy is not None
+            else self.config.vision_feature_select_strategy
+        )
+
+        outputs = self.model(
+            input_ids=input_ids,
+            pixel_values=pixel_values,
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            past_key_values=past_key_values,
+            inputs_embeds=inputs_embeds,
+            vision_feature_layer=vision_feature_layer,
+            vision_feature_select_strategy=vision_feature_select_strategy,
+            use_cache=use_cache,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            cache_position=cache_position,
+            image_sizes=image_sizes,
+            **kwargs,
+        )
+
+        hidden_states = outputs.last_hidden_state
+        # Only compute necessary logits, and do not upcast them to float if we are not computing the loss
+        slice_indices = slice(-logits_to_keep, None) if isinstance(logits_to_keep, int) else logits_to_keep
+        logits = self.lm_head(hidden_states[:, slice_indices, :])
+
+        loss = None
+        if labels is not None:
+            loss = self.loss_function(
+                logits=logits, labels=labels, vocab_size=self.config.text_config.vocab_size, **kwargs
+            )
+
+        aux_loss = None
+        if self.is_moe_model and output_router_logits and labels is not None:
+            aux_loss = load_balancing_loss_func(
+                outputs.router_logits,
+                self.config.text_config.num_experts,
+                self.config.text_config.num_experts_per_tok,
+                attention_mask,
+            )
+            loss += self.config.text_config.router_aux_loss_coef * aux_loss.to(loss.device)
+
+        return InternS1CausalLMOutputWithPast(
+            loss=loss,
+            aux_loss=aux_loss,
+            logits=logits,
+            past_key_values=outputs.past_key_values,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
+            router_logits=outputs.router_logits if self.is_moe_model else None,
+            image_hidden_states=outputs.image_hidden_states,
+        )
+
+    def prepare_inputs_for_generation(
+            self,
+            input_ids,
+            past_key_values=None,
+            inputs_embeds=None,
+            pixel_values=None,
+            attention_mask=None,
+            cache_position=None,
+            logits_to_keep=None,
+            **kwargs,
+    ):
+        # Overwritten -- in specific circumstances we don't want to forward image inputs to the model
+
+        model_inputs = super().prepare_inputs_for_generation(
+            input_ids,
+            past_key_values=past_key_values,
+            inputs_embeds=inputs_embeds,
+            attention_mask=attention_mask,
+            cache_position=cache_position,
+            logits_to_keep=logits_to_keep,
+            **kwargs,
+        )
+
+        if cache_position[0] == 0:
+            # If we're in cached decoding stage, pixel values should be None because input ids do not contain special image token anymore
+            # Otherwise we need pixel values to be passed to model
+            model_inputs["pixel_values"] = pixel_values
+
+        return model_inputs
+
+
+__all__ = [
+    "InternS1VisionPreTrainedModel",
+    "InternS1VisionModel",
+    "InternS1PreTrainedModel",
+    "InternS1Model",
+    "InternS1ForConditionalGeneration",
+]

run.log

@@ -0,0 +1,24 @@
+2026-01-22 06:56:15,580 - __main__ - INFO - Loading model: Kiria-Nozan/Intern-s1-mini-distill-dsv32-11k-samples
+2026-01-22 06:56:15,580 - __main__ - INFO - Output directory: /vast/home/j/jojolee/therapeutic-tuning/results/sft/rejection_sampling_pgb_clin_herg/sft_rejection_sampling_pgb_clin_herg_Intern-s1-mini-distill-dsv32-11k-samples_lr1e-05/2026-01-22_06-56
+2026-01-22 06:56:15,580 - __main__ - INFO - Datasets: ['rejection_sampling_pgb_clin_herg']
+2026-01-22 06:56:16,052 - __main__ - INFO - Loading model 'Kiria-Nozan/Intern-s1-mini-distill-dsv32-11k-samples' with attn_implementation='flash_attention_2'
+2026-01-22 06:56:16,795 - accelerate.utils.modeling - INFO - We will use 90% of the memory on device 0 for storing the model, and 10% for the buffer to avoid OOM. You can set `max_memory` in to a higher value to use more memory (at your own risk).
+2026-01-22 06:56:20,038 - __main__ - INFO - Loading dataset 'rejection_sampling_pgb_clin_herg' from LoaderRegistry...
+2026-01-22 06:56:20,039 - data.loaders.sft.rejection_sampling - INFO - Loading trajectories from task: Pgp_Broccatelli
+2026-01-22 06:56:20,120 - data.loaders.sft.rejection_sampling - INFO -   Loaded 798 examples from Pgp_Broccatelli
+2026-01-22 06:56:20,120 - data.loaders.sft.rejection_sampling - INFO - Loading trajectories from task: ClinTox
+2026-01-22 06:56:20,233 - data.loaders.sft.rejection_sampling - INFO -   Loaded 949 examples from ClinTox
+2026-01-22 06:56:20,233 - data.loaders.sft.rejection_sampling - INFO - Loading trajectories from task: hERG
+2026-01-22 06:56:20,280 - data.loaders.sft.rejection_sampling - INFO -   Loaded 420 examples from hERG
+2026-01-22 06:56:20,694 - data.loaders.sft.rejection_sampling - INFO - Total examples after filtering: 2167
+2026-01-22 06:56:20,695 - __main__ - INFO -   -> Loaded 2167 examples from 'rejection_sampling_pgb_clin_herg'
+2026-01-22 06:56:20,740 - __main__ - INFO - Filtered out 3 traces exceeding ~32768 tokens
+2026-01-22 06:56:20,740 - __main__ - INFO - Total dataset size: 2164 examples
+2026-01-22 06:56:20,740 - __main__ - INFO - Training mode: completion_only
+2026-01-22 06:56:20,740 - __main__ - INFO -   dataset_text_field=None, completion_only_loss=True, assistant_only_loss=False
+2026-01-22 06:57:58,224 - liger_kernel.transformers.monkey_patch - INFO - There are currently no Liger kernels supported for model type: interns1.
+2026-01-22 06:57:58,234 - __main__ - INFO - Verifying dataloader integrity...
+2026-01-22 06:57:58,235 - __main__ - INFO - # of Batches: 249
+2026-01-22 06:58:07,945 - __main__ - INFO - Training batch stats - Avg samples per batch: 8.69, Min: 5, Max: 12
+2026-01-22 06:58:07,945 - __main__ - INFO - Starting training...
+2026-01-22 07:27:44,000 - __main__ - INFO - Pushing model to HuggingFace Hub: jiosephlee/sft_rejection_sampling_pgb_clin_herg_Intern-s1-mini-distill-dsv32-11k-samples_lr1e-05

special_tokens_map.json

@@ -0,0 +1,35 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "context_image_token": "<IMG_CONTEXT>",
+  "end_image_token": "</img>",
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "start_image_token": "<img>",
+  "video_token": "<video>"
+}

tokenization_interns1.py

@@ -0,0 +1,978 @@
+# coding=utf-8
+# Copyright 2025 The Intern team and Shanghai AI Lab team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Tokenization classes for InternS1."""
+
+from typing import Union, Dict, List, Optional, Tuple
+import json
+import os
+from functools import lru_cache
+from abc import ABC, abstractmethod
+import regex as re
+
+import sentencepiece as spm
+from collections import OrderedDict
+
+from transformers.tokenization_utils_base import AddedToken, TextInput
+from transformers.models.qwen2.tokenization_qwen2 import Qwen2Tokenizer
+from transformers.utils import logging
+
+
+logger = logging.get_logger(__name__)
+
+try:
+    from rdkit import Chem
+    from rdkit import RDLogger
+
+    RDLogger.DisableLog("rdApp.error")
+    RDLogger.DisableLog("rdApp.*")
+    RDKIT_AVAILABLE = True
+except ImportError:
+    logger.warning_once(
+        f"If tokenization with SMILES formula is of necessity, please 'pip install RDKit' for better tokenization quality."
+    )
+    RDKIT_AVAILABLE = False
+
+VOCAB_FILES_NAMES = {
+    "vocab_file": "vocab.json",
+    "merges_file": "merges.txt",
+    "sp_model_SMILES": "tokenizer_SMILES.model",
+    "sp_model_IUPAC": "tokenizer_IUPAC.model",
+    "sp_model_FASTA": "tokenizer_FASTA.model",
+}
+
+PRETOKENIZE_REGEX = r"""(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+"""
+
+
+class InternS1CheckModuleMixin(ABC):
+    """
+    Basic auto-detection module.
+
+    Note that short strings are ignored by this module.
+    """
+    def __init__(self, *, min_length: int):
+        self.min_length = min_length
+        self.REGEX = self._build_regex()
+        self.auto_detect_token = []
+        self.truncation = False
+
+    @abstractmethod
+    def _build_regex(self):
+        pass
+
+    @abstractmethod
+    def check_legitimacy(self, candidate: str) -> bool:
+        pass
+
+    def re_split(self, texts: Union[str, List[str]]) -> List[str]:
+        if isinstance(texts, str):
+            texts = [texts]
+
+        total_results = []
+
+        for text in texts:
+            results = []
+            current_pos = 0
+            for match in self.REGEX.finditer(text):
+                candidate = match.group(1)
+
+                if len(candidate) >= self.min_length:
+                    match_start, match_end = match.span(1)
+
+                    if not self.check_legitimacy(candidate):
+                        continue
+
+                    if not self.truncation:
+                        if match_start > 0 and text[match_start - 1].encode("UTF-8").isalpha():
+                            continue
+                        if match_end < len(text) and text[match_end].encode("UTF-8").isalpha():
+                            continue
+
+                    if match_start > current_pos:
+                        non_candidate_part = text[current_pos:match_start]
+                        results.append(non_candidate_part)
+                else:
+                    continue
+
+                results.extend([self.auto_detect_token[0], candidate, self.auto_detect_token[1]])
+                current_pos = match_end
+
+            if current_pos < len(text):
+                remaining_part = text[current_pos:]
+                results.append(remaining_part)
+
+            total_results.extend(results)
+
+        return total_results
+
+
+class FastaCheckModule(InternS1CheckModuleMixin):
+    """
+    Protein sequence auto-detection module.
+
+    Automatically detects protein sequence using regex patterns.
+    """
+    def __init__(self, *, min_length: int = 27):
+        super().__init__(min_length=min_length)
+        self.auto_detect_token = ["<FASTA_AUTO_DETECT>", "</FASTA_AUTO_DETECT>"]
+        self.truncation = True
+
+    def _build_regex(self):
+        return re.compile(r"([A-Z]{" + str(self.min_length) + r",})")
+
+    def check_legitimacy(self, candidate: str):
+        return True
+
+
+bonds = ["-", "=", "#", ":", "/", "\\", ".", "$"]
+organic_symbols = ["B", "C", "N", "O", "P", "S", "F", "Cl", "Br", "I"]
+other_allows = bonds + ["[", "]", "(", ")", ";"]
+aromatic_symbols = ["b", "c", "n", "o", "s", "p"]
+elements = [
+    "H", "He", "Li", "Be", "B", "C", "N", "O", "F", "Ne",
+    "Na", "Mg", "Al", "Si", "P", "S", "Cl", "Ar", "K", "Ca",
+    "Sc", "Ti", "V", "Cr", "Mn", "Fe", "Co", "Ni", "Cu", "Zn",
+    "Ga", "Ge", "As", "Se", "Br", "Kr", "Rb", "Sr", "Y", "Zr",
+    "Nb", "Mo", "Tc", "Ru", "Rh", "Pd", "Ag", "Cd", "In", "Sn",
+    "Sb", "Te", "I", "Xe", "Cs", "Ba", "La", "Ce", "Pr", "Nd",
+    "Pm", "Sm", "Eu", "Gd", "Tb", "Dy", "Ho", "Er", "Tm", "Yb",
+    "Lu", "Hf", "Ta", "W", "Re", "Os", "Ir", "Pt", "Au", "Hg",
+    "Tl", "Pb", "Bi", "Po", "At", "Rn", "Fr", "Ra", "Ac", "Th",
+    "Pa", "U", "Np", "Pu", "Am", "Cm", "Bk", "Cf", "Es", "Fm",
+    "Md", "No", "Lr", "Rf", "Db", "Sg", "Bh", "Hs", "Mt", "Ds",
+    "Rg", "Cn", "Nh", "Fl", "Mc", "Lv", "Ts", "Og"
+]
+
+
+class SmilesCheckModule(InternS1CheckModuleMixin):
+    """
+    SMILES molecular sequence auto-detection module.
+
+    Automatically detects and validates SMILES strings in text using regex patterns
+    or chemical syntax rules. Uses RDKit for precise validation when available,
+    otherwise falls back to rule-based validation.
+    """
+    def __init__(self, *, min_length: int = 10):
+        super().__init__(min_length=min_length)
+        self.auto_detect_token = ["<SMILES_AUTO_DETECT>", "</SMILES_AUTO_DETECT>"]
+        self._SQ_BRACKET_BAN_1 = re.compile(r'(?:[A-GI-Z]|[a-z]){3,}')
+        self._SQ_BRACKET_BAN_2 = re.compile(r'\d{4,}')
+
+    def _build_regex(self):
+        _two_letter_elements = [
+            'Ac', 'Ag', 'Al', 'Am', 'Ar', 'As', 'At', 'Au', 'Ba', 'Be', 'Bh', 'Bi', 'Bk', 'Br', 'Ca', 'Cd',
+            'Ce', 'Cf', 'Cl', 'Cm', 'Cn', 'Co', 'Cr', 'Cs', 'Cu', 'Db', 'Ds', 'Dy', 'Er', 'Es', 'Eu', 'Fe',
+            'Fl', 'Fm', 'Fr', 'Ga', 'Gd', 'Ge', 'He', 'Hf', 'Hg', 'Ho', 'Hs', 'In', 'Ir', 'Kr', 'La', 'Li',
+            'Lr', 'Lu', 'Lv', 'Mc', 'Md', 'Mg', 'Mn', 'Mo', 'Mt', 'Na', 'Nb', 'Nd', 'Ne', 'Nh', 'Ni', 'No',
+            'Np', 'Og', 'Os', 'Pa', 'Pb', 'Pd', 'Pm', 'Po', 'Pr', 'Pt', 'Pu', 'Ra', 'Rb', 'Re', 'Rf', 'Rg',
+            'Rh', 'Rn', 'Ru', 'Sb', 'Sc', 'Se', 'Sg', 'Si', 'Sm', 'Sn', 'Sr', 'Ta', 'Tb', 'Tc', 'Te', 'Th',
+            'Ti', 'Tl', 'Tm', 'Ts', 'Xe', 'Yb', 'Zn', 'Zr'
+        ]
+        _single_letter_elements = [
+            "B", "C", "F", "H", "I", "K", "N", "O", "P", "S", "U", "V", "W", "Y", 'b', 'c', 'n', 'o', 'p', 's'
+        ]
+        all_elements_sorted = sorted(_two_letter_elements + _single_letter_elements, key=lambda x: (-len(x), x))
+        elements_pattern_str = "|".join(all_elements_sorted)
+
+        bracket_atom_pattern_str = r"\[[^\]]+\]"
+        other_single_chars_pattern_str = r"[\(\)\.=\-#@\d\$\%\*:\+\-\/\\]"
+        smiles_unit_pattern = (
+            r"(?:"
+            + bracket_atom_pattern_str
+            + r"|"
+            + elements_pattern_str
+            + r"|"
+            + other_single_chars_pattern_str
+            + r")"
+        )
+        core_sequence_pattern = rf"(?>{smiles_unit_pattern}){{10,}}"
+        constrained_core_sequence_pattern = rf"(?![:.=]){core_sequence_pattern}(?<![:.=])"
+
+        final_regex_str = rf"({constrained_core_sequence_pattern})"
+
+        COMPILED_REGEX = re.compile(final_regex_str)
+        return COMPILED_REGEX
+
+    def check_legitimacy_slow(self, candidate: str) -> bool:
+        """Check legitimacy with RDKit"""
+        if sum(1 for char in candidate if char.encode("UTF-8").isalpha()) < 5:
+            return False
+
+        mol = Chem.MolFromSmiles(candidate)
+        if mol is None:
+            return False
+        else:
+            return True
+
+    def check_legitimacy_fast(self, candidate: str) -> bool:
+        """Check legitimacy with hard rules"""
+        if sum(1 for char in candidate if char.encode("UTF-8").isalpha()) < 5:
+            return False
+
+        if not self.check_rings_and_brackets(candidate):
+            return False
+        else:
+            return True
+
+    def check_legitimacy(self, candidate: str) -> bool:
+        if RDKIT_AVAILABLE:
+            return self.check_legitimacy_slow(candidate)
+        else:
+            return self.check_legitimacy_fast(candidate)
+
+    def check_brackets(self, text):
+        matches = re.findall(r"\[([^\[\]]*)\]", text)
+        for part in matches:
+            if "(" in part or ")" in part:
+                return False
+            if len(part) == 0:
+                return False
+            if part[0] in elements or part[0] in aromatic_symbols or part[:2] in elements:
+                return True
+        return True
+
+    def check_rings_and_brackets(self, text):
+        rings = {}
+        left_sq_bracket, right_sq_bracket = 0, 0
+        left_pt_bracket, right_pt_bracket = 0, 0
+        all_lower = True
+        digits_cnt = 0
+        pos = 0
+        while pos < len(text):
+            step = 0
+            c = text[pos]
+            if ord(c) >= 65 and ord(c) <= 90:
+                all_lower = False
+            if (pos == len(text) - 1 or pos == 0) and c in bonds:
+                return False
+            if pos > 0 and text[pos - 1] in bonds and text[pos] in bonds:
+                return False
+            if c == "[":
+                step = 1
+                left_sq_bracket += 1
+                if left_sq_bracket > right_sq_bracket + 1:
+                    return False
+                if pos == len(text)-1:
+                    return False
+                if ']' not in text[pos+1:]:
+                    return False
+                bracket_span = text[pos+1:text.find(']')]
+
+                if self._SQ_BRACKET_BAN_1.search(bracket_span) or self._SQ_BRACKET_BAN_2.search(bracket_span):
+                    return False
+
+                matches = re.findall(r'\d+', bracket_span)
+                if len(matches)>2:
+                    return False
+            if c == "]":
+                step = 1
+                right_sq_bracket += 1
+                if right_sq_bracket > left_sq_bracket:
+                    return False
+
+            if c == "(":
+                step = 1
+                left_pt_bracket += 1
+            if c == ")":
+                step = 1
+                right_pt_bracket += 1
+                if right_pt_bracket > left_pt_bracket:
+                    return False
+
+            if left_sq_bracket == right_sq_bracket:
+                if c.isdigit():
+                    digits_cnt += 1
+                    step = 1
+                    if (
+                        pos == 0
+                        or (pos == 1 and text[pos - 1] != "%")
+                        or (pos > 1 and text[pos - 1] != "%" and text[pos - 2] != "%")
+                    ):
+                        if c in rings:
+                            if rings[c] == "unclosed":
+                                rings[c] = "closed"
+                            else:
+                                rings[c] = "unclosed"
+                        else:
+                            rings[c] = "unclosed"
+                if c == "%":
+                    if pos >= len(text) - 2 or not text[pos + 1].isdigit() or not text[pos + 2].isdigit():
+                        return False
+                    step = 3
+                    digits_cnt += 1
+                    num = text[pos + 1 : pos + 3]
+                    if num in rings:
+                        if rings[num] == "unclosed":
+                            rings[num] = "closed"
+                        else:
+                            rings[num] = "unclosed"
+                    else:
+                        rings[num] = "unclosed"
+                if step == 0:
+                    if (
+                        pos < len(text) - 1
+                        and text[pos : pos + 2] in organic_symbols + aromatic_symbols + other_allows
+                    ):
+                        step = 2
+                    elif c in organic_symbols + aromatic_symbols + other_allows:
+                        step = 1
+                    else:
+                        return False
+
+            if step == 0:
+                step = 1
+            pos += step
+
+        if left_sq_bracket != right_sq_bracket or any(v == "unclosed" for v in rings.values()):
+            return False
+        if all_lower and digits_cnt < 2:
+            return False
+        return self.check_brackets(text)
+
+
+class InternS1Tokenizer(Qwen2Tokenizer):
+    """
+    Construct an InternS1 tokenizer. Based on byte-level Byte-Pair-Encoding.
+
+    Same with GPT2Tokenizer, this tokenizer has been trained to treat spaces like parts of the tokens so a word will
+    be encoded differently whether it is at the beginning of the sentence (without space) or not:
+
+    ```python
+    >>> from transformers import AutoTokenizer
+
+    >>> tokenizer = AutoTokenizer.from_pretrained("InternS1Tokenizer", trust_remote_code=True)
+    >>> tokenizer("Hello world")["input_ids"]
+    [9707, 1879]
+
+    >>> tokenizer(" Hello world")["input_ids"]
+    [21927, 1879]
+    ```
+    This is expected.
+
+    Include custom extension to support better domain-specific text tokenization, leveraging a separately trained tokenizer model.
+    Users should refer to this superclass [`PreTrainedTokenizer`] for more information regarding those overloaded methods
+
+    Args:
+        vocab_file (`str`):
+            Path to the vocabulary file.
+        merges_file (`str`):
+            Path to the merges file.
+        errors (`str`, *optional*, defaults to `"replace"`):
+            Paradigm to follow when decoding bytes to UTF-8. See
+            [bytes.decode](https://docs.python.org/3/library/stdtypes.html#bytes.decode) for more information.
+        unk_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
+            The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
+            token instead.
+        bos_token (`str`, *optional*):
+            The beginning of sequence token. Not applicable for this tokenizer.
+        eos_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
+            The end of sequence token.
+        pad_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
+            The token used for padding, for example when batching sequences of different lengths.
+        clean_up_tokenization_spaces (`bool`, *optional*, defaults to `False`):
+            Whether or not the model should cleanup the spaces that were added when splitting the input text during the
+            tokenization process. Not applicable to this tokenizer, since tokenization does not add spaces.
+        split_special_tokens (`bool`, *optional*, defaults to `False`):
+            Whether or not the special tokens should be split during the tokenization process. The default behavior is
+            to not split special tokens. This means that if `<|endoftext|>` is the `eos_token`, then `tokenizer.tokenize("<|endoftext|>") =
+            ['<|endoftext|>`]. Otherwise, if `split_special_tokens=True`, then `tokenizer.tokenize("<|endoftext|>")` will be give `['<',
+            '|', 'endo', 'ft', 'ext', '|', '>']`. This argument is only supported for `slow` tokenizers for the moment.
+    """
+
+    vocab_files_names = VOCAB_FILES_NAMES
+    model_input_names = ["input_ids", "attention_mask"]
+
+    def __init__(
+        self,
+        vocab_file,
+        merges_file,
+        errors="replace",
+        unk_token="<|endoftext|>",
+        bos_token=None,
+        eos_token="<|endoftext|>",
+        pad_token="<|endoftext|>",
+        clean_up_tokenization_spaces=False,
+        split_special_tokens=False,
+        **kwargs,
+    ):
+        self.extra_tokenizer_start_mapping = {}
+        self.extra_tokenizer_end_mapping = {}
+        self._extra_special_tokens = []
+
+        self._extra_tokenizer_list = [
+            dict(
+                tokenizer_name="tokenizer_SMILES",
+                tokenizer_path=os.path.join(os.path.dirname(vocab_file), "tokenizer_SMILES.model"),
+                begin_sp_tokens=["<SMILES>", "<SELFIES>"],
+                end_sp_tokens=["</SMILES>", "</SELFIES>"],
+                auto_begin_sp_tokens=["<SMILES_AUTO_DETECT>"],
+                auto_end_sp_tokens=["</SMILES_AUTO_DETECT>"],
+            ),
+            dict(
+                tokenizer_name="tokenizer_IUPAC",
+                tokenizer_path=os.path.join(os.path.dirname(vocab_file), "tokenizer_IUPAC.model"),
+                begin_sp_tokens=["<IUPAC>"],
+                end_sp_tokens=["</IUPAC>"],
+                auto_begin_sp_tokens=[],
+                auto_end_sp_tokens=[],
+            ),
+            dict(
+                tokenizer_name="tokenizer_FASTA",
+                tokenizer_path=os.path.join(os.path.dirname(vocab_file), "tokenizer_FASTA.model"),
+                begin_sp_tokens=[],
+                end_sp_tokens=[],
+                auto_begin_sp_tokens=["<FASTA_AUTO_DETECT>"],
+                auto_end_sp_tokens=["</FASTA_AUTO_DETECT>"],
+            ),
+        ]
+        # Content wrapped in these sp tokens won't be tokenized
+        self.protect_begin_sp_tokens = ["<MOLFORMULA>"]
+        self.protect_end_sp_tokens = ["</MOLFORMULA>"]
+
+        self.auto_begin_sp_tokens = []
+        self.auto_end_sp_tokens = []
+
+        self._unk_token = "<unk>"  # Fall-back
+
+        self.new_sp_token_offset = [26]  # The length of sp token before the start of extra vocab
+        self.tokenizer_mapping = OrderedDict()
+
+        super().__init__(
+            vocab_file=vocab_file,
+            merges_file=merges_file,
+            errors=errors,
+            unk_token=unk_token,
+            bos_token=bos_token,
+            eos_token=eos_token,
+            pad_token=pad_token,
+            clean_up_tokenization_spaces=clean_up_tokenization_spaces,
+            split_special_tokens=split_special_tokens,
+            **kwargs,
+        )
+
+        # keep order for python < 3.7
+        self.tokenizer_mapping = OrderedDict([("tokenizer_original", self.encoder)])
+
+        if self._extra_tokenizer_list is not None:
+            for tokenizer_config in self._extra_tokenizer_list:
+                self._build_extra_tokenizer(tokenizer_config)
+                self._update_special_tokens(tokenizer_config)
+                self._update_logical_special_tokens(tokenizer_config)
+                self.decoder.update(self._build_extra_decoder(tokenizer_config))
+
+        for token in self.protect_begin_sp_tokens:
+            self.tokens_trie.add(token)
+
+        for token in self.protect_end_sp_tokens:
+            self.tokens_trie.add(token)
+
+        self.new_sp_token_offset.append(len(self._added_tokens_decoder) - sum(self.new_sp_token_offset) + len(self._extra_special_tokens))
+        self.check_module_list = [SmilesCheckModule(), FastaCheckModule()]
+
+    @property
+    def vocab_size(self) -> int:
+        """Returns vocab size including extra tokenizer"""
+        total_vocab_size = len(self.encoder)
+        for tokenizer in self.tokenizer_mapping.values():
+            if isinstance(tokenizer, dict):
+                continue
+            else:
+                total_vocab_size += tokenizer.get_piece_size()
+        return total_vocab_size + sum(self.new_sp_token_offset)
+
+    def __len__(self) -> int:
+        """Overload method"""
+        return self.vocab_size
+
+    @property
+    def logical_auto_tokens(self):
+        """Tokens that won't be decoded and only for switching tokenizer"""
+        return self.auto_begin_sp_tokens + self.auto_end_sp_tokens
+
+    @property
+    def extra_tokenizer_bos_keys(self):
+        return self.extra_tokenizer_start_mapping.keys()
+
+    @property
+    def extra_tokenizer_eos_keys(self):
+        return self.extra_tokenizer_end_mapping.keys()
+
+    @property
+    def protect_sp_tokens(self):
+        """Content wrapped by these sp tokens won't apply extra tokenizer"""
+        return self.protect_begin_sp_tokens + self.protect_end_sp_tokens
+
+    def _build_extra_tokenizer(self, tokenizer_config: dict) -> None:
+        """
+        Build domain-specific tokenizers
+        and register them in tokenizer_mapping
+        """
+        _sp_model = spm.SentencePieceProcessor()
+        _sp_model.Load(tokenizer_config["tokenizer_path"])
+        self.tokenizer_mapping.update({tokenizer_config["tokenizer_name"]: _sp_model})
+
+        for begin_sp_token, end_sp_token in zip(
+            tokenizer_config["begin_sp_tokens"], tokenizer_config["end_sp_tokens"]
+        ):
+            self.extra_tokenizer_start_mapping.update({begin_sp_token: tokenizer_config["tokenizer_name"]})
+            self.extra_tokenizer_end_mapping.update({end_sp_token: tokenizer_config["tokenizer_name"]})
+
+        for begin_sp_token, end_sp_token in zip(
+            tokenizer_config["auto_begin_sp_tokens"], tokenizer_config["auto_end_sp_tokens"]
+        ):
+            self.extra_tokenizer_start_mapping.update({begin_sp_token: tokenizer_config["tokenizer_name"]})
+            self.extra_tokenizer_end_mapping.update({end_sp_token: tokenizer_config["tokenizer_name"]})
+
+    def _build_extra_decoder(self, tokenizer_config: dict) -> Dict[int, str]:
+        """Build domain-specific tokenizers' decoder"""
+        extra_decoder = {}
+        sp_model = self.tokenizer_mapping[tokenizer_config["tokenizer_name"]]
+        start_pos = self.vocab_size - sp_model.get_piece_size() - self.new_sp_token_offset[-1]
+        extra_decoder.update(
+            {i: sp_model.id_to_piece(i - start_pos) for i in range(start_pos, start_pos + sp_model.get_piece_size())}
+        )
+        return extra_decoder
+
+    def _update_logical_special_tokens(self, tokenizer_config: dict) -> None:
+        """Update logical special tokens which serve as special token and won't be mapped to a specific token id"""
+        for begin_sp_token, end_sp_token in zip(
+            tokenizer_config["auto_begin_sp_tokens"], tokenizer_config["auto_end_sp_tokens"]
+        ):
+            self.auto_begin_sp_tokens.append(begin_sp_token)
+            self.auto_end_sp_tokens.append(end_sp_token)
+
+            self.tokens_trie.add(begin_sp_token)
+            self.tokens_trie.add(end_sp_token)
+
+    def _update_special_tokens(self, tokenizer_config: dict):
+        """Update special tokens for each modality"""
+        offset = sum(self.new_sp_token_offset[1:]) + len(self.logical_auto_tokens)
+        new_offset = 0
+        for start_key, end_key in zip(
+            list(self.extra_tokenizer_bos_keys)[offset // 2 :], list(self.extra_tokenizer_eos_keys)[offset // 2 :]
+        ):
+            self.tokens_trie.add(start_key)
+
+            if start_key not in tokenizer_config["auto_begin_sp_tokens"]:
+                self._added_tokens_encoder.update({start_key: self.vocab_size + new_offset})
+                self._added_tokens_decoder.update(
+                    {
+                        self.vocab_size + new_offset: AddedToken(
+                            content=start_key,
+                            lstrip=False,
+                            normalized=False,
+                            rstrip=False,
+                            single_word=False,
+                            special=True,
+                        )
+                    }
+                )
+                self.tokens_trie.add(start_key)
+                new_offset += 1
+
+            if end_key not in tokenizer_config["auto_end_sp_tokens"]:
+                self._added_tokens_encoder.update({end_key: self.vocab_size + new_offset})
+                self._added_tokens_decoder.update(
+                    {
+                        self.vocab_size + new_offset: AddedToken(
+                            content=end_key,
+                            lstrip=False,
+                            normalized=False,
+                            rstrip=False,
+                            single_word=False,
+                            special=True,
+                        )
+                    }
+                )
+                self.tokens_trie.add(end_key)
+                new_offset += 1
+        self.new_sp_token_offset.append(new_offset)
+
+    @lru_cache(maxsize=None)  # May cause memory leak
+    def _extra_tokenizer_offset(self, tokenizer_key) -> int:
+        offset = 0
+        for index, (tokenizer_name, tokenizer) in enumerate(self.tokenizer_mapping.items()):
+            if tokenizer_name == tokenizer_key:
+                break
+            else:
+                offset += len(tokenizer) + self.new_sp_token_offset[index]
+        return offset
+
+    def _pop_logical_sp_token(self, extra_tokenizer_stack: list, mapping_name: str) -> None:
+        """Switch tokenizer when it comes to an end sp token"""
+        extra_tokenizer_end_mapping = extra_tokenizer_stack.pop()
+        if extra_tokenizer_end_mapping != self.extra_tokenizer_end_mapping[mapping_name]:
+            logger.warning_once(
+                f"Encounter incorrect nesting of extra tokenizer: {self.extra_tokenizer_end_mapping[mapping_name]} and {extra_tokenizer_end_mapping}"
+            )
+            logger.warning_once("This may lead to unexpected behaviour of the tokenizer, please check your input.")
+
+    def tokenize(self, text: TextInput, **kwargs) -> List[str]:
+        """
+        Converts a string into a sequence of tokens, using the tokenizer.
+
+        It will switch to domain-specific tokenizer once encountering extra/logical sp tokens.
+
+        Args:
+            text: TextInput
+        """
+        split_special_tokens = kwargs.pop("split_special_tokens", self.split_special_tokens)
+
+        text, kwargs = self.prepare_for_tokenization(text, **kwargs)
+
+        if kwargs:
+            logger.warning(f"Keyword arguments {kwargs} not recognized.")
+
+        if hasattr(self, "do_lower_case") and self.do_lower_case:
+            # convert non-special tokens to lowercase. Might be super slow as well?
+            escaped_special_toks = [re.escape(s_tok) for s_tok in (self.all_special_tokens)]
+            escaped_special_toks += [
+                re.escape(s_tok.content)
+                for s_tok in (self._added_tokens_decoder.values())
+                if not s_tok.special and s_tok.normalized
+            ]
+            pattern = r"(" + r"|".join(escaped_special_toks) + r")|" + r"(.+?)"
+            text = re.sub(pattern, lambda m: m.groups()[0] or m.groups()[1].lower(), text)
+
+        if split_special_tokens:
+            no_split_token = []
+            tokens = [text]
+        else:
+            no_split_token = self._added_tokens_encoder.keys()  # don't split on any of the added tokens
+            # "This is something<special_token_1>  else"
+            tokens = self.tokens_trie.split(text)
+
+        # ["This is something", "<special_token_1>", "  else"]
+        for i, token in enumerate(tokens):
+            if token in no_split_token:
+                tok_extended = self._added_tokens_decoder.get(self._added_tokens_encoder[token], None)
+                left = tokens[i - 1] if i > 0 else None
+                right = tokens[i + 1] if i < len(tokens) - 1 else None
+                if isinstance(tok_extended, AddedToken):
+                    if tok_extended.rstrip and right:
+                        # A bit counter-intuitive but we strip the left of the string
+                        # since tok_extended.rstrip means the special token is eating all white spaces on its right
+                        tokens[i + 1] = right.lstrip()
+                    # Strip white spaces on the left
+                    if tok_extended.lstrip and left:
+                        tokens[i - 1] = left.rstrip()  # Opposite here
+                    if tok_extended.single_word and left and left[-1] != " ":
+                        tokens[i - 1] += token
+                        tokens[i] = ""
+                    elif tok_extended.single_word and right and right[0] != " ":
+                        tokens[i + 1] = token + tokens[i + 1]
+                        tokens[i] = ""
+                else:
+                    raise ValueError(
+                        f"{tok_extended} cannot be tokenized because it was not properly added"
+                        f" to the tokenizer. This means that it is not an `AddedToken` but a {type(tok_extended)}"
+                    )
+
+        # ["This is something", "<special_token_1>", "else"]
+        tokenized_text = []
+
+        # Code for Auto Detect
+        if self._extra_tokenizer_list is not None:
+            new_tokens = []
+            not_split_flag = 0
+            for token in tokens:
+                if not token:
+                    continue
+                if token in no_split_token or token in self.protect_sp_tokens:
+                    new_tokens.append(token)
+                    if token in self.extra_tokenizer_bos_keys or token in self.protect_begin_sp_tokens:
+                        not_split_flag += 1  # In case nested sp tokens
+                    elif token in self.extra_tokenizer_eos_keys or token in self.protect_end_sp_tokens:
+                        not_split_flag = max(0, not_split_flag - 1)
+                else:
+                    if not_split_flag:
+                        new_tokens.append(token)
+                    else:
+                        for check_module in self.check_module_list:
+                            token = check_module.re_split(token)
+
+                        new_tokens.extend(token)
+            tokens = new_tokens
+
+        extra_tokenizer_stack = []  # This should be a stack to handle nested extra tokenizer
+
+        for token in tokens:
+            # Need to skip eventual empty (fully stripped) tokens
+            if not token:
+                continue
+            if token in self.protect_sp_tokens:
+                tokenized_text.extend(self._tokenize(token))
+            elif token in no_split_token:
+                tokenized_text.append(token)
+                if token in self.extra_tokenizer_bos_keys:
+                    extra_tokenizer_stack.append(self.extra_tokenizer_start_mapping[token])
+                elif token in self.extra_tokenizer_eos_keys:
+                    if extra_tokenizer_stack:
+                        self._pop_logical_sp_token(extra_tokenizer_stack, token)
+            elif token in self.auto_begin_sp_tokens:
+                tokenized_text.append(token)
+                extra_tokenizer_stack.append(self.extra_tokenizer_start_mapping[token])
+            elif token in self.auto_end_sp_tokens:
+                tokenized_text.append(token)
+                if extra_tokenizer_stack:
+                    self._pop_logical_sp_token(extra_tokenizer_stack, token)
+            else:
+                tokenized_text.extend(self._tokenize(token, extra_tokenizer_stack=extra_tokenizer_stack))
+
+        # ["This", " is", " something", "<special_token_1>", "else"]
+        return tokenized_text
+
+    def _add_tokens(self, new_tokens: Union[List[str], List[AddedToken]], special_tokens: bool = False) -> int:
+        """
+        Modified from `transformers.tokenization_utils._add_tokens`.
+
+        This adaptation supports dynamic tokenizer length due to supplementary tokenizers (e.g., domain-specific or scientific text tokenizers).
+        """
+        added_tokens = 0
+        if new_tokens is None:
+            return added_tokens
+        # TODO this is fairly slow to improve!
+        current_vocab = self.get_vocab().copy()
+        new_idx = max(current_vocab.values()) + 1
+
+        for token in new_tokens:
+            if not isinstance(token, (str, AddedToken)):
+                raise TypeError(f"Token {token} is not a string but a {type(token)}.")
+            if str(token) == "":
+                continue
+            if isinstance(token, str):
+                if token in self._added_tokens_encoder:
+                    continue
+                else:
+                    # very important for fast and slow equivalence!
+                    is_special = token in self.all_special_tokens or special_tokens
+                    token = AddedToken(
+                        token, rstrip=False, lstrip=False, normalized=not is_special, special=is_special
+                    )
+            elif special_tokens:
+                # doing token.special=True changes the normalization! will fix in rust
+                # this is important and the only reason why the AddedTokens in each class are normalized by default
+                token.__setstate__({"special": True, "normalized": token.normalized})
+            if token in self._added_tokens_decoder:
+                continue
+            if not token.special and token.normalized and getattr(self, "do_lower_case", False):
+                # Normalize if requested
+                token.content = token.content.lower()
+            if token.content not in current_vocab:
+                token_index = new_idx + added_tokens
+                current_vocab[token.content] = token_index
+                added_tokens += 1
+                self._extra_special_tokens.append(token)
+            else:
+                token_index = current_vocab[token.content]
+            if token.special and str(token) not in self.all_special_tokens:
+                self._special_tokens_map["additional_special_tokens"].append(token)
+            # the setter automatically updates the reverse map
+            self._added_tokens_decoder[token_index] = token
+            self._added_tokens_encoder[token.content] = token_index
+            if self.verbose:
+                logger.info(f"Adding {token} to the vocabulary")
+        self._update_trie()
+        self._update_total_vocab_size()
+
+        if added_tokens and self.tokenizer_mapping:
+            self.new_sp_token_offset.append(added_tokens)
+
+        return added_tokens
+
+    # Copied from transformers.models.gpt2.tokenization_gpt2.GPT2Tokenizer._tokenize
+    def _tokenize(self, text, **kwargs):
+        """
+        Modified from `transformers.models.gpt2.tokenization_gpt2.GPT2Tokenizer._tokenize`.
+
+        This adaptation supports domain-specific tokenizers.
+        """
+        extra_tokenizer_stack = kwargs.pop("extra_tokenizer_stack", False)
+        if extra_tokenizer_stack:
+            tokenized_text = self.tokenizer_mapping[extra_tokenizer_stack[-1]].encode(text, out_type=str)
+            tokenized_id = self.tokenizer_mapping[extra_tokenizer_stack[-1]].encode(text, out_type=int)
+            final_tokenized_text = []
+            for text_piece, id_piece in zip(tokenized_text, tokenized_id):
+                if id_piece == 0:
+                    final_tokenized_text.extend(self._bpe_tokenize(text_piece))
+                else:
+                    final_tokenized_text.append(text_piece)
+            return final_tokenized_text
+        else:
+            return self._bpe_tokenize(text)
+
+    def _bpe_tokenize(self, text, **kwargs):
+        text = text.replace(
+            "▁", " "
+        )  # This discrepancy stems from differing whitespace treatment in SentencePiece versus BPE tokenization.
+        bpe_tokens = []
+        for token in re.findall(self.pat, text):
+            token = "".join(
+                self.byte_encoder[b] for b in token.encode("utf-8")
+            )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
+            bpe_tokens.extend(bpe_token for bpe_token in self.bpe(token).split(" "))
+        return bpe_tokens
+
+    def convert_tokens_to_ids(self, tokens: Union[str, List[str]]) -> Union[int, List[int]]:
+        """
+        Modified from `transformers.tokenization_utils.PreTrainedTokenzier.convert_tokens_to_ids`.
+
+        Converts a token string (or a sequence of tokens) in a single integer id (or a sequence of ids), using the
+        vocabulary.
+
+        This adaptation supports domain-specific tokenizers.
+
+        Args:
+            tokens (`str` or `List[str]`): One or several token(s) to convert to token id(s).
+
+        Returns:
+            `int` or `List[int]`: The token id or list of token ids.
+        """
+        if tokens is None:
+            return None
+
+        if isinstance(tokens, str):
+            return self._convert_token_to_id_with_added_voc(tokens)
+
+        ids = []
+        extra_tokenizer_stack = []
+
+        for token in tokens:
+            if token not in self.logical_auto_tokens:
+                ids.append(
+                    self._convert_token_to_id_with_added_voc(token, extra_tokenizer_stack=extra_tokenizer_stack)
+                )
+            if token in self.extra_tokenizer_bos_keys:
+                extra_tokenizer_stack.append(self.extra_tokenizer_start_mapping[token])
+            elif token in self.extra_tokenizer_eos_keys:
+                if extra_tokenizer_stack:
+                    self._pop_logical_sp_token(extra_tokenizer_stack, token)
+        return ids
+
+    def _convert_token_to_id_with_added_voc(self, token, **kwargs):
+        """
+        Modified from `transformers.tokenization_utils.PreTrainedTokenzier._convert_token_to_id_with_added_voc`.
+
+        This adaptation supports domain-specific tokenizers.
+        """
+        if token is None:
+            return None
+
+        if token in self._added_tokens_encoder:
+            return self._added_tokens_encoder[token]
+        return self._convert_token_to_id(token, **kwargs)
+
+    def _convert_token_to_id(self, token, **kwargs):
+        """
+        Modified from `transformers.tokenization_utils.PreTrainedTokenzier._convert_token_to_id`.
+
+        Converts a token (str) in an id using the vocab.
+
+        Fall back to original tokenizer once OOV.
+        """
+        extra_tokenizer_stack = kwargs.pop("extra_tokenizer_stack", False)
+        if extra_tokenizer_stack:
+            token_id = self.tokenizer_mapping[extra_tokenizer_stack[-1]].piece_to_id(token)
+            if token_id == self.tokenizer_mapping[extra_tokenizer_stack[-1]].unk_id():
+                return self.encoder.get(token, self.encoder.get(self._unk_token))
+            else:
+                return token_id + self._extra_tokenizer_offset(extra_tokenizer_stack[-1])
+        else:
+            return self.encoder.get(token, self.encoder.get(self._unk_token))
+
+    def _convert_id_to_token(self, index):
+        """Converts an index (integer) in a token (str) using the vocab."""
+        return self.decoder.get(index, "")
+
+    def convert_tokens_to_string(self, tokens):
+        """Converts a sequence of tokens (string) in a single string."""
+        text = "".join(tokens)
+        text = text.replace(
+            "▁", "Ġ"
+        )  # This discrepancy stems from differing whitespace treatment in SentencePiece versus BPE tokenization.
+        text = text.replace("\n", "Ċ")
+        text = bytearray([self.byte_decoder[c] for c in text]).decode("utf-8", errors=self.errors)
+        return text
+
+    def decode(
+        self,
+        token_ids,
+        skip_special_tokens: bool = False,
+        clean_up_tokenization_spaces: Optional[bool] = False,
+        spaces_between_special_tokens: bool = False,
+        **kwargs,
+    ) -> str:
+        # `spaces_between_special_tokens` defaults to True for _decode in slow tokenizers
+        # and cannot be configured elsewhere, but it should default to False for Qwen2Tokenizer
+        return super().decode(
+            token_ids,
+            skip_special_tokens=skip_special_tokens,
+            clean_up_tokenization_spaces=clean_up_tokenization_spaces,
+            spaces_between_special_tokens=spaces_between_special_tokens,
+            **kwargs,
+        )
+
+    # Copied from transformers.models.gpt2.tokenization_gpt2.GPT2Tokenizer.save_vocabulary
+    def save_vocabulary(self, save_directory: str, filename_prefix: Optional[str] = None) -> Tuple[str]:
+        """
+        Modified from `transformers.models.gpt2.tokenization_gpt2.GPT2Tokenizer.save_vocabulary` to support saving custom extension.
+        """
+        if not os.path.isdir(save_directory):
+            logger.error(f"Vocabulary path ({save_directory}) should be a directory")
+            return
+        vocab_file = os.path.join(
+            save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]
+        )
+        merge_file = os.path.join(
+            save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]
+        )
+        sp_model_smiles = os.path.join(
+            save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["sp_model_SMILES"]
+        )
+        sp_model_iupac = os.path.join(
+            save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["sp_model_IUPAC"]
+        )
+        sp_model_fasta = os.path.join(
+            save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["sp_model_FASTA"]
+        )
+
+        with open(vocab_file, "w", encoding="utf-8") as f:
+            f.write(json.dumps(self.encoder, indent=2, sort_keys=True, ensure_ascii=False) + "\n")
+
+        index = 0
+        with open(merge_file, "w", encoding="utf-8") as writer:
+            writer.write("#version: 0.2\n")
+            for bpe_tokens, token_index in sorted(self.bpe_ranks.items(), key=lambda kv: kv[1]):
+                if index != token_index:
+                    logger.warning(
+                        f"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
+                        " Please check that the tokenizer is not corrupted!"
+                    )
+                    index = token_index
+                writer.write(" ".join(bpe_tokens) + "\n")
+                index += 1
+
+        with open(sp_model_smiles, "wb") as f:
+            f.write(self.tokenizer_mapping["tokenizer_SMILES"].serialized_model_proto())
+
+        with open(sp_model_iupac, "wb") as f:
+            f.write(self.tokenizer_mapping["tokenizer_IUPAC"].serialized_model_proto())
+
+        with open(sp_model_fasta, "wb") as f:
+            f.write(self.tokenizer_mapping["tokenizer_FASTA"].serialized_model_proto())
+
+        return vocab_file, merge_file
+
+
+__all__ = ["InternS1Tokenizer"]

tokenizer_FASTA.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6e719023a50767e2da1165925feb3afe77d63702f08d0cd39c4ddadba7cdaaca
+size 5899

tokenizer_IUPAC.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6e719023a50767e2da1165925feb3afe77d63702f08d0cd39c4ddadba7cdaaca
+size 5899

tokenizer_SMILES.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d8dd3252680ab2fedacab7e71b75a48f08d6fbae70a9cc38d355c65ec42fbd0d
+size 3290

tokenizer_config.json

@@ -0,0 +1,434 @@
+{
+  "add_bos_token": false,
+  "add_eos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151665": {
+      "content": "<tool_response>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151666": {
+      "content": "</tool_response>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151667": {
+      "content": "<think>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151668": {
+      "content": "</think>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151925": {
+      "content": "<SMILES>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151926": {
+      "content": "</SMILES>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151927": {
+      "content": "<SELFIES>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151928": {
+      "content": "</SELFIES>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "152441": {
+      "content": "<IUPAC>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "152442": {
+      "content": "</IUPAC>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "152955": {
+      "content": "<FASTA>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "152956": {
+      "content": "</FASTA>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "152957": {
+      "content": "<IMG_CONTEXT>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "152958": {
+      "content": "<img>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "152959": {
+      "content": "</img>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "152960": {
+      "content": "<quad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "152961": {
+      "content": "</quad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "152962": {
+      "content": "<ref>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "152963": {
+      "content": "</ref>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "152964": {
+      "content": "<box>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "152965": {
+      "content": "</box>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "152966": {
+      "content": "<|action_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "152967": {
+      "content": "<|action_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "152968": {
+      "content": "<|interpreter|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "152969": {
+      "content": "<|plugin|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "152970": {
+      "content": "<video>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "auto_map": {
+    "AutoProcessor": "processing_interns1.InternS1Processor",
+    "AutoTokenizer": [
+      "tokenization_interns1.InternS1Tokenizer",
+      null
+    ]
+  },
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "context_image_token": "<IMG_CONTEXT>",
+  "end_image_token": "</img>",
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": {
+    "context_image_token": "<IMG_CONTEXT>",
+    "end_image_token": "</img>",
+    "start_image_token": "<img>",
+    "video_token": "<video>"
+  },
+  "model_max_length": 55000,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "right",
+  "processor_class": "InternS1Processor",
+  "split_special_tokens": false,
+  "start_image_token": "<img>",
+  "tokenizer_class": "InternS1Tokenizer",
+  "unk_token": null,
+  "video_token": "<video>"
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:181a1a8ea7fc8722a71d170e3a1b72b4fc9fe657322b05be89c0eb7a50f92a4c
+size 6737