Add files using upload-large-folder tool

config.json

@@ -0,0 +1,54 @@
+{
+    "_name_or_path": "models/internlm/internlm2_5-7b-chat",
+    "architectures": ["LlavaInternlm2ForCausalLM"],
+    "attn_implementation": "flash_attention_2",
+    "auto_map": {
+        "AutoConfig": "configuration_internlm2.InternLM2Config",
+        "AutoModel": "modeling_internlm2.InternLM2ForCausalLM",
+        "AutoModelForCausalLM": "llava_internlm2.LlavaInternlm2ForCausalLM"
+    },
+    "bias": false,
+    "bos_token_id": 1,
+    "datatype_loss": false,
+    "eos_token_id": 2,
+    "hidden_act": "silu",
+    "hidden_size": 4096,
+    "image_aspect_ratio": "anyres",
+    "image_crop_resolution": null,
+    "image_grid_pinpoints": "[(336, 672), (672, 336), (672, 672), (1008, 336), (336, 1008)]",
+    "image_split_resolution": null,
+    "initializer_range": 0.02,
+    "intermediate_size": 14336,
+    "max_position_embeddings": 32768,
+    "mm_hidden_size": 1024,
+    "mm_patch_merge_type": "spatial_unpad",
+    "mm_projector_lr": null,
+    "mm_projector_type": "mlp2x_gelu",
+    "mm_tunable_parts": "mm_vision_tower,mm_mlp_adapter,mm_language_model",
+    "mm_use_im_patch_token": false,
+    "mm_use_im_start_end": false,
+    "mm_vision_select_feature": "patch",
+    "mm_vision_select_layer": -2,
+    "mm_vision_tower": "/fs-computility/mllm1/shared/hub/models--openai--clip-vit-large-patch14-336/snapshots/ce19dc912ca5cd21c8a653c79e251e808ccabcd1",
+    "mm_vision_tower_lr": 2e-6,
+    "model_type": "internlm2",
+    "num_attention_heads": 32,
+    "num_hidden_layers": 32,
+    "num_key_value_heads": 8,
+    "pad_token_id": 2,
+    "pretraining_tp": 1,
+    "rms_norm_eps": 1e-5,
+    "rope_scaling": {
+        "factor": 2.0,
+        "type": "dynamic"
+    },
+    "rope_theta": 1000000,
+    "tie_word_embeddings": false,
+    "tokenizer_model_max_length": 32768,
+    "tokenizer_padding_side": "right",
+    "torch_dtype": "bfloat16",
+    "transformers_version": "4.37.2",
+    "use_cache": true,
+    "use_mm_proj": true,
+    "vocab_size": 92544
+}

configuration_internlm2.py

@@ -0,0 +1,180 @@
+# coding=utf-8
+# Copyright (c) The InternLM team and The HuggingFace Inc. team. All rights reserved.
+#
+# This code is based on transformers/src/transformers/models/llama/configuration_llama.py
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+""" InternLM2 model configuration"""
+
+from transformers.configuration_utils import PretrainedConfig
+from transformers.utils import logging
+
+logger = logging.get_logger(__name__)
+
+INTERNLM2_PRETRAINED_CONFIG_ARCHIVE_MAP = {}
+
+
+# Modified from transformers.model.llama.configuration_llama.LlamaConfig
+class InternLM2Config(PretrainedConfig):
+    r"""
+    This is the configuration class to store the configuration of a [`InternLM2Model`]. It is used to instantiate
+    an InternLM2 model according to the specified arguments, defining the model architecture. Instantiating a
+    configuration with the defaults will yield a similar configuration to that of the InternLM2-7B.
+
+    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+    documentation from [`PretrainedConfig`] for more information.
+
+
+    Args:
+        vocab_size (`int`, *optional*, defaults to 32000):
+            Vocabulary size of the InternLM2 model. Defines the number of different tokens that can be represented by the
+            `inputs_ids` passed when calling [`InternLM2Model`]
+        hidden_size (`int`, *optional*, defaults to 4096):
+            Dimension of the hidden representations.
+        intermediate_size (`int`, *optional*, defaults to 11008):
+            Dimension of the MLP representations.
+        num_hidden_layers (`int`, *optional*, defaults to 32):
+            Number of hidden layers in the Transformer decoder.
+        num_attention_heads (`int`, *optional*, defaults to 32):
+            Number of attention heads for each attention layer in the Transformer decoder.
+        num_key_value_heads (`int`, *optional*):
+            This is the number of key_value heads that should be used to implement Grouped Query Attention. If
+            `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
+            `num_key_value_heads=1 the model will use Multi Query Attention (MQA) otherwise GQA is used. When
+            converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
+            by meanpooling all the original heads within that group. For more details checkout [this
+            paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to
+            `num_attention_heads`.
+        hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
+            The non-linear activation function (function or string) in the decoder.
+        max_position_embeddings (`int`, *optional*, defaults to 2048):
+            The maximum sequence length that this model might ever be used with. InternLM2 supports up to 32768 tokens.
+        initializer_range (`float`, *optional*, defaults to 0.02):
+            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
+        rms_norm_eps (`float`, *optional*, defaults to 1e-06):
+            The epsilon used by the rms normalization layers.
+        use_cache (`bool`, *optional*, defaults to `True`):
+            Whether or not the model should return the last key/values attentions (not used by all models). Only
+            relevant if `config.is_decoder=True`.
+        pad_token_id (`int`, *optional*):
+            Padding token id.
+        bos_token_id (`int`, *optional*, defaults to 1):
+            Beginning of stream token id.
+        eos_token_id (`int`, *optional*, defaults to 2):
+            End of stream token id.
+        pretraining_tp (`int`, *optional*, defaults to 1):
+            Experimental feature. Tensor parallelism rank used during pretraining. Please refer to [this
+            document](https://huggingface.co/docs/transformers/main/perf_train_gpu_many#tensor-parallelism)
+            to understand more about it. This value is necessary to ensure exact reproducibility
+            of the pretraining results. Please refer to [this
+            issue](https://github.com/pytorch/pytorch/issues/76232).
+        tie_word_embeddings (`bool`, *optional*, defaults to `False`):
+            Whether to tie weight embeddings
+        rope_theta (`float`, *optional*, defaults to 10000.0):
+            The base period of the RoPE embeddings.
+        rope_scaling (`Dict`, *optional*):
+            Dictionary containing the scaling configuration for the RoPE embeddings. Currently supports two scaling
+            strategies: linear and dynamic. Their scaling factor must be a float greater than 1. The expected format is
+            `{"type": strategy name, "factor": scaling factor}`. When using this flag, don't update
+            `max_position_embeddings` to the expected new maximum. See the following thread for more information on how
+            these scaling strategies behave:
+            https://www.reddit.com/r/LocalLLaMA/comments/14mrgpr/dynamically_scaled_rope_further_increases/. This is an
+            experimental feature, subject to breaking API changes in future versions.
+    """
+    _auto_class = "AutoConfig"
+    model_type = "internlm2"
+    keys_to_ignore_at_inference = ["past_key_values"]
+
+    def __init__(  # pylint: disable=W0102
+        self,
+        vocab_size=103168,
+        hidden_size=4096,
+        intermediate_size=11008,
+        num_hidden_layers=32,
+        num_attention_heads=32,
+        num_key_value_heads=None,
+        hidden_act="silu",
+        max_position_embeddings=2048,
+        initializer_range=0.02,
+        rms_norm_eps=1e-6,
+        use_cache=True,
+        pad_token_id=0,
+        bos_token_id=1,
+        eos_token_id=2,
+        pretraining_tp=1,
+        tie_word_embeddings=False,
+        bias=True,
+        rope_theta=10000,
+        rope_scaling=None,
+        attn_implementation=None,
+        **kwargs,
+    ):
+        self.vocab_size = vocab_size
+        self.max_position_embeddings = max_position_embeddings
+        self.hidden_size = hidden_size
+        self.intermediate_size = intermediate_size
+        self.num_hidden_layers = num_hidden_layers
+        self.num_attention_heads = num_attention_heads
+        self.bias = bias
+
+        if num_key_value_heads is None:
+            num_key_value_heads = num_attention_heads
+        self.num_key_value_heads = num_key_value_heads
+
+        self.hidden_act = hidden_act
+        self.initializer_range = initializer_range
+        self.rms_norm_eps = rms_norm_eps
+        self.pretraining_tp = pretraining_tp
+        self.use_cache = use_cache
+        self.rope_theta = rope_theta
+        self.rope_scaling = rope_scaling
+        self._rope_scaling_validation()
+        self.attn_implementation = attn_implementation
+        if self.attn_implementation is None:
+            self.attn_implementation = "eager"
+
+        super().__init__(
+            pad_token_id=pad_token_id,
+            bos_token_id=bos_token_id,
+            eos_token_id=eos_token_id,
+            tie_word_embeddings=tie_word_embeddings,
+            **kwargs,
+        )
+
+    def _rope_scaling_validation(self):
+        """
+        Validate the `rope_scaling` configuration.
+        """
+        if self.rope_scaling is None:
+            return
+
+        if not isinstance(self.rope_scaling, dict) or len(self.rope_scaling) != 2:
+            raise ValueError(
+                "`rope_scaling` must be a dictionary with with two fields, `type` and `factor`, "
+                f"got {self.rope_scaling}"
+            )
+        rope_scaling_type = self.rope_scaling.get("type", None)
+        rope_scaling_factor = self.rope_scaling.get("factor", None)
+        if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic"]:
+            raise ValueError(
+                f"`rope_scaling`'s type field must be one of ['linear', 'dynamic'], got {rope_scaling_type}"
+            )
+        if (
+            rope_scaling_factor is None
+            or not isinstance(rope_scaling_factor, (float, int))
+            or rope_scaling_factor < 1.0
+        ):
+            raise ValueError(
+                f"`rope_scaling`'s factor field must be a number >= 1, got {rope_scaling_factor} "
+                f"of type {type(rope_scaling_factor)}"
+            )

generation_config.json

@@ -0,0 +1,9 @@
+{
+  "bos_token_id": 1,
+  "eos_token_id": [
+    2,
+    92542
+  ],
+  "pad_token_id": 2,
+  "transformers_version": "4.37.2"
+}

llava_internlm2.py

@@ -0,0 +1,242 @@
+#    Copyright 2023 Haotian Liu
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+
+
+from typing import List, Optional, Tuple, Union
+
+import torch
+import torch.nn as nn
+from torch.nn import CrossEntropyLoss
+
+from transformers import AutoConfig, AutoModelForCausalLM
+from llava.model.language_model.internlm2.modeling_internlm2 import InternLM2ForCausalLM, InternLM2Model
+from llava.model.language_model.internlm2.configuration_internlm2 import InternLM2Config
+
+from transformers.modeling_outputs import CausalLMOutputWithPast
+from transformers.generation.utils import GenerateOutput
+
+from llava.model.llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+from llava.utils import rank0_print
+# from ..llava_arch import LlavaMetaModel, LlavaMetaForCausalLM
+
+
+class LlavaInternlm2Config(InternLM2Config):
+    model_type = "llava_internlm2"
+
+
+class LlavaInternlm2Model(LlavaMetaModel, InternLM2Model):
+    config_class = LlavaInternlm2Config
+
+    def __init__(self, config: InternLM2Config):
+        super(LlavaInternlm2Model, self).__init__(config)
+
+
+class LlavaInternlm2ForCausalLM(InternLM2ForCausalLM, LlavaMetaForCausalLM):
+    config_class = LlavaInternlm2Config
+
+    def __init__(self, config):
+        # super(InternLM2ForCausalLM, self).__init__(config)
+        InternLM2ForCausalLM.__init__(self, config)
+        self.model = LlavaInternlm2Model(config)
+        # self.pretraining_tp = config.pretraining_tp
+        self.vocab_size = config.vocab_size
+        self.datatype_loss = config.datatype_loss if hasattr(config, "datatype_loss") else False
+        if self.datatype_loss:
+            rank0_print("Logging per datatype loss")
+        self.lm_head = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
+        # self.output = nn.Linear(config.hidden_size, config.vocab_size, bias=False)
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    def get_model(self):
+        return self.model
+
+    def forward(
+        self,
+        input_ids: torch.LongTensor = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_values: Optional[List[torch.FloatTensor]] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        labels: Optional[torch.LongTensor] = None,
+        use_cache: Optional[bool] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        images: Optional[torch.FloatTensor] = None,
+        image_sizes: Optional[List[List[int]]] = None,
+        modalities: Optional[List[str]] = ["image"],
+        data_type: Optional[str] = "normal",
+        return_dict: Optional[bool] = None,
+        dpo_forward: Optional[bool] = False,
+        cache_position=None,
+    ) -> Union[Tuple, CausalLMOutputWithPast]:
+
+        if inputs_embeds is None:
+            (input_ids, position_ids, attention_mask, past_key_values, inputs_embeds, labels) = (
+                self.prepare_inputs_labels_for_multimodal(
+                    input_ids, position_ids, attention_mask, past_key_values, labels, images, image_sizes
+                )
+            )
+        if not self.datatype_loss:
+            if dpo_forward:
+                outputs = self.model(
+                    input_ids=input_ids,
+                    attention_mask=attention_mask,
+                    position_ids=position_ids,
+                    past_key_values=past_key_values,
+                    inputs_embeds=inputs_embeds,
+                    use_cache=use_cache,
+                    output_attentions=output_attentions,
+                    output_hidden_states=output_hidden_states,
+                    return_dict=return_dict,
+                )
+
+                hidden_states = outputs[0]
+                logits = self.output(hidden_states)
+                return logits, labels
+            else:
+                return super().forward(
+                    input_ids=input_ids,
+                    attention_mask=attention_mask,
+                    position_ids=position_ids,
+                    past_key_values=past_key_values,
+                    inputs_embeds=inputs_embeds,
+                    labels=labels,
+                    use_cache=use_cache,
+                    output_attentions=output_attentions,
+                    output_hidden_states=output_hidden_states,
+                    return_dict=return_dict,
+                )
+        else:
+            output_attentions = (
+                output_attentions if output_attentions is not None else self.config.output_attentions
+            )
+            output_hidden_states = (
+                output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+            )
+            return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+            # decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn)
+            outputs = self.model(
+                input_ids=input_ids,
+                attention_mask=attention_mask,
+                position_ids=position_ids,
+                past_key_values=past_key_values,
+                inputs_embeds=inputs_embeds,
+                use_cache=use_cache,
+                output_attentions=output_attentions,
+                output_hidden_states=output_hidden_states,
+                return_dict=return_dict,
+            )
+
+            hidden_states = outputs[0]
+            logits = self.output(hidden_states)
+            logits = logits.float()
+
+            loss = None
+            per_sample_losses = None
+
+            if labels is not None:
+                # Shift so that tokens < n predict n
+                shift_logits = logits[..., :-1, :].contiguous()
+                shift_labels = labels[..., 1:].contiguous()
+                # Flatten the tokens
+                # loss_fct = CrossEntropyLoss()
+                shift_logits = shift_logits.view(-1, self.config.vocab_size)
+                shift_labels = shift_labels.view(-1)
+                # Enable model parallelism
+                shift_labels = shift_labels.to(shift_logits.device)
+                # loss = loss_fct(shift_logits, shift_labels)
+
+                ##### Compute per sample loss #####
+                # Compute the token-level loss
+                loss_fct = CrossEntropyLoss(reduction="none")  # "none" for token-level losses
+                token_losses = loss_fct(shift_logits, shift_labels)  # Shape: [batch_size * seq_len]
+
+                # Reshape token losses to [batch_size, seq_len - 1]
+                token_losses = token_losses.view(-1, shift_logits.size(0) // inputs_embeds.size(0))
+                # batch_size = inputs_embeds.size(0)
+                # seq_len = inputs_embeds.size(1)
+                # token_losses = token_losses.view(batch_size, seq_len - 1)
+
+                # Mask out padding tokens
+                active_tokens = (shift_labels != -100).view(-1, token_losses.size(1))
+                token_losses *= active_tokens
+
+                # Compute per-sample losses by summing over the sequence length
+                per_sample_losses = token_losses.sum(dim=1) / active_tokens.sum(dim=1).clamp(min=1)
+
+                # Compute overall loss as the mean of per-sample losses
+                loss = per_sample_losses.mean()
+
+            if not return_dict:
+                output = (logits,) + outputs[1:]
+                return (loss,) + output if loss is not None else output
+
+            device = input_ids.device if input_ids is not None else inputs_embeds.device
+            output = CausalLMOutputWithPast(
+                loss=loss,
+                logits=logits,
+                past_key_values=outputs.past_key_values,
+                hidden_states=outputs.hidden_states,
+                attentions=outputs.attentions,
+            )
+            output['logits'] = output['logits'].to(device)
+
+            output["per_sample_losses"] = per_sample_losses  # Include per-sample losses in the output
+
+            return output
+
+    @torch.no_grad()
+    def generate(
+        self,
+        inputs: Optional[torch.Tensor] = None,
+        images: Optional[torch.Tensor] = None,
+        image_sizes: Optional[torch.Tensor] = None,
+        **kwargs,
+    ) -> Union[GenerateOutput, torch.LongTensor]:
+        position_ids = kwargs.pop("position_ids", None)
+        attention_mask = kwargs.pop("attention_mask", None)
+        if "inputs_embeds" in kwargs:
+            raise NotImplementedError("`inputs_embeds` is not supported")
+
+        if images is not None:
+            (inputs, position_ids, attention_mask, _, inputs_embeds, _) = (
+                self.prepare_inputs_labels_for_multimodal(
+                    inputs, position_ids, attention_mask, None, None, images, image_sizes=image_sizes
+                )
+            )
+        else:
+            inputs_embeds = self.get_model().get_input_embeddings()(inputs)
+
+        return super().generate(
+            position_ids=position_ids, attention_mask=attention_mask, inputs_embeds=inputs_embeds, **kwargs
+        )
+
+    def prepare_inputs_for_generation(self, input_ids, past_key_values=None, inputs_embeds=None, **kwargs):
+        images = kwargs.pop("images", None)
+        image_sizes = kwargs.pop("image_sizes", None)
+        inputs = super().prepare_inputs_for_generation(
+            input_ids, past_key_values=past_key_values, inputs_embeds=inputs_embeds, **kwargs
+        )
+        if images is not None:
+            inputs['images'] = images
+        if image_sizes is not None:
+            inputs['image_sizes'] = image_sizes
+        return inputs
+
+
+AutoConfig.register("llava_internlm2", LlavaInternlm2Config)
+AutoModelForCausalLM.register(LlavaInternlm2Config, LlavaInternlm2ForCausalLM)

model-00001-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b11d8af859153e85107f12e24577c1b426e949a83e3ab8cbf0cf2e09ec937b0
+size 4885478896

model-00002-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8f52b47bed4892a966b9afe7ae6c276f12d3e4d26d7b1d06d08baa858ff373dc
+size 4915913424

model-00003-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:79f86d746c312b9b04137d777ae0a226b961f51ac2191365b26c8d2ba7a12438
+size 4998115008

model-00004-of-00004.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a8f338cf107da0432a32c50e616ddc2ddab620bfca46d75e841c2a14b1c0699b
+size 2083100280

model.safetensors.index.json

@@ -0,0 +1,631 @@
+{
+  "metadata": {
+    "total_size": 16882520064
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00004-of-00004.safetensors",
+    "model.image_newline": "model-00001-of-00004.safetensors",
+    "model.layers.0.attention.wo.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.attention.wqkv.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.attention_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.feed_forward.w1.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.feed_forward.w2.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.feed_forward.w3.weight": "model-00001-of-00004.safetensors",
+    "model.layers.0.ffn_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.attention.wo.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.attention.wqkv.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.attention_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.feed_forward.w1.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.feed_forward.w2.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.feed_forward.w3.weight": "model-00001-of-00004.safetensors",
+    "model.layers.1.ffn_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.10.attention.wo.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.attention.wqkv.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.attention_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.feed_forward.w1.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.feed_forward.w2.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.feed_forward.w3.weight": "model-00002-of-00004.safetensors",
+    "model.layers.10.ffn_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.attention.wo.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.attention.wqkv.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.attention_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.feed_forward.w1.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.feed_forward.w2.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.feed_forward.w3.weight": "model-00002-of-00004.safetensors",
+    "model.layers.11.ffn_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.attention.wo.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.attention.wqkv.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.attention_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.feed_forward.w1.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.feed_forward.w2.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.feed_forward.w3.weight": "model-00002-of-00004.safetensors",
+    "model.layers.12.ffn_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.attention.wo.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.attention.wqkv.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.attention_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.feed_forward.w1.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.feed_forward.w2.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.feed_forward.w3.weight": "model-00002-of-00004.safetensors",
+    "model.layers.13.ffn_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.attention.wo.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.attention.wqkv.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.attention_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.feed_forward.w1.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.feed_forward.w2.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.feed_forward.w3.weight": "model-00002-of-00004.safetensors",
+    "model.layers.14.ffn_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.attention.wo.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.attention.wqkv.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.attention_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.feed_forward.w1.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.feed_forward.w2.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.feed_forward.w3.weight": "model-00002-of-00004.safetensors",
+    "model.layers.15.ffn_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.attention.wo.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.attention.wqkv.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.attention_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.feed_forward.w1.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.feed_forward.w2.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.feed_forward.w3.weight": "model-00002-of-00004.safetensors",
+    "model.layers.16.ffn_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.attention.wo.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.attention.wqkv.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.attention_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.feed_forward.w1.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.feed_forward.w2.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.feed_forward.w3.weight": "model-00002-of-00004.safetensors",
+    "model.layers.17.ffn_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.attention.wo.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.attention.wqkv.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.attention_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.feed_forward.w1.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.feed_forward.w2.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.feed_forward.w3.weight": "model-00002-of-00004.safetensors",
+    "model.layers.18.ffn_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.attention.wo.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.attention.wqkv.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.attention_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.feed_forward.w1.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.feed_forward.w2.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.feed_forward.w3.weight": "model-00002-of-00004.safetensors",
+    "model.layers.19.ffn_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.2.attention.wo.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.attention.wqkv.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.attention_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.feed_forward.w1.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.feed_forward.w2.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.feed_forward.w3.weight": "model-00001-of-00004.safetensors",
+    "model.layers.2.ffn_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.20.attention.wo.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.attention.wqkv.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.attention_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.feed_forward.w1.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.feed_forward.w2.weight": "model-00003-of-00004.safetensors",
+    "model.layers.20.feed_forward.w3.weight": "model-00002-of-00004.safetensors",
+    "model.layers.20.ffn_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.attention.wo.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.attention.wqkv.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.attention_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.feed_forward.w1.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.feed_forward.w2.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.feed_forward.w3.weight": "model-00003-of-00004.safetensors",
+    "model.layers.21.ffn_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.attention.wo.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.attention.wqkv.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.attention_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.feed_forward.w1.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.feed_forward.w2.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.feed_forward.w3.weight": "model-00003-of-00004.safetensors",
+    "model.layers.22.ffn_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.attention.wo.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.attention.wqkv.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.attention_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.feed_forward.w1.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.feed_forward.w2.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.feed_forward.w3.weight": "model-00003-of-00004.safetensors",
+    "model.layers.23.ffn_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.attention.wo.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.attention.wqkv.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.attention_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.feed_forward.w1.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.feed_forward.w2.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.feed_forward.w3.weight": "model-00003-of-00004.safetensors",
+    "model.layers.24.ffn_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.attention.wo.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.attention.wqkv.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.attention_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.feed_forward.w1.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.feed_forward.w2.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.feed_forward.w3.weight": "model-00003-of-00004.safetensors",
+    "model.layers.25.ffn_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.attention.wo.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.attention.wqkv.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.attention_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.feed_forward.w1.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.feed_forward.w2.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.feed_forward.w3.weight": "model-00003-of-00004.safetensors",
+    "model.layers.26.ffn_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.attention.wo.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.attention.wqkv.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.attention_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.feed_forward.w1.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.feed_forward.w2.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.feed_forward.w3.weight": "model-00003-of-00004.safetensors",
+    "model.layers.27.ffn_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.attention.wo.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.attention.wqkv.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.attention_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.feed_forward.w1.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.feed_forward.w2.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.feed_forward.w3.weight": "model-00003-of-00004.safetensors",
+    "model.layers.28.ffn_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.attention.wo.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.attention.wqkv.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.attention_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.feed_forward.w1.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.feed_forward.w2.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.feed_forward.w3.weight": "model-00003-of-00004.safetensors",
+    "model.layers.29.ffn_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.3.attention.wo.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.attention.wqkv.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.attention_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.feed_forward.w1.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.feed_forward.w2.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.feed_forward.w3.weight": "model-00001-of-00004.safetensors",
+    "model.layers.3.ffn_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.30.attention.wo.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.attention.wqkv.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.attention_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.feed_forward.w1.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.feed_forward.w2.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.feed_forward.w3.weight": "model-00003-of-00004.safetensors",
+    "model.layers.30.ffn_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.attention.wo.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.attention.wqkv.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.attention_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.feed_forward.w1.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.feed_forward.w2.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.feed_forward.w3.weight": "model-00003-of-00004.safetensors",
+    "model.layers.31.ffn_norm.weight": "model-00003-of-00004.safetensors",
+    "model.layers.4.attention.wo.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.attention.wqkv.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.attention_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.feed_forward.w1.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.feed_forward.w2.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.feed_forward.w3.weight": "model-00001-of-00004.safetensors",
+    "model.layers.4.ffn_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.attention.wo.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.attention.wqkv.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.attention_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.feed_forward.w1.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.feed_forward.w2.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.feed_forward.w3.weight": "model-00001-of-00004.safetensors",
+    "model.layers.5.ffn_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.attention.wo.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.attention.wqkv.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.attention_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.feed_forward.w1.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.feed_forward.w2.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.feed_forward.w3.weight": "model-00001-of-00004.safetensors",
+    "model.layers.6.ffn_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.attention.wo.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.attention.wqkv.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.attention_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.feed_forward.w1.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.feed_forward.w2.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.feed_forward.w3.weight": "model-00001-of-00004.safetensors",
+    "model.layers.7.ffn_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.attention.wo.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.attention.wqkv.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.attention_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.feed_forward.w1.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.feed_forward.w2.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.feed_forward.w3.weight": "model-00001-of-00004.safetensors",
+    "model.layers.8.ffn_norm.weight": "model-00001-of-00004.safetensors",
+    "model.layers.9.attention.wo.weight": "model-00001-of-00004.safetensors",
+    "model.layers.9.attention.wqkv.weight": "model-00001-of-00004.safetensors",
+    "model.layers.9.attention_norm.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.feed_forward.w1.weight": "model-00001-of-00004.safetensors",
+    "model.layers.9.feed_forward.w2.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.feed_forward.w3.weight": "model-00002-of-00004.safetensors",
+    "model.layers.9.ffn_norm.weight": "model-00002-of-00004.safetensors",
+    "model.mm_projector.0.bias": "model-00004-of-00004.safetensors",
+    "model.mm_projector.0.weight": "model-00004-of-00004.safetensors",
+    "model.mm_projector.2.bias": "model-00004-of-00004.safetensors",
+    "model.mm_projector.2.weight": "model-00004-of-00004.safetensors",
+    "model.norm.weight": "model-00003-of-00004.safetensors",
+    "model.tok_embeddings.weight": "model-00001-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.embeddings.class_embedding": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.embeddings.patch_embedding.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.embeddings.position_embedding.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.0.layer_norm1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.0.layer_norm1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.0.layer_norm2.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.0.layer_norm2.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.0.mlp.fc1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.0.mlp.fc1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.0.mlp.fc2.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.0.mlp.fc2.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.0.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.0.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.0.self_attn.out_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.0.self_attn.out_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.0.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.0.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.0.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.0.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.1.layer_norm1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.1.layer_norm1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.1.layer_norm2.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.1.layer_norm2.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.1.mlp.fc1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.1.mlp.fc1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.1.mlp.fc2.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.1.mlp.fc2.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.1.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.1.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.1.self_attn.out_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.1.self_attn.out_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.1.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.1.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.1.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.1.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.10.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.10.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.10.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.10.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.10.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.10.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.10.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.10.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.10.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.10.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.10.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.10.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.10.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.10.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.10.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.10.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.11.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.11.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.11.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.11.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.11.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.11.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.11.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.11.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.11.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.11.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.11.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.11.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.11.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.11.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.11.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.11.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.12.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.12.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.12.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.12.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.12.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.12.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.12.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.12.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.12.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.12.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.12.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.12.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.12.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.12.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.12.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.12.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.13.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.13.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.13.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.13.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.13.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.13.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.13.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.13.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.13.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.13.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.13.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.13.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.13.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.13.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.13.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.13.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.14.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.14.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.14.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.14.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.14.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.14.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.14.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.14.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.14.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.14.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.14.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.14.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.14.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.14.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.14.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.14.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.15.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.15.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.15.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.15.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.15.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.15.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.15.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.15.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.15.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.15.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.15.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.15.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.15.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.15.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.15.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.15.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.16.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.16.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.16.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.16.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.16.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.16.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.16.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.16.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.16.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.16.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.16.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.16.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.16.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.16.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.16.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.16.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.17.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.17.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.17.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.17.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.17.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.17.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.17.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.17.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.17.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.17.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.17.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.17.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.17.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.17.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.17.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.17.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.18.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.18.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.18.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.18.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.18.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.18.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.18.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.18.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.18.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.18.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.18.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.18.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.18.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.18.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.18.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.18.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.19.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.19.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.19.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.19.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.19.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.19.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.19.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.19.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.19.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.19.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.19.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.19.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.19.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.19.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.19.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.19.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.2.layer_norm1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.2.layer_norm1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.2.layer_norm2.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.2.layer_norm2.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.2.mlp.fc1.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.2.mlp.fc1.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.2.mlp.fc2.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.2.mlp.fc2.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.2.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.2.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.2.self_attn.out_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.2.self_attn.out_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.2.self_attn.q_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.2.self_attn.q_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.2.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.2.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.20.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.20.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.20.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.20.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.20.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.20.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.20.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.20.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.20.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.20.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.20.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.20.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.20.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.20.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.20.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.20.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.21.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.21.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.21.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.21.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.21.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.21.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.21.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.21.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.21.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.21.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.21.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.21.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.21.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.21.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.21.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.21.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.22.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.22.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.22.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.22.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.22.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.22.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.22.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.22.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.22.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.22.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.22.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.22.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.22.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.22.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.22.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.22.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.23.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.23.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.23.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.23.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.23.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.23.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.23.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.23.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.23.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.23.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.23.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.23.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.23.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.23.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.23.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.23.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.3.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.3.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.3.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.3.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.3.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.3.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.3.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.3.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.3.self_attn.k_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.3.self_attn.k_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.3.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.3.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.3.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.3.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.3.self_attn.v_proj.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.3.self_attn.v_proj.weight": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.4.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.4.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.4.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.4.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.4.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.4.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.4.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.4.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.4.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.4.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.4.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.4.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.4.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.4.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.4.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.4.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.5.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.5.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.5.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.5.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.5.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.5.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.5.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.5.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.5.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.5.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.5.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.5.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.5.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.5.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.5.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.5.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.6.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.6.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.6.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.6.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.6.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.6.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.6.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.6.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.6.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.6.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.6.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.6.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.6.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.6.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.6.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.6.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.7.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.7.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.7.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.7.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.7.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.7.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.7.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.7.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.7.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.7.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.7.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.7.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.7.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.7.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.7.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.7.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.8.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.8.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.8.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.8.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.8.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.8.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.8.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.8.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.8.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.8.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.8.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.8.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.8.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.8.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.8.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.8.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.9.layer_norm1.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.9.layer_norm1.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.9.layer_norm2.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.9.layer_norm2.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.9.mlp.fc1.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.9.mlp.fc1.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.9.mlp.fc2.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.9.mlp.fc2.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.9.self_attn.k_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.9.self_attn.k_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.9.self_attn.out_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.9.self_attn.out_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.9.self_attn.q_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.9.self_attn.q_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.9.self_attn.v_proj.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.encoder.layers.9.self_attn.v_proj.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.post_layernorm.bias": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.post_layernorm.weight": "model-00004-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.pre_layrnorm.bias": "model-00003-of-00004.safetensors",
+    "model.vision_tower.vision_tower.vision_model.pre_layrnorm.weight": "model-00003-of-00004.safetensors",
+    "output.weight": "model-00004-of-00004.safetensors"
+  }
+}

special_tokens_map.json

@@ -0,0 +1,38 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|action_start|>",
+    "<|action_end|>",
+    "<|interpreter|>",
+    "<|plugin|>"
+  ],
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenization_internlm2.py

@@ -0,0 +1,236 @@
+# coding=utf-8
+# Copyright (c) The InternLM team and The HuggingFace Inc. team. All rights reserved.
+#
+# This code is based on transformers/src/transformers/models/llama/tokenization_llama.py
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Tokenization classes for InternLM."""
+import os
+from shutil import copyfile
+from typing import Any, Dict, List, Optional, Tuple
+
+import sentencepiece as spm
+from transformers.tokenization_utils import PreTrainedTokenizer
+from transformers.utils import logging
+
+logger = logging.get_logger(__name__)
+
+VOCAB_FILES_NAMES = {"vocab_file": "./tokenizer.model"}
+
+PRETRAINED_VOCAB_FILES_MAP = {}
+
+
+# Modified from transformers.model.llama.tokenization_llama.LlamaTokenizer
+class InternLM2Tokenizer(PreTrainedTokenizer):
+    """
+    Construct a InternLM2 tokenizer. Based on byte-level Byte-Pair-Encoding.
+
+    Args:
+        vocab_file (`str`):
+            Path to the vocabulary file.
+    """
+
+    vocab_files_names = VOCAB_FILES_NAMES
+    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
+    model_input_names = ["input_ids", "attention_mask"]
+    _auto_class = "AutoTokenizer"
+
+    def __init__(
+        self,
+        vocab_file,
+        unk_token="<unk>",
+        bos_token="<s>",
+        eos_token="</s>",
+        pad_token="</s>",
+        sp_model_kwargs: Optional[Dict[str, Any]] = None,
+        add_bos_token=True,
+        add_eos_token=False,
+        decode_with_prefix_space=False,
+        clean_up_tokenization_spaces=False,
+        **kwargs,
+    ):
+        self.sp_model_kwargs = {} if sp_model_kwargs is None else sp_model_kwargs
+        self.vocab_file = vocab_file
+        self.add_bos_token = add_bos_token
+        self.add_eos_token = add_eos_token
+        self.decode_with_prefix_space = decode_with_prefix_space
+        self.sp_model = spm.SentencePieceProcessor(**self.sp_model_kwargs)
+        self.sp_model.Load(vocab_file)
+        self._no_prefix_space_tokens = None
+        super().__init__(
+            bos_token=bos_token,
+            eos_token=eos_token,
+            unk_token=unk_token,
+            pad_token=pad_token,
+            clean_up_tokenization_spaces=clean_up_tokenization_spaces,
+            **kwargs,
+        )
+
+    @property
+    def no_prefix_space_tokens(self):
+        if self._no_prefix_space_tokens is None:
+            vocab = self.convert_ids_to_tokens(list(range(self.vocab_size)))
+            self._no_prefix_space_tokens = {i for i, tok in enumerate(vocab) if not tok.startswith("▁")}
+        return self._no_prefix_space_tokens
+
+    @property
+    def vocab_size(self):
+        """Returns vocab size"""
+        return self.sp_model.get_piece_size()
+
+    @property
+    def bos_token_id(self) -> Optional[int]:
+        return self.sp_model.bos_id()
+
+    @property
+    def eos_token_id(self) -> Optional[int]:
+        return self.sp_model.eos_id()
+
+    def get_vocab(self):
+        """Returns vocab as a dict"""
+        vocab = {self.convert_ids_to_tokens(i): i for i in range(self.vocab_size)}
+        vocab.update(self.added_tokens_encoder)
+        return vocab
+
+    def _tokenize(self, text):
+        """Returns a tokenized string."""
+        return self.sp_model.encode(text, out_type=str)
+
+    def _convert_token_to_id(self, token):
+        """Converts a token (str) in an id using the vocab."""
+        return self.sp_model.piece_to_id(token)
+
+    def _convert_id_to_token(self, index):
+        """Converts an index (integer) in a token (str) using the vocab."""
+        token = self.sp_model.IdToPiece(index)
+        return token
+
+    def _maybe_add_prefix_space(self, tokens, decoded):
+        if tokens and tokens[0] not in self.no_prefix_space_tokens:
+            return " " + decoded
+        else:
+            return decoded
+
+    def convert_tokens_to_string(self, tokens):
+        """Converts a sequence of tokens (string) in a single string."""
+        current_sub_tokens = []
+        out_string = ""
+        prev_is_special = False
+        for token in tokens:
+            # make sure that special tokens are not decoded using sentencepiece model
+            if token in self.all_special_tokens:
+                if not prev_is_special:
+                    out_string += " "
+                out_string += self.sp_model.decode(current_sub_tokens) + token
+                prev_is_special = True
+                current_sub_tokens = []
+            else:
+                current_sub_tokens.append(token)
+                prev_is_special = False
+        out_string += self.sp_model.decode(current_sub_tokens)
+        out_string = self.clean_up_tokenization(out_string)
+        out_string = self._maybe_add_prefix_space(tokens=tokens, decoded=out_string)
+        return out_string[1:]
+
+    def save_vocabulary(self, save_directory, filename_prefix: Optional[str] = None) -> Tuple[str]:
+        """
+        Save the vocabulary and special tokens file to a directory.
+
+        Args:
+            save_directory (`str`):
+                The directory in which to save the vocabulary.
+
+        Returns:
+            `Tuple(str)`: Paths to the files saved.
+        """
+        if not os.path.isdir(save_directory):
+            logger.error(f"Vocabulary path ({save_directory}) should be a directory")
+            return
+        out_vocab_file = os.path.join(
+            save_directory, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]
+        )
+
+        if os.path.abspath(self.vocab_file) != os.path.abspath(out_vocab_file) and os.path.isfile(self.vocab_file):
+            copyfile(self.vocab_file, out_vocab_file)
+        elif not os.path.isfile(self.vocab_file):
+            with open(out_vocab_file, "wb") as fi:
+                content_spiece_model = self.sp_model.serialized_model_proto()
+                fi.write(content_spiece_model)
+
+        return (out_vocab_file,)
+
+    def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None):
+        if self.add_bos_token:
+            bos_token_ids = [self.bos_token_id]
+        else:
+            bos_token_ids = []
+
+        output = bos_token_ids + token_ids_0
+
+        if token_ids_1 is not None:
+            output = output + token_ids_1
+
+        if self.add_eos_token:
+            output = output + [self.eos_token_id]
+
+        return output
+
+    def get_special_tokens_mask(
+        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None, already_has_special_tokens: bool = False
+    ) -> List[int]:
+        """
+        Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding
+        special tokens using the tokenizer `prepare_for_model` method.
+
+        Args:
+            token_ids_0 (`List[int]`):
+                List of IDs.
+            token_ids_1 (`List[int]`, *optional*):
+                Optional second list of IDs for sequence pairs.
+            already_has_special_tokens (`bool`, *optional*, defaults to `False`):
+                Whether or not the token list is already formatted with special tokens for the model.
+
+        Returns:
+            `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
+        """
+        if already_has_special_tokens:
+            return super().get_special_tokens_mask(
+                token_ids_0=token_ids_0, token_ids_1=token_ids_1, already_has_special_tokens=True
+            )
+
+        if token_ids_1 is None:
+            return [1] + ([0] * len(token_ids_0)) + [1]
+        return [1] + ([0] * len(token_ids_0)) + [1, 1] + ([0] * len(token_ids_1)) + [1]
+
+    def create_token_type_ids_from_sequences(
+        self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
+    ) -> List[int]:
+        """
+        Create a mask from the two sequences passed to be used in a sequence-pair classification task. T5 does not make
+        use of token type ids, therefore a list of zeros is returned.
+
+        Args:
+            token_ids_0 (`List[int]`):
+                List of IDs.
+            token_ids_1 (`List[int]`, *optional*):
+                Optional second list of IDs for sequence pairs.
+
+        Returns:
+            `List[int]`: List of zeros.
+        """
+        eos = [self.eos_token_id]
+
+        if token_ids_1 is None:
+            return len(token_ids_0 + eos) * [0]
+        return len(token_ids_0 + eos + token_ids_1 + eos) * [0]

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f868398fc4e05ee1e8aeba95ddf18ddcc45b8bce55d5093bead5bbf80429b48b
+size 1477754

tokenizer_config.json

@@ -0,0 +1,99 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "92538": {
+      "content": "<|plugin|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "92539": {
+      "content": "<|interpreter|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "92540": {
+      "content": "<|action_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "92541": {
+      "content": "<|action_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "92542": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "92543": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|action_start|>",
+    "<|action_end|>",
+    "<|interpreter|>",
+    "<|plugin|>"
+  ],
+  "auto_map": {
+    "AutoTokenizer": [
+      "tokenization_internlm2.InternLM2Tokenizer",
+      null
+    ]
+  },
+  "bos_token": "<s>",
+  "chat_template": "{{ bos_token }}{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "model_max_length": 32768,
+  "pad_token": "</s>",
+  "padding_side": "right",
+  "tokenizer_class": "InternLM2Tokenizer",
+  "unk_token": "<unk>"
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:69649ba3f3a80b494a682545ea472ba821288c2aff4b418db2314ea18d4c607c
+size 7160

training_log_20250116_215545.txt

@@ -0,0 +1,856 @@
+[2025-01-16 21:55:52,608] torch.distributed.run: [WARNING]
+[2025-01-16 21:55:52,608] torch.distributed.run: [WARNING] *****************************************
+[2025-01-16 21:55:52,608] torch.distributed.run: [WARNING] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.
+[2025-01-16 21:55:52,608] torch.distributed.run: [WARNING] *****************************************
+The cache for model files in Transformers v4.22.0 has been updated. Migrating your old cache. This is a one-time only operation. You can interrupt this and resume the migration later on by calling `transformers.utils.move_cache()`.
+
+0it [00:00, ?it/s]
+0it [00:00, ?it/s]
+/cpfs02/user/zhaoxiangyu/miniconda3/envs/llava/lib/python3.10/site-packages/_distutils_hack/__init__.py:54: UserWarning: Reliance on distutils from stdlib is deprecated. Users must rely on setuptools to provide the distutils module. Avoid importing distutils or import setuptools first, and avoid setting SETUPTOOLS_USE_DISTUTILS=stdlib. Register concerns at https://github.com/pypa/setuptools/issues/new?template=distutils-deprecation.yml
+  warnings.warn(
+/cpfs02/user/zhaoxiangyu/miniconda3/envs/llava/lib/python3.10/site-packages/_distutils_hack/__init__.py:54: UserWarning: Reliance on distutils from stdlib is deprecated. Users must rely on setuptools to provide the distutils module. Avoid importing distutils or import setuptools first, and avoid setting SETUPTOOLS_USE_DISTUTILS=stdlib. Register concerns at https://github.com/pypa/setuptools/issues/new?template=distutils-deprecation.yml
+  warnings.warn(
+/cpfs02/user/zhaoxiangyu/miniconda3/envs/llava/lib/python3.10/site-packages/_distutils_hack/__init__.py:54: UserWarning: Reliance on distutils from stdlib is deprecated. Users must rely on setuptools to provide the distutils module. Avoid importing distutils or import setuptools first, and avoid setting SETUPTOOLS_USE_DISTUTILS=stdlib. Register concerns at https://github.com/pypa/setuptools/issues/new?template=distutils-deprecation.yml
+  warnings.warn(
+/cpfs02/user/zhaoxiangyu/miniconda3/envs/llava/lib/python3.10/site-packages/_distutils_hack/__init__.py:54: UserWarning: Reliance on distutils from stdlib is deprecated. Users must rely on setuptools to provide the distutils module. Avoid importing distutils or import setuptools first, and avoid setting SETUPTOOLS_USE_DISTUTILS=stdlib. Register concerns at https://github.com/pypa/setuptools/issues/new?template=distutils-deprecation.yml
+  warnings.warn(
+/cpfs02/user/zhaoxiangyu/miniconda3/envs/llava/lib/python3.10/site-packages/_distutils_hack/__init__.py:54: UserWarning: Reliance on distutils from stdlib is deprecated. Users must rely on setuptools to provide the distutils module. Avoid importing distutils or import setuptools first, and avoid setting SETUPTOOLS_USE_DISTUTILS=stdlib. Register concerns at https://github.com/pypa/setuptools/issues/new?template=distutils-deprecation.yml
+  warnings.warn(
+/cpfs02/user/zhaoxiangyu/miniconda3/envs/llava/lib/python3.10/site-packages/_distutils_hack/__init__.py:54: UserWarning: Reliance on distutils from stdlib is deprecated. Users must rely on setuptools to provide the distutils module. Avoid importing distutils or import setuptools first, and avoid setting SETUPTOOLS_USE_DISTUTILS=stdlib. Register concerns at https://github.com/pypa/setuptools/issues/new?template=distutils-deprecation.yml
+  warnings.warn(
+/cpfs02/user/zhaoxiangyu/miniconda3/envs/llava/lib/python3.10/site-packages/_distutils_hack/__init__.py:54: UserWarning: Reliance on distutils from stdlib is deprecated. Users must rely on setuptools to provide the distutils module. Avoid importing distutils or import setuptools first, and avoid setting SETUPTOOLS_USE_DISTUTILS=stdlib. Register concerns at https://github.com/pypa/setuptools/issues/new?template=distutils-deprecation.yml
+  warnings.warn(
+/cpfs02/user/zhaoxiangyu/miniconda3/envs/llava/lib/python3.10/site-packages/_distutils_hack/__init__.py:54: UserWarning: Reliance on distutils from stdlib is deprecated. Users must rely on setuptools to provide the distutils module. Avoid importing distutils or import setuptools first, and avoid setting SETUPTOOLS_USE_DISTUTILS=stdlib. Register concerns at https://github.com/pypa/setuptools/issues/new?template=distutils-deprecation.yml
+  warnings.warn(
+[2025-01-16 21:56:07,205] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
+[2025-01-16 21:56:07,205] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
+[2025-01-16 21:56:07,205] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
+[2025-01-16 21:56:07,205] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
+[2025-01-16 21:56:07,205] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
+[2025-01-16 21:56:07,205] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
+[2025-01-16 21:56:07,205] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
+[2025-01-16 21:56:07,205] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
+df: df: /root/.triton/autotune/root/.triton/autotunedf: /root/.triton/autotune: 没有那个文件或目录: 没有那个文件或目录
+: 没有那个文件或目录
+
+df: /root/.triton/autotunedf: /root/.triton/autotune: 没有那个文件或目录
+: 没有那个文件或目录
+df: /root/.triton/autotune: 没有那个文件或目录
+ [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH
+ [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH
+ [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH
+ [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH
+ [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH
+ [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH
+ [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH
+ [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH
+ [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.1
+ [WARNING]  using untested triton version (2.1.0), only 1.0.0 is known to be compatible
+ [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.1
+ [WARNING]  using untested triton version (2.1.0), only 1.0.0 is known to be compatible
+ [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.1
+ [WARNING]  using untested triton version (2.1.0), only 1.0.0 is known to be compatible
+ [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.1
+ [WARNING]  using untested triton version (2.1.0), only 1.0.0 is known to be compatible
+ [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.1
+ [WARNING]  using untested triton version (2.1.0), only 1.0.0 is known to be compatible
+ [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.1
+ [WARNING]  using untested triton version (2.1.0), only 1.0.0 is known to be compatible
+ [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.1
+ [WARNING]  using untested triton version (2.1.0), only 1.0.0 is known to be compatible
+ [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.1
+ [WARNING]  using untested triton version (2.1.0), only 1.0.0 is known to be compatible
+[2025-01-16 21:56:24,924] [INFO] [comm.py:637:init_distributed] cdb=None
+[2025-01-16 21:56:24,924] [INFO] [comm.py:637:init_distributed] cdb=None
+[2025-01-16 21:56:24,925] [INFO] [comm.py:637:init_distributed] cdb=None
+[2025-01-16 21:56:24,925] [INFO] [comm.py:637:init_distributed] cdb=None
+[2025-01-16 21:56:24,925] [INFO] [comm.py:637:init_distributed] cdb=None
+[2025-01-16 21:56:24,925] [INFO] [comm.py:637:init_distributed] cdb=None
+[2025-01-16 21:56:24,925] [INFO] [comm.py:637:init_distributed] cdb=None
+[2025-01-16 21:56:24,926] [INFO] [comm.py:637:init_distributed] cdb=None
+01/16/2025 21:56:24 - WARNING - llava.train.train - Process rank: 0, device: cuda:0, n_gpu: 1distributed training: True, 16-bits training: False
+01/16/2025 21:56:24 - INFO - llava.train.train - Training/evaluation parameters TrainingArguments(
+_n_gpu=1,
+adafactor=False,
+adam_beta1=0.9,
+adam_beta2=0.999,
+adam_epsilon=1e-08,
+auto_find_batch_size=False,
+bf16=True,
+bf16_full_eval=False,
+bits=16,
+cache_dir=None,
+data_seed=None,
+dataloader_drop_last=False,
+dataloader_num_workers=4,
+dataloader_persistent_workers=False,
+dataloader_pin_memory=True,
+ddp_backend=None,
+ddp_broadcast_buffers=None,
+ddp_bucket_cap_mb=None,
+ddp_find_unused_parameters=None,
+ddp_timeout=1800,
+debug=[],
+deepspeed=./scripts/zero3.json,
+disable_tqdm=False,
+dispatch_batches=None,
+do_eval=False,
+do_predict=False,
+do_train=False,
+double_quant=True,
+eval_accumulation_steps=None,
+eval_delay=0,
+eval_steps=None,
+evaluation_strategy=no,
+fp16=False,
+fp16_backend=auto,
+fp16_full_eval=False,
+fp16_opt_level=O1,
+freeze_mm_mlp_adapter=False,
+fsdp=[],
+fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_grad_ckpt': False},
+fsdp_min_num_params=0,
+fsdp_transformer_layer_cls_to_wrap=None,
+full_determinism=False,
+gradient_accumulation_steps=2,
+gradient_checkpointing=True,
+gradient_checkpointing_kwargs=None,
+greater_is_better=None,
+group_by_length=False,
+group_by_modality_length=True,
+half_precision_backend=auto,
+hub_always_push=False,
+hub_model_id=None,
+hub_private_repo=False,
+hub_strategy=every_save,
+hub_token=<HUB_TOKEN>,
+ignore_data_skip=False,
+include_inputs_for_metrics=False,
+include_num_input_tokens_seen=False,
+include_tokens_per_second=False,
+jit_mode_eval=False,
+label_names=None,
+label_smoothing_factor=0.0,
+learning_rate=2e-05,
+length_column_name=length,
+load_best_model_at_end=False,
+local_rank=0,
+log_level=passive,
+log_level_replica=warning,
+log_on_each_node=True,
+logging_dir=./checkpoints/llavaAR4-internlm2_5-7b-sft-llavanext-notext-kn-infpolishmd-detail-knins40k-creationme10kfixed-chart11kmerge-tqa8k-info28kgpt/runs/Jan16_21-56-24_dlc1w85u6fy2xvlb-worker-0,
+logging_first_step=False,
+logging_nan_inf_filter=True,
+logging_steps=1.0,
+logging_strategy=steps,
+lora_alpha=16,
+lora_bias=none,
+lora_dropout=0.05,
+lora_enable=False,
+lora_r=64,
+lora_weight_path=,
+lr_scheduler_kwargs={},
+lr_scheduler_type=cosine,
+max_grad_norm=1.0,
+max_steps=-1,
+metric_for_best_model=None,
+mm_projector_lr=None,
+mm_vision_tower_lr=2e-06,
+model_max_length=32768,
+mp_parameters=,
+mpt_attn_impl=triton,
+neftune_noise_alpha=None,
+no_cuda=False,
+num_train_epochs=1.0,
+optim=adamw_torch,
+optim_args=None,
+output_dir=./checkpoints/llavaAR4-internlm2_5-7b-sft-llavanext-notext-kn-infpolishmd-detail-knins40k-creationme10kfixed-chart11kmerge-tqa8k-info28kgpt,
+overwrite_output_dir=False,
+past_index=-1,
+per_device_eval_batch_size=4,
+per_device_train_batch_size=4,
+prediction_loss_only=False,
+push_to_hub=False,
+push_to_hub_model_id=None,
+push_to_hub_organization=None,
+push_to_hub_token=<PUSH_TO_HUB_TOKEN>,
+quant_type=nf4,
+ray_scope=last,
+remove_unused_columns=False,
+report_to=['wandb'],
+resume_from_checkpoint=None,
+run_name=llavaAR4-internlm2_5-7b-sft-llavanext-notext-kn-infpolishmd-detail-knins40k-creationme10kfixed-chart11kmerge-tqa8k-info28kgpt,
+save_on_each_node=False,
+save_only_model=False,
+save_safetensors=True,
+save_steps=10000,
+save_strategy=steps,
+save_total_limit=1,
+seed=42,
+skip_memory_metrics=True,
+split_batches=False,
+tf32=True,
+torch_compile=False,
+torch_compile_backend=None,
+torch_compile_mode=None,
+torchdynamo=None,
+tpu_metrics_debug=False,
+tpu_num_cores=None,
+use_cpu=False,
+use_ipex=False,
+use_legacy_prediction_loop=False,
+use_mps_device=False,
+warmup_ratio=0.03,
+warmup_steps=0,
+weight_decay=0.0,
+)
+01/16/2025 21:56:24 - INFO - llava.train.train - Training/evaluation parameters DataArguments(data_path=None, meta_path='playground/meta_json/llavanext_sample/llava_next_notext_inf37kpolishmd_de35k_know40k_knins40k_creation10kfixed_chart11kmerge_tqa8k_info28k_gpt.json', lazy_preprocess=True, is_multimodal=False, image_folder=None, image_aspect_ratio='anyres', image_grid_pinpoints='[(336, 672), (672, 336), (672, 672), (1008, 336), (336, 1008)]', image_crop_resolution=None, image_split_resolution=None, use_data_resampling=False)
+[INFO|configuration_utils.py:727] 2025-01-16 21:56:24,953 >> loading configuration file models/internlm/internlm2_5-7b-chat/config.json
+[INFO|configuration_utils.py:727] 2025-01-16 21:56:24,975 >> loading configuration file models/internlm/internlm2_5-7b-chat/config.json
+[INFO|configuration_utils.py:792] 2025-01-16 21:56:24,976 >> Model config InternLM2Config {
+  "_name_or_path": "models/internlm/internlm2_5-7b-chat",
+  "architectures": [
+    "InternLM2ForCausalLM"
+  ],
+  "attn_implementation": "eager",
+  "auto_map": {
+    "AutoConfig": "configuration_internlm2.InternLM2Config",
+    "AutoModel": "modeling_internlm2.InternLM2ForCausalLM",
+    "AutoModelForCausalLM": "modeling_internlm2.InternLM2ForCausalLM"
+  },
+  "bias": false,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 32768,
+  "model_type": "internlm2",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "pad_token_id": 2,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": {
+    "factor": 2.0,
+    "type": "dynamic"
+  },
+  "rope_theta": 1000000,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.37.2",
+  "use_cache": true,
+  "vocab_size": 92544
+}
+
+[WARNING|modeling_utils.py:2918] 2025-01-16 21:56:24,979 >> The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
+[INFO|modeling_utils.py:3473] 2025-01-16 21:56:25,020 >> loading weights file models/internlm/internlm2_5-7b-chat/model.safetensors.index.json
+[INFO|modeling_utils.py:1426] 2025-01-16 21:56:25,023 >> Instantiating LlavaInternlm2ForCausalLM model under default dtype torch.bfloat16.
+[INFO|modeling_utils.py:3582] 2025-01-16 21:56:25,023 >> Detected DeepSpeed ZeRO-3: activating zero.init() for this model
+[INFO|configuration_utils.py:826] 2025-01-16 21:56:25,030 >> Generate config GenerationConfig {
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "pad_token_id": 2
+}
+
+01/16/2025 21:56:27 - WARNING - llava.train.train - Process rank: 2, device: cuda:2, n_gpu: 1distributed training: True, 16-bits training: False
+01/16/2025 21:56:27 - WARNING - llava.train.train - Process rank: 3, device: cuda:3, n_gpu: 1distributed training: True, 16-bits training: False
+01/16/2025 21:56:27 - WARNING - llava.train.train - Process rank: 7, device: cuda:7, n_gpu: 1distributed training: True, 16-bits training: False
+[WARNING|modeling_utils.py:2918] 2025-01-16 21:56:27,414 >> The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
+[WARNING|modeling_utils.py:2918] 2025-01-16 21:56:27,415 >> The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
+[WARNING|modeling_utils.py:2918] 2025-01-16 21:56:27,426 >> The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
+01/16/2025 21:56:27 - WARNING - llava.train.train - Process rank: 1, device: cuda:1, n_gpu: 1distributed training: True, 16-bits training: False
+[WARNING|modeling_utils.py:2918] 2025-01-16 21:56:27,443 >> The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
+01/16/2025 21:56:27 - WARNING - llava.train.train - Process rank: 4, device: cuda:4, n_gpu: 1distributed training: True, 16-bits training: False
+01/16/2025 21:56:27 - WARNING - llava.train.train - Process rank: 6, device: cuda:6, n_gpu: 1distributed training: True, 16-bits training: False
+[WARNING|modeling_utils.py:2918] 2025-01-16 21:56:27,472 >> The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
+[WARNING|modeling_utils.py:2918] 2025-01-16 21:56:27,479 >> The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
+01/16/2025 21:56:27 - WARNING - llava.train.train - Process rank: 5, device: cuda:5, n_gpu: 1distributed training: True, 16-bits training: False
+[WARNING|modeling_utils.py:2918] 2025-01-16 21:56:27,787 >> The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
+dlc1w85u6fy2xvlb-worker-0:74:74 [1] NCCL INFO cudaDriverVersion 12010
+dlc1w85u6fy2xvlb-worker-0:80:80 [7] NCCL INFO cudaDriverVersion 12010
+dlc1w85u6fy2xvlb-worker-0:75:75 [2] NCCL INFO cudaDriverVersion 12010
+dlc1w85u6fy2xvlb-worker-0:78:78 [5] NCCL INFO cudaDriverVersion 12010
+dlc1w85u6fy2xvlb-worker-0:79:79 [6] NCCL INFO cudaDriverVersion 12010
+dlc1w85u6fy2xvlb-worker-0:77:77 [4] NCCL INFO cudaDriverVersion 12010
+dlc1w85u6fy2xvlb-worker-0:73:73 [0] NCCL INFO cudaDriverVersion 12010
+dlc1w85u6fy2xvlb-worker-0:76:76 [3] NCCL INFO cudaDriverVersion 12010
+dlc1w85u6fy2xvlb-worker-0:80:80 [7] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth
+dlc1w85u6fy2xvlb-worker-0:77:77 [4] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth
+dlc1w85u6fy2xvlb-worker-0:78:78 [5] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth
+dlc1w85u6fy2xvlb-worker-0:75:75 [2] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth
+dlc1w85u6fy2xvlb-worker-0:76:76 [3] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth
+dlc1w85u6fy2xvlb-worker-0:74:74 [1] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth
+dlc1w85u6fy2xvlb-worker-0:73:73 [0] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth
+dlc1w85u6fy2xvlb-worker-0:79:79 [6] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth
+dlc1w85u6fy2xvlb-worker-0:78:78 [5] NCCL INFO Bootstrap : Using eth0:22.8.45.198<0>
+dlc1w85u6fy2xvlb-worker-0:77:77 [4] NCCL INFO Bootstrap : Using eth0:22.8.45.198<0>
+dlc1w85u6fy2xvlb-worker-0:80:80 [7] NCCL INFO Bootstrap : Using eth0:22.8.45.198<0>
+dlc1w85u6fy2xvlb-worker-0:77:77 [4] NCCL INFO Plugin name set by env to libnccl-net-none.so
+dlc1w85u6fy2xvlb-worker-0:78:78 [5] NCCL INFO Plugin name set by env to libnccl-net-none.so
+dlc1w85u6fy2xvlb-worker-0:80:80 [7] NCCL INFO Plugin name set by env to libnccl-net-none.so
+dlc1w85u6fy2xvlb-worker-0:74:74 [1] NCCL INFO Bootstrap : Using eth0:22.8.45.198<0>
+dlc1w85u6fy2xvlb-worker-0:74:74 [1] NCCL INFO Plugin name set by env to libnccl-net-none.so
+dlc1w85u6fy2xvlb-worker-0:76:76 [3] NCCL INFO Bootstrap : Using eth0:22.8.45.198<0>
+dlc1w85u6fy2xvlb-worker-0:79:79 [6] NCCL INFO Bootstrap : Using eth0:22.8.45.198<0>
+dlc1w85u6fy2xvlb-worker-0:76:76 [3] NCCL INFO Plugin name set by env to libnccl-net-none.so
+dlc1w85u6fy2xvlb-worker-0:73:73 [0] NCCL INFO Bootstrap : Using eth0:22.8.45.198<0>
+dlc1w85u6fy2xvlb-worker-0:79:79 [6] NCCL INFO Plugin name set by env to libnccl-net-none.so
+dlc1w85u6fy2xvlb-worker-0:73:73 [0] NCCL INFO Plugin name set by env to libnccl-net-none.so
+dlc1w85u6fy2xvlb-worker-0:75:75 [2] NCCL INFO Bootstrap : Using eth0:22.8.45.198<0>
+dlc1w85u6fy2xvlb-worker-0:75:75 [2] NCCL INFO Plugin name set by env to libnccl-net-none.so
+dlc1w85u6fy2xvlb-worker-0:74:74 [1] NCCL INFO NET/Plugin : Plugin load (libnccl-net-none.so) returned 2 : libnccl-net-none.so: cannot open shared object file: No such file or directory
+dlc1w85u6fy2xvlb-worker-0:78:78 [5] NCCL INFO NET/Plugin : Plugin load (libnccl-net-none.so) returned 2 : libnccl-net-none.so: cannot open shared object file: No such file or directory
+dlc1w85u6fy2xvlb-worker-0:75:75 [2] NCCL INFO NET/Plugin : Plugin load (libnccl-net-none.so) returned 2 : libnccl-net-none.so: cannot open shared object file: No such file or directory
+dlc1w85u6fy2xvlb-worker-0:79:79 [6] NCCL INFO NET/Plugin : Plugin load (libnccl-net-none.so) returned 2 : libnccl-net-none.so: cannot open shared object file: No such file or directory
+dlc1w85u6fy2xvlb-worker-0:74:74 [1] NCCL INFO NET/Plugin : No plugin found, using internal implementation
+dlc1w85u6fy2xvlb-worker-0:78:78 [5] NCCL INFO NET/Plugin : No plugin found, using internal implementation
+dlc1w85u6fy2xvlb-worker-0:79:79 [6] NCCL INFO NET/Plugin : No plugin found, using internal implementation
+dlc1w85u6fy2xvlb-worker-0:75:75 [2] NCCL INFO NET/Plugin : No plugin found, using internal implementation
+dlc1w85u6fy2xvlb-worker-0:80:80 [7] NCCL INFO NET/Plugin : Plugin load (libnccl-net-none.so) returned 2 : libnccl-net-none.so: cannot open shared object file: No such file or directory
+dlc1w85u6fy2xvlb-worker-0:77:77 [4] NCCL INFO NET/Plugin : Plugin load (libnccl-net-none.so) returned 2 : libnccl-net-none.so: cannot open shared object file: No such file or directory
+dlc1w85u6fy2xvlb-worker-0:80:80 [7] NCCL INFO NET/Plugin : No plugin found, using internal implementation
+dlc1w85u6fy2xvlb-worker-0:77:77 [4] NCCL INFO NET/Plugin : No plugin found, using internal implementation
+dlc1w85u6fy2xvlb-worker-0:73:73 [0] NCCL INFO NET/Plugin : Plugin load (libnccl-net-none.so) returned 2 : libnccl-net-none.so: cannot open shared object file: No such file or directory
+dlc1w85u6fy2xvlb-worker-0:76:76 [3] NCCL INFO NET/Plugin : Plugin load (libnccl-net-none.so) returned 2 : libnccl-net-none.so: cannot open shared object file: No such file or directory
+dlc1w85u6fy2xvlb-worker-0:73:73 [0] NCCL INFO NET/Plugin : No plugin found, using internal implementation
+dlc1w85u6fy2xvlb-worker-0:76:76 [3] NCCL INFO NET/Plugin : No plugin found, using internal implementation
+dlc1w85u6fy2xvlb-worker-0:74:318 [1] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth
+dlc1w85u6fy2xvlb-worker-0:75:317 [2] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth
+dlc1w85u6fy2xvlb-worker-0:80:320 [7] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth
+dlc1w85u6fy2xvlb-worker-0:76:316 [3] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth
+dlc1w85u6fy2xvlb-worker-0:78:321 [5] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth
+dlc1w85u6fy2xvlb-worker-0:73:323 [0] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth
+dlc1w85u6fy2xvlb-worker-0:74:318 [1] NCCL INFO NCCL_IB_HCA set to mlx5
+dlc1w85u6fy2xvlb-worker-0:75:317 [2] NCCL INFO NCCL_IB_HCA set to mlx5
+dlc1w85u6fy2xvlb-worker-0:78:321 [5] NCCL INFO NCCL_IB_HCA set to mlx5
+dlc1w85u6fy2xvlb-worker-0:80:320 [7] NCCL INFO NCCL_IB_HCA set to mlx5
+dlc1w85u6fy2xvlb-worker-0:76:316 [3] NCCL INFO NCCL_IB_HCA set to mlx5
+dlc1w85u6fy2xvlb-worker-0:73:323 [0] NCCL INFO NCCL_IB_HCA set to mlx5
+dlc1w85u6fy2xvlb-worker-0:77:319 [4] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth
+dlc1w85u6fy2xvlb-worker-0:77:319 [4] NCCL INFO NCCL_IB_HCA set to mlx5
+dlc1w85u6fy2xvlb-worker-0:79:322 [6] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth
+dlc1w85u6fy2xvlb-worker-0:79:322 [6] NCCL INFO NCCL_IB_HCA set to mlx5
+dlc1w85u6fy2xvlb-worker-0:73:323 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [2]mlx5_2:1/RoCE [3]mlx5_3:1/RoCE [RO]; OOB eth0:22.8.45.198<0>
+dlc1w85u6fy2xvlb-worker-0:73:323 [0] NCCL INFO Using network IB
+dlc1w85u6fy2xvlb-worker-0:75:317 [2] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [2]mlx5_2:1/RoCE [3]mlx5_3:1/RoCE [RO]; OOB eth0:22.8.45.198<0>
+dlc1w85u6fy2xvlb-worker-0:75:317 [2] NCCL INFO Using network IB
+dlc1w85u6fy2xvlb-worker-0:78:321 [5] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [2]mlx5_2:1/RoCE [3]mlx5_3:1/RoCE [RO]; OOB eth0:22.8.45.198<0>
+dlc1w85u6fy2xvlb-worker-0:78:321 [5] NCCL INFO Using network IB
+dlc1w85u6fy2xvlb-worker-0:74:318 [1] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [2]mlx5_2:1/RoCE [3]mlx5_3:1/RoCE [RO]; OOB eth0:22.8.45.198<0>
+dlc1w85u6fy2xvlb-worker-0:74:318 [1] NCCL INFO Using network IB
+dlc1w85u6fy2xvlb-worker-0:76:316 [3] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [2]mlx5_2:1/RoCE [3]mlx5_3:1/RoCE [RO]; OOB eth0:22.8.45.198<0>
+dlc1w85u6fy2xvlb-worker-0:76:316 [3] NCCL INFO Using network IB
+dlc1w85u6fy2xvlb-worker-0:77:319 [4] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [2]mlx5_2:1/RoCE [3]mlx5_3:1/RoCE [RO]; OOB eth0:22.8.45.198<0>
+dlc1w85u6fy2xvlb-worker-0:77:319 [4] NCCL INFO Using network IB
+dlc1w85u6fy2xvlb-worker-0:80:320 [7] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [2]mlx5_2:1/RoCE [3]mlx5_3:1/RoCE [RO]; OOB eth0:22.8.45.198<0>
+dlc1w85u6fy2xvlb-worker-0:80:320 [7] NCCL INFO Using network IB
+dlc1w85u6fy2xvlb-worker-0:79:322 [6] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [2]mlx5_2:1/RoCE [3]mlx5_3:1/RoCE [RO]; OOB eth0:22.8.45.198<0>
+dlc1w85u6fy2xvlb-worker-0:79:322 [6] NCCL INFO Using network IB
+dlc1w85u6fy2xvlb-worker-0:76:316 [3] NCCL INFO comm 0x9b7bdc70 rank 11 nranks 16 cudaDev 3 nvmlDev 3 busId 40 commId 0xe730164ea45002c1 - Init START
+dlc1w85u6fy2xvlb-worker-0:79:322 [6] NCCL INFO comm 0x9aa4c060 rank 14 nranks 16 cudaDev 6 nvmlDev 6 busId 70 commId 0xe730164ea45002c1 - Init START
+dlc1w85u6fy2xvlb-worker-0:78:321 [5] NCCL INFO comm 0x9b39f9e0 rank 13 nranks 16 cudaDev 5 nvmlDev 5 busId 60 commId 0xe730164ea45002c1 - Init START
+dlc1w85u6fy2xvlb-worker-0:77:319 [4] NCCL INFO comm 0x9b9e2970 rank 12 nranks 16 cudaDev 4 nvmlDev 4 busId 50 commId 0xe730164ea45002c1 - Init START
+dlc1w85u6fy2xvlb-worker-0:80:320 [7] NCCL INFO comm 0x9bccd080 rank 15 nranks 16 cudaDev 7 nvmlDev 7 busId 80 commId 0xe730164ea45002c1 - Init START
+dlc1w85u6fy2xvlb-worker-0:73:323 [0] NCCL INFO comm 0x9aabc5f0 rank 8 nranks 16 cudaDev 0 nvmlDev 0 busId 10 commId 0xe730164ea45002c1 - Init START
+dlc1w85u6fy2xvlb-worker-0:75:317 [2] NCCL INFO comm 0x9bcdb060 rank 10 nranks 16 cudaDev 2 nvmlDev 2 busId 30 commId 0xe730164ea45002c1 - Init START
+dlc1w85u6fy2xvlb-worker-0:74:318 [1] NCCL INFO comm 0x9a893920 rank 9 nranks 16 cudaDev 1 nvmlDev 1 busId 20 commId 0xe730164ea45002c1 - Init START
+dlc1w85u6fy2xvlb-worker-0:79:322 [6] NCCL INFO NVLS multicast support is not available on dev 6
+dlc1w85u6fy2xvlb-worker-0:76:316 [3] NCCL INFO Setting affinity for GPU 3 to ffffffff,ffffffff,ffffffff
+dlc1w85u6fy2xvlb-worker-0:76:316 [3] NCCL INFO NVLS multicast support is not available on dev 3
+dlc1w85u6fy2xvlb-worker-0:80:320 [7] NCCL INFO NVLS multicast support is not available on dev 7
+dlc1w85u6fy2xvlb-worker-0:74:318 [1] NCCL INFO Setting affinity for GPU 1 to ffffffff,ffffffff,ffffffff
+dlc1w85u6fy2xvlb-worker-0:74:318 [1] NCCL INFO NVLS multicast support is not available on dev 1
+dlc1w85u6fy2xvlb-worker-0:78:321 [5] NCCL INFO NVLS multicast support is not available on dev 5
+dlc1w85u6fy2xvlb-worker-0:73:323 [0] NCCL INFO Setting affinity for GPU 0 to ffffffff,ffffffff,ffffffff
+dlc1w85u6fy2xvlb-worker-0:73:323 [0] NCCL INFO NVLS multicast support is not available on dev 0
+dlc1w85u6fy2xvlb-worker-0:77:319 [4] NCCL INFO NVLS multicast support is not available on dev 4
+dlc1w85u6fy2xvlb-worker-0:75:317 [2] NCCL INFO Setting affinity for GPU 2 to ffffffff,ffffffff,ffffffff
+dlc1w85u6fy2xvlb-worker-0:75:317 [2] NCCL INFO NVLS multicast support is not available on dev 2
+dlc1w85u6fy2xvlb-worker-0:77:319 [4] NCCL INFO NCCL_MIN_NCHANNELS set by environment to 4.
+dlc1w85u6fy2xvlb-worker-0:77:319 [4] NCCL INFO Trees [0] 13/-1/-1->12->11 [1] 13/-1/-1->12->11 [2] 13/-1/-1->12->4 [3] 13/-1/-1->12->11 [4] 13/-1/-1->12->11 [5] 13/-1/-1->12->11 [6] 13/4/-1->12->-1 [7] 13/-1/-1->12->11
+dlc1w85u6fy2xvlb-worker-0:77:319 [4] NCCL INFO P2P Chunksize set to 131072
+dlc1w85u6fy2xvlb-worker-0:78:321 [5] NCCL INFO NCCL_MIN_NCHANNELS set by environment to 4.
+dlc1w85u6fy2xvlb-worker-0:80:320 [7] NCCL INFO NCCL_MIN_NCHANNELS set by environment to 4.
+dlc1w85u6fy2xvlb-worker-0:79:322 [6] NCCL INFO NCCL_MIN_NCHANNELS set by environment to 4.
+dlc1w85u6fy2xvlb-worker-0:78:321 [5] NCCL INFO Trees [0] 14/-1/-1->13->12 [1] 14/-1/-1->13->12 [2] 14/-1/-1->13->12 [3] -1/-1/-1->13->12 [4] 14/-1/-1->13->12 [5] 14/-1/-1->13->12 [6] 14/-1/-1->13->12 [7] -1/-1/-1->13->12
+dlc1w85u6fy2xvlb-worker-0:80:320 [7] NCCL INFO Trees [0] -1/-1/-1->15->14 [1] 8/-1/-1->15->14 [2] 8/-1/-1->15->14 [3] 8/-1/-1->15->14 [4] -1/-1/-1->15->14 [5] 8/-1/-1->15->14 [6] 8/-1/-1->15->14 [7] 8/-1/-1->15->14
+dlc1w85u6fy2xvlb-worker-0:78:321 [5] NCCL INFO P2P Chunksize set to 131072
+dlc1w85u6fy2xvlb-worker-0:80:320 [7] NCCL INFO P2P Chunksize set to 131072
+dlc1w85u6fy2xvlb-worker-0:79:322 [6] NCCL INFO Trees [0] 15/-1/-1->14->13 [1] 15/-1/-1->14->13 [2] 15/-1/-1->14->13 [3] 15/-1/-1->14->6 [4] 15/-1/-1->14->13 [5] 15/-1/-1->14->13 [6] 15/-1/-1->14->13 [7] 15/6/-1->14->-1
+dlc1w85u6fy2xvlb-worker-0:76:316 [3] NCCL INFO NCCL_MIN_NCHANNELS set by environment to 4.
+dlc1w85u6fy2xvlb-worker-0:79:322 [6] NCCL INFO P2P Chunksize set to 131072
+dlc1w85u6fy2xvlb-worker-0:75:317 [2] NCCL INFO NCCL_MIN_NCHANNELS set by environment to 4.
+dlc1w85u6fy2xvlb-worker-0:75:317 [2] NCCL INFO Trees [0] 11/-1/-1->10->9 [1] 11/-1/-1->10->2 [2] 11/-1/-1->10->9 [3] 11/-1/-1->10->9 [4] 11/-1/-1->10->9 [5] 11/2/-1->10->-1 [6] 11/-1/-1->10->9 [7] 11/-1/-1->10->9
+dlc1w85u6fy2xvlb-worker-0:76:316 [3] NCCL INFO Trees [0] 12/-1/-1->11->10 [1] 12/-1/-1->11->10 [2] -1/-1/-1->11->10 [3] 12/-1/-1->11->10 [4] 12/-1/-1->11->10 [5] 12/-1/-1->11->10 [6] -1/-1/-1->11->10 [7] 12/-1/-1->11->10
+dlc1w85u6fy2xvlb-worker-0:75:317 [2] NCCL INFO P2P Chunksize set to 131072
+dlc1w85u6fy2xvlb-worker-0:76:316 [3] NCCL INFO P2P Chunksize set to 131072
+dlc1w85u6fy2xvlb-worker-0:74:318 [1] NCCL INFO NCCL_MIN_NCHANNELS set by environment to 4.
+dlc1w85u6fy2xvlb-worker-0:73:323 [0] NCCL INFO NCCL_MIN_NCHANNELS set by environment to 4.
+dlc1w85u6fy2xvlb-worker-0:74:318 [1] NCCL INFO Trees [0] 10/-1/-1->9->8 [1] -1/-1/-1->9->8 [2] 10/-1/-1->9->8 [3] 10/-1/-1->9->8 [4] 10/-1/-1->9->8 [5] -1/-1/-1->9->8 [6] 10/-1/-1->9->8 [7] 10/-1/-1->9->8
+dlc1w85u6fy2xvlb-worker-0:73:323 [0] NCCL INFO Trees [0] 9/-1/-1->8->0 [1] 9/-1/-1->8->15 [2] 9/-1/-1->8->15 [3] 9/-1/-1->8->15 [4] 9/0/-1->8->-1 [5] 9/-1/-1->8->15 [6] 9/-1/-1->8->15 [7] 9/-1/-1->8->15
+dlc1w85u6fy2xvlb-worker-0:74:318 [1] NCCL INFO P2P Chunksize set to 131072
+dlc1w85u6fy2xvlb-worker-0:73:323 [0] NCCL INFO P2P Chunksize set to 131072
+dlc1w85u6fy2xvlb-worker-0:77:319 [4] NCCL INFO Channel 03/0 : 12[4] -> 15[7] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:75:317 [2] NCCL INFO Channel 01/0 : 10[2] -> 15[7] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:73:323 [0] NCCL INFO Channel 01/0 : 8[0] -> 11[3] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:77:319 [4] NCCL INFO Channel 07/0 : 12[4] -> 15[7] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:74:318 [1] NCCL INFO Channel 00/0 : 9[1] -> 0[0] [send] via NET/IB/0/GDRDMA
+dlc1w85u6fy2xvlb-worker-0:79:322 [6] NCCL INFO Channel 03/0 : 7[7] -> 14[6] [receive] via NET/IB/3/GDRDMA
+dlc1w85u6fy2xvlb-worker-0:74:318 [1] NCCL INFO Channel 04/0 : 9[1] -> 0[0] [send] via NET/IB/0/GDRDMA
+dlc1w85u6fy2xvlb-worker-0:79:322 [6] NCCL INFO Channel 07/0 : 7[7] -> 14[6] [receive] via NET/IB/3/GDRDMA
+dlc1w85u6fy2xvlb-worker-0:75:317 [2] NCCL INFO Channel 05/0 : 10[2] -> 15[7] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:73:323 [0] NCCL INFO Channel 05/0 : 8[0] -> 11[3] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:73:323 [0] NCCL INFO Channel 03/0 : 8[0] -> 13[5] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:73:323 [0] NCCL INFO Channel 07/0 : 8[0] -> 13[5] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:77:319 [4] NCCL INFO Channel 02/0 : 5[5] -> 12[4] [receive] via NET/IB/2/GDRDMA
+dlc1w85u6fy2xvlb-worker-0:77:319 [4] NCCL INFO Channel 06/0 : 5[5] -> 12[4] [receive] via NET/IB/2/GDRDMA
+dlc1w85u6fy2xvlb-worker-0:76:316 [3] NCCL INFO Channel 01/0 : 11[3] -> 2[2] [send] via NET/IB/1/GDRDMA
+dlc1w85u6fy2xvlb-worker-0:76:316 [3] NCCL INFO Channel 05/0 : 11[3] -> 2[2] [send] via NET/IB/1/GDRDMA
+dlc1w85u6fy2xvlb-worker-0:73:323 [0] NCCL INFO Channel 00/0 : 1[1] -> 8[0] [receive] via NET/IB/0/GDRDMA
+dlc1w85u6fy2xvlb-worker-0:75:317 [2] NCCL INFO Channel 01/0 : 3[3] -> 10[2] [receive] via NET/IB/1/GDRDMA
+dlc1w85u6fy2xvlb-worker-0:73:323 [0] NCCL INFO Channel 04/0 : 1[1] -> 8[0] [receive] via NET/IB/0/GDRDMA
+dlc1w85u6fy2xvlb-worker-0:73:323 [0] NCCL INFO Channel 00/0 : 8[0] -> 15[7] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:75:317 [2] NCCL INFO Channel 05/0 : 3[3] -> 10[2] [receive] via NET/IB/1/GDRDMA
+dlc1w85u6fy2xvlb-worker-0:73:323 [0] NCCL INFO Channel 02/0 : 8[0] -> 15[7] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:77:319 [4] NCCL INFO Channel 01/0 : 12[4] -> 9[1] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:73:323 [0] NCCL INFO Channel 04/0 : 8[0] -> 15[7] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:78:321 [5] NCCL INFO Channel 02/0 : 13[5] -> 4[4] [send] via NET/IB/2/GDRDMA
+dlc1w85u6fy2xvlb-worker-0:78:321 [5] NCCL INFO Channel 06/0 : 13[5] -> 4[4] [send] via NET/IB/2/GDRDMA
+dlc1w85u6fy2xvlb-worker-0:79:322 [6] NCCL INFO Channel 03/0 : 14[6] -> 11[3] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:80:320 [7] NCCL INFO Channel 03/0 : 15[7] -> 6[6] [send] via NET/IB/3/GDRDMA
+dlc1w85u6fy2xvlb-worker-0:80:320 [7] NCCL INFO Channel 07/0 : 15[7] -> 6[6] [send] via NET/IB/3/GDRDMA
+dlc1w85u6fy2xvlb-worker-0:77:319 [4] NCCL INFO Channel 05/0 : 12[4] -> 9[1] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:73:323 [0] NCCL INFO Channel 06/0 : 8[0] -> 15[7] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:80:320 [7] NCCL INFO Channel 00/0 : 15[7] -> 14[6] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:79:322 [6] NCCL INFO Channel 07/0 : 14[6] -> 11[3] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:80:320 [7] NCCL INFO Channel 01/0 : 15[7] -> 14[6] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:80:320 [7] NCCL INFO Channel 02/0 : 15[7] -> 14[6] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:80:320 [7] NCCL INFO Channel 04/0 : 15[7] -> 14[6] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:75:317 [2] NCCL INFO Channel 00/0 : 10[2] -> 9[1] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:80:320 [7] NCCL INFO Channel 05/0 : 15[7] -> 14[6] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:75:317 [2] NCCL INFO Channel 02/0 : 10[2] -> 9[1] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:78:321 [5] NCCL INFO Channel 00/0 : 13[5] -> 12[4] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:80:320 [7] NCCL INFO Channel 06/0 : 15[7] -> 14[6] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:76:316 [3] NCCL INFO Channel 00/0 : 11[3] -> 10[2] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:75:317 [2] NCCL INFO Channel 03/0 : 10[2] -> 9[1] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:78:321 [5] NCCL INFO Channel 01/0 : 13[5] -> 12[4] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:76:316 [3] NCCL INFO Channel 02/0 : 11[3] -> 10[2] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:75:317 [2] NCCL INFO Channel 04/0 : 10[2] -> 9[1] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:78:321 [5] NCCL INFO Channel 03/0 : 13[5] -> 12[4] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:79:322 [6] NCCL INFO Channel 00/0 : 14[6] -> 13[5] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:76:316 [3] NCCL INFO Channel 03/0 : 11[3] -> 10[2] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:75:317 [2] NCCL INFO Channel 06/0 : 10[2] -> 9[1] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:78:321 [5] NCCL INFO Channel 04/0 : 13[5] -> 12[4] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:79:322 [6] NCCL INFO Channel 01/0 : 14[6] -> 13[5] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:76:316 [3] NCCL INFO Channel 04/0 : 11[3] -> 10[2] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:75:317 [2] NCCL INFO Channel 07/0 : 10[2] -> 9[1] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:78:321 [5] NCCL INFO Channel 05/0 : 13[5] -> 12[4] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:79:322 [6] NCCL INFO Channel 02/0 : 14[6] -> 13[5] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:76:316 [3] NCCL INFO Channel 06/0 : 11[3] -> 10[2] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:78:321 [5] NCCL INFO Channel 07/0 : 13[5] -> 12[4] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:77:319 [4] NCCL INFO Channel 00/0 : 12[4] -> 11[3] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:79:322 [6] NCCL INFO Channel 04/0 : 14[6] -> 13[5] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:76:316 [3] NCCL INFO Channel 07/0 : 11[3] -> 10[2] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:74:318 [1] NCCL INFO Channel 01/0 : 9[1] -> 8[0] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:77:319 [4] NCCL INFO Channel 02/0 : 12[4] -> 11[3] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:79:322 [6] NCCL INFO Channel 05/0 : 14[6] -> 13[5] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:74:318 [1] NCCL INFO Channel 02/0 : 9[1] -> 8[0] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:77:319 [4] NCCL INFO Channel 04/0 : 12[4] -> 11[3] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:79:322 [6] NCCL INFO Channel 06/0 : 14[6] -> 13[5] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:74:318 [1] NCCL INFO Channel 03/0 : 9[1] -> 8[0] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:77:319 [4] NCCL INFO Channel 06/0 : 12[4] -> 11[3] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:74:318 [1] NCCL INFO Channel 05/0 : 9[1] -> 8[0] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:74:318 [1] NCCL INFO Channel 06/0 : 9[1] -> 8[0] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:78:357 [5] NCCL INFO NCCL_IB_QPS_PER_CONNECTION set by environment to 8.
+dlc1w85u6fy2xvlb-worker-0:74:318 [1] NCCL INFO Channel 07/0 : 9[1] -> 8[0] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:76:360 [3] NCCL INFO NCCL_IB_QPS_PER_CONNECTION set by environment to 8.
+dlc1w85u6fy2xvlb-worker-0:77:356 [4] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3.
+dlc1w85u6fy2xvlb-worker-0:77:356 [4] NCCL INFO NCCL_IB_QPS_PER_CONNECTION set by environment to 8.
+dlc1w85u6fy2xvlb-worker-0:74:363 [1] NCCL INFO NCCL_IB_QPS_PER_CONNECTION set by environment to 8.
+dlc1w85u6fy2xvlb-worker-0:78:357 [5] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3.
+dlc1w85u6fy2xvlb-worker-0:76:360 [3] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3.
+dlc1w85u6fy2xvlb-worker-0:80:358 [7] NCCL INFO NCCL_IB_QPS_PER_CONNECTION set by environment to 8.
+dlc1w85u6fy2xvlb-worker-0:77:356 [4] NCCL INFO NCCL_IB_TC set by environment to 136.
+dlc1w85u6fy2xvlb-worker-0:77:356 [4] NCCL INFO NCCL_IB_SL set by environment to 5.
+dlc1w85u6fy2xvlb-worker-0:77:356 [4] NCCL INFO NCCL_IB_TIMEOUT set by environment to 22.
+dlc1w85u6fy2xvlb-worker-0:74:363 [1] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3.
+dlc1w85u6fy2xvlb-worker-0:75:359 [2] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3.
+dlc1w85u6fy2xvlb-worker-0:75:359 [2] NCCL INFO NCCL_IB_QPS_PER_CONNECTION set by environment to 8.
+dlc1w85u6fy2xvlb-worker-0:80:358 [7] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3.
+dlc1w85u6fy2xvlb-worker-0:79:361 [6] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3.
+dlc1w85u6fy2xvlb-worker-0:79:361 [6] NCCL INFO NCCL_IB_QPS_PER_CONNECTION set by environment to 8.
+dlc1w85u6fy2xvlb-worker-0:75:359 [2] NCCL INFO NCCL_IB_TC set by environment to 136.
+dlc1w85u6fy2xvlb-worker-0:75:359 [2] NCCL INFO NCCL_IB_SL set by environment to 5.
+dlc1w85u6fy2xvlb-worker-0:75:359 [2] NCCL INFO NCCL_IB_TIMEOUT set by environment to 22.
+dlc1w85u6fy2xvlb-worker-0:76:360 [3] NCCL INFO NCCL_IB_TC set by environment to 136.
+dlc1w85u6fy2xvlb-worker-0:76:360 [3] NCCL INFO NCCL_IB_SL set by environment to 5.
+dlc1w85u6fy2xvlb-worker-0:76:360 [3] NCCL INFO NCCL_IB_TIMEOUT set by environment to 22.
+dlc1w85u6fy2xvlb-worker-0:78:357 [5] NCCL INFO NCCL_IB_TC set by environment to 136.
+dlc1w85u6fy2xvlb-worker-0:78:357 [5] NCCL INFO NCCL_IB_SL set by environment to 5.
+dlc1w85u6fy2xvlb-worker-0:78:357 [5] NCCL INFO NCCL_IB_TIMEOUT set by environment to 22.
+dlc1w85u6fy2xvlb-worker-0:79:361 [6] NCCL INFO NCCL_IB_TC set by environment to 136.
+dlc1w85u6fy2xvlb-worker-0:79:361 [6] NCCL INFO NCCL_IB_SL set by environment to 5.
+dlc1w85u6fy2xvlb-worker-0:73:362 [0] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3.
+dlc1w85u6fy2xvlb-worker-0:73:362 [0] NCCL INFO NCCL_IB_QPS_PER_CONNECTION set by environment to 8.
+dlc1w85u6fy2xvlb-worker-0:79:361 [6] NCCL INFO NCCL_IB_TIMEOUT set by environment to 22.
+dlc1w85u6fy2xvlb-worker-0:74:363 [1] NCCL INFO NCCL_IB_TC set by environment to 136.
+dlc1w85u6fy2xvlb-worker-0:74:363 [1] NCCL INFO NCCL_IB_SL set by environment to 5.
+dlc1w85u6fy2xvlb-worker-0:74:363 [1] NCCL INFO NCCL_IB_TIMEOUT set by environment to 22.
+dlc1w85u6fy2xvlb-worker-0:80:358 [7] NCCL INFO NCCL_IB_TC set by environment to 136.
+dlc1w85u6fy2xvlb-worker-0:80:358 [7] NCCL INFO NCCL_IB_SL set by environment to 5.
+dlc1w85u6fy2xvlb-worker-0:80:358 [7] NCCL INFO NCCL_IB_TIMEOUT set by environment to 22.
+dlc1w85u6fy2xvlb-worker-0:73:362 [0] NCCL INFO NCCL_IB_TC set by environment to 136.
+dlc1w85u6fy2xvlb-worker-0:73:362 [0] NCCL INFO NCCL_IB_SL set by environment to 5.
+dlc1w85u6fy2xvlb-worker-0:73:362 [0] NCCL INFO NCCL_IB_TIMEOUT set by environment to 22.
+dlc1w85u6fy2xvlb-worker-0:76:316 [3] NCCL INFO Connected all rings
+dlc1w85u6fy2xvlb-worker-0:80:320 [7] NCCL INFO Connected all rings
+dlc1w85u6fy2xvlb-worker-0:78:321 [5] NCCL INFO Connected all rings
+dlc1w85u6fy2xvlb-worker-0:74:318 [1] NCCL INFO Connected all rings
+dlc1w85u6fy2xvlb-worker-0:76:316 [3] NCCL INFO Channel 00/0 : 11[3] -> 12[4] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:76:316 [3] NCCL INFO Channel 01/0 : 11[3] -> 12[4] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:76:316 [3] NCCL INFO Channel 03/0 : 11[3] -> 12[4] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:77:319 [4] NCCL INFO Connected all rings
+dlc1w85u6fy2xvlb-worker-0:76:316 [3] NCCL INFO Channel 04/0 : 11[3] -> 12[4] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:79:322 [6] NCCL INFO Connected all rings
+dlc1w85u6fy2xvlb-worker-0:76:316 [3] NCCL INFO Channel 05/0 : 11[3] -> 12[4] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:73:323 [0] NCCL INFO Connected all rings
+dlc1w85u6fy2xvlb-worker-0:73:323 [0] NCCL INFO Channel 00/0 : 8[0] -> 9[1] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:75:317 [2] NCCL INFO Connected all rings
+dlc1w85u6fy2xvlb-worker-0:76:316 [3] NCCL INFO Channel 07/0 : 11[3] -> 12[4] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:73:323 [0] NCCL INFO Channel 01/0 : 8[0] -> 9[1] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:78:321 [5] NCCL INFO Channel 00/0 : 13[5] -> 14[6] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:73:323 [0] NCCL INFO Channel 02/0 : 8[0] -> 9[1] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:78:321 [5] NCCL INFO Channel 01/0 : 13[5] -> 14[6] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:73:323 [0] NCCL INFO Channel 03/0 : 8[0] -> 9[1] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:74:318 [1] NCCL INFO Channel 00/0 : 9[1] -> 10[2] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:78:321 [5] NCCL INFO Channel 02/0 : 13[5] -> 14[6] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:73:323 [0] NCCL INFO Channel 04/0 : 8[0] -> 9[1] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:74:318 [1] NCCL INFO Channel 02/0 : 9[1] -> 10[2] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:78:321 [5] NCCL INFO Channel 04/0 : 13[5] -> 14[6] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:77:319 [4] NCCL INFO Channel 00/0 : 12[4] -> 13[5] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:73:323 [0] NCCL INFO Channel 05/0 : 8[0] -> 9[1] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:74:318 [1] NCCL INFO Channel 03/0 : 9[1] -> 10[2] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:78:321 [5] NCCL INFO Channel 05/0 : 13[5] -> 14[6] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:79:322 [6] NCCL INFO Channel 00/0 : 14[6] -> 15[7] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:77:319 [4] NCCL INFO Channel 01/0 : 12[4] -> 13[5] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:73:323 [0] NCCL INFO Channel 06/0 : 8[0] -> 9[1] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:74:318 [1] NCCL INFO Channel 04/0 : 9[1] -> 10[2] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:78:321 [5] NCCL INFO Channel 06/0 : 13[5] -> 14[6] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:79:322 [6] NCCL INFO Channel 01/0 : 14[6] -> 15[7] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:75:317 [2] NCCL INFO Channel 00/0 : 10[2] -> 11[3] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:77:319 [4] NCCL INFO Channel 02/0 : 12[4] -> 13[5] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:73:323 [0] NCCL INFO Channel 07/0 : 8[0] -> 9[1] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:74:318 [1] NCCL INFO Channel 06/0 : 9[1] -> 10[2] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:79:322 [6] NCCL INFO Channel 02/0 : 14[6] -> 15[7] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:75:317 [2] NCCL INFO Channel 01/0 : 10[2] -> 11[3] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:77:319 [4] NCCL INFO Channel 03/0 : 12[4] -> 13[5] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:74:318 [1] NCCL INFO Channel 07/0 : 9[1] -> 10[2] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:79:322 [6] NCCL INFO Channel 03/0 : 14[6] -> 15[7] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:75:317 [2] NCCL INFO Channel 02/0 : 10[2] -> 11[3] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:77:319 [4] NCCL INFO Channel 04/0 : 12[4] -> 13[5] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:73:323 [0] NCCL INFO Channel 01/0 : 8[0] -> 15[7] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:79:322 [6] NCCL INFO Channel 04/0 : 14[6] -> 15[7] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:75:317 [2] NCCL INFO Channel 03/0 : 10[2] -> 11[3] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:77:319 [4] NCCL INFO Channel 05/0 : 12[4] -> 13[5] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:73:323 [0] NCCL INFO Channel 03/0 : 8[0] -> 15[7] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:79:322 [6] NCCL INFO Channel 05/0 : 14[6] -> 15[7] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:75:317 [2] NCCL INFO Channel 04/0 : 10[2] -> 11[3] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:77:319 [4] NCCL INFO Channel 06/0 : 12[4] -> 13[5] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:79:322 [6] NCCL INFO Channel 06/0 : 14[6] -> 15[7] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:73:323 [0] NCCL INFO Channel 05/0 : 8[0] -> 15[7] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:75:317 [2] NCCL INFO Channel 05/0 : 10[2] -> 11[3] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:77:319 [4] NCCL INFO Channel 07/0 : 12[4] -> 13[5] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:79:322 [6] NCCL INFO Channel 07/0 : 14[6] -> 15[7] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:73:323 [0] NCCL INFO Channel 07/0 : 8[0] -> 15[7] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:75:317 [2] NCCL INFO Channel 06/0 : 10[2] -> 11[3] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:77:319 [4] NCCL INFO Channel 02/0 : 4[4] -> 12[4] [receive] via NET/IB/2/GDRDMA
+dlc1w85u6fy2xvlb-worker-0:77:319 [4] NCCL INFO Channel 06/0 : 4[4] -> 12[4] [receive] via NET/IB/2/GDRDMA
+dlc1w85u6fy2xvlb-worker-0:77:319 [4] NCCL INFO Channel 02/0 : 12[4] -> 4[4] [send] via NET/IB/2/GDRDMA
+dlc1w85u6fy2xvlb-worker-0:77:319 [4] NCCL INFO Channel 06/0 : 12[4] -> 4[4] [send] via NET/IB/2/GDRDMA
+dlc1w85u6fy2xvlb-worker-0:78:321 [5] NCCL INFO Channel 02/0 : 13[5] -> 12[4] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:79:322 [6] NCCL INFO Channel 03/0 : 6[6] -> 14[6] [receive] via NET/IB/3/GDRDMA
+dlc1w85u6fy2xvlb-worker-0:79:322 [6] NCCL INFO Channel 07/0 : 6[6] -> 14[6] [receive] via NET/IB/3/GDRDMA
+dlc1w85u6fy2xvlb-worker-0:79:322 [6] NCCL INFO Channel 03/0 : 14[6] -> 6[6] [send] via NET/IB/3/GDRDMA
+dlc1w85u6fy2xvlb-worker-0:79:322 [6] NCCL INFO Channel 07/0 : 14[6] -> 6[6] [send] via NET/IB/3/GDRDMA
+dlc1w85u6fy2xvlb-worker-0:75:317 [2] NCCL INFO Channel 07/0 : 10[2] -> 11[3] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:78:321 [5] NCCL INFO Channel 06/0 : 13[5] -> 12[4] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:74:318 [1] NCCL INFO Channel 00/0 : 9[1] -> 8[0] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:75:317 [2] NCCL INFO Channel 01/0 : 2[2] -> 10[2] [receive] via NET/IB/1/GDRDMA
+dlc1w85u6fy2xvlb-worker-0:75:317 [2] NCCL INFO Channel 05/0 : 2[2] -> 10[2] [receive] via NET/IB/1/GDRDMA
+dlc1w85u6fy2xvlb-worker-0:75:317 [2] NCCL INFO Channel 01/0 : 10[2] -> 2[2] [send] via NET/IB/1/GDRDMA
+dlc1w85u6fy2xvlb-worker-0:75:317 [2] NCCL INFO Channel 05/0 : 10[2] -> 2[2] [send] via NET/IB/1/GDRDMA
+dlc1w85u6fy2xvlb-worker-0:77:319 [4] NCCL INFO Channel 01/0 : 12[4] -> 11[3] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:74:318 [1] NCCL INFO Channel 04/0 : 9[1] -> 8[0] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:77:319 [4] NCCL INFO Channel 03/0 : 12[4] -> 11[3] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:77:319 [4] NCCL INFO Channel 05/0 : 12[4] -> 11[3] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:80:320 [7] NCCL INFO Channel 01/0 : 15[7] -> 8[0] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:73:323 [0] NCCL INFO Channel 00/0 : 0[0] -> 8[0] [receive] via NET/IB/0/GDRDMA
+dlc1w85u6fy2xvlb-worker-0:73:323 [0] NCCL INFO Channel 04/0 : 0[0] -> 8[0] [receive] via NET/IB/0/GDRDMA
+dlc1w85u6fy2xvlb-worker-0:73:323 [0] NCCL INFO Channel 00/0 : 8[0] -> 0[0] [send] via NET/IB/0/GDRDMA
+dlc1w85u6fy2xvlb-worker-0:73:323 [0] NCCL INFO Channel 04/0 : 8[0] -> 0[0] [send] via NET/IB/0/GDRDMA
+dlc1w85u6fy2xvlb-worker-0:77:319 [4] NCCL INFO Channel 07/0 : 12[4] -> 11[3] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:80:320 [7] NCCL INFO Channel 02/0 : 15[7] -> 8[0] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:76:316 [3] NCCL INFO Channel 01/0 : 11[3] -> 10[2] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:80:320 [7] NCCL INFO Channel 03/0 : 15[7] -> 8[0] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:76:316 [3] NCCL INFO Channel 05/0 : 11[3] -> 10[2] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:80:320 [7] NCCL INFO Channel 05/0 : 15[7] -> 8[0] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:80:320 [7] NCCL INFO Channel 06/0 : 15[7] -> 8[0] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:80:320 [7] NCCL INFO Channel 07/0 : 15[7] -> 8[0] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:80:320 [7] NCCL INFO Channel 03/0 : 15[7] -> 14[6] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:80:320 [7] NCCL INFO Channel 07/0 : 15[7] -> 14[6] via P2P/IPC/read
+dlc1w85u6fy2xvlb-worker-0:78:321 [5] NCCL INFO Connected all trees
+dlc1w85u6fy2xvlb-worker-0:78:321 [5] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512
+dlc1w85u6fy2xvlb-worker-0:78:321 [5] NCCL INFO 8 coll channels, 0 nvls channels, 8 p2p channels, 2 p2p channels per peer
+dlc1w85u6fy2xvlb-worker-0:76:316 [3] NCCL INFO Connected all trees
+dlc1w85u6fy2xvlb-worker-0:76:316 [3] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512
+dlc1w85u6fy2xvlb-worker-0:76:316 [3] NCCL INFO 8 coll channels, 0 nvls channels, 8 p2p channels, 2 p2p channels per peer
+dlc1w85u6fy2xvlb-worker-0:74:318 [1] NCCL INFO Connected all trees
+dlc1w85u6fy2xvlb-worker-0:74:318 [1] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512
+dlc1w85u6fy2xvlb-worker-0:74:318 [1] NCCL INFO 8 coll channels, 0 nvls channels, 8 p2p channels, 2 p2p channels per peer
+dlc1w85u6fy2xvlb-worker-0:80:320 [7] NCCL INFO Connected all trees
+dlc1w85u6fy2xvlb-worker-0:80:320 [7] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512
+dlc1w85u6fy2xvlb-worker-0:80:320 [7] NCCL INFO 8 coll channels, 0 nvls channels, 8 p2p channels, 2 p2p channels per peer
+dlc1w85u6fy2xvlb-worker-0:77:319 [4] NCCL INFO Connected all trees
+dlc1w85u6fy2xvlb-worker-0:77:319 [4] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512
+dlc1w85u6fy2xvlb-worker-0:77:319 [4] NCCL INFO 8 coll channels, 0 nvls channels, 8 p2p channels, 2 p2p channels per peer
+dlc1w85u6fy2xvlb-worker-0:75:317 [2] NCCL INFO Connected all trees
+dlc1w85u6fy2xvlb-worker-0:75:317 [2] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512
+dlc1w85u6fy2xvlb-worker-0:75:317 [2] NCCL INFO 8 coll channels, 0 nvls channels, 8 p2p channels, 2 p2p channels per peer
+dlc1w85u6fy2xvlb-worker-0:79:322 [6] NCCL INFO Connected all trees
+dlc1w85u6fy2xvlb-worker-0:79:322 [6] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512
+dlc1w85u6fy2xvlb-worker-0:79:322 [6] NCCL INFO 8 coll channels, 0 nvls channels, 8 p2p channels, 2 p2p channels per peer
+dlc1w85u6fy2xvlb-worker-0:73:323 [0] NCCL INFO Connected all trees
+dlc1w85u6fy2xvlb-worker-0:73:323 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512
+dlc1w85u6fy2xvlb-worker-0:73:323 [0] NCCL INFO 8 coll channels, 0 nvls channels, 8 p2p channels, 2 p2p channels per peer
+dlc1w85u6fy2xvlb-worker-0:74:318 [1] NCCL INFO comm 0x9a893920 rank 9 nranks 16 cudaDev 1 nvmlDev 1 busId 20 commId 0xe730164ea45002c1 - Init COMPLETE
+dlc1w85u6fy2xvlb-worker-0:78:321 [5] NCCL INFO comm 0x9b39f9e0 rank 13 nranks 16 cudaDev 5 nvmlDev 5 busId 60 commId 0xe730164ea45002c1 - Init COMPLETE
+dlc1w85u6fy2xvlb-worker-0:76:316 [3] NCCL INFO comm 0x9b7bdc70 rank 11 nranks 16 cudaDev 3 nvmlDev 3 busId 40 commId 0xe730164ea45002c1 - Init COMPLETE
+dlc1w85u6fy2xvlb-worker-0:80:320 [7] NCCL INFO comm 0x9bccd080 rank 15 nranks 16 cudaDev 7 nvmlDev 7 busId 80 commId 0xe730164ea45002c1 - Init COMPLETE
+dlc1w85u6fy2xvlb-worker-0:77:319 [4] NCCL INFO comm 0x9b9e2970 rank 12 nranks 16 cudaDev 4 nvmlDev 4 busId 50 commId 0xe730164ea45002c1 - Init COMPLETE
+dlc1w85u6fy2xvlb-worker-0:73:323 [0] NCCL INFO comm 0x9aabc5f0 rank 8 nranks 16 cudaDev 0 nvmlDev 0 busId 10 commId 0xe730164ea45002c1 - Init COMPLETE
+dlc1w85u6fy2xvlb-worker-0:75:317 [2] NCCL INFO comm 0x9bcdb060 rank 10 nranks 16 cudaDev 2 nvmlDev 2 busId 30 commId 0xe730164ea45002c1 - Init COMPLETE
+dlc1w85u6fy2xvlb-worker-0:79:322 [6] NCCL INFO comm 0x9aa4c060 rank 14 nranks 16 cudaDev 6 nvmlDev 6 busId 70 commId 0xe730164ea45002c1 - Init COMPLETE
+
+Loading checkpoint shards:   0%|          | 0/8 [00:00<?, ?it/s]
+Loading checkpoint shards:   0%|          | 0/8 [00:00<?, ?it/s]
+Loading checkpoint shards:   0%|          | 0/8 [00:00<?, ?it/s]
+Loading checkpoint shards:   0%|          | 0/8 [00:00<?, ?it/s]
+Loading checkpoint shards:   0%|          | 0/8 [00:00<?, ?it/s]
+Loading checkpoint shards:   0%|          | 0/8 [00:00<?, ?it/s]
+Loading checkpoint shards:   0%|          | 0/8 [00:00<?, ?it/s]
+Loading checkpoint shards:   0%|          | 0/8 [00:00<?, ?it/s]
+Loading checkpoint shards:  12%|█▎        | 1/8 [00:00<00:01,  4.69it/s]
+Loading checkpoint shards:  12%|█▎        | 1/8 [00:00<00:01,  4.43it/s]
+Loading checkpoint shards:  12%|█▎        | 1/8 [00:00<00:01,  4.53it/s]
+Loading checkpoint shards:  12%|█▎        | 1/8 [00:00<00:01,  4.60it/s]
+Loading checkpoint shards:  12%|█▎        | 1/8 [00:00<00:01,  4.43it/s]
+Loading checkpoint shards:  12%|█▎        | 1/8 [00:00<00:01,  4.50it/s]
+Loading checkpoint shards:  12%|█▎        | 1/8 [00:00<00:01,  5.10it/s]
+Loading checkpoint shards:  12%|█▎        | 1/8 [00:00<00:01,  4.90it/s]
+Loading checkpoint shards:  25%|██▌       | 2/8 [00:00<00:01,  4.94it/s]
+Loading checkpoint shards:  25%|██▌       | 2/8 [00:00<00:01,  4.32it/s]
+Loading checkpoint shards:  25%|██▌       | 2/8 [00:00<00:01,  4.40it/s]
+Loading checkpoint shards:  25%|██▌       | 2/8 [00:00<00:01,  4.40it/s]
+Loading checkpoint shards:  25%|██▌       | 2/8 [00:00<00:01,  4.35it/s]
+Loading checkpoint shards:  25%|██▌       | 2/8 [00:00<00:01,  4.30it/s]
+Loading checkpoint shards:  25%|██▌       | 2/8 [00:00<00:01,  4.32it/s]
+Loading checkpoint shards:  25%|██▌       | 2/8 [00:00<00:01,  4.84it/s]
+Loading checkpoint shards:  38%|███▊      | 3/8 [00:01<00:04,  1.21it/s]
+Loading checkpoint shards:  38%|███▊      | 3/8 [00:02<00:04,  1.19it/s]
+Loading checkpoint shards:  38%|███▊      | 3/8 [00:02<00:04,  1.18it/s]
+Loading checkpoint shards:  38%|███▊      | 3/8 [00:02<00:04,  1.19it/s]
+Loading checkpoint shards:  38%|███▊      | 3/8 [00:02<00:04,  1.19it/s]
+Loading checkpoint shards:  38%|███▊      | 3/8 [00:02<00:04,  1.18it/s]
+Loading checkpoint shards:  38%|███▊      | 3/8 [00:01<00:04,  1.20it/s]
+Loading checkpoint shards:  38%|███▊      | 3/8 [00:02<00:04,  1.18it/s]
+Loading checkpoint shards:  50%|█████     | 4/8 [00:03<00:04,  1.07s/it]
+Loading checkpoint shards:  50%|█████     | 4/8 [00:03<00:04,  1.07s/it]
+Loading checkpoint shards:  50%|█████     | 4/8 [00:03<00:04,  1.06s/it]
+Loading checkpoint shards:  50%|█████     | 4/8 [00:03<00:04,  1.07s/it]
+Loading checkpoint shards:  50%|█████     | 4/8 [00:03<00:04,  1.07s/it]
+Loading checkpoint shards:  50%|█████     | 4/8 [00:03<00:04,  1.07s/it]
+Loading checkpoint shards:  50%|█████     | 4/8 [00:03<00:04,  1.06s/it]
+Loading checkpoint shards:  50%|█████     | 4/8 [00:03<00:04,  1.07s/it]
+Loading checkpoint shards:  62%|██████▎   | 5/8 [00:04<00:03,  1.12s/it]
+Loading checkpoint shards:  62%|██████▎   | 5/8 [00:04<00:03,  1.12s/it]
+Loading checkpoint shards:  62%|██████▎   | 5/8 [00:04<00:03,  1.12s/it]
+Loading checkpoint shards:  62%|██████▎   | 5/8 [00:04<00:03,  1.12s/it]
+Loading checkpoint shards:  62%|██████▎   | 5/8 [00:04<00:03,  1.12s/it]
+Loading checkpoint shards:  62%|██████▎   | 5/8 [00:04<00:03,  1.12s/it]
+Loading checkpoint shards:  62%|██████▎   | 5/8 [00:04<00:03,  1.11s/it]
+Loading checkpoint shards:  62%|██████▎   | 5/8 [00:04<00:03,  1.12s/it]
+Loading checkpoint shards:  75%|███████▌  | 6/8 [00:05<00:02,  1.16s/it]
+Loading checkpoint shards:  75%|███████▌  | 6/8 [00:05<00:02,  1.17s/it]
+Loading checkpoint shards:  75%|███████▌  | 6/8 [00:05<00:02,  1.17s/it]
+Loading checkpoint shards:  75%|███████▌  | 6/8 [00:05<00:02,  1.17s/it]
+Loading checkpoint shards:  75%|███████▌  | 6/8 [00:05<00:02,  1.17s/it]
+Loading checkpoint shards:  75%|███████▌  | 6/8 [00:05<00:02,  1.17s/it]
+Loading checkpoint shards:  75%|███████▌  | 6/8 [00:05<00:02,  1.16s/it]
+Loading checkpoint shards:  75%|███████▌  | 6/8 [00:05<00:02,  1.16s/it]
+Loading checkpoint shards:  88%|████████▊ | 7/8 [00:06<00:01,  1.12s/it]
+Loading checkpoint shards:  88%|████████▊ | 7/8 [00:06<00:01,  1.13s/it]
+Loading checkpoint shards:  88%|████████▊ | 7/8 [00:06<00:01,  1.13s/it]
+Loading checkpoint shards:  88%|████████▊ | 7/8 [00:06<00:01,  1.13s/it]
+Loading checkpoint shards:  88%|████████▊ | 7/8 [00:06<00:01,  1.13s/it]
+Loading checkpoint shards:  88%|████████▊ | 7/8 [00:06<00:01,  1.13s/it]
+Loading checkpoint shards:  88%|████████▊ | 7/8 [00:06<00:01,  1.12s/it]/fs-computility/mllm1/shared/hub/
+Loading checkpoint shards:  88%|████████▊ | 7/8 [00:06<00:01,  1.12s/it]
+Loading checkpoint shards: 100%|██████████| 8/8 [00:07<00:00,  1.10it/s]
+Loading checkpoint shards: 100%|██████████| 8/8 [00:07<00:00,  1.09it/s]
+[WARNING|modeling_utils.py:4352] 2025-01-16 21:56:38,453 >> Some weights of LlavaInternlm2ForCausalLM were not initialized from the model checkpoint at models/internlm/internlm2_5-7b-chat and are newly initialized: ['lm_head.weight']
+You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.
+
+Loading checkpoint shards: 100%|██████████| 8/8 [00:07<00:00,  1.09it/s]
+Loading checkpoint shards: 100%|██████████| 8/8 [00:07<00:00,  1.08it/s]
+[WARNING|modeling_utils.py:4352] 2025-01-16 21:56:38,453 >> Some weights of LlavaInternlm2ForCausalLM were not initialized from the model checkpoint at models/internlm/internlm2_5-7b-chat and are newly initialized: ['lm_head.weight']
+You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.
+
+Loading checkpoint shards: 100%|██████████| 8/8 [00:07<00:00,  1.09it/s]
+Loading checkpoint shards: 100%|██████████| 8/8 [00:07<00:00,  1.08it/s]
+[WARNING|modeling_utils.py:4352] 2025-01-16 21:56:38,455 >> Some weights of LlavaInternlm2ForCausalLM were not initialized from the model checkpoint at models/internlm/internlm2_5-7b-chat and are newly initialized: ['lm_head.weight']
+You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.
+
+Loading checkpoint shards: 100%|██████████| 8/8 [00:07<00:00,  1.09it/s]
+Loading checkpoint shards: 100%|██████████| 8/8 [00:07<00:00,  1.09it/s]
+Loading checkpoint shards: 100%|██████████| 8/8 [00:07<00:00,  1.08it/s]
+
+Loading checkpoint shards: 100%|█████���████| 8/8 [00:07<00:00,  1.08it/s]
+[WARNING|modeling_utils.py:4352] 2025-01-16 21:56:38,456 >> Some weights of LlavaInternlm2ForCausalLM were not initialized from the model checkpoint at models/internlm/internlm2_5-7b-chat and are newly initialized: ['lm_head.weight']
+You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.
+[WARNING|modeling_utils.py:4352] 2025-01-16 21:56:38,456 >> Some weights of LlavaInternlm2ForCausalLM were not initialized from the model checkpoint at models/internlm/internlm2_5-7b-chat and are newly initialized: ['lm_head.weight']
+You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.
+
+Loading checkpoint shards: 100%|██████████| 8/8 [00:07<00:00,  1.09it/s]
+Loading checkpoint shards: 100%|██████████| 8/8 [00:07<00:00,  1.08it/s]
+[WARNING|modeling_utils.py:4352] 2025-01-16 21:56:38,458 >> Some weights of LlavaInternlm2ForCausalLM were not initialized from the model checkpoint at models/internlm/internlm2_5-7b-chat and are newly initialized: ['lm_head.weight']
+You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.
+/fs-computility/mllm1/shared/hub/
+Loading checkpoint shards: 100%|██████████| 8/8 [00:07<00:00,  1.10it/s]
+Loading checkpoint shards: 100%|██████████| 8/8 [00:07<00:00,  1.09it/s]
+[INFO|modeling_utils.py:4350] 2025-01-16 21:56:38,462 >> All model checkpoint weights were used when initializing LlavaInternlm2ForCausalLM.
+
+[WARNING|modeling_utils.py:4352] 2025-01-16 21:56:38,462 >> Some weights of LlavaInternlm2ForCausalLM were not initialized from the model checkpoint at models/internlm/internlm2_5-7b-chat and are newly initialized: ['lm_head.weight']
+You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.
+
+Loading checkpoint shards: 100%|██████████| 8/8 [00:07<00:00,  1.09it/s]
+Loading checkpoint shards: 100%|██████████| 8/8 [00:07<00:00,  1.08it/s]
+[WARNING|modeling_utils.py:4352] 2025-01-16 21:56:38,471 >> Some weights of LlavaInternlm2ForCausalLM were not initialized from the model checkpoint at models/internlm/internlm2_5-7b-chat and are newly initialized: ['lm_head.weight']
+You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.
+[INFO|configuration_utils.py:779] 2025-01-16 21:56:38,530 >> loading configuration file models/internlm/internlm2_5-7b-chat/generation_config.json
+Using tokenizer from models/internlm/internlm2_5-7b-chat
+using cache dir None
+Using tokenizer from models/internlm/internlm2_5-7b-chat
+using cache dir None
+Using tokenizer from models/internlm/internlm2_5-7b-chat
+using cache dir None
+[INFO|configuration_utils.py:826] 2025-01-16 21:56:38,530 >> Generate config GenerationConfig {
+  "bos_token_id": 1,/fs-computility/mllm1/shared/hub/
+  "eos_token_id": [
+    2,/fs-computility/mllm1/shared/hub/
+    92542
+  ],
+  "pad_token_id": 2/fs-computility/mllm1/shared/hub/
+}
+
+Using tokenizer from models/internlm/internlm2_5-7b-chat
+using cache dir None
+Using tokenizer from models/internlm/internlm2_5-7b-chat
+using cache dir None
+Using tokenizer from models/internlm/internlm2_5-7b-chat
+using cache dir None
+Using tokenizer from models/internlm/internlm2_5-7b-chat
+using cache dir None
+Using tokenizer from models/internlm/internlm2_5-7b-chat
+using cache dir None
+[INFO|tokenization_utils_base.py:2025] 2025-01-16 21:56:38,548 >> loading file ./tokenizer.model
+[INFO|tokenization_utils_base.py:2025] 2025-01-16 21:56:38,548 >> loading file added_tokens.json
+[INFO|tokenization_utils_base.py:2025] 2025-01-16 21:56:38,548 >> loading file special_tokens_map.json
+[INFO|tokenization_utils_base.py:2025] 2025-01-16 21:56:38,548 >> loading file tokenizer_config.json
+[INFO|tokenization_utils_base.py:2025] 2025-01-16 21:56:38,548 >> loading file tokenizer.json
+01/16/2025 21:56:38 - WARNING - llava.train.train - Using conversation template: Conversation(system='<|im_start|>system\nYou are a helpful assistant. ', roles=('<|im_start|>user\n', '<|im_start|>assistant\n'), messages=[], offset=0, sep_style=<SeparatorStyle.MPT: 3>, sep='<|im_end|>', sep2=None, version='internlm_v2', mm_system=None, skip_next=False)
+01/16/2025 21:56:38 - WARNING - llava.train.train - Using conversation template: Conversation(system='<|im_start|>system\nYou are a helpful assistant. ', roles=('<|im_start|>user\n', '<|im_start|>assistant\n'), messages=[], offset=0, sep_style=<SeparatorStyle.MPT: 3>, sep='<|im_end|>', sep2=None, version='internlm_v2', mm_system=None, skip_next=False)
+01/16/2025 21:56:38 - WARNING - llava.train.train - Using conversation template: Conversation(system='<|im_start|>system\nYou are a helpful assistant. ', roles=('<|im_start|>user\n', '<|im_start|>assistant\n'), messages=[], offset=0, sep_style=<SeparatorStyle.MPT: 3>, sep='<|im_end|>', sep2=None, version='internlm_v2', mm_system=None, skip_next=False)
+01/16/2025 21:56:38 - WARNING - llava.train.train - Using conversation template: Conversation(system='<|im_start|>system\nYou are a helpful assistant. ', roles=('<|im_start|>user\n', '<|im_start|>assistant\n'), messages=[], offset=0, sep_style=<SeparatorStyle.MPT: 3>, sep='<|im_end|>', sep2=None, version='internlm_v2', mm_system=None, skip_next=False)
+01/16/2025 21:56:38 - WARNING - llava.train.train - Using conversation template: Conversation(system='<|im_start|>system\nYou are a helpful assistant. ', roles=('<|im_start|>user\n', '<|im_start|>assistant\n'), messages=[], offset=0, sep_style=<SeparatorStyle.MPT: 3>, sep='<|im_end|>', sep2=None, version='internlm_v2', mm_system=None, skip_next=False)
+01/16/2025 21:56:38 - WARNING - llava.train.train - Using conversation template: Conversation(system='<|im_start|>system\nYou are a helpful assistant. ', roles=('<|im_start|>user\n', '<|im_start|>assistant\n'), messages=[], offset=0, sep_style=<SeparatorStyle.MPT: 3>, sep='<|im_end|>', sep2=None, version='internlm_v2', mm_system=None, skip_next=False)
+01/16/2025 21:56:38 - WARNING - llava.train.train - Using conversation template: Conversation(system='<|im_start|>system\nYou are a helpful assistant. ', roles=('<|im_start|>user\n', '<|im_start|>assistant\n'), messages=[], offset=0, sep_style=<SeparatorStyle.MPT: 3>, sep='<|im_end|>', sep2=None, version='internlm_v2', mm_system=None, skip_next=False)
+01/16/2025 21:56:38 - INFO - llava.train.train - Using conversation template: Conversation(system='<|im_start|>system\nYou are a helpful assistant. ', roles=('<|im_start|>user\n', '<|im_start|>assistant\n'), messages=[], offset=0, sep_style=<SeparatorStyle.MPT: 3>, sep='<|im_end|>', sep2=None, version='internlm_v2', mm_system=None, skip_next=False)
+[INFO|image_processing_utils.py:373] 2025-01-16 21:56:38,791 >> loading configuration file /fs-computility/mllm1/shared/hub/models--openai--clip-vit-large-patch14-336/snapshots/ce19dc912ca5cd21c8a653c79e251e808ccabcd1/preprocessor_config.json
+[INFO|image_processing_utils.py:738] 2025-01-16 21:56:38,791 >> size should be a dictionary on of the following set of keys: ({'height', 'width'}, {'shortest_edge'}, {'longest_edge', 'shortest_edge'}, {'longest_edge'}), got 336. Converted to {'shortest_edge': 336}.
+[INFO|image_processing_utils.py:738] 2025-01-16 21:56:38,791 >> crop_size should be a dictionary on of the following set of keys: ({'height', 'width'}, {'shortest_edge'}, {'longest_edge', 'shortest_edge'}, {'longest_edge'}), got 336. Converted to {'height': 336, 'width': 336}.
+[INFO|image_processing_utils.py:425] 2025-01-16 21:56:38,791 >> Image processor CLIPImageProcessor {
+  "crop_size": {
+    "height": 336,
+    "width": 336
+  },
+  "do_center_crop": true,
+  "do_convert_rgb": true,
+  "do_normalize": true,
+  "do_rescale": true,
+  "do_resize": true,
+  "image_mean": [
+    0.48145466,
+    0.4578275,
+    0.40821073
+  ],
+  "image_processor_type": "CLIPImageProcessor",
+  "image_std": [
+    0.26862954,
+    0.26130258,
+    0.27577711
+  ],
+  "resample": 3,
+  "rescale_factor": 0.00392156862745098,
+  "size": {
+    "shortest_edge": 336
+  }
+}
+
+[INFO|configuration_utils.py:727] 2025-01-16 21:56:38,798 >> loading configuration file /fs-computility/mllm1/shared/hub/models--openai--clip-vit-large-patch14-336/snapshots/ce19dc912ca5cd21c8a653c79e251e808ccabcd1/config.json
+[INFO|configuration_utils.py:792] 2025-01-16 21:56:38,798 >> Model config CLIPVisionConfig {
+  "attention_dropout": 0.0,
+  "dropout": 0.0,
+  "hidden_act": "quick_gelu",
+  "hidden_size": 1024,
+  "image_size": 336,
+  "initializer_factor": 1.0,
+  "initializer_range": 0.02,
+  "intermediate_size": 4096,
+  "layer_norm_eps": 1e-05,
+  "model_type": "clip_vision_model",
+  "num_attention_heads": 16,
+  "num_channels": 3,
+  "num_hidden_layers": 24,
+  "patch_size": 14,
+  "projection_dim": 768,
+  "transformers_version": "4.37.2"
+}
+
+[INFO|modeling_utils.py:3473] 2025-01-16 21:56:38,799 >> loading weights file /fs-computility/mllm1/shared/hub/models--openai--clip-vit-large-patch14-336/snapshots/ce19dc912ca5cd21c8a653c79e251e808ccabcd1/pytorch_model.bin
+[INFO|modeling_utils.py:3582] 2025-01-16 21:56:42,256 >> Detected DeepSpeed ZeRO-3: activating zero.init() for this model
+[INFO|modeling_utils.py:4340] 2025-01-16 21:56:43,384 >> Some weights of the model checkpoint at /fs-computility/mllm1/shared/hub/models--openai--clip-vit-large-patch14-336/snapshots/ce19dc912ca5cd21c8a653c79e251e808ccabcd1 were not used when initializing CLIPVisionModel: ['logit_scale', 'text_model.embeddings.position_embedding.weight', 'text_model.embeddings.position_ids', 'text_model.embeddings.token_embedding.weight', 'text_model.encoder.layers.0.layer_norm1.bias', 'text_model.encoder.layers.0.layer_norm1.weight', 'text_model.encoder.layers.0.layer_norm2.bias', 'text_model.encoder.layers.0.layer_norm2.weight', 'text_model.encoder.layers.0.mlp.fc1.bias', 'text_model.encoder.layers.0.mlp.fc1.weight', 'text_model.encoder.layers.0.mlp.fc2.bias', 'text_model.encoder.layers.0.mlp.fc2.weight', 'text_model.encoder.layers.0.self_attn.k_proj.bias', 'text_model.encoder.layers.0.self_attn.k_proj.weight', 'text_model.encoder.layers.0.self_attn.out_proj.bias', 'text_model.encoder.layers.0.self_attn.out_proj.weight', 'text_model.encoder.layers.0.self_attn.q_proj.bias', 'text_model.encoder.layers.0.self_attn.q_proj.weight', 'text_model.encoder.layers.0.self_attn.v_proj.bias', 'text_model.encoder.layers.0.self_attn.v_proj.weight', 'text_model.encoder.layers.1.layer_norm1.bias', 'text_model.encoder.layers.1.layer_norm1.weight', 'text_model.encoder.layers.1.layer_norm2.bias', 'text_model.encoder.layers.1.layer_norm2.weight', 'text_model.encoder.layers.1.mlp.fc1.bias', 'text_model.encoder.layers.1.mlp.fc1.weight', 'text_model.encoder.layers.1.mlp.fc2.bias', 'text_model.encoder.layers.1.mlp.fc2.weight', 'text_model.encoder.layers.1.self_attn.k_proj.bias', 'text_model.encoder.layers.1.self_attn.k_proj.weight', 'text_model.encoder.layers.1.self_attn.out_proj.bias', 'text_model.encoder.layers.1.self_attn.out_proj.weight', 'text_model.encoder.layers.1.self_attn.q_proj.bias', 'text_model.encoder.layers.1.self_attn.q_proj.weight', 'text_model.encoder.layers.1.self_attn.v_proj.bias', 'text_model.encoder.layers.1.self_attn.v_proj.weight', 'text_model.encoder.layers.10.layer_norm1.bias', 'text_model.encoder.layers.10.layer_norm1.weight', 'text_model.encoder.layers.10.layer_norm2.bias', 'text_model.encoder.layers.10.layer_norm2.weight', 'text_model.encoder.layers.10.mlp.fc1.bias', 'text_model.encoder.layers.10.mlp.fc1.weight', 'text_model.encoder.layers.10.mlp.fc2.bias', 'text_model.encoder.layers.10.mlp.fc2.weight', 'text_model.encoder.layers.10.self_attn.k_proj.bias', 'text_model.encoder.layers.10.self_attn.k_proj.weight', 'text_model.encoder.layers.10.self_attn.out_proj.bias', 'text_model.encoder.layers.10.self_attn.out_proj.weight', 'text_model.encoder.layers.10.self_attn.q_proj.bias', 'text_model.encoder.layers.10.self_attn.q_proj.weight', 'text_model.encoder.layers.10.self_attn.v_proj.bias', 'text_model.encoder.layers.10.self_attn.v_proj.weight', 'text_model.encoder.layers.11.layer_norm1.bias', 'text_model.encoder.layers.11.layer_norm1.weight', 'text_model.encoder.layers.11.layer_norm2.bias', 'text_model.encoder.layers.11.layer_norm2.weight', 'text_model.encoder.layers.11.mlp.fc1.bias', 'text_model.encoder.layers.11.mlp.fc1.weight', 'text_model.encoder.layers.11.mlp.fc2.bias', 'text_model.encoder.layers.11.mlp.fc2.weight', 'text_model.encoder.layers.11.self_attn.k_proj.bias', 'text_model.encoder.layers.11.self_attn.k_proj.weight', 'text_model.encoder.layers.11.self_attn.out_proj.bias', 'text_model.encoder.layers.11.self_attn.out_proj.weight', 'text_model.encoder.layers.11.self_attn.q_proj.bias', 'text_model.encoder.layers.11.self_attn.q_proj.weight', 'text_model.encoder.layers.11.self_attn.v_proj.bias', 'text_model.encoder.layers.11.self_attn.v_proj.weight', 'text_model.encoder.layers.2.layer_norm1.bias', 'text_model.encoder.layers.2.layer_norm1.weight', 'text_model.encoder.layers.2.layer_norm2.bias', 'text_model.encoder.layers.2.layer_norm2.weight', 'text_model.encoder.layers.2.mlp.fc1.bias', 'text_model.encoder.layers.2.mlp.fc1.weight', 'text_model.encoder.layers.2.mlp.fc2.bias', 'text_model.encoder.layers.2.mlp.fc2.weight', 'text_model.encoder.layers.2.self_attn.k_proj.bias', 'text_model.encoder.layers.2.self_attn.k_proj.weight', 'text_model.encoder.layers.2.self_attn.out_proj.bias', 'text_model.encoder.layers.2.self_attn.out_proj.weight', 'text_model.encoder.layers.2.self_attn.q_proj.bias', 'text_model.encoder.layers.2.self_attn.q_proj.weight', 'text_model.encoder.layers.2.self_attn.v_proj.bias', 'text_model.encoder.layers.2.self_attn.v_proj.weight', 'text_model.encoder.layers.3.layer_norm1.bias', 'text_model.encoder.layers.3.layer_norm1.weight', 'text_model.encoder.layers.3.layer_norm2.bias', 'text_model.encoder.layers.3.layer_norm2.weight', 'text_model.encoder.layers.3.mlp.fc1.bias', 'text_model.encoder.layers.3.mlp.fc1.weight', 'text_model.encoder.layers.3.mlp.fc2.bias', 'text_model.encoder.layers.3.mlp.fc2.weight', 'text_model.encoder.layers.3.self_attn.k_proj.bias', 'text_model.encoder.layers.3.self_attn.k_proj.weight', 'text_model.encoder.layers.3.self_attn.out_proj.bias', 'text_model.encoder.layers.3.self_attn.out_proj.weight', 'text_model.encoder.layers.3.self_attn.q_proj.bias', 'text_model.encoder.layers.3.self_attn.q_proj.weight', 'text_model.encoder.layers.3.self_attn.v_proj.bias', 'text_model.encoder.layers.3.self_attn.v_proj.weight', 'text_model.encoder.layers.4.layer_norm1.bias', 'text_model.encoder.layers.4.layer_norm1.weight', 'text_model.encoder.layers.4.layer_norm2.bias', 'text_model.encoder.layers.4.layer_norm2.weight', 'text_model.encoder.layers.4.mlp.fc1.bias', 'text_model.encoder.layers.4.mlp.fc1.weight', 'text_model.encoder.layers.4.mlp.fc2.bias', 'text_model.encoder.layers.4.mlp.fc2.weight', 'text_model.encoder.layers.4.self_attn.k_proj.bias', 'text_model.encoder.layers.4.self_attn.k_proj.weight', 'text_model.encoder.layers.4.self_attn.out_proj.bias', 'text_model.encoder.layers.4.self_attn.out_proj.weight', 'text_model.encoder.layers.4.self_attn.q_proj.bias', 'text_model.encoder.layers.4.self_attn.q_proj.weight', 'text_model.encoder.layers.4.self_attn.v_proj.bias', 'text_model.encoder.layers.4.self_attn.v_proj.weight', 'text_model.encoder.layers.5.layer_norm1.bias', 'text_model.encoder.layers.5.layer_norm1.weight', 'text_model.encoder.layers.5.layer_norm2.bias', 'text_model.encoder.layers.5.layer_norm2.weight', 'text_model.encoder.layers.5.mlp.fc1.bias', 'text_model.encoder.layers.5.mlp.fc1.weight', 'text_model.encoder.layers.5.mlp.fc2.bias', 'text_model.encoder.layers.5.mlp.fc2.weight', 'text_model.encoder.layers.5.self_attn.k_proj.bias', 'text_model.encoder.layers.5.self_attn.k_proj.weight', 'text_model.encoder.layers.5.self_attn.out_proj.bias', 'text_model.encoder.layers.5.self_attn.out_proj.weight', 'text_model.encoder.layers.5.self_attn.q_proj.bias', 'text_model.encoder.layers.5.self_attn.q_proj.weight', 'text_model.encoder.layers.5.self_attn.v_proj.bias', 'text_model.encoder.layers.5.self_attn.v_proj.weight', 'text_model.encoder.layers.6.layer_norm1.bias', 'text_model.encoder.layers.6.layer_norm1.weight', 'text_model.encoder.layers.6.layer_norm2.bias', 'text_model.encoder.layers.6.layer_norm2.weight', 'text_model.encoder.layers.6.mlp.fc1.bias', 'text_model.encoder.layers.6.mlp.fc1.weight', 'text_model.encoder.layers.6.mlp.fc2.bias', 'text_model.encoder.layers.6.mlp.fc2.weight', 'text_model.encoder.layers.6.self_attn.k_proj.bias', 'text_model.encoder.layers.6.self_attn.k_proj.weight', 'text_model.encoder.layers.6.self_attn.out_proj.bias', 'text_model.encoder.layers.6.self_attn.out_proj.weight', 'text_model.encoder.layers.6.self_attn.q_proj.bias', 'text_model.encoder.layers.6.self_attn.q_proj.weight', 'text_model.encoder.layers.6.self_attn.v_proj.bias', 'text_model.encoder.layers.6.self_attn.v_proj.weight', 'text_model.encoder.layers.7.layer_norm1.bias', 'text_model.encoder.layers.7.layer_norm1.weight', 'text_model.encoder.layers.7.layer_norm2.bias', 'text_model.encoder.layers.7.layer_norm2.weight', 'text_model.encoder.layers.7.mlp.fc1.bias', 'text_model.encoder.layers.7.mlp.fc1.weight', 'text_model.encoder.layers.7.mlp.fc2.bias', 'text_model.encoder.layers.7.mlp.fc2.weight', 'text_model.encoder.layers.7.self_attn.k_proj.bias', 'text_model.encoder.layers.7.self_attn.k_proj.weight', 'text_model.encoder.layers.7.self_attn.out_proj.bias', 'text_model.encoder.layers.7.self_attn.out_proj.weight', 'text_model.encoder.layers.7.self_attn.q_proj.bias', 'text_model.encoder.layers.7.self_attn.q_proj.weight', 'text_model.encoder.layers.7.self_attn.v_proj.bias', 'text_model.encoder.layers.7.self_attn.v_proj.weight', 'text_model.encoder.layers.8.layer_norm1.bias', 'text_model.encoder.layers.8.layer_norm1.weight', 'text_model.encoder.layers.8.layer_norm2.bias', 'text_model.encoder.layers.8.layer_norm2.weight', 'text_model.encoder.layers.8.mlp.fc1.bias', 'text_model.encoder.layers.8.mlp.fc1.weight', 'text_model.encoder.layers.8.mlp.fc2.bias', 'text_model.encoder.layers.8.mlp.fc2.weight', 'text_model.encoder.layers.8.self_attn.k_proj.bias', 'text_model.encoder.layers.8.self_attn.k_proj.weight', 'text_model.encoder.layers.8.self_attn.out_proj.bias', 'text_model.encoder.layers.8.self_attn.out_proj.weight', 'text_model.encoder.layers.8.self_attn.q_proj.bias', 'text_model.encoder.layers.8.self_attn.q_proj.weight', 'text_model.encoder.layers.8.self_attn.v_proj.bias', 'text_model.encoder.layers.8.self_attn.v_proj.weight', 'text_model.encoder.layers.9.layer_norm1.bias', 'text_model.encoder.layers.9.layer_norm1.weight', 'text_model.encoder.layers.9.layer_norm2.bias', 'text_model.encoder.layers.9.layer_norm2.weight', 'text_model.encoder.layers.9.mlp.fc1.bias', 'text_model.encoder.layers.9.mlp.fc1.weight', 'text_model.encoder.layers.9.mlp.fc2.bias', 'text_model.encoder.layers.9.mlp.fc2.weight', 'text_model.encoder.layers.9.self_attn.k_proj.bias', 'text_model.encoder.layers.9.self_attn.k_proj.weight', 'text_model.encoder.layers.9.self_attn.out_proj.bias', 'text_model.encoder.layers.9.self_attn.out_proj.weight', 'text_model.encoder.layers.9.self_attn.q_proj.bias', 'text_model.encoder.layers.9.self_attn.q_proj.weight', 'text_model.encoder.layers.9.self_attn.v_proj.bias', 'text_model.encoder.layers.9.self_attn.v_proj.weight', 'text_model.final_layer_norm.bias', 'text_model.final_layer_norm.weight', 'text_projection.weight', 'visual_projection.weight']
+- This IS expected if you are initializing CLIPVisionModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).
+- This IS NOT expected if you are initializing CLIPVisionModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).
+[INFO|modeling_utils.py:4358] 2025-01-16 21:56:43,384 >> All the weights of CLIPVisionModel were initialized from the model checkpoint at /fs-computility/mllm1/shared/hub/models--openai--clip-vit-large-patch14-336/snapshots/ce19dc912ca5cd21c8a653c79e251e808ccabcd1.
+If your task is similar to the task the model of the checkpoint was trained on, you can already use CLIPVisionModel for predictions without further training.
+01/16/2025 21:57:01 - INFO - llava.train.train - Add dataset: llava-next-sft-notext with length: 738601, data type: normal, seed: 0
+01/16/2025 21:57:04 - INFO - llava.train.train - Add dataset: knowledge_gqa9k_art1500_cc3m30k with length: 40813, data type: know, seed: 1
+01/16/2025 21:57:07 - INFO - llava.train.train - Add dataset: Inferencial_flickr7k_cc3m30k_polished_md with length: 37117, data type: inf_polishmd, seed: 2
+01/16/2025 21:57:10 - INFO - llava.train.train - Add dataset: Detail_flickr7k_cc3m28k with length: 35313, data type: detail, seed: 3
+01/16/2025 21:57:14 - INFO - llava.train.train - Add dataset: Knowledge_instruct40k with length: 40218, data type: know_ins, seed: 4
+01/16/2025 21:57:17 - INFO - llava.train.train - Add dataset: Creation10k_fixed with length: 9698, data type: creation, seed: 5
+01/16/2025 21:57:21 - INFO - llava.train.train - Add dataset: Chartqa_generate_11k_gpt_qwen_merge with length: 11160, data type: chart, seed: 6
+01/16/2025 21:57:24 - INFO - llava.train.train - Add dataset: Tqa_detail_qwengenerate_multi8k_gpt with length: 8391, data type: tqa, seed: 7
+01/16/2025 21:57:27 - INFO - llava.train.train - Add dataset: Infovqa_single_gpt with length: 23068, data type: info, seed: 8
+[INFO|trainer.py:571] 2025-01-16 21:57:27,893 >> Using auto half precision backend
+[INFO|trainer.py:1721] 2025-01-16 21:58:16,446 >> ***** Running training *****
+[INFO|trainer.py:1722] 2025-01-16 21:58:16,446 >>   Num examples = 944,379
+[INFO|trainer.py:1723] 2025-01-16 21:58:16,446 >>   Num Epochs = 1
+[INFO|trainer.py:1724] 2025-01-16 21:58:16,446 >>   Instantaneous batch size per device = 4
+[INFO|trainer.py:1727] 2025-01-16 21:58:16,446 >>   Total train batch size (w. parallel, distributed & accumulation) = 128
+[INFO|trainer.py:1728] 2025-01-16 21:58:16,447 >>   Gradient Accumulation steps = 2
+[INFO|trainer.py:1729] 2025-01-16 21:58:16,447 >>   Total optimization steps = 7,378
+[INFO|trainer.py:1730] 2025-01-16 21:58:16,448 >>   Number of trainable parameters = 8,441,260,032

training_log_20250116_222338.txt

@@ -0,0 +1,852 @@
+[2025-01-16 22:23:44,292] torch.distributed.run: [WARNING]
+[2025-01-16 22:23:44,292] torch.distributed.run: [WARNING] *****************************************
+[2025-01-16 22:23:44,292] torch.distributed.run: [WARNING] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed.
+[2025-01-16 22:23:44,292] torch.distributed.run: [WARNING] *****************************************
+/cpfs02/user/zhaoxiangyu/miniconda3/envs/llava/lib/python3.10/site-packages/_distutils_hack/__init__.py:54: UserWarning: Reliance on distutils from stdlib is deprecated. Users must rely on setuptools to provide the distutils module. Avoid importing distutils or import setuptools first, and avoid setting SETUPTOOLS_USE_DISTUTILS=stdlib. Register concerns at https://github.com/pypa/setuptools/issues/new?template=distutils-deprecation.yml
+  warnings.warn(
+/cpfs02/user/zhaoxiangyu/miniconda3/envs/llava/lib/python3.10/site-packages/_distutils_hack/__init__.py:54: UserWarning: Reliance on distutils from stdlib is deprecated. Users must rely on setuptools to provide the distutils module. Avoid importing distutils or import setuptools first, and avoid setting SETUPTOOLS_USE_DISTUTILS=stdlib. Register concerns at https://github.com/pypa/setuptools/issues/new?template=distutils-deprecation.yml
+  warnings.warn(
+/cpfs02/user/zhaoxiangyu/miniconda3/envs/llava/lib/python3.10/site-packages/_distutils_hack/__init__.py:54: UserWarning: Reliance on distutils from stdlib is deprecated. Users must rely on setuptools to provide the distutils module. Avoid importing distutils or import setuptools first, and avoid setting SETUPTOOLS_USE_DISTUTILS=stdlib. Register concerns at https://github.com/pypa/setuptools/issues/new?template=distutils-deprecation.yml
+  warnings.warn(
+/cpfs02/user/zhaoxiangyu/miniconda3/envs/llava/lib/python3.10/site-packages/_distutils_hack/__init__.py:54: UserWarning: Reliance on distutils from stdlib is deprecated. Users must rely on setuptools to provide the distutils module. Avoid importing distutils or import setuptools first, and avoid setting SETUPTOOLS_USE_DISTUTILS=stdlib. Register concerns at https://github.com/pypa/setuptools/issues/new?template=distutils-deprecation.yml
+  warnings.warn(
+/cpfs02/user/zhaoxiangyu/miniconda3/envs/llava/lib/python3.10/site-packages/_distutils_hack/__init__.py:54: UserWarning: Reliance on distutils from stdlib is deprecated. Users must rely on setuptools to provide the distutils module. Avoid importing distutils or import setuptools first, and avoid setting SETUPTOOLS_USE_DISTUTILS=stdlib. Register concerns at https://github.com/pypa/setuptools/issues/new?template=distutils-deprecation.yml
+  warnings.warn(
+/cpfs02/user/zhaoxiangyu/miniconda3/envs/llava/lib/python3.10/site-packages/_distutils_hack/__init__.py:54: UserWarning: Reliance on distutils from stdlib is deprecated. Users must rely on setuptools to provide the distutils module. Avoid importing distutils or import setuptools first, and avoid setting SETUPTOOLS_USE_DISTUTILS=stdlib. Register concerns at https://github.com/pypa/setuptools/issues/new?template=distutils-deprecation.yml
+  warnings.warn(
+/cpfs02/user/zhaoxiangyu/miniconda3/envs/llava/lib/python3.10/site-packages/_distutils_hack/__init__.py:54: UserWarning: Reliance on distutils from stdlib is deprecated. Users must rely on setuptools to provide the distutils module. Avoid importing distutils or import setuptools first, and avoid setting SETUPTOOLS_USE_DISTUTILS=stdlib. Register concerns at https://github.com/pypa/setuptools/issues/new?template=distutils-deprecation.yml
+  warnings.warn(
+/cpfs02/user/zhaoxiangyu/miniconda3/envs/llava/lib/python3.10/site-packages/_distutils_hack/__init__.py:54: UserWarning: Reliance on distutils from stdlib is deprecated. Users must rely on setuptools to provide the distutils module. Avoid importing distutils or import setuptools first, and avoid setting SETUPTOOLS_USE_DISTUTILS=stdlib. Register concerns at https://github.com/pypa/setuptools/issues/new?template=distutils-deprecation.yml
+  warnings.warn(
+[2025-01-16 22:23:58,303] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
+[2025-01-16 22:23:58,303] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
+[2025-01-16 22:23:58,303] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
+[2025-01-16 22:23:58,303] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
+[2025-01-16 22:23:58,303] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
+[2025-01-16 22:23:58,303] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
+[2025-01-16 22:23:58,303] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
+[2025-01-16 22:23:58,303] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)
+df: df: df: df: df: /root/.triton/autotune/root/.triton/autotune/root/.triton/autotune/root/.triton/autotune/root/.triton/autotunedf: /root/.triton/autotune: 没有那个文件或目录
+: 没有那个文件或目录
+: 没有那个文件或目录: 没有那个文件或目录
+
+: 没有那个文件或目录
+: 没有那个文件或目录
+ [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH
+ [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH
+ [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH
+ [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH
+ [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH
+ [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH
+ [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH
+ [WARNING]  Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH
+ [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.1
+ [WARNING]  using untested triton version (2.1.0), only 1.0.0 is known to be compatible
+ [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.1
+ [WARNING]  using untested triton version (2.1.0), only 1.0.0 is known to be compatible
+ [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.1
+ [WARNING]  using untested triton version (2.1.0), only 1.0.0 is known to be compatible
+ [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.1
+ [WARNING]  using untested triton version (2.1.0), only 1.0.0 is known to be compatible
+ [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.1
+ [WARNING]  using untested triton version (2.1.0), only 1.0.0 is known to be compatible
+ [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.1
+ [WARNING]  using untested triton version (2.1.0), only 1.0.0 is known to be compatible
+ [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.1
+ [WARNING]  using untested triton version (2.1.0), only 1.0.0 is known to be compatible
+ [WARNING]  sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.1
+ [WARNING]  using untested triton version (2.1.0), only 1.0.0 is known to be compatible
+[2025-01-16 22:24:14,156] [INFO] [comm.py:637:init_distributed] cdb=None
+[2025-01-16 22:24:14,156] [INFO] [comm.py:637:init_distributed] cdb=None
+[2025-01-16 22:24:14,157] [INFO] [comm.py:637:init_distributed] cdb=None
+[2025-01-16 22:24:14,157] [INFO] [comm.py:637:init_distributed] cdb=None
+[2025-01-16 22:24:14,157] [INFO] [comm.py:637:init_distributed] cdb=None
+[2025-01-16 22:24:14,157] [INFO] [comm.py:637:init_distributed] cdb=None
+[2025-01-16 22:24:14,157] [INFO] [comm.py:637:init_distributed] cdb=None
+[2025-01-16 22:24:14,157] [INFO] [comm.py:637:init_distributed] cdb=None
+01/16/2025 22:24:14 - WARNING - llava.train.train - Process rank: 0, device: cuda:0, n_gpu: 1distributed training: True, 16-bits training: False
+01/16/2025 22:24:14 - INFO - llava.train.train - Training/evaluation parameters TrainingArguments(
+_n_gpu=1,
+adafactor=False,
+adam_beta1=0.9,
+adam_beta2=0.999,
+adam_epsilon=1e-08,
+auto_find_batch_size=False,
+bf16=True,
+bf16_full_eval=False,
+bits=16,
+cache_dir=None,
+data_seed=None,
+dataloader_drop_last=False,
+dataloader_num_workers=4,
+dataloader_persistent_workers=False,
+dataloader_pin_memory=True,
+ddp_backend=None,
+ddp_broadcast_buffers=None,
+ddp_bucket_cap_mb=None,
+ddp_find_unused_parameters=None,
+ddp_timeout=1800,
+debug=[],
+deepspeed=./scripts/zero3.json,
+disable_tqdm=False,
+dispatch_batches=None,
+do_eval=False,
+do_predict=False,
+do_train=False,
+double_quant=True,
+eval_accumulation_steps=None,
+eval_delay=0,
+eval_steps=None,
+evaluation_strategy=no,
+fp16=False,
+fp16_backend=auto,
+fp16_full_eval=False,
+fp16_opt_level=O1,
+freeze_mm_mlp_adapter=False,
+fsdp=[],
+fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_grad_ckpt': False},
+fsdp_min_num_params=0,
+fsdp_transformer_layer_cls_to_wrap=None,
+full_determinism=False,
+gradient_accumulation_steps=2,
+gradient_checkpointing=True,
+gradient_checkpointing_kwargs=None,
+greater_is_better=None,
+group_by_length=False,
+group_by_modality_length=True,
+half_precision_backend=auto,
+hub_always_push=False,
+hub_model_id=None,
+hub_private_repo=False,
+hub_strategy=every_save,
+hub_token=<HUB_TOKEN>,
+ignore_data_skip=False,
+include_inputs_for_metrics=False,
+include_num_input_tokens_seen=False,
+include_tokens_per_second=False,
+jit_mode_eval=False,
+label_names=None,
+label_smoothing_factor=0.0,
+learning_rate=2e-05,
+length_column_name=length,
+load_best_model_at_end=False,
+local_rank=0,
+log_level=passive,
+log_level_replica=warning,
+log_on_each_node=True,
+logging_dir=./checkpoints/llavaAR4-internlm2_5-7b-sft-llavanext-notext-kn-infpolishmd-detail-knins40k-creationme10kfixed-chart11kmerge-tqa8k-info28kgpt/runs/Jan16_22-24-14_dlc1abaccnl2nzws-worker-0,
+logging_first_step=False,
+logging_nan_inf_filter=True,
+logging_steps=1.0,
+logging_strategy=steps,
+lora_alpha=16,
+lora_bias=none,
+lora_dropout=0.05,
+lora_enable=False,
+lora_r=64,
+lora_weight_path=,
+lr_scheduler_kwargs={},
+lr_scheduler_type=cosine,
+max_grad_norm=1.0,
+max_steps=-1,
+metric_for_best_model=None,
+mm_projector_lr=None,
+mm_vision_tower_lr=2e-06,
+model_max_length=32768,
+mp_parameters=,
+mpt_attn_impl=triton,
+neftune_noise_alpha=None,
+no_cuda=False,
+num_train_epochs=1.0,
+optim=adamw_torch,
+optim_args=None,
+output_dir=./checkpoints/llavaAR4-internlm2_5-7b-sft-llavanext-notext-kn-infpolishmd-detail-knins40k-creationme10kfixed-chart11kmerge-tqa8k-info28kgpt,
+overwrite_output_dir=False,
+past_index=-1,
+per_device_eval_batch_size=4,
+per_device_train_batch_size=4,
+prediction_loss_only=False,
+push_to_hub=False,
+push_to_hub_model_id=None,
+push_to_hub_organization=None,
+push_to_hub_token=<PUSH_TO_HUB_TOKEN>,
+quant_type=nf4,
+ray_scope=last,
+remove_unused_columns=False,
+report_to=['wandb'],
+resume_from_checkpoint=None,
+run_name=llavaAR4-internlm2_5-7b-sft-llavanext-notext-kn-infpolishmd-detail-knins40k-creationme10kfixed-chart11kmerge-tqa8k-info28kgpt,
+save_on_each_node=False,
+save_only_model=False,
+save_safetensors=True,
+save_steps=10000,
+save_strategy=steps,
+save_total_limit=1,
+seed=42,
+skip_memory_metrics=True,
+split_batches=False,
+tf32=True,
+torch_compile=False,
+torch_compile_backend=None,
+torch_compile_mode=None,
+torchdynamo=None,
+tpu_metrics_debug=False,
+tpu_num_cores=None,
+use_cpu=False,
+use_ipex=False,
+use_legacy_prediction_loop=False,
+use_mps_device=False,
+warmup_ratio=0.03,
+warmup_steps=0,
+weight_decay=0.0,
+)
+01/16/2025 22:24:14 - INFO - llava.train.train - Training/evaluation parameters DataArguments(data_path=None, meta_path='playground/meta_json/llavanext_sample/llava_next_notext_inf37kpolishmd_de35k_know40k_knins40k_creation10kfixed_chart11kmerge_tqa8k_info28k_gpt.json', lazy_preprocess=True, is_multimodal=False, image_folder=None, image_aspect_ratio='anyres', image_grid_pinpoints='[(336, 672), (672, 336), (672, 672), (1008, 336), (336, 1008)]', image_crop_resolution=None, image_split_resolution=None, use_data_resampling=False)
+[INFO|configuration_utils.py:727] 2025-01-16 22:24:14,198 >> loading configuration file models/internlm/internlm2_5-7b-chat/config.json
+[INFO|configuration_utils.py:727] 2025-01-16 22:24:14,220 >> loading configuration file models/internlm/internlm2_5-7b-chat/config.json
+[INFO|configuration_utils.py:792] 2025-01-16 22:24:14,221 >> Model config InternLM2Config {
+  "_name_or_path": "models/internlm/internlm2_5-7b-chat",
+  "architectures": [
+    "InternLM2ForCausalLM"
+  ],
+  "attn_implementation": "eager",
+  "auto_map": {
+    "AutoConfig": "configuration_internlm2.InternLM2Config",
+    "AutoModel": "modeling_internlm2.InternLM2ForCausalLM",
+    "AutoModelForCausalLM": "modeling_internlm2.InternLM2ForCausalLM"
+  },
+  "bias": false,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 32768,
+  "model_type": "internlm2",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "pad_token_id": 2,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": {
+    "factor": 2.0,
+    "type": "dynamic"
+  },
+  "rope_theta": 1000000,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.37.2",
+  "use_cache": true,
+  "vocab_size": 92544
+}
+
+[WARNING|modeling_utils.py:2918] 2025-01-16 22:24:14,225 >> The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
+[INFO|modeling_utils.py:3473] 2025-01-16 22:24:14,230 >> loading weights file models/internlm/internlm2_5-7b-chat/model.safetensors.index.json
+[INFO|modeling_utils.py:1426] 2025-01-16 22:24:14,232 >> Instantiating LlavaInternlm2ForCausalLM model under default dtype torch.bfloat16.
+[INFO|modeling_utils.py:3582] 2025-01-16 22:24:14,232 >> Detected DeepSpeed ZeRO-3: activating zero.init() for this model
+[INFO|configuration_utils.py:826] 2025-01-16 22:24:14,238 >> Generate config GenerationConfig {
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "pad_token_id": 2
+}
+
+01/16/2025 22:24:16 - WARNING - llava.train.train - Process rank: 5, device: cuda:5, n_gpu: 1distributed training: True, 16-bits training: False
+[WARNING|modeling_utils.py:2918] 2025-01-16 22:24:16,509 >> The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
+01/16/2025 22:24:16 - WARNING - llava.train.train - Process rank: 3, device: cuda:3, n_gpu: 1distributed training: True, 16-bits training: False
+01/16/2025 22:24:16 - WARNING - llava.train.train - Process rank: 6, device: cuda:6, n_gpu: 1distributed training: True, 16-bits training: False
+01/16/2025 22:24:16 - WARNING - llava.train.train - Process rank: 2, device: cuda:2, n_gpu: 1distributed training: True, 16-bits training: False
+01/16/2025 22:24:16 - WARNING - llava.train.train - Process rank: 4, device: cuda:4, n_gpu: 1distributed training: True, 16-bits training: False
+01/16/2025 22:24:16 - WARNING - llava.train.train - Process rank: 1, device: cuda:1, n_gpu: 1distributed training: True, 16-bits training: False
+[WARNING|modeling_utils.py:2918] 2025-01-16 22:24:16,539 >> The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
+[WARNING|modeling_utils.py:2918] 2025-01-16 22:24:16,541 >> The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
+[WARNING|modeling_utils.py:2918] 2025-01-16 22:24:16,545 >> The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
+[WARNING|modeling_utils.py:2918] 2025-01-16 22:24:16,549 >> The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
+[WARNING|modeling_utils.py:2918] 2025-01-16 22:24:16,553 >> The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
+01/16/2025 22:24:16 - WARNING - llava.train.train - Process rank: 7, device: cuda:7, n_gpu: 1distributed training: True, 16-bits training: False
+[WARNING|modeling_utils.py:2918] 2025-01-16 22:24:16,603 >> The argument `trust_remote_code` is to be used with Auto classes. It has no effect here and is ignored.
+dlc1abaccnl2nzws-worker-0:78:78 [6] NCCL INFO cudaDriverVersion 12010
+dlc1abaccnl2nzws-worker-0:76:76 [4] NCCL INFO cudaDriverVersion 12010
+dlc1abaccnl2nzws-worker-0:78:78 [6] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth
+dlc1abaccnl2nzws-worker-0:76:76 [4] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth
+dlc1abaccnl2nzws-worker-0:76:76 [4] NCCL INFO Bootstrap : Using eth0:22.8.38.24<0>
+dlc1abaccnl2nzws-worker-0:78:78 [6] NCCL INFO Bootstrap : Using eth0:22.8.38.24<0>
+dlc1abaccnl2nzws-worker-0:78:78 [6] NCCL INFO Plugin name set by env to libnccl-net-none.so
+dlc1abaccnl2nzws-worker-0:76:76 [4] NCCL INFO Plugin name set by env to libnccl-net-none.so
+dlc1abaccnl2nzws-worker-0:76:76 [4] NCCL INFO NET/Plugin : Plugin load (libnccl-net-none.so) returned 2 : libnccl-net-none.so: cannot open shared object file: No such file or directory
+dlc1abaccnl2nzws-worker-0:78:78 [6] NCCL INFO NET/Plugin : Plugin load (libnccl-net-none.so) returned 2 : libnccl-net-none.so: cannot open shared object file: No such file or directory
+dlc1abaccnl2nzws-worker-0:78:78 [6] NCCL INFO NET/Plugin : No plugin found, using internal implementation
+dlc1abaccnl2nzws-worker-0:76:76 [4] NCCL INFO NET/Plugin : No plugin found, using internal implementation
+dlc1abaccnl2nzws-worker-0:75:75 [3] NCCL INFO cudaDriverVersion 12010
+dlc1abaccnl2nzws-worker-0:75:75 [3] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth
+dlc1abaccnl2nzws-worker-0:75:75 [3] NCCL INFO Bootstrap : Using eth0:22.8.38.24<0>
+dlc1abaccnl2nzws-worker-0:75:75 [3] NCCL INFO Plugin name set by env to libnccl-net-none.so
+dlc1abaccnl2nzws-worker-0:75:75 [3] NCCL INFO NET/Plugin : Plugin load (libnccl-net-none.so) returned 2 : libnccl-net-none.so: cannot open shared object file: No such file or directory
+dlc1abaccnl2nzws-worker-0:75:75 [3] NCCL INFO NET/Plugin : No plugin found, using internal implementation
+dlc1abaccnl2nzws-worker-0:77:77 [5] NCCL INFO cudaDriverVersion 12010
+dlc1abaccnl2nzws-worker-0:77:77 [5] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth
+dlc1abaccnl2nzws-worker-0:77:77 [5] NCCL INFO Bootstrap : Using eth0:22.8.38.24<0>
+dlc1abaccnl2nzws-worker-0:77:77 [5] NCCL INFO Plugin name set by env to libnccl-net-none.so
+dlc1abaccnl2nzws-worker-0:77:77 [5] NCCL INFO NET/Plugin : Plugin load (libnccl-net-none.so) returned 2 : libnccl-net-none.so: cannot open shared object file: No such file or directory
+dlc1abaccnl2nzws-worker-0:77:77 [5] NCCL INFO NET/Plugin : No plugin found, using internal implementation
+dlc1abaccnl2nzws-worker-0:79:79 [7] NCCL INFO cudaDriverVersion 12010
+dlc1abaccnl2nzws-worker-0:79:79 [7] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth
+dlc1abaccnl2nzws-worker-0:79:79 [7] NCCL INFO Bootstrap : Using eth0:22.8.38.24<0>
+dlc1abaccnl2nzws-worker-0:79:79 [7] NCCL INFO Plugin name set by env to libnccl-net-none.so
+dlc1abaccnl2nzws-worker-0:79:79 [7] NCCL INFO NET/Plugin : Plugin load (libnccl-net-none.so) returned 2 : libnccl-net-none.so: cannot open shared object file: No such file or directory
+dlc1abaccnl2nzws-worker-0:79:79 [7] NCCL INFO NET/Plugin : No plugin found, using internal implementation
+dlc1abaccnl2nzws-worker-0:74:74 [2] NCCL INFO cudaDriverVersion 12010
+dlc1abaccnl2nzws-worker-0:74:74 [2] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth
+dlc1abaccnl2nzws-worker-0:74:74 [2] NCCL INFO Bootstrap : Using eth0:22.8.38.24<0>
+dlc1abaccnl2nzws-worker-0:74:74 [2] NCCL INFO Plugin name set by env to libnccl-net-none.so
+dlc1abaccnl2nzws-worker-0:74:74 [2] NCCL INFO NET/Plugin : Plugin load (libnccl-net-none.so) returned 2 : libnccl-net-none.so: cannot open shared object file: No such file or directory
+dlc1abaccnl2nzws-worker-0:74:74 [2] NCCL INFO NET/Plugin : No plugin found, using internal implementation
+dlc1abaccnl2nzws-worker-0:73:73 [1] NCCL INFO cudaDriverVersion 12010
+dlc1abaccnl2nzws-worker-0:73:73 [1] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth
+dlc1abaccnl2nzws-worker-0:73:73 [1] NCCL INFO Bootstrap : Using eth0:22.8.38.24<0>
+dlc1abaccnl2nzws-worker-0:73:73 [1] NCCL INFO Plugin name set by env to libnccl-net-none.so
+dlc1abaccnl2nzws-worker-0:73:73 [1] NCCL INFO NET/Plugin : Plugin load (libnccl-net-none.so) returned 2 : libnccl-net-none.so: cannot open shared object file: No such file or directory
+dlc1abaccnl2nzws-worker-0:73:73 [1] NCCL INFO NET/Plugin : No plugin found, using internal implementation
+dlc1abaccnl2nzws-worker-0:78:314 [6] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth
+dlc1abaccnl2nzws-worker-0:76:315 [4] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth
+dlc1abaccnl2nzws-worker-0:78:314 [6] NCCL INFO NCCL_IB_HCA set to mlx5
+dlc1abaccnl2nzws-worker-0:76:315 [4] NCCL INFO NCCL_IB_HCA set to mlx5
+dlc1abaccnl2nzws-worker-0:72:72 [0] NCCL INFO cudaDriverVersion 12010
+dlc1abaccnl2nzws-worker-0:72:72 [0] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth
+dlc1abaccnl2nzws-worker-0:72:72 [0] NCCL INFO Bootstrap : Using eth0:22.8.38.24<0>
+dlc1abaccnl2nzws-worker-0:72:72 [0] NCCL INFO Plugin name set by env to libnccl-net-none.so
+dlc1abaccnl2nzws-worker-0:72:72 [0] NCCL INFO NET/Plugin : Plugin load (libnccl-net-none.so) returned 2 : libnccl-net-none.so: cannot open shared object file: No such file or directory
+dlc1abaccnl2nzws-worker-0:72:72 [0] NCCL INFO NET/Plugin : No plugin found, using internal implementation
+dlc1abaccnl2nzws-worker-0:75:316 [3] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth
+dlc1abaccnl2nzws-worker-0:75:316 [3] NCCL INFO NCCL_IB_HCA set to mlx5
+dlc1abaccnl2nzws-worker-0:77:317 [5] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth
+dlc1abaccnl2nzws-worker-0:77:317 [5] NCCL INFO NCCL_IB_HCA set to mlx5
+dlc1abaccnl2nzws-worker-0:79:321 [7] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth
+dlc1abaccnl2nzws-worker-0:79:321 [7] NCCL INFO NCCL_IB_HCA set to mlx5
+dlc1abaccnl2nzws-worker-0:74:323 [2] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth
+dlc1abaccnl2nzws-worker-0:74:323 [2] NCCL INFO NCCL_IB_HCA set to mlx5
+dlc1abaccnl2nzws-worker-0:73:324 [1] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth
+dlc1abaccnl2nzws-worker-0:73:324 [1] NCCL INFO NCCL_IB_HCA set to mlx5
+dlc1abaccnl2nzws-worker-0:72:326 [0] NCCL INFO NCCL_SOCKET_IFNAME set by environment to eth
+dlc1abaccnl2nzws-worker-0:72:326 [0] NCCL INFO NCCL_IB_HCA set to mlx5
+dlc1abaccnl2nzws-worker-0:76:315 [4] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [2]mlx5_2:1/RoCE [3]mlx5_3:1/RoCE [RO]; OOB eth0:22.8.38.24<0>
+dlc1abaccnl2nzws-worker-0:76:315 [4] NCCL INFO Using network IB
+dlc1abaccnl2nzws-worker-0:75:316 [3] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [2]mlx5_2:1/RoCE [3]mlx5_3:1/RoCE [RO]; OOB eth0:22.8.38.24<0>
+dlc1abaccnl2nzws-worker-0:75:316 [3] NCCL INFO Using network IB
+dlc1abaccnl2nzws-worker-0:78:314 [6] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [2]mlx5_2:1/RoCE [3]mlx5_3:1/RoCE [RO]; OOB eth0:22.8.38.24<0>
+dlc1abaccnl2nzws-worker-0:78:314 [6] NCCL INFO Using network IB
+dlc1abaccnl2nzws-worker-0:77:317 [5] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [2]mlx5_2:1/RoCE [3]mlx5_3:1/RoCE [RO]; OOB eth0:22.8.38.24<0>
+dlc1abaccnl2nzws-worker-0:77:317 [5] NCCL INFO Using network IB
+dlc1abaccnl2nzws-worker-0:79:321 [7] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [2]mlx5_2:1/RoCE [3]mlx5_3:1/RoCE [RO]; OOB eth0:22.8.38.24<0>
+dlc1abaccnl2nzws-worker-0:79:321 [7] NCCL INFO Using network IB
+dlc1abaccnl2nzws-worker-0:73:324 [1] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [2]mlx5_2:1/RoCE [3]mlx5_3:1/RoCE [RO]; OOB eth0:22.8.38.24<0>
+dlc1abaccnl2nzws-worker-0:73:324 [1] NCCL INFO Using network IB
+dlc1abaccnl2nzws-worker-0:74:323 [2] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [2]mlx5_2:1/RoCE [3]mlx5_3:1/RoCE [RO]; OOB eth0:22.8.38.24<0>
+dlc1abaccnl2nzws-worker-0:74:323 [2] NCCL INFO Using network IB
+dlc1abaccnl2nzws-worker-0:72:326 [0] NCCL INFO NET/IB : Using [0]mlx5_0:1/RoCE [1]mlx5_1:1/RoCE [2]mlx5_2:1/RoCE [3]mlx5_3:1/RoCE [RO]; OOB eth0:22.8.38.24<0>
+dlc1abaccnl2nzws-worker-0:72:326 [0] NCCL INFO Using network IB
+dlc1abaccnl2nzws-worker-0:79:321 [7] NCCL INFO comm 0x9afcfd70 rank 15 nranks 16 cudaDev 7 nvmlDev 7 busId 80 commId 0x13a7e6351c9956c9 - Init START
+dlc1abaccnl2nzws-worker-0:72:326 [0] NCCL INFO comm 0x9baa4e10 rank 8 nranks 16 cudaDev 0 nvmlDev 0 busId 10 commId 0x13a7e6351c9956c9 - Init START
+dlc1abaccnl2nzws-worker-0:78:314 [6] NCCL INFO comm 0x9a052210 rank 14 nranks 16 cudaDev 6 nvmlDev 6 busId 70 commId 0x13a7e6351c9956c9 - Init START
+dlc1abaccnl2nzws-worker-0:77:317 [5] NCCL INFO comm 0x99f6a330 rank 13 nranks 16 cudaDev 5 nvmlDev 5 busId 60 commId 0x13a7e6351c9956c9 - Init START
+dlc1abaccnl2nzws-worker-0:74:323 [2] NCCL INFO comm 0x9b2bc4b0 rank 10 nranks 16 cudaDev 2 nvmlDev 2 busId 30 commId 0x13a7e6351c9956c9 - Init START
+dlc1abaccnl2nzws-worker-0:73:324 [1] NCCL INFO comm 0x9a987d60 rank 9 nranks 16 cudaDev 1 nvmlDev 1 busId 20 commId 0x13a7e6351c9956c9 - Init START
+dlc1abaccnl2nzws-worker-0:76:315 [4] NCCL INFO comm 0x9a914480 rank 12 nranks 16 cudaDev 4 nvmlDev 4 busId 50 commId 0x13a7e6351c9956c9 - Init START
+dlc1abaccnl2nzws-worker-0:75:316 [3] NCCL INFO comm 0x9b8645b0 rank 11 nranks 16 cudaDev 3 nvmlDev 3 busId 40 commId 0x13a7e6351c9956c9 - Init START
+dlc1abaccnl2nzws-worker-0:77:317 [5] NCCL INFO NVLS multicast support is not available on dev 5
+dlc1abaccnl2nzws-worker-0:79:321 [7] NCCL INFO NVLS multicast support is not available on dev 7
+dlc1abaccnl2nzws-worker-0:76:315 [4] NCCL INFO NVLS multicast support is not available on dev 4
+dlc1abaccnl2nzws-worker-0:75:316 [3] NCCL INFO Setting affinity for GPU 3 to ffffffff,ffffffff,ffffffff
+dlc1abaccnl2nzws-worker-0:75:316 [3] NCCL INFO NVLS multicast support is not available on dev 3
+dlc1abaccnl2nzws-worker-0:78:314 [6] NCCL INFO NVLS multicast support is not available on dev 6
+dlc1abaccnl2nzws-worker-0:74:323 [2] NCCL INFO Setting affinity for GPU 2 to ffffffff,ffffffff,ffffffff
+dlc1abaccnl2nzws-worker-0:74:323 [2] NCCL INFO NVLS multicast support is not available on dev 2
+dlc1abaccnl2nzws-worker-0:72:326 [0] NCCL INFO Setting affinity for GPU 0 to ffffffff,ffffffff,ffffffff
+dlc1abaccnl2nzws-worker-0:72:326 [0] NCCL INFO NVLS multicast support is not available on dev 0
+dlc1abaccnl2nzws-worker-0:73:324 [1] NCCL INFO Setting affinity for GPU 1 to ffffffff,ffffffff,ffffffff
+dlc1abaccnl2nzws-worker-0:73:324 [1] NCCL INFO NVLS multicast support is not available on dev 1
+dlc1abaccnl2nzws-worker-0:72:326 [0] NCCL INFO NCCL_MIN_NCHANNELS set by environment to 4.
+dlc1abaccnl2nzws-worker-0:79:321 [7] NCCL INFO NCCL_MIN_NCHANNELS set by environment to 4.
+dlc1abaccnl2nzws-worker-0:72:326 [0] NCCL INFO Trees [0] 9/-1/-1->8->0 [1] 9/-1/-1->8->15 [2] 9/-1/-1->8->15 [3] 9/-1/-1->8->15 [4] 9/0/-1->8->-1 [5] 9/-1/-1->8->15 [6] 9/-1/-1->8->15 [7] 9/-1/-1->8->15
+dlc1abaccnl2nzws-worker-0:78:314 [6] NCCL INFO NCCL_MIN_NCHANNELS set by environment to 4.
+dlc1abaccnl2nzws-worker-0:79:321 [7] NCCL INFO Trees [0] -1/-1/-1->15->14 [1] 8/-1/-1->15->14 [2] 8/-1/-1->15->14 [3] 8/-1/-1->15->14 [4] -1/-1/-1->15->14 [5] 8/-1/-1->15->14 [6] 8/-1/-1->15->14 [7] 8/-1/-1->15->14
+dlc1abaccnl2nzws-worker-0:72:326 [0] NCCL INFO P2P Chunksize set to 131072
+dlc1abaccnl2nzws-worker-0:79:321 [7] NCCL INFO P2P Chunksize set to 131072
+dlc1abaccnl2nzws-worker-0:77:317 [5] NCCL INFO NCCL_MIN_NCHANNELS set by environment to 4.
+dlc1abaccnl2nzws-worker-0:78:314 [6] NCCL INFO Trees [0] 15/-1/-1->14->13 [1] 15/-1/-1->14->13 [2] 15/-1/-1->14->13 [3] 15/-1/-1->14->6 [4] 15/-1/-1->14->13 [5] 15/-1/-1->14->13 [6] 15/-1/-1->14->13 [7] 15/6/-1->14->-1
+dlc1abaccnl2nzws-worker-0:73:324 [1] NCCL INFO NCCL_MIN_NCHANNELS set by environment to 4.
+dlc1abaccnl2nzws-worker-0:78:314 [6] NCCL INFO P2P Chunksize set to 131072
+dlc1abaccnl2nzws-worker-0:74:323 [2] NCCL INFO NCCL_MIN_NCHANNELS set by environment to 4.
+dlc1abaccnl2nzws-worker-0:77:317 [5] NCCL INFO Trees [0] 14/-1/-1->13->12 [1] 14/-1/-1->13->12 [2] 14/-1/-1->13->12 [3] -1/-1/-1->13->12 [4] 14/-1/-1->13->12 [5] 14/-1/-1->13->12 [6] 14/-1/-1->13->12 [7] -1/-1/-1->13->12
+dlc1abaccnl2nzws-worker-0:77:317 [5] NCCL INFO P2P Chunksize set to 131072
+dlc1abaccnl2nzws-worker-0:73:324 [1] NCCL INFO Trees [0] 10/-1/-1->9->8 [1] -1/-1/-1->9->8 [2] 10/-1/-1->9->8 [3] 10/-1/-1->9->8 [4] 10/-1/-1->9->8 [5] -1/-1/-1->9->8 [6] 10/-1/-1->9->8 [7] 10/-1/-1->9->8
+dlc1abaccnl2nzws-worker-0:74:323 [2] NCCL INFO Trees [0] 11/-1/-1->10->9 [1] 11/-1/-1->10->2 [2] 11/-1/-1->10->9 [3] 11/-1/-1->10->9 [4] 11/-1/-1->10->9 [5] 11/2/-1->10->-1 [6] 11/-1/-1->10->9 [7] 11/-1/-1->10->9
+dlc1abaccnl2nzws-worker-0:75:316 [3] NCCL INFO NCCL_MIN_NCHANNELS set by environment to 4.
+dlc1abaccnl2nzws-worker-0:73:324 [1] NCCL INFO P2P Chunksize set to 131072
+dlc1abaccnl2nzws-worker-0:74:323 [2] NCCL INFO P2P Chunksize set to 131072
+dlc1abaccnl2nzws-worker-0:76:315 [4] NCCL INFO NCCL_MIN_NCHANNELS set by environment to 4.
+dlc1abaccnl2nzws-worker-0:75:316 [3] NCCL INFO Trees [0] 12/-1/-1->11->10 [1] 12/-1/-1->11->10 [2] -1/-1/-1->11->10 [3] 12/-1/-1->11->10 [4] 12/-1/-1->11->10 [5] 12/-1/-1->11->10 [6] -1/-1/-1->11->10 [7] 12/-1/-1->11->10
+dlc1abaccnl2nzws-worker-0:75:316 [3] NCCL INFO P2P Chunksize set to 131072
+dlc1abaccnl2nzws-worker-0:76:315 [4] NCCL INFO Trees [0] 13/-1/-1->12->11 [1] 13/-1/-1->12->11 [2] 13/-1/-1->12->4 [3] 13/-1/-1->12->11 [4] 13/-1/-1->12->11 [5] 13/-1/-1->12->11 [6] 13/4/-1->12->-1 [7] 13/-1/-1->12->11
+dlc1abaccnl2nzws-worker-0:76:315 [4] NCCL INFO P2P Chunksize set to 131072
+dlc1abaccnl2nzws-worker-0:72:326 [0] NCCL INFO Channel 01/0 : 8[0] -> 11[3] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:74:323 [2] NCCL INFO Channel 01/0 : 10[2] -> 15[7] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:76:315 [4] NCCL INFO Channel 03/0 : 12[4] -> 15[7] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:73:324 [1] NCCL INFO Channel 00/0 : 9[1] -> 0[0] [send] via NET/IB/0/GDRDMA
+dlc1abaccnl2nzws-worker-0:78:314 [6] NCCL INFO Channel 03/0 : 7[7] -> 14[6] [receive] via NET/IB/3/GDRDMA
+dlc1abaccnl2nzws-worker-0:78:314 [6] NCCL INFO Channel 07/0 : 7[7] -> 14[6] [receive] via NET/IB/3/GDRDMA
+dlc1abaccnl2nzws-worker-0:73:324 [1] NCCL INFO Channel 04/0 : 9[1] -> 0[0] [send] via NET/IB/0/GDRDMA
+dlc1abaccnl2nzws-worker-0:72:326 [0] NCCL INFO Channel 05/0 : 8[0] -> 11[3] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:74:323 [2] NCCL INFO Channel 05/0 : 10[2] -> 15[7] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:76:315 [4] NCCL INFO Channel 07/0 : 12[4] -> 15[7] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:72:326 [0] NCCL INFO Channel 03/0 : 8[0] -> 13[5] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:72:326 [0] NCCL INFO Channel 07/0 : 8[0] -> 13[5] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:75:316 [3] NCCL INFO Channel 01/0 : 11[3] -> 2[2] [send] via NET/IB/1/GDRDMA
+dlc1abaccnl2nzws-worker-0:75:316 [3] NCCL INFO Channel 05/0 : 11[3] -> 2[2] [send] via NET/IB/1/GDRDMA
+dlc1abaccnl2nzws-worker-0:76:315 [4] NCCL INFO Channel 02/0 : 5[5] -> 12[4] [receive] via NET/IB/2/GDRDMA
+dlc1abaccnl2nzws-worker-0:76:315 [4] NCCL INFO Channel 06/0 : 5[5] -> 12[4] [receive] via NET/IB/2/GDRDMA
+dlc1abaccnl2nzws-worker-0:72:326 [0] NCCL INFO Channel 00/0 : 1[1] -> 8[0] [receive] via NET/IB/0/GDRDMA
+dlc1abaccnl2nzws-worker-0:72:326 [0] NCCL INFO Channel 04/0 : 1[1] -> 8[0] [receive] via NET/IB/0/GDRDMA
+dlc1abaccnl2nzws-worker-0:72:326 [0] NCCL INFO Channel 00/0 : 8[0] -> 15[7] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:72:326 [0] NCCL INFO Channel 02/0 : 8[0] -> 15[7] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:77:317 [5] NCCL INFO Channel 02/0 : 13[5] -> 4[4] [send] via NET/IB/2/GDRDMA
+dlc1abaccnl2nzws-worker-0:77:317 [5] NCCL INFO Channel 06/0 : 13[5] -> 4[4] [send] via NET/IB/2/GDRDMA
+dlc1abaccnl2nzws-worker-0:72:326 [0] NCCL INFO Channel 04/0 : 8[0] -> 15[7] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:76:315 [4] NCCL INFO Channel 01/0 : 12[4] -> 9[1] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:79:321 [7] NCCL INFO Channel 03/0 : 15[7] -> 6[6] [send] via NET/IB/3/GDRDMA
+dlc1abaccnl2nzws-worker-0:79:321 [7] NCCL INFO Channel 07/0 : 15[7] -> 6[6] [send] via NET/IB/3/GDRDMA
+dlc1abaccnl2nzws-worker-0:72:326 [0] NCCL INFO Channel 06/0 : 8[0] -> 15[7] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:78:314 [6] NCCL INFO Channel 03/0 : 14[6] -> 11[3] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:74:323 [2] NCCL INFO Channel 01/0 : 3[3] -> 10[2] [receive] via NET/IB/1/GDRDMA
+dlc1abaccnl2nzws-worker-0:74:323 [2] NCCL INFO Channel 05/0 : 3[3] -> 10[2] [receive] via NET/IB/1/GDRDMA
+dlc1abaccnl2nzws-worker-0:76:315 [4] NCCL INFO Channel 05/0 : 12[4] -> 9[1] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:79:321 [7] NCCL INFO Channel 00/0 : 15[7] -> 14[6] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:78:314 [6] NCCL INFO Channel 07/0 : 14[6] -> 11[3] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:79:321 [7] NCCL INFO Channel 01/0 : 15[7] -> 14[6] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:79:321 [7] NCCL INFO Channel 02/0 : 15[7] -> 14[6] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:79:321 [7] NCCL INFO Channel 04/0 : 15[7] -> 14[6] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:77:317 [5] NCCL INFO Channel 00/0 : 13[5] -> 12[4] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:79:321 [7] NCCL INFO Channel 05/0 : 15[7] -> 14[6] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:77:317 [5] NCCL INFO Channel 01/0 : 13[5] -> 12[4] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:79:321 [7] NCCL INFO Channel 06/0 : 15[7] -> 14[6] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:74:323 [2] NCCL INFO Channel 00/0 : 10[2] -> 9[1] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:75:316 [3] NCCL INFO Channel 00/0 : 11[3] -> 10[2] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:77:317 [5] NCCL INFO Channel 03/0 : 13[5] -> 12[4] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:74:323 [2] NCCL INFO Channel 02/0 : 10[2] -> 9[1] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:77:317 [5] NCCL INFO Channel 04/0 : 13[5] -> 12[4] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:75:316 [3] NCCL INFO Channel 02/0 : 11[3] -> 10[2] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:74:323 [2] NCCL INFO Channel 03/0 : 10[2] -> 9[1] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:77:317 [5] NCCL INFO Channel 05/0 : 13[5] -> 12[4] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:75:316 [3] NCCL INFO Channel 03/0 : 11[3] -> 10[2] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:78:314 [6] NCCL INFO Channel 00/0 : 14[6] -> 13[5] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:74:323 [2] NCCL INFO Channel 04/0 : 10[2] -> 9[1] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:77:317 [5] NCCL INFO Channel 07/0 : 13[5] -> 12[4] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:75:316 [3] NCCL INFO Channel 04/0 : 11[3] -> 10[2] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:78:314 [6] NCCL INFO Channel 01/0 : 14[6] -> 13[5] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:74:323 [2] NCCL INFO Channel 06/0 : 10[2] -> 9[1] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:75:316 [3] NCCL INFO Channel 06/0 : 11[3] -> 10[2] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:78:314 [6] NCCL INFO Channel 02/0 : 14[6] -> 13[5] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:74:323 [2] NCCL INFO Channel 07/0 : 10[2] -> 9[1] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:76:315 [4] NCCL INFO Channel 00/0 : 12[4] -> 11[3] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:75:316 [3] NCCL INFO Channel 07/0 : 11[3] -> 10[2] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:73:324 [1] NCCL INFO Channel 01/0 : 9[1] -> 8[0] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:78:314 [6] NCCL INFO Channel 04/0 : 14[6] -> 13[5] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:76:315 [4] NCCL INFO Channel 02/0 : 12[4] -> 11[3] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:73:324 [1] NCCL INFO Channel 02/0 : 9[1] -> 8[0] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:78:314 [6] NCCL INFO Channel 05/0 : 14[6] -> 13[5] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:76:315 [4] NCCL INFO Channel 04/0 : 12[4] -> 11[3] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:78:314 [6] NCCL INFO Channel 06/0 : 14[6] -> 13[5] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:73:324 [1] NCCL INFO Channel 03/0 : 9[1] -> 8[0] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:76:315 [4] NCCL INFO Channel 06/0 : 12[4] -> 11[3] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:73:324 [1] NCCL INFO Channel 05/0 : 9[1] -> 8[0] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:73:324 [1] NCCL INFO Channel 06/0 : 9[1] -> 8[0] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:73:324 [1] NCCL INFO Channel 07/0 : 9[1] -> 8[0] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:77:357 [5] NCCL INFO NCCL_IB_QPS_PER_CONNECTION set by environment to 8.
+dlc1abaccnl2nzws-worker-0:75:360 [3] NCCL INFO NCCL_IB_QPS_PER_CONNECTION set by environment to 8.
+dlc1abaccnl2nzws-worker-0:73:359 [1] NCCL INFO NCCL_IB_QPS_PER_CONNECTION set by environment to 8.
+dlc1abaccnl2nzws-worker-0:77:357 [5] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3.
+dlc1abaccnl2nzws-worker-0:75:360 [3] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3.
+dlc1abaccnl2nzws-worker-0:79:355 [7] NCCL INFO NCCL_IB_QPS_PER_CONNECTION set by environment to 8.
+dlc1abaccnl2nzws-worker-0:74:358 [2] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3.
+dlc1abaccnl2nzws-worker-0:73:359 [1] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3.
+dlc1abaccnl2nzws-worker-0:74:358 [2] NCCL INFO NCCL_IB_QPS_PER_CONNECTION set by environment to 8.
+dlc1abaccnl2nzws-worker-0:76:361 [4] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3.
+dlc1abaccnl2nzws-worker-0:76:361 [4] NCCL INFO NCCL_IB_QPS_PER_CONNECTION set by environment to 8.
+dlc1abaccnl2nzws-worker-0:74:358 [2] NCCL INFO NCCL_IB_TC set by environment to 136.
+dlc1abaccnl2nzws-worker-0:74:358 [2] NCCL INFO NCCL_IB_SL set by environment to 5.
+dlc1abaccnl2nzws-worker-0:74:358 [2] NCCL INFO NCCL_IB_TIMEOUT set by environment to 22.
+dlc1abaccnl2nzws-worker-0:79:355 [7] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3.
+dlc1abaccnl2nzws-worker-0:76:361 [4] NCCL INFO NCCL_IB_TC set by environment to 136.
+dlc1abaccnl2nzws-worker-0:76:361 [4] NCCL INFO NCCL_IB_SL set by environment to 5.
+dlc1abaccnl2nzws-worker-0:76:361 [4] NCCL INFO NCCL_IB_TIMEOUT set by environment to 22.
+dlc1abaccnl2nzws-worker-0:75:360 [3] NCCL INFO NCCL_IB_TC set by environment to 136.
+dlc1abaccnl2nzws-worker-0:75:360 [3] NCCL INFO NCCL_IB_SL set by environment to 5.
+dlc1abaccnl2nzws-worker-0:75:360 [3] NCCL INFO NCCL_IB_TIMEOUT set by environment to 22.
+dlc1abaccnl2nzws-worker-0:77:357 [5] NCCL INFO NCCL_IB_TC set by environment to 136.
+dlc1abaccnl2nzws-worker-0:77:357 [5] NCCL INFO NCCL_IB_SL set by environment to 5.
+dlc1abaccnl2nzws-worker-0:77:357 [5] NCCL INFO NCCL_IB_TIMEOUT set by environment to 22.
+dlc1abaccnl2nzws-worker-0:72:354 [0] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3.
+dlc1abaccnl2nzws-worker-0:72:354 [0] NCCL INFO NCCL_IB_QPS_PER_CONNECTION set by environment to 8.
+dlc1abaccnl2nzws-worker-0:78:356 [6] NCCL INFO NCCL_IB_GID_INDEX set by environment to 3.
+dlc1abaccnl2nzws-worker-0:78:356 [6] NCCL INFO NCCL_IB_QPS_PER_CONNECTION set by environment to 8.
+dlc1abaccnl2nzws-worker-0:79:355 [7] NCCL INFO NCCL_IB_TC set by environment to 136.
+dlc1abaccnl2nzws-worker-0:79:355 [7] NCCL INFO NCCL_IB_SL set by environment to 5.
+dlc1abaccnl2nzws-worker-0:72:354 [0] NCCL INFO NCCL_IB_TC set by environment to 136.
+dlc1abaccnl2nzws-worker-0:72:354 [0] NCCL INFO NCCL_IB_SL set by environment to 5.
+dlc1abaccnl2nzws-worker-0:72:354 [0] NCCL INFO NCCL_IB_TIMEOUT set by environment to 22.
+dlc1abaccnl2nzws-worker-0:79:355 [7] NCCL INFO NCCL_IB_TIMEOUT set by environment to 22.
+dlc1abaccnl2nzws-worker-0:73:359 [1] NCCL INFO NCCL_IB_TC set by environment to 136.
+dlc1abaccnl2nzws-worker-0:73:359 [1] NCCL INFO NCCL_IB_SL set by environment to 5.
+dlc1abaccnl2nzws-worker-0:73:359 [1] NCCL INFO NCCL_IB_TIMEOUT set by environment to 22.
+dlc1abaccnl2nzws-worker-0:78:356 [6] NCCL INFO NCCL_IB_TC set by environment to 136.
+dlc1abaccnl2nzws-worker-0:78:356 [6] NCCL INFO NCCL_IB_SL set by environment to 5.
+dlc1abaccnl2nzws-worker-0:78:356 [6] NCCL INFO NCCL_IB_TIMEOUT set by environment to 22.
+dlc1abaccnl2nzws-worker-0:77:317 [5] NCCL INFO Connected all rings
+dlc1abaccnl2nzws-worker-0:75:316 [3] NCCL INFO Connected all rings
+dlc1abaccnl2nzws-worker-0:73:324 [1] NCCL INFO Connected all rings
+dlc1abaccnl2nzws-worker-0:79:321 [7] NCCL INFO Connected all rings
+dlc1abaccnl2nzws-worker-0:77:317 [5] NCCL INFO Channel 00/0 : 13[5] -> 14[6] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:77:317 [5] NCCL INFO Channel 01/0 : 13[5] -> 14[6] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:77:317 [5] NCCL INFO Channel 02/0 : 13[5] -> 14[6] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:77:317 [5] NCCL INFO Channel 04/0 : 13[5] -> 14[6] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:74:323 [2] NCCL INFO Connected all rings
+dlc1abaccnl2nzws-worker-0:77:317 [5] NCCL INFO Channel 05/0 : 13[5] -> 14[6] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:77:317 [5] NCCL INFO Channel 06/0 : 13[5] -> 14[6] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:76:315 [4] NCCL INFO Connected all rings
+dlc1abaccnl2nzws-worker-0:75:316 [3] NCCL INFO Channel 00/0 : 11[3] -> 12[4] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:78:314 [6] NCCL INFO Connected all rings
+dlc1abaccnl2nzws-worker-0:72:326 [0] NCCL INFO Connected all rings
+dlc1abaccnl2nzws-worker-0:72:326 [0] NCCL INFO Channel 00/0 : 8[0] -> 9[1] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:75:316 [3] NCCL INFO Channel 01/0 : 11[3] -> 12[4] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:73:324 [1] NCCL INFO Channel 00/0 : 9[1] -> 10[2] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:72:326 [0] NCCL INFO Channel 01/0 : 8[0] -> 9[1] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:75:316 [3] NCCL INFO Channel 03/0 : 11[3] -> 12[4] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:73:324 [1] NCCL INFO Channel 02/0 : 9[1] -> 10[2] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:72:326 [0] NCCL INFO Channel 02/0 : 8[0] -> 9[1] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:75:316 [3] NCCL INFO Channel 04/0 : 11[3] -> 12[4] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:73:324 [1] NCCL INFO Channel 03/0 : 9[1] -> 10[2] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:72:326 [0] NCCL INFO Channel 03/0 : 8[0] -> 9[1] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:75:316 [3] NCCL INFO Channel 05/0 : 11[3] -> 12[4] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:74:323 [2] NCCL INFO Channel 00/0 : 10[2] -> 11[3] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:73:324 [1] NCCL INFO Channel 04/0 : 9[1] -> 10[2] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:72:326 [0] NCCL INFO Channel 04/0 : 8[0] -> 9[1] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:75:316 [3] NCCL INFO Channel 07/0 : 11[3] -> 12[4] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:74:323 [2] NCCL INFO Channel 01/0 : 10[2] -> 11[3] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:73:324 [1] NCCL INFO Channel 06/0 : 9[1] -> 10[2] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:76:315 [4] NCCL INFO Channel 00/0 : 12[4] -> 13[5] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:72:326 [0] NCCL INFO Channel 05/0 : 8[0] -> 9[1] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:78:314 [6] NCCL INFO Channel 00/0 : 14[6] -> 15[7] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:74:323 [2] NCCL INFO Channel 02/0 : 10[2] -> 11[3] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:73:324 [1] NCCL INFO Channel 07/0 : 9[1] -> 10[2] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:76:315 [4] NCCL INFO Channel 01/0 : 12[4] -> 13[5] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:72:326 [0] NCCL INFO Channel 06/0 : 8[0] -> 9[1] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:78:314 [6] NCCL INFO Channel 01/0 : 14[6] -> 15[7] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:74:323 [2] NCCL INFO Channel 03/0 : 10[2] -> 11[3] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:76:315 [4] NCCL INFO Channel 02/0 : 12[4] -> 13[5] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:72:326 [0] NCCL INFO Channel 07/0 : 8[0] -> 9[1] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:78:314 [6] NCCL INFO Channel 02/0 : 14[6] -> 15[7] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:74:323 [2] NCCL INFO Channel 04/0 : 10[2] -> 11[3] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:76:315 [4] NCCL INFO Channel 03/0 : 12[4] -> 13[5] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:72:326 [0] NCCL INFO Channel 01/0 : 8[0] -> 15[7] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:78:314 [6] NCCL INFO Channel 03/0 : 14[6] -> 15[7] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:74:323 [2] NCCL INFO Channel 05/0 : 10[2] -> 11[3] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:76:315 [4] NCCL INFO Channel 04/0 : 12[4] -> 13[5] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:72:326 [0] NCCL INFO Channel 03/0 : 8[0] -> 15[7] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:78:314 [6] NCCL INFO Channel 04/0 : 14[6] -> 15[7] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:74:323 [2] NCCL INFO Channel 06/0 : 10[2] -> 11[3] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:76:315 [4] NCCL INFO Channel 05/0 : 12[4] -> 13[5] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:72:326 [0] NCCL INFO Channel 05/0 : 8[0] -> 15[7] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:78:314 [6] NCCL INFO Channel 05/0 : 14[6] -> 15[7] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:74:323 [2] NCCL INFO Channel 07/0 : 10[2] -> 11[3] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:76:315 [4] NCCL INFO Channel 06/0 : 12[4] -> 13[5] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:72:326 [0] NCCL INFO Channel 07/0 : 8[0] -> 15[7] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:78:314 [6] NCCL INFO Channel 06/0 : 14[6] -> 15[7] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:76:315 [4] NCCL INFO Channel 07/0 : 12[4] -> 13[5] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:74:323 [2] NCCL INFO Channel 01/0 : 2[2] -> 10[2] [receive] via NET/IB/1/GDRDMA
+dlc1abaccnl2nzws-worker-0:73:324 [1] NCCL INFO Channel 00/0 : 9[1] -> 8[0] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:74:323 [2] NCCL INFO Channel 05/0 : 2[2] -> 10[2] [receive] via NET/IB/1/GDRDMA
+dlc1abaccnl2nzws-worker-0:74:323 [2] NCCL INFO Channel 01/0 : 10[2] -> 2[2] [send] via NET/IB/1/GDRDMA
+dlc1abaccnl2nzws-worker-0:74:323 [2] NCCL INFO Channel 05/0 : 10[2] -> 2[2] [send] via NET/IB/1/GDRDMA
+dlc1abaccnl2nzws-worker-0:78:314 [6] NCCL INFO Channel 07/0 : 14[6] -> 15[7] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:73:324 [1] NCCL INFO Channel 04/0 : 9[1] -> 8[0] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:76:315 [4] NCCL INFO Channel 02/0 : 4[4] -> 12[4] [receive] via NET/IB/2/GDRDMA
+dlc1abaccnl2nzws-worker-0:76:315 [4] NCCL INFO Channel 06/0 : 4[4] -> 12[4] [receive] via NET/IB/2/GDRDMA
+dlc1abaccnl2nzws-worker-0:76:315 [4] NCCL INFO Channel 02/0 : 12[4] -> 4[4] [send] via NET/IB/2/GDRDMA
+dlc1abaccnl2nzws-worker-0:76:315 [4] NCCL INFO Channel 06/0 : 12[4] -> 4[4] [send] via NET/IB/2/GDRDMA
+dlc1abaccnl2nzws-worker-0:77:317 [5] NCCL INFO Channel 02/0 : 13[5] -> 12[4] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:78:314 [6] NCCL INFO Channel 03/0 : 6[6] -> 14[6] [receive] via NET/IB/3/GDRDMA
+dlc1abaccnl2nzws-worker-0:78:314 [6] NCCL INFO Channel 07/0 : 6[6] -> 14[6] [receive] via NET/IB/3/GDRDMA
+dlc1abaccnl2nzws-worker-0:78:314 [6] NCCL INFO Channel 03/0 : 14[6] -> 6[6] [send] via NET/IB/3/GDRDMA
+dlc1abaccnl2nzws-worker-0:78:314 [6] NCCL INFO Channel 07/0 : 14[6] -> 6[6] [send] via NET/IB/3/GDRDMA
+dlc1abaccnl2nzws-worker-0:77:317 [5] NCCL INFO Channel 06/0 : 13[5] -> 12[4] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:75:316 [3] NCCL INFO Channel 01/0 : 11[3] -> 10[2] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:76:315 [4] NCCL INFO Channel 01/0 : 12[4] -> 11[3] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:79:321 [7] NCCL INFO Channel 01/0 : 15[7] -> 8[0] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:72:326 [0] NCCL INFO Channel 00/0 : 0[0] -> 8[0] [receive] via NET/IB/0/GDRDMA
+dlc1abaccnl2nzws-worker-0:72:326 [0] NCCL INFO Channel 04/0 : 0[0] -> 8[0] [receive] via NET/IB/0/GDRDMA
+dlc1abaccnl2nzws-worker-0:72:326 [0] NCCL INFO Channel 00/0 : 8[0] -> 0[0] [send] via NET/IB/0/GDRDMA
+dlc1abaccnl2nzws-worker-0:72:326 [0] NCCL INFO Channel 04/0 : 8[0] -> 0[0] [send] via NET/IB/0/GDRDMA
+dlc1abaccnl2nzws-worker-0:75:316 [3] NCCL INFO Channel 05/0 : 11[3] -> 10[2] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:76:315 [4] NCCL INFO Channel 03/0 : 12[4] -> 11[3] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:79:321 [7] NCCL INFO Channel 02/0 : 15[7] -> 8[0] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:76:315 [4] NCCL INFO Channel 05/0 : 12[4] -> 11[3] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:79:321 [7] NCCL INFO Channel 03/0 : 15[7] -> 8[0] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:76:315 [4] NCCL INFO Channel 07/0 : 12[4] -> 11[3] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:79:321 [7] NCCL INFO Channel 05/0 : 15[7] -> 8[0] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:79:321 [7] NCCL INFO Channel 06/0 : 15[7] -> 8[0] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:79:321 [7] NCCL INFO Channel 07/0 : 15[7] -> 8[0] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:79:321 [7] NCCL INFO Channel 03/0 : 15[7] -> 14[6] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:79:321 [7] NCCL INFO Channel 07/0 : 15[7] -> 14[6] via P2P/IPC/read
+dlc1abaccnl2nzws-worker-0:77:317 [5] NCCL INFO Connected all trees
+dlc1abaccnl2nzws-worker-0:77:317 [5] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512
+dlc1abaccnl2nzws-worker-0:77:317 [5] NCCL INFO 8 coll channels, 0 nvls channels, 8 p2p channels, 2 p2p channels per peer
+dlc1abaccnl2nzws-worker-0:75:316 [3] NCCL INFO Connected all trees
+dlc1abaccnl2nzws-worker-0:75:316 [3] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512
+dlc1abaccnl2nzws-worker-0:75:316 [3] NCCL INFO 8 coll channels, 0 nvls channels, 8 p2p channels, 2 p2p channels per peer
+dlc1abaccnl2nzws-worker-0:73:324 [1] NCCL INFO Connected all trees
+dlc1abaccnl2nzws-worker-0:73:324 [1] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512
+dlc1abaccnl2nzws-worker-0:73:324 [1] NCCL INFO 8 coll channels, 0 nvls channels, 8 p2p channels, 2 p2p channels per peer
+dlc1abaccnl2nzws-worker-0:79:321 [7] NCCL INFO Connected all trees
+dlc1abaccnl2nzws-worker-0:79:321 [7] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512
+dlc1abaccnl2nzws-worker-0:79:321 [7] NCCL INFO 8 coll channels, 0 nvls channels, 8 p2p channels, 2 p2p channels per peer
+dlc1abaccnl2nzws-worker-0:76:315 [4] NCCL INFO Connected all trees
+dlc1abaccnl2nzws-worker-0:76:315 [4] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512
+dlc1abaccnl2nzws-worker-0:76:315 [4] NCCL INFO 8 coll channels, 0 nvls channels, 8 p2p channels, 2 p2p channels per peer
+dlc1abaccnl2nzws-worker-0:74:323 [2] NCCL INFO Connected all trees
+dlc1abaccnl2nzws-worker-0:74:323 [2] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512
+dlc1abaccnl2nzws-worker-0:74:323 [2] NCCL INFO 8 coll channels, 0 nvls channels, 8 p2p channels, 2 p2p channels per peer
+dlc1abaccnl2nzws-worker-0:78:314 [6] NCCL INFO Connected all trees
+dlc1abaccnl2nzws-worker-0:78:314 [6] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512
+dlc1abaccnl2nzws-worker-0:78:314 [6] NCCL INFO 8 coll channels, 0 nvls channels, 8 p2p channels, 2 p2p channels per peer
+dlc1abaccnl2nzws-worker-0:72:326 [0] NCCL INFO Connected all trees
+dlc1abaccnl2nzws-worker-0:72:326 [0] NCCL INFO threadThresholds 8/8/64 | 128/8/64 | 512 | 512
+dlc1abaccnl2nzws-worker-0:72:326 [0] NCCL INFO 8 coll channels, 0 nvls channels, 8 p2p channels, 2 p2p channels per peer
+dlc1abaccnl2nzws-worker-0:78:314 [6] NCCL INFO comm 0x9a052210 rank 14 nranks 16 cudaDev 6 nvmlDev 6 busId 70 commId 0x13a7e6351c9956c9 - Init COMPLETE
+dlc1abaccnl2nzws-worker-0:77:317 [5] NCCL INFO comm 0x99f6a330 rank 13 nranks 16 cudaDev 5 nvmlDev 5 busId 60 commId 0x13a7e6351c9956c9 - Init COMPLETE
+dlc1abaccnl2nzws-worker-0:79:321 [7] NCCL INFO comm 0x9afcfd70 rank 15 nranks 16 cudaDev 7 nvmlDev 7 busId 80 commId 0x13a7e6351c9956c9 - Init COMPLETE
+dlc1abaccnl2nzws-worker-0:73:324 [1] NCCL INFO comm 0x9a987d60 rank 9 nranks 16 cudaDev 1 nvmlDev 1 busId 20 commId 0x13a7e6351c9956c9 - Init COMPLETE
+dlc1abaccnl2nzws-worker-0:75:316 [3] NCCL INFO comm 0x9b8645b0 rank 11 nranks 16 cudaDev 3 nvmlDev 3 busId 40 commId 0x13a7e6351c9956c9 - Init COMPLETE
+dlc1abaccnl2nzws-worker-0:74:323 [2] NCCL INFO comm 0x9b2bc4b0 rank 10 nranks 16 cudaDev 2 nvmlDev 2 busId 30 commId 0x13a7e6351c9956c9 - Init COMPLETE
+dlc1abaccnl2nzws-worker-0:76:315 [4] NCCL INFO comm 0x9a914480 rank 12 nranks 16 cudaDev 4 nvmlDev 4 busId 50 commId 0x13a7e6351c9956c9 - Init COMPLETE
+dlc1abaccnl2nzws-worker-0:72:326 [0] NCCL INFO comm 0x9baa4e10 rank 8 nranks 16 cudaDev 0 nvmlDev 0 busId 10 commId 0x13a7e6351c9956c9 - Init COMPLETE
+
+Loading checkpoint shards:   0%|          | 0/8 [00:00<?, ?it/s]
+Loading checkpoint shards:   0%|          | 0/8 [00:00<?, ?it/s]
+Loading checkpoint shards:   0%|          | 0/8 [00:00<?, ?it/s]
+Loading checkpoint shards:   0%|          | 0/8 [00:00<?, ?it/s]
+Loading checkpoint shards:   0%|          | 0/8 [00:00<?, ?it/s]
+Loading checkpoint shards:   0%|          | 0/8 [00:00<?, ?it/s]
+Loading checkpoint shards:   0%|          | 0/8 [00:00<?, ?it/s]
+Loading checkpoint shards:   0%|          | 0/8 [00:00<?, ?it/s]
+Loading checkpoint shards:  12%|█▎        | 1/8 [00:00<00:01,  4.53it/s]
+Loading checkpoint shards:  12%|█▎        | 1/8 [00:00<00:01,  4.50it/s]
+Loading checkpoint shards:  12%|█▎        | 1/8 [00:00<00:01,  4.18it/s]
+Loading checkpoint shards:  12%|█▎        | 1/8 [00:00<00:01,  4.61it/s]
+Loading checkpoint shards:  12%|█▎        | 1/8 [00:00<00:01,  4.67it/s]
+Loading checkpoint shards:  12%|█▎        | 1/8 [00:00<00:01,  4.41it/s]
+Loading checkpoint shards:  12%|█▎        | 1/8 [00:00<00:01,  4.60it/s]
+Loading checkpoint shards:  12%|█▎        | 1/8 [00:00<00:01,  4.57it/s]
+Loading checkpoint shards:  25%|██▌       | 2/8 [00:00<00:01,  4.54it/s]
+Loading checkpoint shards:  25%|██▌       | 2/8 [00:00<00:01,  4.40it/s]
+Loading checkpoint shards:  25%|██▌       | 2/8 [00:00<00:01,  4.49it/s]
+Loading checkpoint shards:  25%|██▌       | 2/8 [00:00<00:01,  4.51it/s]
+Loading checkpoint shards:  25%|██▌       | 2/8 [00:00<00:01,  4.25it/s]
+Loading checkpoint shards:  25%|██▌       | 2/8 [00:00<00:01,  4.37it/s]
+Loading checkpoint shards:  25%|██▌       | 2/8 [00:00<00:01,  4.48it/s]
+Loading checkpoint shards:  25%|██▌       | 2/8 [00:00<00:01,  4.39it/s]
+Loading checkpoint shards:  38%|███▊      | 3/8 [00:01<00:04,  1.24it/s]
+Loading checkpoint shards:  38%|███▊      | 3/8 [00:01<00:04,  1.25it/s]
+Loading checkpoint shards:  38%|███▊      | 3/8 [00:01<00:04,  1.24it/s]
+Loading checkpoint shards:  38%|███▊      | 3/8 [00:01<00:04,  1.24it/s]
+Loading checkpoint shards:  38%|███▊      | 3/8 [00:01<00:04,  1.25it/s]
+Loading checkpoint shards:  38%|███▊      | 3/8 [00:01<00:04,  1.25it/s]
+Loading checkpoint shards:  38%|███▊      | 3/8 [00:01<00:04,  1.24it/s]
+Loading checkpoint shards:  38%|███▊      | 3/8 [00:01<00:04,  1.24it/s]
+Loading checkpoint shards:  50%|█████     | 4/8 [00:03<00:04,  1.06s/it]
+Loading checkpoint shards:  50%|█████     | 4/8 [00:03<00:04,  1.06s/it]
+Loading checkpoint shards:  50%|█████     | 4/8 [00:03<00:04,  1.06s/it]
+Loading checkpoint shards:  50%|█████     | 4/8 [00:03<00:04,  1.06s/it]
+Loading checkpoint shards:  50%|█████     | 4/8 [00:03<00:04,  1.07s/it]
+Loading checkpoint shards:  50%|█████     | 4/8 [00:03<00:04,  1.06s/it]
+Loading checkpoint shards:  50%|█████     | 4/8 [00:03<00:04,  1.06s/it]
+Loading checkpoint shards:  50%|█████     | 4/8 [00:03<00:04,  1.06s/it]
+Loading checkpoint shards:  62%|██████▎   | 5/8 [00:04<00:03,  1.12s/it]
+Loading checkpoint shards:  62%|██████▎   | 5/8 [00:04<00:03,  1.12s/it]
+Loading checkpoint shards:  62%|██████▎   | 5/8 [00:04<00:03,  1.12s/it]
+Loading checkpoint shards:  62%|██████▎   | 5/8 [00:04<00:03,  1.12s/it]
+Loading checkpoint shards:  62%|██████▎   | 5/8 [00:04<00:03,  1.12s/it]
+Loading checkpoint shards:  62%|██████▎   | 5/8 [00:04<00:03,  1.12s/it]
+Loading checkpoint shards:  62%|██████▎   | 5/8 [00:04<00:03,  1.12s/it]
+Loading checkpoint shards:  62%|██████▎   | 5/8 [00:04<00:03,  1.12s/it]
+Loading checkpoint shards:  75%|███████▌  | 6/8 [00:05<00:02,  1.18s/it]
+Loading checkpoint shards:  75%|███████▌  | 6/8 [00:05<00:02,  1.18s/it]
+Loading checkpoint shards:  75%|███████▌  | 6/8 [00:05<00:02,  1.18s/it]
+Loading checkpoint shards:  75%|███████▌  | 6/8 [00:05<00:02,  1.18s/it]
+Loading checkpoint shards:  75%|███████▌  | 6/8 [00:05<00:02,  1.18s/it]
+Loading checkpoint shards:  75%|███████▌  | 6/8 [00:05<00:02,  1.18s/it]
+Loading checkpoint shards:  75%|███████▌  | 6/8 [00:05<00:02,  1.18s/it]
+Loading checkpoint shards:  75%|███████▌  | 6/8 [00:05<00:02,  1.18s/it]
+Loading checkpoint shards:  88%|████████▊ | 7/8 [00:07<00:01,  1.20s/it]
+Loading checkpoint shards:  88%|████████▊ | 7/8 [00:07<00:01,  1.20s/it]
+Loading checkpoint shards:  88%|████████▊ | 7/8 [00:07<00:01,  1.20s/it]
+Loading checkpoint shards:  88%|████████▊ | 7/8 [00:07<00:01,  1.20s/it]
+Loading checkpoint shards:  88%|████████▊ | 7/8 [00:07<00:01,  1.20s/it]
+Loading checkpoint shards:  88%|████████▊ | 7/8 [00:07<00:01,  1.20s/it]
+Loading checkpoint shards:  88%|████████▊ | 7/8 [00:07<00:01,  1.20s/it]
+Loading checkpoint shards:  88%|████████▊ | 7/8 [00:07<00:01,  1.20s/it]/fs-computility/mllm1/shared/hub/
+Loading checkpoint shards: 100%|██████████| 8/8 [00:07<00:00,  1.02it/s]
+Loading checkpoint shards: 100%|██████████| 8/8 [00:07<00:00,  1.04it/s]
+[WARNING|modeling_utils.py:4352] 2025-01-16 22:24:26,316 >> Some weights of LlavaInternlm2ForCausalLM were not initialized from the model checkpoint at models/internlm/internlm2_5-7b-chat and are newly initialized: ['lm_head.weight']
+You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.
+Using tokenizer from models/internlm/internlm2_5-7b-chat
+using cache dir None
+
+Loading checkpoint shards: 100%|██████████| 8/8 [00:07<00:00,  1.02it/s]
+Loading checkpoint shards: 100%|██████████| 8/8 [00:07<00:00,  1.04it/s]
+[WARNING|modeling_utils.py:4352] 2025-01-16 22:24:26,325 >> Some weights of LlavaInternlm2ForCausalLM were not initialized from the model checkpoint at models/internlm/internlm2_5-7b-chat and are newly initialized: ['lm_head.weight']
+You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.
+
+Loading checkpoint shards: 100%|██████████| 8/8 [00:07<00:00,  1.02it/s]
+Loading checkpoint shards: 100%|██████████| 8/8 [00:07<00:00,  1.04it/s]
+[WARNING|modeling_utils.py:4352] 2025-01-16 22:24:26,326 >> Some weights of LlavaInternlm2ForCausalLM were not initialized from the model checkpoint at models/internlm/internlm2_5-7b-chat and are newly initialized: ['lm_head.weight']
+You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.
+
+Loading checkpoint shards: 100%|██████████| 8/8 [00:07<00:00,  1.02it/s]
+Loading checkpoint shards: 100%|██████████| 8/8 [00:07<00:00,  1.04it/s]
+[WARNING|modeling_utils.py:4352] 2025-01-16 22:24:26,327 >> Some weights of LlavaInternlm2ForCausalLM were not initialized from the model checkpoint at models/internlm/internlm2_5-7b-chat and are newly initialized: ['lm_head.weight']
+You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.
+
+Loading checkpoint shards: 100%|██████████| 8/8 [00:07<00:00,  1.02it/s]
+Loading checkpoint shards: 100%|██████████| 8/8 [00:07<00:00,  1.04it/s]
+[WARNING|modeling_utils.py:4352] 2025-01-16 22:24:26,328 >> Some weights of LlavaInternlm2ForCausalLM were not initialized from the model checkpoint at models/internlm/internlm2_5-7b-chat and are newly initialized: ['lm_head.weight']
+You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.
+
+Loading checkpoint shards: 100%|██████████| 8/8 [00:07<00:00,  1.02it/s]
+Loading checkpoint shards: 100%|██████████| 8/8 [00:07<00:00,  1.04it/s]
+[INFO|modeling_utils.py:4350] 2025-01-16 22:24:26,328 >> All model checkpoint weights were used when initializing LlavaInternlm2ForCausalLM.
+
+[WARNING|modeling_utils.py:4352] 2025-01-16 22:24:26,328 >> Some weights of LlavaInternl/fs-computility/mllm1/shared/hub/ the model checkpoint at models/internlm/internlm2_5-7b-chat and are newly initialized: ['lm_head.weight']
+You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.
+
+Loading checkpoint shards: 100%|██████████| 8/8 [00:07<00:00,  1.02it/s]
+Loading checkpoint shards: 100%|██████████| 8/8 [00:07<00:00,  1.04it/s]
+[WARNING|modeling_utils.py:4352] 2025-01-16 22:24:26,329 >> Some weights of LlavaInternlm2ForCausalLM were not initialized from the model checkpoint at models/internlm/internlm2_5-7b-chat and are newly initialized: ['lm_head.weight']
+You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.
+Using tokenizer from models/internlm/internlm2_5-7b-chat
+using cache dir None
+
+Loading checkpoint shards: 100%|██████████| 8/8 [00:07<00:00,  1.02it/s]
+Loading checkpoint shards: 100%|██████████| 8/8 [00:07<00:00,  1.04it/s]
+Using tokenizer from models/internlm/internlm2_5-7b-chat
+using cache dir None
+[WARNING|modeling_utils.py:4352] 2025-01-16 22:24:26,330 >> Some weights of LlavaInternlm2ForCausalLM were not initialized from the model checkpoint at models/internlm/internlm2_5-7b-chat and are newly initialized: ['lm_head.weight']
+You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.
+Using tokenizer from models/internlm/internlm2_5-7b-chat
+using cache dir None
+Using tokenizer from models/internlm/internlm2_5-7b-chat
+using cache dir None
+[INFO|configuration_utils.py:779] 2025-01-16 22:24:26,332 >> loading configura/fs-computility/mllm1/shared/hub/b-chat/generation_config.json
+[INFO|configuration_utils.py:826] 2025-01-16 22:24:26,332 >> Generate config GenerationConfig {
+  "bos_token_id": 1,/fs-computility/mllm1/shared/hub/
+  "eos_token_id": [
+    2,
+    92542/fs-computility/mllm1/shared/hub/
+  ],
+  "pad_token_id": 2
+}
+
+Using tokenizer from models/internlm/internlm2_5-7b-chat
+using cache dir None
+Using tokenizer from models/internlm/internlm2_5-7b-chat
+using cache dir None
+Using tokenizer from models/internlm/internlm2_5-7b-chat
+using cache dir None
+[INFO|tokenization_utils_base.py:2025] 2025-01-16 22:24:26,343 >> loading file ./tokenizer.model
+[INFO|tokenization_utils_base.py:2025] 2025-01-16 22:24:26,343 >> loading file added_tokens.json
+[INFO|tokenization_utils_base.py:2025] 2025-01-16 22:24:26,344 >> loading file special_tokens_map.json
+[INFO|tokenization_utils_base.py:2025] 2025-01-16 22:24:26,344 >> loading file tokenizer_config.json
+[INFO|tokenization_utils_base.py:2025] 2025-01-16 22:24:26,344 >> loading file tokenizer.json
+01/16/2025 22:24:26 - WARNING - llava.train.train - Using conversation template: Conversation(system='<|im_start|>system\nYou are a helpful assistant. ', roles=('<|im_start|>user\n', '<|im_start|>assistant\n'), messages=[], offset=0, sep_style=<SeparatorStyle.MPT: 3>, sep='<|im_end|>', sep2=None, version='internlm_v2', mm_system=None, skip_next=False)
+01/16/2025 22:24:26 - WARNING - llava.train.train - Using conversation template: Conversation(system='<|im_start|>system\nYou are a helpful assistant. ', roles=('<|im_start|>user\n', '<|im_start|>assistant\n'), messages=[], offset=0, sep_style=<SeparatorStyle.MPT: 3>, sep='<|im_end|>', sep2=None, version='internlm_v2', mm_system=None, skip_next=False)
+01/16/2025 22:24:26 - WARNING - llava.train.train - Using conversation template: Conversation(system='<|im_start|>system\nYou are a helpful assistant. ', roles=('<|im_start|>user\n', '<|im_start|>assistant\n'), messages=[], offset=0, sep_style=<SeparatorStyle.MPT: 3>, sep='<|im_end|>', sep2=None, version='internlm_v2', mm_system=None, skip_next=False)
+01/16/2025 22:24:26 - WARNING - llava.train.train - Using conversation template: Conversation(system='<|im_start|>system\nYou are a helpful assistant. ', roles=('<|im_start|>user\n', '<|im_start|>assistant\n'), messages=[], offset=0, sep_style=<SeparatorStyle.MPT: 3>, sep='<|im_end|>', sep2=None, version='internlm_v2', mm_system=None, skip_next=False)
+01/16/2025 22:24:26 - WARNING - llava.train.train - Using conversation template: Conversation(system='<|im_start|>system\nYou are a helpful assistant. ', roles=('<|im_start|>user\n', '<|im_start|>assistant\n'), messages=[], offset=0, sep_style=<SeparatorStyle.MPT: 3>, sep='<|im_end|>', sep2=None, version='internlm_v2', mm_system=None, skip_next=False)
+01/16/2025 22:24:26 - WARNING - llava.train.train - Using conversation template: Conversation(system='<|im_start|>system\nYou are a helpful assistant. ', roles=('<|im_start|>user\n', '<|im_start|>assistant\n'), messages=[], offset=0, sep_style=<SeparatorStyle.MPT: 3>, sep='<|im_end|>', sep2=None, version='internlm_v2', mm_system=None, skip_next=False)
+01/16/2025 22:24:26 - INFO - llava.train.train - Using conversation template: Conversation(system='<|im_start|>system\nYou are a helpful assistant. ', roles=('<|im_start|>user\n', '<|im_start|>assistant\n'), messages=[], offset=0, sep_style=<SeparatorStyle.MPT: 3>, sep='<|im_end|>', sep2=None, version='internlm_v2', mm_system=None, skip_next=False)
+[INFO|image_processing_utils.py:373] 2025-01-16 22:24:26,548 >> loading configuration file /fs-computility/mllm1/shared/hub/models--openai--clip-vit-large-patch14-336/snapshots/ce19dc912ca5cd21c8a653c79e251e808ccabcd1/preprocessor_config.json
+[INFO|image_processing_utils.py:738] 2025-01-16 22:24:26,548 >> size should be a dictionary on of the following set of keys: ({'width', 'height'}, {'shortest_edge'}, {'longest_edge', 'shortest_edge'}, {'longest_edge'}), got 336. Converted to {'shortest_edge': 336}.
+[INFO|image_processing_utils.py:738] 2025-01-16 22:24:26,548 >> crop_size should be a dictionary on of the following set of keys: ({'width', 'height'}, {'shortest_edge'}, {'longest_edge', 'shortest_edge'}, {'longest_edge'}), got 336. Converted to {'height': 336, 'width': 336}.
+[INFO|image_processing_utils.py:425] 2025-01-16 22:24:26,548 >> Image processor CLIPImageProcessor {
+  "crop_size": {
+    "height": 336,
+    "width": 336
+  },
+  "do_center_crop": true,
+  "do_convert_rgb": true,
+  "do_normalize": true,
+  "do_rescale": true,
+  "do_resize": true,
+  "image_mean": [
+    0.48145466,
+    0.4578275,
+    0.40821073
+  ],
+  "image_processor_type": "CLIPImageProcessor",
+  "image_std": [
+    0.26862954,
+    0.26130258,
+    0.27577711
+  ],
+  "resample": 3,
+  "rescale_factor": 0.00392156862745098,
+  "size": {
+    "shortest_edge": 336
+  }
+}
+
+01/16/2025 22:24:26 - WARNING - llava.train.train - Using conversation template: Conversation(system='<|im_start|>system\nYou are a helpful assistant. ', roles=('<|im_start|>user\n', '<|im_start|>assistant\n'), messages=[], offset=0, sep_style=<SeparatorStyle.MPT: 3>, sep='<|im_end|>', sep2=None, version='internlm_v2', mm_system=None, skip_next=False)
+[INFO|configuration_utils.py:727] 2025-01-16 22:24:26,554 >> loading configuration file /fs-computility/mllm1/shared/hub/models--openai--clip-vit-large-patch14-336/snapshots/ce19dc912ca5cd21c8a653c79e251e808ccabcd1/config.json
+[INFO|configuration_utils.py:792] 2025-01-16 22:24:26,555 >> Model config CLIPVisionConfig {
+  "attention_dropout": 0.0,
+  "dropout": 0.0,
+  "hidden_act": "quick_gelu",
+  "hidden_size": 1024,
+  "image_size": 336,
+  "initializer_factor": 1.0,
+  "initializer_range": 0.02,
+  "intermediate_size": 4096,
+  "layer_norm_eps": 1e-05,
+  "model_type": "clip_vision_model",
+  "num_attention_heads": 16,
+  "num_channels": 3,
+  "num_hidden_layers": 24,
+  "patch_size": 14,
+  "projection_dim": 768,
+  "transformers_version": "4.37.2"
+}
+
+[INFO|modeling_utils.py:3473] 2025-01-16 22:24:26,556 >> loading weights file /fs-computility/mllm1/shared/hub/models--openai--clip-vit-large-patch14-336/snapshots/ce19dc912ca5cd21c8a653c79e251e808ccabcd1/pytorch_model.bin
+[INFO|modeling_utils.py:3582] 2025-01-16 22:24:29,943 >> Detected DeepSpeed ZeRO-3: activating zero.init() for this model
+[INFO|modeling_utils.py:4340] 2025-01-16 22:24:31,911 >> Some weights of the model checkpoint at /fs-computility/mllm1/shared/hub/models--openai--clip-vit-large-patch14-336/snapshots/ce19dc912ca5cd21c8a653c79e251e808ccabcd1 were not used when initializing CLIPVisionModel: ['logit_scale', 'text_model.embeddings.position_embedding.weight', 'text_model.embeddings.position_ids', 'text_model.embeddings.token_embedding.weight', 'text_model.encoder.layers.0.layer_norm1.bias', 'text_model.encoder.layers.0.layer_norm1.weight', 'text_model.encoder.layers.0.layer_norm2.bias', 'text_model.encoder.layers.0.layer_norm2.weight', 'text_model.encoder.layers.0.mlp.fc1.bias', 'text_model.encoder.layers.0.mlp.fc1.weight', 'text_model.encoder.layers.0.mlp.fc2.bias', 'text_model.encoder.layers.0.mlp.fc2.weight', 'text_model.encoder.layers.0.self_attn.k_proj.bias', 'text_model.encoder.layers.0.self_attn.k_proj.weight', 'text_model.encoder.layers.0.self_attn.out_proj.bias', 'text_model.encoder.layers.0.self_attn.out_proj.weight', 'text_model.encoder.layers.0.self_attn.q_proj.bias', 'text_model.encoder.layers.0.self_attn.q_proj.weight', 'text_model.encoder.layers.0.self_attn.v_proj.bias', 'text_model.encoder.layers.0.self_attn.v_proj.weight', 'text_model.encoder.layers.1.layer_norm1.bias', 'text_model.encoder.layers.1.layer_norm1.weight', 'text_model.encoder.layers.1.layer_norm2.bias', 'text_model.encoder.layers.1.layer_norm2.weight', 'text_model.encoder.layers.1.mlp.fc1.bias', 'text_model.encoder.layers.1.mlp.fc1.weight', 'text_model.encoder.layers.1.mlp.fc2.bias', 'text_model.encoder.layers.1.mlp.fc2.weight', 'text_model.encoder.layers.1.self_attn.k_proj.bias', 'text_model.encoder.layers.1.self_attn.k_proj.weight', 'text_model.encoder.layers.1.self_attn.out_proj.bias', 'text_model.encoder.layers.1.self_attn.out_proj.weight', 'text_model.encoder.layers.1.self_attn.q_proj.bias', 'text_model.encoder.layers.1.self_attn.q_proj.weight', 'text_model.encoder.layers.1.self_attn.v_proj.bias', 'text_model.encoder.layers.1.self_attn.v_proj.weight', 'text_model.encoder.layers.10.layer_norm1.bias', 'text_model.encoder.layers.10.layer_norm1.weight', 'text_model.encoder.layers.10.layer_norm2.bias', 'text_model.encoder.layers.10.layer_norm2.weight', 'text_model.encoder.layers.10.mlp.fc1.bias', 'text_model.encoder.layers.10.mlp.fc1.weight', 'text_model.encoder.layers.10.mlp.fc2.bias', 'text_model.encoder.layers.10.mlp.fc2.weight', 'text_model.encoder.layers.10.self_attn.k_proj.bias', 'text_model.encoder.layers.10.self_attn.k_proj.weight', 'text_model.encoder.layers.10.self_attn.out_proj.bias', 'text_model.encoder.layers.10.self_attn.out_proj.weight', 'text_model.encoder.layers.10.self_attn.q_proj.bias', 'text_model.encoder.layers.10.self_attn.q_proj.weight', 'text_model.encoder.layers.10.self_attn.v_proj.bias', 'text_model.encoder.layers.10.self_attn.v_proj.weight', 'text_model.encoder.layers.11.layer_norm1.bias', 'text_model.encoder.layers.11.layer_norm1.weight', 'text_model.encoder.layers.11.layer_norm2.bias', 'text_model.encoder.layers.11.layer_norm2.weight', 'text_model.encoder.layers.11.mlp.fc1.bias', 'text_model.encoder.layers.11.mlp.fc1.weight', 'text_model.encoder.layers.11.mlp.fc2.bias', 'text_model.encoder.layers.11.mlp.fc2.weight', 'text_model.encoder.layers.11.self_attn.k_proj.bias', 'text_model.encoder.layers.11.self_attn.k_proj.weight', 'text_model.encoder.layers.11.self_attn.out_proj.bias', 'text_model.encoder.layers.11.self_attn.out_proj.weight', 'text_model.encoder.layers.11.self_attn.q_proj.bias', 'text_model.encoder.layers.11.self_attn.q_proj.weight', 'text_model.encoder.layers.11.self_attn.v_proj.bias', 'text_model.encoder.layers.11.self_attn.v_proj.weight', 'text_model.encoder.layers.2.layer_norm1.bias', 'text_model.encoder.layers.2.layer_norm1.weight', 'text_model.encoder.layers.2.layer_norm2.bias', 'text_model.encoder.layers.2.layer_norm2.weight', 'text_model.encoder.layers.2.mlp.fc1.bias', 'text_model.encoder.layers.2.mlp.fc1.weight', 'text_model.encoder.layers.2.mlp.fc2.bias', 'text_model.encoder.layers.2.mlp.fc2.weight', 'text_model.encoder.layers.2.self_attn.k_proj.bias', 'text_model.encoder.layers.2.self_attn.k_proj.weight', 'text_model.encoder.layers.2.self_attn.out_proj.bias', 'text_model.encoder.layers.2.self_attn.out_proj.weight', 'text_model.encoder.layers.2.self_attn.q_proj.bias', 'text_model.encoder.layers.2.self_attn.q_proj.weight', 'text_model.encoder.layers.2.self_attn.v_proj.bias', 'text_model.encoder.layers.2.self_attn.v_proj.weight', 'text_model.encoder.layers.3.layer_norm1.bias', 'text_model.encoder.layers.3.layer_norm1.weight', 'text_model.encoder.layers.3.layer_norm2.bias', 'text_model.encoder.layers.3.layer_norm2.weight', 'text_model.encoder.layers.3.mlp.fc1.bias', 'text_model.encoder.layers.3.mlp.fc1.weight', 'text_model.encoder.layers.3.mlp.fc2.bias', 'text_model.encoder.layers.3.mlp.fc2.weight', 'text_model.encoder.layers.3.self_attn.k_proj.bias', 'text_model.encoder.layers.3.self_attn.k_proj.weight', 'text_model.encoder.layers.3.self_attn.out_proj.bias', 'text_model.encoder.layers.3.self_attn.out_proj.weight', 'text_model.encoder.layers.3.self_attn.q_proj.bias', 'text_model.encoder.layers.3.self_attn.q_proj.weight', 'text_model.encoder.layers.3.self_attn.v_proj.bias', 'text_model.encoder.layers.3.self_attn.v_proj.weight', 'text_model.encoder.layers.4.layer_norm1.bias', 'text_model.encoder.layers.4.layer_norm1.weight', 'text_model.encoder.layers.4.layer_norm2.bias', 'text_model.encoder.layers.4.layer_norm2.weight', 'text_model.encoder.layers.4.mlp.fc1.bias', 'text_model.encoder.layers.4.mlp.fc1.weight', 'text_model.encoder.layers.4.mlp.fc2.bias', 'text_model.encoder.layers.4.mlp.fc2.weight', 'text_model.encoder.layers.4.self_attn.k_proj.bias', 'text_model.encoder.layers.4.self_attn.k_proj.weight', 'text_model.encoder.layers.4.self_attn.out_proj.bias', 'text_model.encoder.layers.4.self_attn.out_proj.weight', 'text_model.encoder.layers.4.self_attn.q_proj.bias', 'text_model.encoder.layers.4.self_attn.q_proj.weight', 'text_model.encoder.layers.4.self_attn.v_proj.bias', 'text_model.encoder.layers.4.self_attn.v_proj.weight', 'text_model.encoder.layers.5.layer_norm1.bias', 'text_model.encoder.layers.5.layer_norm1.weight', 'text_model.encoder.layers.5.layer_norm2.bias', 'text_model.encoder.layers.5.layer_norm2.weight', 'text_model.encoder.layers.5.mlp.fc1.bias', 'text_model.encoder.layers.5.mlp.fc1.weight', 'text_model.encoder.layers.5.mlp.fc2.bias', 'text_model.encoder.layers.5.mlp.fc2.weight', 'text_model.encoder.layers.5.self_attn.k_proj.bias', 'text_model.encoder.layers.5.self_attn.k_proj.weight', 'text_model.encoder.layers.5.self_attn.out_proj.bias', 'text_model.encoder.layers.5.self_attn.out_proj.weight', 'text_model.encoder.layers.5.self_attn.q_proj.bias', 'text_model.encoder.layers.5.self_attn.q_proj.weight', 'text_model.encoder.layers.5.self_attn.v_proj.bias', 'text_model.encoder.layers.5.self_attn.v_proj.weight', 'text_model.encoder.layers.6.layer_norm1.bias', 'text_model.encoder.layers.6.layer_norm1.weight', 'text_model.encoder.layers.6.layer_norm2.bias', 'text_model.encoder.layers.6.layer_norm2.weight', 'text_model.encoder.layers.6.mlp.fc1.bias', 'text_model.encoder.layers.6.mlp.fc1.weight', 'text_model.encoder.layers.6.mlp.fc2.bias', 'text_model.encoder.layers.6.mlp.fc2.weight', 'text_model.encoder.layers.6.self_attn.k_proj.bias', 'text_model.encoder.layers.6.self_attn.k_proj.weight', 'text_model.encoder.layers.6.self_attn.out_proj.bias', 'text_model.encoder.layers.6.self_attn.out_proj.weight', 'text_model.encoder.layers.6.self_attn.q_proj.bias', 'text_model.encoder.layers.6.self_attn.q_proj.weight', 'text_model.encoder.layers.6.self_attn.v_proj.bias', 'text_model.encoder.layers.6.self_attn.v_proj.weight', 'text_model.encoder.layers.7.layer_norm1.bias', 'text_model.encoder.layers.7.layer_norm1.weight', 'text_model.encoder.layers.7.layer_norm2.bias', 'text_model.encoder.layers.7.layer_norm2.weight', 'text_model.encoder.layers.7.mlp.fc1.bias', 'text_model.encoder.layers.7.mlp.fc1.weight', 'text_model.encoder.layers.7.mlp.fc2.bias', 'text_model.encoder.layers.7.mlp.fc2.weight', 'text_model.encoder.layers.7.self_attn.k_proj.bias', 'text_model.encoder.layers.7.self_attn.k_proj.weight', 'text_model.encoder.layers.7.self_attn.out_proj.bias', 'text_model.encoder.layers.7.self_attn.out_proj.weight', 'text_model.encoder.layers.7.self_attn.q_proj.bias', 'text_model.encoder.layers.7.self_attn.q_proj.weight', 'text_model.encoder.layers.7.self_attn.v_proj.bias', 'text_model.encoder.layers.7.self_attn.v_proj.weight', 'text_model.encoder.layers.8.layer_norm1.bias', 'text_model.encoder.layers.8.layer_norm1.weight', 'text_model.encoder.layers.8.layer_norm2.bias', 'text_model.encoder.layers.8.layer_norm2.weight', 'text_model.encoder.layers.8.mlp.fc1.bias', 'text_model.encoder.layers.8.mlp.fc1.weight', 'text_model.encoder.layers.8.mlp.fc2.bias', 'text_model.encoder.layers.8.mlp.fc2.weight', 'text_model.encoder.layers.8.self_attn.k_proj.bias', 'text_model.encoder.layers.8.self_attn.k_proj.weight', 'text_model.encoder.layers.8.self_attn.out_proj.bias', 'text_model.encoder.layers.8.self_attn.out_proj.weight', 'text_model.encoder.layers.8.self_attn.q_proj.bias', 'text_model.encoder.layers.8.self_attn.q_proj.weight', 'text_model.encoder.layers.8.self_attn.v_proj.bias', 'text_model.encoder.layers.8.self_attn.v_proj.weight', 'text_model.encoder.layers.9.layer_norm1.bias', 'text_model.encoder.layers.9.layer_norm1.weight', 'text_model.encoder.layers.9.layer_norm2.bias', 'text_model.encoder.layers.9.layer_norm2.weight', 'text_model.encoder.layers.9.mlp.fc1.bias', 'text_model.encoder.layers.9.mlp.fc1.weight', 'text_model.encoder.layers.9.mlp.fc2.bias', 'text_model.encoder.layers.9.mlp.fc2.weight', 'text_model.encoder.layers.9.self_attn.k_proj.bias', 'text_model.encoder.layers.9.self_attn.k_proj.weight', 'text_model.encoder.layers.9.self_attn.out_proj.bias', 'text_model.encoder.layers.9.self_attn.out_proj.weight', 'text_model.encoder.layers.9.self_attn.q_proj.bias', 'text_model.encoder.layers.9.self_attn.q_proj.weight', 'text_model.encoder.layers.9.self_attn.v_proj.bias', 'text_model.encoder.layers.9.self_attn.v_proj.weight', 'text_model.final_layer_norm.bias', 'text_model.final_layer_norm.weight', 'text_projection.weight', 'visual_projection.weight']
+- This IS expected if you are initializing CLIPVisionModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).
+- This IS NOT expected if you are initializing CLIPVisionModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).
+[INFO|modeling_utils.py:4358] 2025-01-16 22:24:31,912 >> All the weights of CLIPVisionModel were initialized from the model checkpoint at /fs-computility/mllm1/shared/hub/models--openai--clip-vit-large-patch14-336/snapshots/ce19dc912ca5cd21c8a653c79e251e808ccabcd1.
+If your task is similar to the task the model of the checkpoint was trained on, you can already use CLIPVisionModel for predictions without further training.
+01/16/2025 22:24:50 - INFO - llava.train.train - Add dataset: llava-next-sft-notext with length: 738601, data type: normal, seed: 0
+01/16/2025 22:24:53 - INFO - llava.train.train - Add dataset: knowledge_gqa9k_art1500_cc3m30k with length: 40813, data type: know, seed: 1
+01/16/2025 22:24:56 - INFO - llava.train.train - Add dataset: Inferencial_flickr7k_cc3m30k_polished_md with length: 37117, data type: inf_polishmd, seed: 2
+01/16/2025 22:24:59 - INFO - llava.train.train - Add dataset: Detail_flickr7k_cc3m28k with length: 35313, data type: detail, seed: 3
+01/16/2025 22:25:03 - INFO - llava.train.train - Add dataset: Knowledge_instruct40k with length: 40218, data type: know_ins, seed: 4
+01/16/2025 22:25:06 - INFO - llava.train.train - Add dataset: Creation10k_fixed with length: 9698, data type: creation, seed: 5
+01/16/2025 22:25:10 - INFO - llava.train.train - Add dataset: Chartqa_generate_11k_gpt_qwen_merge with length: 11160, data type: chart, seed: 6
+01/16/2025 22:25:13 - INFO - llava.train.train - Add dataset: Tqa_detail_qwengenerate_multi8k_gpt with length: 8391, data type: tqa, seed: 7
+01/16/2025 22:25:17 - INFO - llava.train.train - Add dataset: Infovqa_single_gpt with length: 23068, data type: info, seed: 8
+[INFO|trainer.py:571] 2025-01-16 22:25:17,060 >> Using auto half precision backend
+[INFO|trainer.py:1721] 2025-01-16 22:26:02,469 >> ***** Running training *****
+[INFO|trainer.py:1722] 2025-01-16 22:26:02,469 >>   Num examples = 944,379
+[INFO|trainer.py:1723] 2025-01-16 22:26:02,469 >>   Num Epochs = 1
+[INFO|trainer.py:1724] 2025-01-16 22:26:02,469 >>   Instantaneous batch size per device = 4
+[INFO|trainer.py:1727] 2025-01-16 22:26:02,469 >>   Total train batch size (w. parallel, distributed & accumulation) = 128
+[INFO|trainer.py:1728] 2025-01-16 22:26:02,469 >>   Gradient Accumulation steps = 2
+[INFO|trainer.py:1729] 2025-01-16 22:26:02,469 >>   Total optimization steps = 7,378
+[INFO|trainer.py:1730] 2025-01-16 22:26:02,471 >>   Number of trainable parameters = 8,441,260,032