Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

config.json

@@ -0,0 +1,76 @@
+{
+  "architectures": [
+    "ForwardVisualTokensArchForCausalLM"
+  ],
+  "auto_map": {
+    "AutoConfig": "modeling_forward_visual_tokens_llava_arch.ForwardVisualTokensArchConfig",
+    "AutoModel": "modeling_forward_visual_tokens_llava_arch.ForwardVisualTokensArchModel",
+    "AutoModelForCausalLM": "modeling_forward_visual_tokens_llava_arch.ForwardVisualTokensArchForCausalLM"
+  },
+  "attention_dropout": 0.0,
+  "dtype": "float32",
+  "eos_token_id": [
+    151645,
+    151643
+  ],
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "image_token_id": 151655,
+  "initializer_range": 0.02,
+  "intermediate_size": 22016,
+  "layer_types": [
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 32768,
+  "max_window_layers": 28,
+  "model_type": "forward_visual_tokens_llava_arch",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 32,
+  "p_processor_name_or_path": "Qwen/Qwen3-4B",
+  "pad_token_id": 151643,
+  "perceiver_name_or_path": "team6013/DPA-LLaVA-0.6B",
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 10000.0,
+  "sliding_window": null,
+  "t_tokenizer_name_or_path": "Qwen/Qwen2.5-VL-3B-Instruct",
+  "thinker_name_or_path": "Qwen/Qwen3-4B",
+  "tie_word_embeddings": false,
+  "transformers_version": "4.57.1",
+  "use_cache": true,
+  "use_sliding_window": false,
+  "visual_bandwidth": 1,
+  "vocab_size": 151936
+}

generation_config.json

@@ -0,0 +1,4 @@
+{
+  "_from_model_config": true,
+  "transformers_version": "4.57.1"
+}

modeling_forward_visual_tokens_llava_arch.py

@@ -0,0 +1,655 @@
+import torch
+from PIL import Image
+from typing import Optional, Union
+import json
+import os
+from datetime import datetime
+from transformers import (
+    Qwen2_5_VLForConditionalGeneration,
+    AutoTokenizer,
+    AutoProcessor,
+    Qwen3ForCausalLM,
+    Qwen3Config
+)
+from transformers import Qwen2PreTrainedModel
+from transformers.generation import GenerationMixin
+from transformers.processing_utils import Unpack
+from transformers.utils import is_torchdynamo_compiling, ModelOutput
+from transformers.models.qwen2_5_vl.modeling_qwen2_5_vl import (
+    Qwen2_5_VLModelOutputWithPast,
+)
+from .modeling_llava_baseline import LLaVABaselineModelForConditionalGeneration, LLaVABaselineConfig
+# Compatibility fix: KwargsForCausalLM doesn't exist in newer transformers versions
+# try:
+#     from transformers.models.qwen2_5_vl.modeling_qwen2_5_vl import KwargsForCausalLM
+# except ImportError:
+#     from transformers.models.qwen2_5_vl.modeling_qwen2_5_vl import TransformersKwargs as KwargsForCausalLM
+from transformers.modeling_outputs import CausalLMOutputWithPast
+
+from dataclasses import dataclass
+
+from transformers.utils import auto_docstring
+from transformers import Qwen2Config
+
+IMG_START_ID = 151652
+IMG_PAD_ID = 151655
+IMG_END_ID = 151653
+
+IMG_THINKER_PAD_ID = 151655
+IMG_THINKER_START_ID = 151652
+IMG_THINKER_END_ID = 151653
+
+
+
+class ForwardVisualTokensArchConfig(Qwen2Config):
+    model_type = "forward_visual_tokens_llava_arch"
+    keys_to_ignore_at_inference = ["past_key_values"]
+
+    def __init__(
+        self,
+        use_cache=True,
+        perceiver_name_or_path="../LLaVA-baseline-checkpoint-6000",
+        thinker_name_or_path="Qwen/Qwen3-4B",
+        t_tokenizer_name_or_path="../dpa_qwen3_tokenizer",
+        p_tokenizer_name_or_path="../dpa_qwen25_processor",
+        visual_bandwidth=1,
+        **kwargs,
+    ):
+        self.use_cache = use_cache
+        self.perceiver_name_or_path = perceiver_name_or_path
+        self.thinker_name_or_path = thinker_name_or_path
+        self.t_tokenizer_name_or_path = t_tokenizer_name_or_path
+        self.p_processor_name_or_path = p_tokenizer_name_or_path
+
+        self.visual_bandwidth = visual_bandwidth
+        self.image_token_id = IMG_PAD_ID
+
+        super().__init__(**kwargs)
+
+
+class ForwardVisualTokensArchPreTrainedModel(Qwen2PreTrainedModel):
+    config_class = ForwardVisualTokensArchConfig
+
+
+def add_special_tokens(tkz):
+    additional_special_tokens = [f"<im_msg-{i}>" for i in range(128)]
+    tkz.add_special_tokens({"additional_special_tokens": additional_special_tokens})
+    mapping = {
+        tok: tkz._convert_token_to_id_with_added_voc(tok)
+        for tok in additional_special_tokens
+    }
+    return tkz, mapping
+
+
+@dataclass
+@auto_docstring(
+    custom_intro="""
+    Base class for Llava outputs, with hidden states and attentions.
+    """
+)
+class ForwardVisualTokensArchOutputWithPast(ModelOutput):
+    r"""
+    past_key_values (`tuple(tuple(torch.FloatTensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
+        Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape
+        `(batch_size, num_heads, sequence_length, embed_size_per_head)`)
+
+        Contains pre-computed hidden-states (key and values in the self-attention blocks) that can be used (see
+        `past_key_values` input) to speed up sequential decoding.
+    rope_deltas (`torch.LongTensor` of shape `(batch_size, )`, *optional*):
+        The rope index difference between sequence length and multimodal rope.
+    """
+
+    past_key_values: Optional[list[torch.FloatTensor]] = None
+    hidden_states: Optional[tuple[torch.FloatTensor]] = None
+    attentions: Optional[tuple[torch.FloatTensor]] = None
+    logits: Optional[tuple[torch.FloatTensor]] = None
+
+
+class ForwardVisualTokensArchModel(ForwardVisualTokensArchPreTrainedModel, GenerationMixin):
+    def __init__(self, config: ForwardVisualTokensArchConfig):
+        super().__init__(config)
+
+        assert self.config.perceiver_name_or_path is not None
+        assert self.config.thinker_name_or_path is not None
+
+        assert self.config.p_processor_name_or_path is not None
+        assert self.config.t_tokenizer_name_or_path is not None
+
+        perceiver_config = LLaVABaselineConfig.from_pretrained(
+            self.config.perceiver_name_or_path,
+        )
+        self.perceiver = LLaVABaselineModelForConditionalGeneration(perceiver_config)
+        # self.perceiver.gradient_checkpointing_enable()
+
+        self.p_processor = AutoProcessor.from_pretrained(
+            self.config.p_processor_name_or_path
+        )
+        self.p_processor.tokenizer.padding_side = "left"
+
+        thinker_config = Qwen3Config.from_pretrained(self.config.thinker_name_or_path)
+        self.thinker = Qwen3ForCausalLM(thinker_config)
+        # self.thinker.gradient_checkpointing_enable()
+
+        self.t_tokenizer = AutoTokenizer.from_pretrained(
+            self.config.t_tokenizer_name_or_path, padding_side="left"
+        )
+
+        self.linear_align_dim = torch.nn.Sequential(
+            torch.nn.Linear(#因为是嫁接的模型，必须直接访问里面的language model的config才是真实维度
+                self.perceiver.model.vlm.language_model.config.hidden_size, self.perceiver.model.vlm.language_model.config.hidden_size
+            ),
+            torch.nn.ReLU(),
+            torch.nn.Linear(
+                self.perceiver.model.vlm.language_model.config.hidden_size, self.thinker.config.hidden_size
+            ),
+        )
+
+        self.config: ForwardVisualTokensArchConfig
+
+    def get_visual_message_tokens(self):
+        size = self.config.visual_bandwidth
+        tokens = [f"<im_msg-{i}>" for i in range(size)]
+        return tokens
+
+    def get_visual_message_token_ids(self, model):
+        tokens = self.get_visual_message_tokens()
+        if model == "p":
+            ids = self.p_processor.tokenizer.convert_tokens_to_ids(tokens)
+        elif model == "t":
+            ids = self.t_tokenizer.convert_tokens_to_ids(tokens)
+        else:
+            raise NotImplementedError
+        return ids
+
+    def get_visual_message(self):
+        message = "".join(self.get_visual_message_tokens())
+        return message
+
+    def chat(self, images, msgs, *args, **kwargs):
+        assert len(images) == len(msgs)
+        assert args == ()
+        assert "max_new_tokens" not in kwargs
+
+        # p_prompt_template = 'Encode the image into {num_feat} tokens, including information related to the question. Here is the question: {question}'
+        p_prompt_template = "{question}"
+        questions = []
+        p_images = []
+        p_texts = []
+
+        for i in range(len(images)):
+            image = images[i]
+            msg_list = msgs[i]
+
+            # print(f'Image-{i}: {image}')
+            # print(f'Msg-{i}: {msg_list}')
+
+            if not (len(msg_list) == 1 and msg_list[0]["role"] == "user"):
+                raise ValueError(
+                    f"Each message list must contain a single user dictionary. Error at index {i}."
+                )
+
+            pil_image = (
+                Image.open(image).convert("RGB") if isinstance(image, str) else image
+            )
+            p_images.append(pil_image)
+
+            question = msg_list[0]["content"]
+            questions.append(question)
+
+            p_message = [
+                {
+                    "role": "user",
+                    "content": [
+                        {"type": "image", "image": image},
+                        {
+                            "type": "text",
+                            "text": p_prompt_template.format(question=question),
+                        },
+                        # {'type': 'text', 'text': p_prompt_template.format(num_feat=self.config.visual_bandwidth,
+                        #                                                   question=question)}
+                    ],
+                }
+                # {'role': 'assisstant', 'content': [
+                #     {'type': 'text', 'text': self.get_visual_message()}
+                # ]}
+            ]
+            # print(f'P-Message-{i}: {p_message}')
+            p_texts.append(
+                self.p_processor.apply_chat_template(
+                    p_message, tokenize=False, add_generation_prompt=False
+                )
+            )
+
+        # print(f'{p_texts=}')
+        perceiver_inputs = self.p_processor(
+            text=p_texts,
+            images=p_images,
+            padding=True,
+            return_tensors="pt",
+        ).to(self.device)
+
+        # print('Token IDs of perceiver inputs',
+        #       perceiver_inputs['input_ids'].tolist())
+        # print('Tokens of perceiver inputs', [
+        #       self.p_processor.tokenizer.convert_ids_to_tokens(ids) for ids in perceiver_inputs['input_ids']])
+
+        # t_prompt_template = '{question} Image: ' + self.get_visual_message()
+        t_prompt_template = "<image>{question}"
+        t_texts = []
+        for i in range(len(questions)):
+            prompt = t_prompt_template.format(question=questions[i])
+
+            p_input_ids = perceiver_inputs["input_ids"][i].tolist()
+            img_start_idx = p_input_ids.index(IMG_START_ID)
+            img_end_idx = p_input_ids.index(IMG_END_ID)
+
+            assert img_start_idx < img_end_idx
+
+            prompt = prompt.replace(
+                "<image>",
+                "<|vision_start|>"
+                + "<|image_pad|>" * (img_end_idx - img_start_idx - 1)
+                + "<|vision_end|>",
+            )
+            message = [
+                {"role": "user", "content": prompt},
+                # {"role": "assistant", "content": "<think>\n\n</think>\n\n"}
+            ]
+            t_texts.append(
+                self.t_tokenizer.apply_chat_template(
+                    message,
+                    tokenize=False,
+                    add_generation_prompt=True,
+                    enable_thinking=True,
+                    # message, tokenize=False, add_generation_prompt=True, enable_thinking=False
+                    # ))
+                )
+                + "<think>\n\n"
+            )
+            # print(f'\n\n##T-Message-{i}: {t_texts[-1]}')
+
+        model_inputs_t = self.t_tokenizer(
+            t_texts, return_tensors="pt", padding=True
+        ).to(self.thinker.device)
+
+        model_inputs_t["input_ids_of_perceiver"] = perceiver_inputs["input_ids"]
+        model_inputs_t["attention_mask_of_perceiver"] = perceiver_inputs[
+            "attention_mask"
+        ]
+        model_inputs_t["pixel_values"] = perceiver_inputs["pixel_values"]
+        model_inputs_t["image_grid_thw"] = perceiver_inputs["image_grid_thw"]
+
+        # print(
+        #     f'Thinker generation config: {self.thinker.generation_config.to_dict()}')
+        thinker_generation_params = kwargs.get("thinker_generation_params", {})
+        thinker_generation_params["max_new_tokens"] = thinker_generation_params.get(
+            "max_new_tokens", 32768
+        )
+
+        assert model_inputs_t["pixel_values"] is not None
+
+        with torch.inference_mode():
+            generated_ids_t = self.generate(
+                **model_inputs_t,
+                **thinker_generation_params,
+                eos_token_id=self.t_tokenizer.eos_token_id,
+            )
+        # print(f'Thinker output ids: {generated_ids_t}')
+        # print(
+        #     f'Thinker output toks: {[self.t_tokenizer.convert_ids_to_tokens(ids) for ids in generated_ids_t]}')
+
+        final_responses = []
+        for i in range(len(msgs)):
+            output_ids = generated_ids_t[i][len(model_inputs_t.input_ids[i]) :].tolist()
+            try:
+                # 寻找 </think> token (151668)
+                index = len(output_ids) - output_ids[::-1].index(151668)
+                print(
+                    f"len output_ids: {len(output_ids)}, subtract {output_ids[::-1].index(151668)}"
+                )
+            except ValueError:
+                index = 0
+
+            thinking_content = self.t_tokenizer.decode(
+                output_ids[:index], skip_special_tokens=True
+            ).strip("\n")
+            # print(f"\n\n##Thinking content-{i}: {thinking_content}")
+
+            print(f"content ids: {output_ids[index:]}")
+
+            content = self.t_tokenizer.decode(
+                output_ids[index:], skip_special_tokens=True
+            ).strip("\n")
+            final_responses.append(content)
+            # print(f"\n\n##Answer content-{i}: {content}")
+
+        # return [x[0] for x in self.generate([image], [msgs], *args, **kwargs)]
+        return final_responses
+
+    # NOTE: All inputs should be considered as inputs to thinker
+    #       The thinker consumes multimodal data by calling perceiver
+    def prepare_inputs_for_generation(
+        self,
+        input_ids,
+        past_key_values=None,
+        input_ids_of_perceiver=None,
+        attention_mask_of_perceiver=None,
+        attention_mask=None,
+        inputs_embeds=None,
+        cache_position=None,
+        position_ids=None,
+        use_cache=True,
+        pixel_values=None,
+        pixel_values_videos=None,
+        image_grid_thw=None,
+        video_grid_thw=None,
+        second_per_grid_ts=None,
+        **kwargs,
+    ):
+        # Overwritten -- in specific circumstances we don't want to forward image inputs to the model
+        assert pixel_values is not None
+        model_inputs = super().prepare_inputs_for_generation(
+            input_ids,
+            attention_mask=attention_mask,
+            input_ids_of_perceiver=input_ids_of_perceiver,
+            attention_mask_of_perceiver=attention_mask_of_perceiver,
+            past_key_values=past_key_values,
+            inputs_embeds=inputs_embeds,
+            cache_position=cache_position,
+            position_ids=position_ids,
+            pixel_values=pixel_values,
+            pixel_values_videos=pixel_values_videos,
+            image_grid_thw=image_grid_thw,
+            video_grid_thw=video_grid_thw,
+            second_per_grid_ts=second_per_grid_ts,
+            use_cache=use_cache,
+            **kwargs,
+        )
+        # print(f'\n@@@@ prepare inputs for generation', f'##${model_inputs["pixel_values"].shape}$##', flush=True)
+
+        # # Qwen2-5-VL position_ids are prepareed with rope_deltas in forward
+        # model_inputs["position_ids"] = None
+
+        assert model_inputs["pixel_values"] is not None
+        if cache_position[0] != 0:
+            # print(f'Cache hit, skip pixel values encoding', flush=True)
+            model_inputs["pixel_values"] = None
+            # model_inputs["pixel_values_videos"] = None
+
+        return model_inputs
+
+    @auto_docstring
+    def forward(
+        self,
+        input_ids: torch.LongTensor = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        input_ids_of_perceiver: torch.LongTensor = None,
+        attention_mask_of_perceiver: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_values: Optional[list[torch.FloatTensor]] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        use_cache: Optional[bool] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        return_dict: Optional[bool] = None,
+        pixel_values: Optional[torch.Tensor] = None,
+        pixel_values_videos: Optional[torch.FloatTensor] = None,
+        image_grid_thw: Optional[torch.LongTensor] = None,
+        video_grid_thw: Optional[torch.LongTensor] = None,
+        rope_deltas: Optional[torch.LongTensor] = None,
+        cache_position: Optional[torch.LongTensor] = None,
+        second_per_grid_ts: Optional[torch.Tensor] = None,
+        **kwargs,
+    ) -> Union[tuple, Qwen2_5_VLModelOutputWithPast]:
+
+        t_input_ids = input_ids
+        del input_ids
+                
+        if inputs_embeds is None:
+            inputs_embeds = self.thinker.get_input_embeddings()(t_input_ids)
+
+            if pixel_values is not None:
+                p_msg_st_id = IMG_START_ID
+                p_msg_ed_id = IMG_END_ID
+                p_msg_st_list = []
+                p_msg_ed_list = []
+                
+                # Iterate over batch: each element may contain multiple images (in packing mode)
+                for batch_idx, perceiver_sample_input_ids in enumerate(input_ids_of_perceiver):
+                    # Find ALL image start/end tokens in this (potentially packed) sequence
+                    st_indices = (perceiver_sample_input_ids == p_msg_st_id).nonzero(
+                        as_tuple=True
+                    )[0]
+                    ed_indices = (perceiver_sample_input_ids == p_msg_ed_id).nonzero(
+                        as_tuple=True
+                    )[0]
+                    samples = (perceiver_sample_input_ids == 151644).nonzero(
+                        as_tuple=True
+                    )[0]
+                    
+                    # In packing mode: multiple images per sequence (len(st_indices) = pack_size)
+                    # In non-packing mode: one image per sequence (len(st_indices) = 1)
+                    assert len(st_indices) >= 1, f"No start token found in perceiver input {batch_idx}"
+                    assert len(ed_indices) >= 1, f"No end token found in perceiver input {batch_idx}"
+                    assert len(st_indices) == len(ed_indices), f"Mismatched start/end tokens in batch {batch_idx}"
+                    
+                    # Collect start/end positions for all images in this batch element
+                    for st, ed in zip(st_indices, ed_indices):
+                        p_msg_st_list.append(st)
+                        p_msg_ed_list.append(ed)
+
+                # Prepare perceiver inputs
+                perceiver_kwargs = {
+                    'input_ids': input_ids_of_perceiver,
+                    'pixel_values': pixel_values,
+                    'attention_mask': attention_mask_of_perceiver,
+                    'image_grid_thw': image_grid_thw,
+                    'output_hidden_states': True,
+                }
+                
+                # TEMPORARY: Disable position_ids for perceiver to debug hang issue
+                # Add position_ids if available (for packing support)
+                position_ids_of_perceiver = kwargs.get('position_ids_of_perceiver')
+                if position_ids_of_perceiver is not None:
+                    perceiver_kwargs['position_ids'] = position_ids_of_perceiver
+                
+                out = self.perceiver(**perceiver_kwargs)
+
+                # only keep last layer hidden states, release other layers
+                last_layer_hiddens = out.hidden_states[-1]
+                # print(f"Perceiver last_layer_hiddens shape: {last_layer_hiddens.shape}")
+
+                # 释放不需要的中间变量，但保留梯度
+                if hasattr(out, "hidden_states"):
+                    del out.hidden_states  # 释放其他层的隐藏状态
+                if hasattr(out, "attentions"):
+                    del out.attentions  # 释放注意力权重
+
+                # Extract visual features from all images
+                # p_msg_st_list and p_msg_ed_list contain positions for all images in order
+                # We need to track which batch element each position belongs to
+                batch_msg = []
+                img_idx = 0  # Track which image we're processing
+                for batch_idx, perceiver_sample_input_ids in enumerate(input_ids_of_perceiver):
+                    # Find how many images are in this batch element
+                    st_indices = (perceiver_sample_input_ids == p_msg_st_id).nonzero(as_tuple=True)[0]
+                    num_images_in_batch = len(st_indices)
+                    
+                    # Extract features for each image in this batch element
+                    for _ in range(num_images_in_batch):
+                        st = p_msg_st_list[img_idx]
+                        ed = p_msg_ed_list[img_idx]
+                        # Extract from the correct batch element's hidden states
+                        msg_feat = last_layer_hiddens[batch_idx, st : ed + 1, :]
+                        batch_msg.append(msg_feat)
+                        img_idx += 1
+                # print(f"Extracted {len(batch_msg)} image features from {input_ids_of_perceiver.shape[0]} batch elements")
+
+                image_features = torch.cat(batch_msg, dim=0)
+                image_features = self.linear_align_dim(image_features)
+
+                n_msg_features = image_features.shape[0]
+                msg_mask = (
+                    (t_input_ids == IMG_THINKER_START_ID)
+                    | (t_input_ids == IMG_THINKER_END_ID)
+                    | (t_input_ids == IMG_THINKER_PAD_ID)
+                )
+                n_msg_tokens = msg_mask.sum()
+
+                if not is_torchdynamo_compiling() and n_msg_tokens != n_msg_features:
+                    raise ValueError(
+                        f"Image features and image tokens do not match: tokens: {n_msg_tokens}, features {n_msg_features}"
+                    )
+
+                mask_unsqueezed = msg_mask.unsqueeze(-1)
+                mask_expanded = mask_unsqueezed.expand_as(inputs_embeds)
+
+                image_mask = mask_expanded.to(inputs_embeds.device)
+                image_features = image_features.to(
+                    inputs_embeds.device, inputs_embeds.dtype
+                )
+
+                inputs_embeds = inputs_embeds.masked_scatter(image_mask, image_features)
+
+                del last_layer_hiddens, batch_msg, mask_expanded, mask_unsqueezed
+
+        outputs = self.thinker(
+            input_ids=None,
+            position_ids=position_ids,
+            attention_mask=attention_mask,
+            past_key_values=past_key_values,
+            inputs_embeds=inputs_embeds,
+            use_cache=use_cache,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=True,
+            cache_position=cache_position,
+            **kwargs,
+        )
+
+        output = ForwardVisualTokensArchOutputWithPast(
+            past_key_values=outputs.past_key_values,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
+            logits=outputs.logits,
+        )
+        
+        return output if return_dict else output.to_tuple()
+
+
+class ForwardVisualTokensArchForCausalLM(ForwardVisualTokensArchPreTrainedModel, GenerationMixin):
+    def __init__(self, config: ForwardVisualTokensArchConfig):
+        super().__init__(config)
+        self.model = ForwardVisualTokensArchModel(config)
+        self.vocab_size = config.vocab_size
+
+        self.lm_head = self.model.thinker.lm_head
+        # del self.model.thinker.lm_head
+
+        self.config.eos_token_id = self.model.thinker.generation_config.eos_token_id
+        if self.model.t_tokenizer.pad_token_id is None:
+            self.model.t_tokenizer.pad_token = self.model.t_tokenizer.eos_token
+
+        self.config.pad_token_id = self.model.t_tokenizer.pad_token_id
+        print(
+            f"Config eos_token_id: {self.config.eos_token_id}, pad_token_id: {self.config.pad_token_id}"
+        )
+
+        self.post_init()
+
+    def get_input_embeddings(self):
+        return self.model.thinker.get_input_embeddings()
+
+    def set_input_embeddings(self, value):
+        self.model.thinker.set_input_embeddings(value)
+
+    def _register_perceiver_embedding_gradient_hook(self):
+        try:
+            embedding_layer = self.model.perceiver.get_input_embeddings()
+            print(
+                f"Successfully located Perceiver's embedding layer: {embedding_layer}"
+            )
+
+            trainable_token_ids = self.model.get_visual_message_token_ids("p")
+            if not trainable_token_ids:
+                print(
+                    "WARNING: No trainable token IDs found for Perceiver. Hook will not be effective."
+                )
+                return
+
+            print(f"Target trainable token IDs for Perceiver: {trainable_token_ids}")
+
+            vocab_size, _ = embedding_layer.weight.shape
+            mask = torch.zeros_like(embedding_layer.weight)
+
+            for token_id in trainable_token_ids:
+                mask[token_id, :] = 1.0
+
+            def grad_mask_hook(grad):
+                return grad.mul_(mask)
+
+            embedding_layer.weight.register_hook(grad_mask_hook)
+
+            print("=" * 70)
+            print("SUCCESS: PERCEIVER embedding gradient hook has been registered.")
+            print(
+                f"Only embeddings for the following Perceiver token IDs will be updated: {trainable_token_ids}"
+            )
+            print("This message should only appear ONCE at the beginning of training.")
+            print("=" * 70)
+
+        except Exception as e:
+            print(
+                f"ERROR: Failed to register Perceiver embedding gradient hook. Reason: {e}"
+            )
+
+    # def get_output_embeddings(self):
+    #     return self.model.thinker.get_output_embeddings()
+
+    def forward(
+        self,
+        input_ids: torch.LongTensor = None,
+        labels: Optional[torch.LongTensor] = None,
+        **kwargs,
+    ) -> Union[tuple, CausalLMOutputWithPast]:
+        # For lora training
+        kwargs['return_dict'] = True
+        return_dict = kwargs.get("return_dict", True)
+        
+        # print('------------------------------------------------', flush=True)
+        # print(f'input_ids: {input_ids.shape}', flush=True)
+        outputs = self.model(
+            input_ids=input_ids,
+            # index=index,
+            # return_dict=True,
+            **kwargs,
+        )
+
+        logits = outputs.logits
+        loss = None
+
+        if labels is not None:
+            loss = self.loss_function(
+                logits=logits,
+                labels=labels,
+                vocab_size=self.config.vocab_size,
+                **kwargs,
+            )
+
+        if not return_dict:
+            output = (logits,) + outputs[1:]
+            return ((loss,) + output) if loss is not None else output
+
+        return CausalLMOutputWithPast(
+            loss=loss,
+            logits=logits,
+            past_key_values=outputs.past_key_values,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
+        )
+
+    def prepare_inputs_for_generation(self, input_ids, **kwargs):
+        return self.model.prepare_inputs_for_generation(input_ids, **kwargs)
+
+    def chat(self, images, msgs, *args, **kwargs):
+        return self.model.chat(images, msgs, *args, **kwargs)

modeling_llava_baseline.py

@@ -0,0 +1,711 @@
+from typing import Callable, List, Optional, Tuple, Union
+
+import torch
+import torch.distributed as dist
+import torch.nn as nn
+import transformers.models.qwen2_5_vl.modeling_qwen2_5_vl as qwen25
+import transformers.models.qwen3.modeling_qwen3 as qwen3
+from transformers import (Qwen2_5_VLModel, Qwen2Config,
+                          Qwen2PreTrainedModel, AutoConfig)
+from transformers.cache_utils import Cache, DynamicCache
+from transformers.configuration_utils import PretrainedConfig
+from transformers.generation import GenerationMixin
+from transformers.masking_utils import (ALL_MASK_ATTENTION_FUNCTIONS,
+                                        BlockMask,
+                                        _is_torch_greater_or_equal_than_2_6,
+                                        and_masks,
+                                        causal_mask_function,
+                                        or_masks,
+                                        packed_sequence_mask_function)
+from transformers.modeling_flash_attention_utils import FlashAttentionKwargs
+from transformers.modeling_outputs import BaseModelOutputWithPast
+from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS
+# from transformers.models.qwen3.modeling_qwen3 import Qwen3Attention, Qwen3Model, eager_attention_forward
+# from transformers.models.qwen2_5_vl.modeling_qwen2_5_vl import Qwen2_5_VLCausalLMOutputWithPast, Qwen2_5_VLRotaryEmbedding, apply_multimodal_rotary_pos_emb
+from transformers.processing_utils import Unpack
+from transformers.utils import auto_docstring
+from transformers.utils.deprecation import deprecate_kwarg
+try:
+    from transformers.masking_utils import _is_torch_xpu_available
+except:
+    _is_torch_xpu_available = False
+from transformers.masking_utils import sliding_window_causal_mask_function
+
+
+def find_packed_sequence_indices(position_ids: torch.Tensor) -> torch.Tensor:
+    """
+    Find the indices of the sequence to which each new query token in the sequence belongs when using packed
+    tensor format (i.e. several sequences packed in the same batch dimension).
+
+    Args:
+        position_ids (`torch.Tensor`)
+            A 2D tensor of shape (batch_size, query_length) indicating the positions of each token in the sequences.
+
+    Returns:
+        A 2D tensor where each similar integer indicates that the tokens belong to the same sequence. For example, if we
+        pack 3 sequences of 2, 3 and 1 tokens respectively along a single batch dim, this will return [[0, 0, 1, 1, 1, 2]].
+    """
+    # What separate different sequences is when 2 consecutive positions_ids are separated by more than 1. So
+    # taking the diff (by prepending the first value - 1 to keep correct indexing) and applying cumsum to the result
+    # gives exactly the sequence indices
+    # Note that we assume that a single sequence cannot span several batch dimensions, i.e. 1 single sequence
+    # cannot be part of the end of the first batch dim and the start of the 2nd one for example
+    first_dummy_value = position_ids[:, :1] - 1  # We just need the diff on this first value to be 1
+    position_diff = torch.diff(position_ids, prepend=first_dummy_value, dim=-1)
+    packed_sequence_mask = (position_diff < 0).cumsum(-1)
+
+    # Here it would be nice to return None if we did not detect packed sequence format, i.e. if `packed_sequence_mask[:, -1] == 0`
+    # but it causes issues with export
+    return packed_sequence_mask
+
+
+def _preprocess_mask_arguments(
+    config: PretrainedConfig,
+    input_embeds: torch.Tensor,
+    attention_mask: Optional[Union[torch.Tensor, BlockMask]],
+    cache_position: torch.Tensor,
+    past_key_values: Optional[Cache],
+    position_ids: Optional[torch.Tensor],
+    layer_idx: Optional[int],
+) -> tuple[bool, Optional[Union[torch.Tensor, BlockMask]], int, int]:
+    """
+    Perform some common pre-processing of the mask arguments we get from the modeling code. Mostly determine the
+    key-value length and offsets, and if we should early exit or not.
+
+    Args:
+        config (`PretrainedConfig`):
+            The model config.
+        input_embeds (`torch.Tensor`):
+            The input embeddings of shape (batch_size, query_length, hidden_dim). This is used only to infer the
+            batch size, query length and dtype.
+        attention_mask (`torch.Tensor`, optional):
+            The 2D attention mask corresponding to padded tokens of shape (batch_size, number_of_seen_tokens+q_length).
+            It can also be an already prepared 4D mask, in which case it is returned as-is.
+        cache_position (`torch.Tensor`):
+            A tensor of shape (query_length,) indicating the current indices of the input sequence elements.
+        past_key_values (`Cache`, optional):
+            The past key values, if we use a cache.
+        position_ids (`torch.Tensor`, optional)
+            A 2D tensor of shape (batch_size, query_length) indicating the positions of each token in the sequences.
+        layer_idx (`int`, optional):
+            If `past_key_values` is not None, this is the layer index of the cache from which to get the key-value
+            length and offset. Indeed, for hybrid caches, different layers may return different lengths.
+
+    Returns:
+        early_exit (`bool`):
+            Whether we should early exit mask creation, and return the mask as-is.
+        attention_mask (`torch.Tensor` or `BlockMask` or `None`):
+            The attention mask to either return immediately, or to use in downstream mask creation.
+        packed_sequence_mask (`torch.Tensor`, optional):
+            In case we detected packed sequence format, this is a tensor where each similar integer indicates that
+            the tokens belong to the same sequence.
+        kv_length (`int`):
+            The size that the key and value states will have during the attention computation.
+        kv_offset (`int`):
+            An offset to indicate at which first position the key and values states will refer to.
+    """
+    # If the mask is already 4D, simply return as-is (it was already prepared, or it is custom)
+    if isinstance(attention_mask, (torch.Tensor, BlockMask)) and len(attention_mask.shape) == 4:
+        return True, attention_mask, None, None, None
+
+    # For TGI/vLLM backends, or other custom attention without equivalent mask creation: we don't need a mask!
+    # Note: it's not ideal to check the `_global_mapping` attribute instead of the object itself, however otherwise
+    # full graph dynamo tracing (i.e. torch.export or compile with `fullgraph=True`) will fail on Python<3.11
+    # with `torch._dynamo.exc.Unsupported: 'inline in skipfiles:Mapping.__contains__ | __contains__, skipped
+    # according trace_rules.lookup SKIP_DIRS'` -- can be removed when we require Python>=3.11
+    if config._attn_implementation not in ALL_MASK_ATTENTION_FUNCTIONS._global_mapping:
+        return True, None, None, None, None
+
+    # Move the mask to correct device, and potentially switch dtype for efficiency
+    if attention_mask is not None and attention_mask.ndim == 2:
+        attention_mask = attention_mask.to(device=cache_position.device, dtype=torch.bool)
+
+    # If using a cache, it can give all information about mask sizes based on seen tokens
+    if past_key_values is not None:
+        kv_length, kv_offset = past_key_values.get_mask_sizes(cache_position, layer_idx)
+    # Otherwise, the sizes are simply the input sizes
+    else:
+        kv_length, kv_offset = input_embeds.shape[1], 0
+
+    # We check the position_ids for potential packed sequence format (only if the 2D attention mask is explicitly None,
+    # and we don't have past_key_values, i.e. generally a training setup)
+    packed_sequence_mask = None
+    if position_ids is not None and attention_mask is None and past_key_values is None:
+        batch_size = input_embeds.shape[0]
+        # The position ids are sometimes just unsqueezed, without being expanded
+        if batch_size != position_ids.shape[0]:
+            position_ids = position_ids.expand(batch_size, -1)
+        packed_sequence_mask = find_packed_sequence_indices(position_ids)
+
+    return False, attention_mask, packed_sequence_mask, kv_length, kv_offset
+
+
+def create_causal_mask(
+    config: PretrainedConfig,
+    input_embeds: torch.Tensor,
+    attention_mask: Optional[torch.Tensor],
+    cache_position: torch.Tensor,
+    past_key_values: Optional[Cache],
+    position_ids: Optional[torch.Tensor] = None,
+    or_mask_function: Optional[Callable] = None,
+    and_mask_function: Optional[Callable] = None,
+) -> Optional[Union[torch.Tensor, BlockMask]]:
+    """
+    Create a standard causal mask based on the attention implementation used (stored in the config). If `past_key_values`
+    has an hybrid cache structure, this function will return the mask corresponding to one of the "full_attention" layers (to align
+    to what is needed in the `modeling_xxx.py` files).
+
+    Args:
+        config (`PretrainedConfig`):
+            The model config.
+        input_embeds (`torch.Tensor`):
+            The input embeddings of shape (batch_size, query_length, hidden_dim). This is used only to infer the
+            batch size, query length and dtype.
+        attention_mask (`torch.Tensor`, optional):
+            The 2D attention mask corresponding to padded tokens of shape (batch_size, number_of_seen_tokens+q_length).
+            It can also be an already prepared 4D mask, in which case it is returned as-is.
+        cache_position (`torch.Tensor`):
+            A tensor of shape (query_length,) indicating the current indices of the input sequence elements.
+        past_key_values (`Cache`, optional):
+            The past key values, if we use a cache.
+        position_ids (`torch.Tensor`, optional)
+            A 2D tensor of shape (batch_size, query_length) indicating the positions of each token in the sequences.
+        or_mask_function (`Callable`, optional):
+            An optional mask function to combine with the causal mask function (by doing the union of both). This is
+            useful to easily overlay another mask on top of the causal one, for example for image tokens handling.
+        and_mask_function (`Callable`, optional):
+            An optional mask function to combine with the causal mask function (by doing the intersection of both). This is
+            useful to easily overlay another mask on top of the causal one, for example for image tokens handling.
+    """
+    # If we have an hybrid cache structure, here we want to create the mask for the full layers
+    if hasattr(past_key_values, "is_sliding") and False in past_key_values.is_sliding:
+        layer_idx = past_key_values.is_sliding.index(False)
+    else:
+        layer_idx = 0
+
+    early_exit, attention_mask, packed_sequence_mask, kv_length, kv_offset = _preprocess_mask_arguments(
+        config, input_embeds, attention_mask, cache_position, past_key_values, position_ids, layer_idx
+    )
+    if early_exit:
+        return attention_mask
+
+    batch_size, dtype = input_embeds.shape[0], input_embeds.dtype
+    mask_factory_function = causal_mask_function
+    mask_interface = ALL_MASK_ATTENTION_FUNCTIONS[config._attn_implementation]
+
+    # Do not allow skip if we are compiling (this is to match BC)
+    # TODO: cyril -> probably revisit and remove this, but a lot of tests rely on it
+    if _is_torch_xpu_available:
+        allow_is_causal_skip = True
+    else:
+        allow_is_causal_skip = not getattr(past_key_values, "is_compileable", False)
+
+    # Allow slight deviations from causal mask
+    # Note that it is very important to apply this before any other deviations of the mask (such as packed sequence mask,
+    # padding mask, etc) as the resulting mask may otherwise not be correct!
+    if or_mask_function is not None:
+        if not _is_torch_greater_or_equal_than_2_6:
+            raise ValueError("Using `or_mask_function` or `and_mask_function` arguments require torch>=2.6")
+        mask_factory_function = or_masks(mask_factory_function, or_mask_function)
+        allow_is_causal_skip = False
+    if and_mask_function is not None:
+        if not _is_torch_greater_or_equal_than_2_6:
+            raise ValueError("Using `or_mask_function` or `and_mask_function` arguments require torch>=2.6")
+        mask_factory_function = and_masks(mask_factory_function, and_mask_function)
+        allow_is_causal_skip = False
+
+    # If we detected packing format
+    if packed_sequence_mask is not None and _is_torch_greater_or_equal_than_2_6:
+        mask_factory_function = and_masks(mask_factory_function, packed_sequence_mask_function(packed_sequence_mask))
+        allow_is_causal_skip = False
+
+    # We now create the mask
+    causal_mask = mask_interface(
+        batch_size=batch_size,
+        cache_position=cache_position,
+        kv_length=kv_length,
+        kv_offset=kv_offset,
+        mask_function=mask_factory_function,
+        attention_mask=attention_mask,
+        allow_is_causal_skip=allow_is_causal_skip,  # additional kwarg for sdpa
+        dtype=dtype,  # Additional kwarg for eager
+        config=config,  # Pass the config as well, in case someone wants to easily have their own mask_interface
+    )
+    return causal_mask
+
+
+def create_sliding_window_causal_mask(
+    config: PretrainedConfig,
+    input_embeds: torch.Tensor,
+    attention_mask: Optional[torch.Tensor],
+    cache_position: torch.Tensor,
+    past_key_values: Optional[Cache],
+    position_ids: Optional[torch.Tensor] = None,
+    or_mask_function: Optional[Callable] = None,
+    and_mask_function: Optional[Callable] = None,
+) -> Optional[Union[torch.Tensor, BlockMask]]:
+    """
+    Create a sliding window causal mask based on the attention implementation used (stored in the config). This type
+    of attention pattern was mostly democratized by Mistral. If `past_key_values` has an hybrid cache structure, this
+    function will return the mask corresponding to one of the "sliding_attention" layers (to align to what is needed in the
+    `modeling_xxx.py` files).
+
+    Args:
+        config (`PretrainedConfig`):
+            The model config.
+        input_embeds (`torch.Tensor`):
+            The input embeddings of shape (batch_size, query_length, hidden_dim). This is used only to infer the
+            batch size, query length and dtype.
+        attention_mask (`torch.Tensor`, optional):
+            The 2D attention mask corresponding to padded tokens of shape (batch_size, number_of_seen_tokens+q_length).
+            It can also be an already prepared 4D mask, in which case it is returned as-is.
+        cache_position (`torch.Tensor`):
+            A tensor of shape (query_length,) indicating the current indices of the input sequence elements.
+        past_key_values (`Cache`, optional):
+            The past key values, if we use a cache.
+        position_ids (`torch.Tensor`, optional)
+            A 2D tensor of shape (batch_size, query_length) indicating the positions of each token in the sequences.
+        or_mask_function (`Callable`, optional):
+            An optional mask function to combine with the sliding causal mask function (by doing the union of both). This is
+            useful to easily overlay another mask on top of the sliding causal one, for example for image tokens handling.
+        and_mask_function (`Callable`, optional):
+            An optional mask function to combine with the sliding causal mask function (by doing the intersection of both). This is
+            useful to easily overlay another mask on top of the sliding causal one, for example for image tokens handling.
+    """
+    # If we have an hybrid cache structure, here we want to create the mask for the sliding layers
+    if hasattr(past_key_values, "is_sliding") and True in past_key_values.is_sliding:
+        layer_idx = past_key_values.is_sliding.index(True)
+    else:
+        layer_idx = 0
+
+    early_exit, attention_mask, packed_sequence_mask, kv_length, kv_offset = _preprocess_mask_arguments(
+        config, input_embeds, attention_mask, cache_position, past_key_values, position_ids, layer_idx
+    )
+    if early_exit:
+        return attention_mask
+
+    sliding_window = getattr(config, "sliding_window", None)
+    if sliding_window is None:
+        raise ValueError("Could not find a `sliding_window` argument in the config, or it is not set")
+
+    batch_size, dtype = input_embeds.shape[0], input_embeds.dtype
+    mask_factory_function = sliding_window_causal_mask_function(sliding_window)
+    mask_interface = ALL_MASK_ATTENTION_FUNCTIONS[config._attn_implementation]
+
+    # Do not allow skip if we are compiling (this is to match BC)
+    # TODO: cyril -> probably revisit and remove this, but a lot of tests rely on it
+    allow_is_causal_skip = not getattr(past_key_values, "is_compileable", False)
+
+    # Allow slight deviations from causal mask
+    # Note that it is very important to apply this before any other deviations of the mask (such as packed sequence mask,
+    # padding mask, etc) as the resulting mask may otherwise not be correct!
+    if or_mask_function is not None:
+        if not _is_torch_greater_or_equal_than_2_6:
+            raise ValueError("Using `or_mask_function` or `and_mask_function` arguments require torch>=2.6")
+        mask_factory_function = or_masks(mask_factory_function, or_mask_function)
+        allow_is_causal_skip = False
+    if and_mask_function is not None:
+        if not _is_torch_greater_or_equal_than_2_6:
+            raise ValueError("Using `or_mask_function` or `and_mask_function` arguments require torch>=2.6")
+        mask_factory_function = and_masks(mask_factory_function, and_mask_function)
+        allow_is_causal_skip = False
+
+    # If we detected packing format
+    if packed_sequence_mask is not None and _is_torch_greater_or_equal_than_2_6:
+        mask_factory_function = and_masks(mask_factory_function, packed_sequence_mask_function(packed_sequence_mask))
+        allow_is_causal_skip = False
+
+    # We now create the mask
+    causal_mask = mask_interface(
+        batch_size=batch_size,
+        cache_position=cache_position,
+        kv_length=kv_length,
+        kv_offset=kv_offset,
+        mask_function=mask_factory_function,
+        attention_mask=attention_mask,
+        allow_is_causal_skip=allow_is_causal_skip,  # additional kwarg for sdpa
+        local_size=sliding_window,  # Additional kwarg for sdpa
+        dtype=dtype,  # Additional kwarg for eager
+        config=config,  # Pass the config as well, in case someone wants to easily have their own mask_interface
+    )
+    return causal_mask
+
+class Qwen3Attention(qwen3.Qwen3Attention):
+    @deprecate_kwarg("past_key_value", new_name="past_key_values", version="4.58")
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        position_embeddings: Tuple[torch.Tensor, torch.Tensor],
+        attention_mask: Optional[torch.Tensor],
+        past_key_values: Optional[Cache] = None,
+        cache_position: Optional[torch.LongTensor] = None,
+        **kwargs: Unpack[FlashAttentionKwargs],
+    ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]:
+
+        input_shape = hidden_states.shape[:-1]
+        hidden_shape = (*input_shape, -1, self.head_dim)
+
+        query_states = self.q_norm(self.q_proj(
+            hidden_states).view(hidden_shape)).transpose(1, 2)
+        key_states = self.k_norm(self.k_proj(
+            hidden_states).view(hidden_shape)).transpose(1, 2)
+        value_states = self.v_proj(hidden_states).view(
+            hidden_shape).transpose(1, 2)
+
+        # 获取 3D 的 cos 和 sin，用于多模态 RoPE
+        cos, sin = position_embeddings
+
+        # 调用多模态的 RoPE 函数
+        mrope_section = self.rope_scaling["mrope_section"]
+        query_states, key_states = qwen25.apply_multimodal_rotary_pos_emb(
+            query_states, key_states, cos, sin, mrope_section
+        )
+
+        if past_key_values is not None:
+            cache_kwargs = {"sin": sin, "cos": cos,
+                            "cache_position": cache_position}
+            key_states, value_states = past_key_values.update(
+                key_states, value_states, self.layer_idx, cache_kwargs)
+
+        attention_interface: Callable = qwen3.eager_attention_forward
+        if self.config._attn_implementation != "eager":
+            if self.config._attn_implementation == "sdpa" and kwargs.get("output_attentions", False):
+                assert False, (
+                    "`torch.nn.functional.scaled_dot_product_attention` does not support `output_attentions=True`. Falling back to "
+                    'eager attention. This warning can be removed using the argument `attn_implementation="eager"` when loading the model.'
+                )
+            else:
+                attention_interface = ALL_ATTENTION_FUNCTIONS[self.config._attn_implementation]
+
+        attn_output, attn_weights = attention_interface(
+            self,
+            query_states,
+            key_states,
+            value_states,
+            attention_mask,
+            dropout=0.0 if not self.training else self.attention_dropout,
+            scaling=self.scaling,
+            sliding_window=self.sliding_window,
+            **kwargs,
+        )
+
+        attn_output = attn_output.reshape(*input_shape, -1).contiguous()
+        attn_output = self.o_proj(attn_output)
+        return attn_output, attn_weights
+
+
+class Qwen3DecoderLayer(qwen3.Qwen3DecoderLayer):
+    def __init__(self, config: qwen3.Qwen3Config, layer_idx: int):
+        super().__init__(config, layer_idx)
+        self.self_attn = Qwen3Attention(config=config, layer_idx=layer_idx)
+
+
+class Qwen3Model(qwen3.Qwen3PreTrainedModel):
+    def __init__(self, config: qwen3.Qwen3Config):
+        super().__init__(config)
+        self.padding_idx = config.pad_token_id
+        self.vocab_size = config.vocab_size
+
+        self.embed_tokens = nn.Embedding(
+            config.vocab_size, config.hidden_size, self.padding_idx)
+        self.layers = nn.ModuleList(
+            [Qwen3DecoderLayer(config, layer_idx)
+             for layer_idx in range(config.num_hidden_layers)]
+        )
+        self.norm = qwen3.Qwen3RMSNorm(
+            config.hidden_size, eps=config.rms_norm_eps)
+        self.rotary_emb = qwen3.Qwen3RotaryEmbedding(config=config)
+        self.gradient_checkpointing = False
+        self.has_sliding_layers = "sliding_attention" in self.config.layer_types
+
+        # Initialize weights and apply final processing
+        self.post_init()
+        
+    def get_input_embeddings(self):
+        """
+        For transformers library version compatability.
+        """
+        return self.embed_tokens
+
+    @auto_docstring
+    def forward(
+        self,
+        input_ids: Optional[torch.LongTensor] = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_values: Optional[Cache] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        use_cache: Optional[bool] = None,
+        cache_position: Optional[torch.LongTensor] = None,
+        **kwargs: Unpack,
+    ):
+        if (input_ids is None) ^ (inputs_embeds is not None):
+            raise ValueError(
+                "You must specify exactly one of input_ids or inputs_embeds")
+
+        if inputs_embeds is None:
+            inputs_embeds = self.embed_tokens(input_ids)
+
+        if use_cache and past_key_values is None:
+            past_key_values = DynamicCache(config=self.config)
+
+        if cache_position is None:
+            past_seen_tokens = past_key_values.get_seq_length(
+            ) if past_key_values is not None else 0
+            cache_position = torch.arange(
+                past_seen_tokens, past_seen_tokens + inputs_embeds.shape[1], device=inputs_embeds.device
+            )
+
+        if position_ids is None:
+            position_ids = cache_position.view(1, 1, -1).expand(3, inputs_embeds.shape[0], -1)
+        elif position_ids.ndim == 2:
+            position_ids = position_ids[None, ...].expand(3, position_ids.shape[0], -1)
+
+        if position_ids.ndim == 3 and position_ids.shape[0] == 4:
+            position_ids = position_ids[1:]
+            t_position_ids = position_ids[1]
+        else:
+            t_position_ids = position_ids[0]
+
+
+        if not isinstance(causal_mask_mapping := attention_mask, dict):
+            mask_kwargs = {
+                "config": self.config,
+                "input_embeds": inputs_embeds,
+                "attention_mask": attention_mask,
+                "cache_position": cache_position,
+                "past_key_values": past_key_values,
+                "position_ids": t_position_ids,
+            }
+            causal_mask_mapping = {
+                "full_attention": create_causal_mask(**mask_kwargs),
+            }
+            if self.has_sliding_layers:
+                causal_mask_mapping["sliding_attention"] = create_sliding_window_causal_mask(
+                    **mask_kwargs)
+
+        hidden_states = inputs_embeds
+        all_hidden_states = ()
+
+        # create position embeddings to be shared across the decoder layers
+        position_embeddings = self.rotary_emb(hidden_states, position_ids)
+
+        for decoder_layer in self.layers[: self.config.num_hidden_layers]:
+            all_hidden_states += (hidden_states,)
+            hidden_states = decoder_layer(
+                hidden_states,
+                attention_mask=causal_mask_mapping[decoder_layer.attention_type],
+                position_ids=position_ids,
+                past_key_values=past_key_values,
+                use_cache=use_cache,
+                cache_position=cache_position,
+                position_embeddings=position_embeddings,
+                **kwargs,
+            )
+            if isinstance(hidden_states, tuple):
+                hidden_states = hidden_states[0]
+
+        hidden_states = self.norm(hidden_states)
+        all_hidden_states += (hidden_states,)
+        return BaseModelOutputWithPast(
+            last_hidden_state=hidden_states,
+            hidden_states=all_hidden_states,# for transformers library version compatability
+            past_key_values=past_key_values if use_cache else None,
+        )
+
+
+
+class LLaVABaselineConfig(Qwen2Config):
+    model_type = "llava_baseline"
+    keys_to_ignore_at_inference = ["past_key_values"]
+
+    def __init__(self,
+                 vit_path='Qwen/Qwen2.5-VL-3B-Instruct',
+                 llm_path='Qwen/Qwen3-4B',
+                 **kwargs):
+        self.vit_path = vit_path
+        self.llm_path = llm_path
+        super().__init__(**kwargs)
+
+        # Remove text_config and vision_config if they exist as dicts
+        # to prevent GenerationConfig from trying to call .to_dict() on them
+        if hasattr(self, 'text_config') and isinstance(self.text_config, dict):
+            delattr(self, 'text_config')
+        if hasattr(self, 'vision_config') and isinstance(self.vision_config, dict):
+            delattr(self, 'vision_config')
+
+
+class LLaVABaselinePreTrainedModel(Qwen2PreTrainedModel):
+    config_class = LLaVABaselineConfig
+
+
+class LLaVABaselineModel(LLaVABaselinePreTrainedModel):
+    def __init__(self, config: LLaVABaselineConfig):
+        super().__init__(config)
+        # self.vlm = Qwen2_5_VLModel.from_pretrained(
+        #     config.vit_path, low_cpu_mem_usage=True)
+        # self.vlm.language_model = Qwen3Model.from_pretrained(config.llm_path)
+        vlm_config = AutoConfig.from_pretrained(config.vit_path)
+        language_config = AutoConfig.from_pretrained(config.llm_path)
+        self.vlm = Qwen2_5_VLModel(vlm_config)
+        self.vlm.language_model = Qwen3Model(language_config)
+        self.vlm.language_model.rotary_emb = qwen25.Qwen2_5_VLRotaryEmbedding(
+            config=config)
+
+        # Set rope_scaling for each attention layer
+        for layer in self.vlm.language_model.layers:
+            layer.self_attn.rope_scaling = self.vlm.config.rope_scaling
+
+        # Adapt patch merger MLP output dimension to match LLM hidden size
+        llm_hidden_size = self.vlm.language_model.config.hidden_size
+        patch_merger = self.vlm.visual.merger
+        mlp_input_dim = patch_merger.hidden_size
+        original_output_dim = patch_merger.mlp[2].out_features
+        if original_output_dim != llm_hidden_size:
+            new_mlp = nn.Sequential(
+                nn.Linear(mlp_input_dim, mlp_input_dim),
+                nn.GELU(),
+                nn.Linear(mlp_input_dim, llm_hidden_size)
+            )
+            patch_merger.mlp = new_mlp
+
+        self.config: LLaVABaselineConfig
+
+    def forward(self, *args, **kwargs):
+        return self.vlm.forward(*args, **kwargs)
+
+
+class LLaVABaselineModelForConditionalGeneration(LLaVABaselinePreTrainedModel, GenerationMixin):
+    def __init__(self, config: LLaVABaselineConfig):
+        super().__init__(config)
+        self.model = LLaVABaselineModel(config)
+        self.lm_head = nn.Linear(self.model.vlm.language_model.config.hidden_size,
+                                 self.model.vlm.language_model.config.vocab_size, bias=False)
+
+        self.post_init()
+
+    def tie_weights(self):
+        """
+        Tie the weights between the input embeddings and the output embeddings.
+        """
+        if getattr(self.model.vlm.language_model.config.get_text_config(decoder=True), "tie_word_embeddings", True):
+            output_embeddings = self.get_output_embeddings()
+            if output_embeddings is not None:
+                self._tie_or_clone_weights(output_embeddings, self.get_input_embeddings())
+
+    def get_input_embeddings(self):
+        return self.model.vlm.get_input_embeddings()
+
+    def set_input_embeddings(self, value):
+        self.model.vlm.set_input_embeddings(value)
+
+    def get_output_embeddings(self):
+        return self.lm_head
+
+    def set_output_embeddings(self, new_embeddings):
+        self.lm_head = new_embeddings
+
+    def set_decoder(self, decoder):
+        self.model = decoder
+
+    def get_decoder(self):
+        return self.model
+
+    @property
+    def language_model(self):
+        return self.model.vlm.language_model
+
+    @property
+    def visual(self):
+        return self.model.vlm.visual
+
+    def forward(
+        self,
+        input_ids: torch.LongTensor = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_values: Optional[List[torch.FloatTensor]] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        labels: Optional[torch.LongTensor] = None,
+        use_cache: Optional[bool] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        return_dict: Optional[bool] = None,
+        pixel_values: Optional[torch.Tensor] = None,
+        pixel_values_videos: Optional[torch.FloatTensor] = None,
+        image_grid_thw: Optional[torch.LongTensor] = None,
+        video_grid_thw: Optional[torch.LongTensor] = None,
+        rope_deltas: Optional[torch.LongTensor] = None,
+        cache_position: Optional[torch.LongTensor] = None,
+        second_per_grid_ts: Optional[torch.Tensor] = None,
+        **kwargs
+    ) -> Union[Tuple, qwen25.Qwen2_5_VLCausalLMOutputWithPast]:
+        output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions
+        output_hidden_states = (
+            output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
+        )
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+
+        outputs = self.model(
+            input_ids=input_ids,
+            pixel_values=pixel_values,
+            pixel_values_videos=pixel_values_videos,
+            image_grid_thw=image_grid_thw,
+            video_grid_thw=video_grid_thw,
+            second_per_grid_ts=second_per_grid_ts,
+            position_ids=position_ids,
+            attention_mask=attention_mask,
+            past_key_values=past_key_values,
+            inputs_embeds=inputs_embeds,
+            use_cache=use_cache,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+            cache_position=cache_position,
+        )
+
+        hidden_states = outputs[0]
+        logits = self.lm_head(hidden_states)
+
+        loss = None
+        if labels is not None:
+            loss = self.loss_function(
+                logits=logits, labels=labels, vocab_size=self.config.vocab_size)
+
+            rank = dist.get_rank() if dist.is_initialized() else 'N/A'
+            num_items = (labels != -100).sum().item()
+            loss_sum = loss.item() * num_items
+
+        if not return_dict:
+            output = (logits,) + outputs[1:]
+            return (loss,) + output if loss is not None else output
+
+        return qwen25.Qwen2_5_VLCausalLMOutputWithPast(
+            loss=loss,
+            logits=logits,
+            past_key_values=outputs.past_key_values,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
+            rope_deltas=outputs.rope_deltas,
+        )
+
+    def prepare_inputs_for_generation(
+        self,
+        input_ids,
+        past_key_values=None,
+        attention_mask=None,
+        inputs_embeds=None,
+        cache_position=None,
+        position_ids=None,
+        use_cache=True,
+        pixel_values=None,
+        pixel_values_videos=None,
+        image_grid_thw=None,
+        video_grid_thw=None,
+        second_per_grid_ts=None,
+        **kwargs,
+    ):
+        return self.model.vlm.prepare_inputs_for_generation(input_ids, **kwargs)
+
+
+__all__ = ["LLaVABaselineModelForConditionalGeneration", "LLaVABaselineConfig"]

preprocessor_config.json

@@ -0,0 +1,19 @@
+{
+  "min_pixels": 3136,
+  "max_pixels": 12845056,
+  "patch_size": 14,
+  "temporal_patch_size": 2,
+  "merge_size": 2,
+  "image_mean": [
+    0.48145466,
+    0.4578275,
+    0.40821073
+  ],
+  "image_std": [
+    0.26862954,
+    0.26130258,
+    0.27577711
+  ],
+  "image_processor_type": "Qwen2VLImageProcessor",
+  "processor_class": "Qwen2_5_VLProcessor"
+}

pytorch_model-00001-of-00005.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2cb4b5a69b8c7ab2a446595d62dfd92b6c7142dad8c0f000b39f3f03fb3e0ede
+size 4992065246

pytorch_model-00002-of-00005.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7ae793f0f200ffa51af4d2bf12a38fe6ebbd55d967e7d0cfe86b0dc9404e4cf8
+size 4936657797

pytorch_model-00003-of-00005.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5f889c92122d11edb6663c5ed2fab30cacca751c85ec03e20484801cd8badf08
+size 4944341612

pytorch_model-00004-of-00005.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3e7f0197f41c99a829818cb095760bfe58a8ea4ad4d0e2daec3c9e8725eb495e
+size 4944341564

pytorch_model-00005-of-00005.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e09c1732837392a24dedba2ec50efa725f4a2a1be71ce8f7ed16aab38d3a4bfe
+size 1325524675

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:78dd0349b2245dd29033a8de5342142774df4631a4255d552ccf58b52c2da943
+size 11446736

tokenizer_config.json

@@ -0,0 +1,1379 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151665": {
+      "content": "<tool_response>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151666": {
+      "content": "</tool_response>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151667": {
+      "content": "<think>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151668": {
+      "content": "</think>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151669": {
+      "content": "<im_msg-0>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151670": {
+      "content": "<im_msg-1>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151671": {
+      "content": "<im_msg-2>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151672": {
+      "content": "<im_msg-3>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151673": {
+      "content": "<im_msg-4>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151674": {
+      "content": "<im_msg-5>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151675": {
+      "content": "<im_msg-6>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151676": {
+      "content": "<im_msg-7>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151677": {
+      "content": "<im_msg-8>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151678": {
+      "content": "<im_msg-9>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151679": {
+      "content": "<im_msg-10>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151680": {
+      "content": "<im_msg-11>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151681": {
+      "content": "<im_msg-12>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151682": {
+      "content": "<im_msg-13>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151683": {
+      "content": "<im_msg-14>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151684": {
+      "content": "<im_msg-15>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151685": {
+      "content": "<im_msg-16>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151686": {
+      "content": "<im_msg-17>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151687": {
+      "content": "<im_msg-18>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151688": {
+      "content": "<im_msg-19>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151689": {
+      "content": "<im_msg-20>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151690": {
+      "content": "<im_msg-21>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151691": {
+      "content": "<im_msg-22>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151692": {
+      "content": "<im_msg-23>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151693": {
+      "content": "<im_msg-24>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151694": {
+      "content": "<im_msg-25>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151695": {
+      "content": "<im_msg-26>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151696": {
+      "content": "<im_msg-27>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151697": {
+      "content": "<im_msg-28>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151698": {
+      "content": "<im_msg-29>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151699": {
+      "content": "<im_msg-30>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151700": {
+      "content": "<im_msg-31>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151701": {
+      "content": "<im_msg-32>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151702": {
+      "content": "<im_msg-33>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151703": {
+      "content": "<im_msg-34>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151704": {
+      "content": "<im_msg-35>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151705": {
+      "content": "<im_msg-36>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151706": {
+      "content": "<im_msg-37>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151707": {
+      "content": "<im_msg-38>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151708": {
+      "content": "<im_msg-39>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151709": {
+      "content": "<im_msg-40>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151710": {
+      "content": "<im_msg-41>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151711": {
+      "content": "<im_msg-42>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151712": {
+      "content": "<im_msg-43>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151713": {
+      "content": "<im_msg-44>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151714": {
+      "content": "<im_msg-45>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151715": {
+      "content": "<im_msg-46>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151716": {
+      "content": "<im_msg-47>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151717": {
+      "content": "<im_msg-48>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151718": {
+      "content": "<im_msg-49>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151719": {
+      "content": "<im_msg-50>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151720": {
+      "content": "<im_msg-51>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151721": {
+      "content": "<im_msg-52>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151722": {
+      "content": "<im_msg-53>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151723": {
+      "content": "<im_msg-54>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151724": {
+      "content": "<im_msg-55>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151725": {
+      "content": "<im_msg-56>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151726": {
+      "content": "<im_msg-57>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151727": {
+      "content": "<im_msg-58>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151728": {
+      "content": "<im_msg-59>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151729": {
+      "content": "<im_msg-60>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151730": {
+      "content": "<im_msg-61>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151731": {
+      "content": "<im_msg-62>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151732": {
+      "content": "<im_msg-63>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151733": {
+      "content": "<im_msg-64>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151734": {
+      "content": "<im_msg-65>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151735": {
+      "content": "<im_msg-66>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151736": {
+      "content": "<im_msg-67>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151737": {
+      "content": "<im_msg-68>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151738": {
+      "content": "<im_msg-69>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151739": {
+      "content": "<im_msg-70>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151740": {
+      "content": "<im_msg-71>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151741": {
+      "content": "<im_msg-72>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151742": {
+      "content": "<im_msg-73>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151743": {
+      "content": "<im_msg-74>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151744": {
+      "content": "<im_msg-75>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151745": {
+      "content": "<im_msg-76>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151746": {
+      "content": "<im_msg-77>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151747": {
+      "content": "<im_msg-78>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151748": {
+      "content": "<im_msg-79>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151749": {
+      "content": "<im_msg-80>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151750": {
+      "content": "<im_msg-81>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151751": {
+      "content": "<im_msg-82>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151752": {
+      "content": "<im_msg-83>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151753": {
+      "content": "<im_msg-84>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151754": {
+      "content": "<im_msg-85>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151755": {
+      "content": "<im_msg-86>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151756": {
+      "content": "<im_msg-87>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151757": {
+      "content": "<im_msg-88>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151758": {
+      "content": "<im_msg-89>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151759": {
+      "content": "<im_msg-90>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151760": {
+      "content": "<im_msg-91>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151761": {
+      "content": "<im_msg-92>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151762": {
+      "content": "<im_msg-93>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151763": {
+      "content": "<im_msg-94>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151764": {
+      "content": "<im_msg-95>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151765": {
+      "content": "<im_msg-96>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151766": {
+      "content": "<im_msg-97>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151767": {
+      "content": "<im_msg-98>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151768": {
+      "content": "<im_msg-99>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151769": {
+      "content": "<im_msg-100>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151770": {
+      "content": "<im_msg-101>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151771": {
+      "content": "<im_msg-102>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151772": {
+      "content": "<im_msg-103>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151773": {
+      "content": "<im_msg-104>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151774": {
+      "content": "<im_msg-105>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151775": {
+      "content": "<im_msg-106>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151776": {
+      "content": "<im_msg-107>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151777": {
+      "content": "<im_msg-108>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151778": {
+      "content": "<im_msg-109>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151779": {
+      "content": "<im_msg-110>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151780": {
+      "content": "<im_msg-111>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151781": {
+      "content": "<im_msg-112>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151782": {
+      "content": "<im_msg-113>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151783": {
+      "content": "<im_msg-114>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151784": {
+      "content": "<im_msg-115>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151785": {
+      "content": "<im_msg-116>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151786": {
+      "content": "<im_msg-117>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151787": {
+      "content": "<im_msg-118>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151788": {
+      "content": "<im_msg-119>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151789": {
+      "content": "<im_msg-120>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151790": {
+      "content": "<im_msg-121>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151791": {
+      "content": "<im_msg-122>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151792": {
+      "content": "<im_msg-123>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151793": {
+      "content": "<im_msg-124>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151794": {
+      "content": "<im_msg-125>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151795": {
+      "content": "<im_msg-126>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151796": {
+      "content": "<im_msg-127>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<im_msg-0>",
+    "<im_msg-1>",
+    "<im_msg-2>",
+    "<im_msg-3>",
+    "<im_msg-4>",
+    "<im_msg-5>",
+    "<im_msg-6>",
+    "<im_msg-7>",
+    "<im_msg-8>",
+    "<im_msg-9>",
+    "<im_msg-10>",
+    "<im_msg-11>",
+    "<im_msg-12>",
+    "<im_msg-13>",
+    "<im_msg-14>",
+    "<im_msg-15>",
+    "<im_msg-16>",
+    "<im_msg-17>",
+    "<im_msg-18>",
+    "<im_msg-19>",
+    "<im_msg-20>",
+    "<im_msg-21>",
+    "<im_msg-22>",
+    "<im_msg-23>",
+    "<im_msg-24>",
+    "<im_msg-25>",
+    "<im_msg-26>",
+    "<im_msg-27>",
+    "<im_msg-28>",
+    "<im_msg-29>",
+    "<im_msg-30>",
+    "<im_msg-31>",
+    "<im_msg-32>",
+    "<im_msg-33>",
+    "<im_msg-34>",
+    "<im_msg-35>",
+    "<im_msg-36>",
+    "<im_msg-37>",
+    "<im_msg-38>",
+    "<im_msg-39>",
+    "<im_msg-40>",
+    "<im_msg-41>",
+    "<im_msg-42>",
+    "<im_msg-43>",
+    "<im_msg-44>",
+    "<im_msg-45>",
+    "<im_msg-46>",
+    "<im_msg-47>",
+    "<im_msg-48>",
+    "<im_msg-49>",
+    "<im_msg-50>",
+    "<im_msg-51>",
+    "<im_msg-52>",
+    "<im_msg-53>",
+    "<im_msg-54>",
+    "<im_msg-55>",
+    "<im_msg-56>",
+    "<im_msg-57>",
+    "<im_msg-58>",
+    "<im_msg-59>",
+    "<im_msg-60>",
+    "<im_msg-61>",
+    "<im_msg-62>",
+    "<im_msg-63>",
+    "<im_msg-64>",
+    "<im_msg-65>",
+    "<im_msg-66>",
+    "<im_msg-67>",
+    "<im_msg-68>",
+    "<im_msg-69>",
+    "<im_msg-70>",
+    "<im_msg-71>",
+    "<im_msg-72>",
+    "<im_msg-73>",
+    "<im_msg-74>",
+    "<im_msg-75>",
+    "<im_msg-76>",
+    "<im_msg-77>",
+    "<im_msg-78>",
+    "<im_msg-79>",
+    "<im_msg-80>",
+    "<im_msg-81>",
+    "<im_msg-82>",
+    "<im_msg-83>",
+    "<im_msg-84>",
+    "<im_msg-85>",
+    "<im_msg-86>",
+    "<im_msg-87>",
+    "<im_msg-88>",
+    "<im_msg-89>",
+    "<im_msg-90>",
+    "<im_msg-91>",
+    "<im_msg-92>",
+    "<im_msg-93>",
+    "<im_msg-94>",
+    "<im_msg-95>",
+    "<im_msg-96>",
+    "<im_msg-97>",
+    "<im_msg-98>",
+    "<im_msg-99>",
+    "<im_msg-100>",
+    "<im_msg-101>",
+    "<im_msg-102>",
+    "<im_msg-103>",
+    "<im_msg-104>",
+    "<im_msg-105>",
+    "<im_msg-106>",
+    "<im_msg-107>",
+    "<im_msg-108>",
+    "<im_msg-109>",
+    "<im_msg-110>",
+    "<im_msg-111>",
+    "<im_msg-112>",
+    "<im_msg-113>",
+    "<im_msg-114>",
+    "<im_msg-115>",
+    "<im_msg-116>",
+    "<im_msg-117>",
+    "<im_msg-118>",
+    "<im_msg-119>",
+    "<im_msg-120>",
+    "<im_msg-121>",
+    "<im_msg-122>",
+    "<im_msg-123>",
+    "<im_msg-124>",
+    "<im_msg-125>",
+    "<im_msg-126>",
+    "<im_msg-127>"
+  ],
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 131072,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "left",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}