Upload folder using huggingface_hub

config.json

@@ -2,6 +2,11 @@
   "architectures": [
     "DeepQwenVLForCausalLM"
   ],
+  "auto_map": {
+    "AutoConfig": "model.DeepQwenVLConfig",
+    "AutoModel": "model.DeepQwenVLForCausalLM",
+    "AutoModelForCausalLM": "model.DeepQwenVLForCausalLM"
+  },
   "attention_dropout": 0.0,
   "bos_token_id": 151643,
   "deepseek_vision_hidden_size": 2048,

data.py

@@ -0,0 +1,758 @@
+from typing import List, Dict, Optional, Tuple
+from PIL import Image, ImageOps, ImageDraw, ImageFont
+import torch
+import torch.nn as nn
+from torchvision import transforms
+from transformers import TextStreamer
+from transformers.tokenization_utils import PreTrainedTokenizer as T
+from abc import ABC
+import re
+import numpy as np
+
+
+def load_image(image_path):
+    try:
+        image = Image.open(image_path)
+        corrected_image = ImageOps.exif_transpose(image)
+
+        return corrected_image
+        
+    except Exception as e:
+        print(f"error: {e}")
+
+        return None
+
+
+def re_match(text):
+    pattern = r'(<\|ref\|>(.*?)<\|/ref\|><\|det\|>(.*?)<\|/det\|>)'
+    matches = re.findall(pattern, text, re.DOTALL)
+
+    # pattern1 = r'<\|ref\|>.*?<\|/ref\|>\n'
+    # new_text1 = re.sub(pattern1, '', text, flags=re.DOTALL)
+
+    mathes_image = []
+    mathes_other = []
+    for a_match in matches:
+        if '<|ref|>image<|/ref|>' in a_match[0]:
+            mathes_image.append(a_match[0])
+        else:
+            mathes_other.append(a_match[0])
+    return matches, mathes_image, mathes_other
+
+
+def extract_coordinates_and_label(ref_text, image_width, image_height):
+
+    try:
+        label_type = ref_text[1]
+        cor_list = eval(ref_text[2])
+    except Exception as e:
+        print(e)
+        return None
+
+    return (label_type, cor_list)
+
+
+def draw_bounding_boxes(image, refs, ouput_path):
+
+    image_width, image_height = image.size
+    
+    img_draw = image.copy()
+    draw = ImageDraw.Draw(img_draw)
+
+    overlay = Image.new('RGBA', img_draw.size, (0, 0, 0, 0))
+    draw2 = ImageDraw.Draw(overlay)
+    
+    font = ImageFont.load_default()
+
+    img_idx = 0
+    
+    for i, ref in enumerate(refs):
+        try:
+            result = extract_coordinates_and_label(ref, image_width, image_height)
+            if result:
+                label_type, points_list = result
+                
+                color = (np.random.randint(0, 200), np.random.randint(0, 200), np.random.randint(0, 255))
+
+                color_a = color + (20, )
+                for points in points_list:
+                    x1, y1, x2, y2 = points
+
+                    x1 = int(x1 / 999 * image_width)
+                    y1 = int(y1 / 999 * image_height)
+
+                    x2 = int(x2 / 999 * image_width)
+                    y2 = int(y2 / 999 * image_height)
+
+                    if label_type == 'image':
+                        try:
+                            cropped = image.crop((x1, y1, x2, y2))
+                            cropped.save(f"{ouput_path}/images/{img_idx}.jpg")
+                        except Exception as e:
+                            print(e)
+                            pass
+                        img_idx += 1
+                        
+                    try:
+                        if label_type == 'title':
+                            draw.rectangle([x1, y1, x2, y2], outline=color, width=4)
+                            draw2.rectangle([x1, y1, x2, y2], fill=color_a, outline=(0, 0, 0, 0), width=1)
+                        else:
+                            draw.rectangle([x1, y1, x2, y2], outline=color, width=2)
+                            draw2.rectangle([x1, y1, x2, y2], fill=color_a, outline=(0, 0, 0, 0), width=1)
+                        text_x = x1
+                        text_y = max(0, y1 - 15)
+                            
+                        
+                        text_bbox = draw.textbbox((0, 0), label_type, font=font)
+                        text_width = text_bbox[2] - text_bbox[0]
+                        text_height = text_bbox[3] - text_bbox[1]
+                        draw.rectangle([text_x, text_y, text_x + text_width, text_y + text_height], 
+                                    fill=(255, 255, 255, 30))
+                        
+                        draw.text((text_x, text_y), label_type, font=font, fill=color)
+                    except:
+                        pass
+        except:
+            continue
+    img_draw.paste(overlay, (0, 0), overlay)
+    return img_draw
+
+
+def process_image_with_refs(image, ref_texts, output_path):
+
+    result_image = draw_bounding_boxes(image, ref_texts, output_path)
+    
+    return result_image
+
+
+def find_closest_aspect_ratio(aspect_ratio, target_ratios, width, height, image_size):
+    best_ratio_diff = float('inf')
+    best_ratio = (1, 1)
+    area = width * height
+    for ratio in target_ratios:
+        target_aspect_ratio = ratio[0] / ratio[1]
+        ratio_diff = abs(aspect_ratio - target_aspect_ratio)
+        if ratio_diff < best_ratio_diff:
+            best_ratio_diff = ratio_diff
+            best_ratio = ratio
+        elif ratio_diff == best_ratio_diff:
+            if area > 0.5 * image_size * image_size * ratio[0] * ratio[1]:
+                best_ratio = ratio
+
+    # print(f'width: {width}, height: {height}, best_ratio: {best_ratio}')
+    return best_ratio
+
+
+def dynamic_preprocess(image, min_num=2, max_num=9, image_size=640, use_thumbnail=False):
+    orig_width, orig_height = image.size
+    aspect_ratio = orig_width / orig_height
+
+    # calculate the existing image aspect ratio
+    target_ratios = set((i, j) for n in range(min_num, max_num + 1) for i in range(1, n + 1) for j in range(1, n + 1) if i * j <= max_num and i * j >= min_num)
+    
+    target_ratios = sorted(target_ratios, key=lambda x: x[0] * x[1])
+    # print(f"target_ratios: {target_ratios}")
+
+    target_aspect_ratio = find_closest_aspect_ratio(
+        aspect_ratio, 
+        target_ratios, 
+        orig_width,
+        orig_height, 
+        image_size
+    )
+    # print(f"target_aspect_ratio: {target_aspect_ratio}")
+
+    # calculate the target width and height
+    target_width = image_size * target_aspect_ratio[0]
+    target_height = image_size * target_aspect_ratio[1]
+    blocks = target_aspect_ratio[0] * target_aspect_ratio[1]
+
+    # resize the image
+    resized_img = image.resize((target_width, target_height))
+    processed_images = []
+    for i in range(blocks):
+        box = (
+            (i % (target_width // image_size)) * image_size,
+            (i // (target_width // image_size)) * image_size,
+            ((i % (target_width // image_size)) + 1) * image_size,
+            ((i // (target_width // image_size)) + 1) * image_size
+        )
+        # split the image
+        split_img = resized_img.crop(box)
+        processed_images.append(split_img)
+
+    assert len(processed_images) == blocks
+    # print(f"Number of processed images: {len(processed_images)}, Blocks: {blocks}")
+
+    if use_thumbnail and len(processed_images) != 1:
+        thumbnail_img = image.resize((image_size, image_size))
+        processed_images.append(thumbnail_img)
+    return processed_images, target_aspect_ratio
+
+
+def normalize_transform(mean, std):
+    if mean is None and std is None:
+        transform = None
+    elif mean is None and std is not None:
+        mean = [0.] * len(std)
+        transform = transforms.Normalize(mean=mean, std=std)
+    elif mean is not None and std is None:
+        std = [1.] * len(mean)
+        transform = transforms.Normalize(mean=mean, std=std)
+    else:
+        transform = transforms.Normalize(mean=mean, std=std)
+
+    return transform
+
+def format_messages(
+        tokenizer: T,
+        conversations: List[Dict[str, str]],
+        system_prompt: str = "",
+):
+    if system_prompt is not None and system_prompt != "":
+        sys_prompt = {
+            "role": "system",
+            "content": system_prompt,
+        }
+        conversations = [sys_prompt] + conversations
+
+    sft_prompt = tokenizer.apply_chat_template(
+        conversations,
+    )
+
+    return sft_prompt
+
+
+def text_encode(tokenizer, text: str, bos: bool = True, eos: bool = False):
+    """
+    Encode text with optional BOS/EOS tokens.
+    
+    Note: Qwen2VL tokenizer has bos_token_id=None, so we skip BOS for Qwen.
+    The chat template handles special tokens automatically.
+    """
+    t = tokenizer.encode(text, add_special_tokens=False)
+    bos_id = tokenizer.bos_token_id
+    eos_id = tokenizer.eos_token_id
+
+    # Only add BOS if tokenizer has one AND bos=True
+    if bos and bos_id is not None:
+        t = [bos_id] + t
+    
+    # Only add EOS if tokenizer has one AND eos=True
+    if eos and eos_id is not None:
+        t = t + [eos_id]
+
+    return t
+
+def load_pil_images(conversations: List[Dict[str, str]]) -> List[Image.Image]:
+    pil_images = []
+
+    for message in conversations:
+        pil_image = None
+    
+        if message["role"].lower() == "user":
+            if isinstance(message["content"], List):
+                for d in message["content"]:
+                    if d.get("type", "") == "image":
+                        # Support both "image" (Qwen format) and "data" keys
+                        image_path = d.get("image") or d.get("data", "")
+                        pil_image = load_image(image_path)
+
+            elif isinstance(message["content"], Dict):
+                if message["content"].get("type", "") == "image":
+                    # Support both "image" (Qwen format) and "data" keys
+                    image_path = message["content"].get("image") or message["content"].get("data", "")
+                    pil_image = load_image(image_path)
+
+            if pil_image is not None:
+                pil_images.append(pil_image)
+
+    return pil_images
+
+
+class BaseTransform(ABC):
+
+    def set_rng(self, *args, **kwargs):
+        pass
+
+    def __call__(self, *args, **kwargs) -> torch.Tensor:
+        pass
+
+    @property
+    def default_shape(self):
+        raise NotImplementedError
+
+
+class BasicImageTransform(BaseTransform):
+    def __init__(
+        self, 
+        mean: Optional[Tuple[float, float, float]] = (0.5, 0.5, 0.5),
+        std: Optional[Tuple[float, float, float]] = (0.5, 0.5, 0.5),
+        normalize: bool = True
+    ):
+        self.mean = mean
+        self.std = std
+    
+        transform_pipelines = [
+            transforms.ToTensor()
+        ]
+
+        normalize = normalize_transform(mean, std) if normalize else nn.Identity()
+        if normalize is not None:
+            transform_pipelines.append(normalize)
+
+        self.transform = transforms.Compose(transform_pipelines)
+    
+    def __call__(self, x):
+        x = self.transform(x)
+        return x
+
+class NoEOSTextStreamer(TextStreamer):
+    
+    def on_finalized_text(self, text: str, stream_end: bool = False):
+        eos_text = self.tokenizer.decode([self.tokenizer.eos_token_id], skip_special_tokens=False)
+        text = text.replace(eos_text, "\n")
+        print(text, flush=True, end="")
+
+
+
+
+# @title Create datacollator
+
+import torch
+import math
+from dataclasses import dataclass
+from typing import Dict, List, Any, Tuple
+from PIL import Image, ImageOps
+from torch.nn.utils.rnn import pad_sequence
+import io
+
+# Use local functions (Qwen-compatible) instead of DeepSeek's versions
+# from deepseek_ocr.modeling_deepseekocr import (
+#     format_messages,
+#     text_encode,
+#     BasicImageTransform,
+#     dynamic_preprocess,
+# )
+
+
+@dataclass
+class DeepQwenDataCollator:
+    """
+    Data collator for DeepQwen model using Qwen2VL tokenizer.
+    
+    This collator processes images using DeepSeek OCR's dynamic cropping algorithm
+    while maintaining compatibility with Qwen2VL's tokenization format.
+    
+    Key token mappings (Qwen2VL):
+        - image_token: <|image_pad|> (id=151655)
+        - vision_start: <|vision_start|> (id=151652)
+        - vision_end: <|vision_end|> (id=151653)
+        - eos_token: <|im_end|> (id=151645)
+        - NO bos_token (bos_token_id is None)
+    
+    Args:
+        tokenizer: Qwen2VL Tokenizer
+        model: Model
+        image_size: Size for image patches (default: 640)
+        base_size: Size for global view (default: 1024)
+        crop_mode: Whether to use dynamic cropping for large images
+        train_on_responses_only: If True, only train on assistant responses (mask user prompts)
+    """
+    tokenizer: T
+    model: Any
+    image_size: int = 640
+    base_size: int = 1024
+    crop_mode: bool = True
+    train_on_responses_only: bool = True
+
+    def __init__(
+        self,
+        tokenizer,
+        model,
+        image_size: int = 640,
+        base_size: int = 1024,
+        crop_mode: bool = True,
+        train_on_responses_only: bool = True,
+        max_length: int = None,
+    ):
+        self.tokenizer = tokenizer
+        self.model = model
+        self.image_size = image_size
+        self.base_size = base_size
+        self.crop_mode = crop_mode
+        self.dtype = model.dtype  # Get dtype from model
+        self.train_on_responses_only = train_on_responses_only
+        self.max_length = max_length  # None means no truncation
+        
+        # Qwen2VL specific token IDs
+        # <|image_pad|> = 151655
+        self.image_token_id = getattr(tokenizer, 'image_token_id', None)
+        if self.image_token_id is None:
+            # Fallback: try to get from added_tokens or use default Qwen2VL value
+            self.image_token_id = 151655  # Qwen2VL's <|image_pad|>
+        
+        self.image_token = tokenizer.decode([self.image_token_id], skip_special_tokens=False)
+        
+        # Vision wrapper tokens for Qwen2VL format
+        self.vision_start_token_id = getattr(tokenizer, 'vision_start_token_id', 151652)
+        self.vision_end_token_id = getattr(tokenizer, 'vision_end_token_id', 151653)
+
+        self.image_transform = BasicImageTransform(
+            mean=(0.5, 0.5, 0.5),
+            std=(0.5, 0.5, 0.5),
+            normalize=True
+        )
+        self.patch_size = 16
+        self.downsample_ratio = 4
+
+        # Qwen2VL has NO bos_token (bos_token_id is None)
+        # The chat template handles conversation formatting
+        self.bos_id = tokenizer.bos_token_id  # Will be None for Qwen2VL
+        self.eos_id = tokenizer.eos_token_id  # 151645 for Qwen2VL
+        self.pad_token_id = tokenizer.pad_token_id  # 151643 for Qwen2VL
+
+    def deserialize_image(self, image_data) -> Image.Image:
+        """Convert image data (bytes dict, PIL Image, or file path) to PIL Image in RGB mode"""
+        if isinstance(image_data, Image.Image):
+            return image_data.convert("RGB")
+        elif isinstance(image_data, str):
+            # File path - load lazily
+            image = load_image(image_data)
+            if image is None:
+                raise ValueError(f"Failed to load image from path: {image_data}")
+            return image.convert("RGB")
+        elif isinstance(image_data, dict) and 'bytes' in image_data:
+            image_bytes = image_data['bytes']
+            image = Image.open(io.BytesIO(image_bytes))
+            return image.convert("RGB")
+        else:
+            raise ValueError(f"Unsupported image format: {type(image_data)}")
+
+    def calculate_image_token_count(self, image: Image.Image, crop_ratio: Tuple[int, int]) -> int:
+        """Calculate the number of tokens this image will generate"""
+        num_queries = math.ceil((self.image_size // self.patch_size) / self.downsample_ratio)
+        num_queries_base = math.ceil((self.base_size // self.patch_size) / self.downsample_ratio)
+
+        width_crop_num, height_crop_num = crop_ratio
+
+        if self.crop_mode:
+            img_tokens = num_queries_base * num_queries_base + 1
+            if width_crop_num > 1 or height_crop_num > 1:
+                img_tokens += (num_queries * width_crop_num + 1) * (num_queries * height_crop_num)
+        else:
+            img_tokens = num_queries * num_queries + 1
+
+        return img_tokens
+
+    def process_image(self, image: Image.Image) -> Tuple[List, List, List, List, Tuple[int, int]]:
+        """
+        Process a single image based on crop_mode and size thresholds
+
+        Returns:
+            Tuple of (images_list, images_crop_list, images_spatial_crop, tokenized_image, crop_ratio)
+        """
+        images_list = []
+        images_crop_list = []
+        images_spatial_crop = []
+
+        if self.crop_mode:
+            # Determine crop ratio based on image size
+            if image.size[0] <= 640 and image.size[1] <= 640:
+                crop_ratio = (1, 1)
+                images_crop_raw = []
+            else:
+                images_crop_raw, crop_ratio = dynamic_preprocess(
+                    image, min_num=2, max_num=9,
+                    image_size=self.image_size, use_thumbnail=False
+                )
+
+            # Process global view with padding
+            global_view = ImageOps.pad(
+                image, (self.base_size, self.base_size),
+                color=tuple(int(x * 255) for x in self.image_transform.mean)
+            )
+            images_list.append(self.image_transform(global_view).to(self.dtype))
+
+            width_crop_num, height_crop_num = crop_ratio
+            images_spatial_crop.append([width_crop_num, height_crop_num])
+
+            # Process local views (crops) if applicable
+            if width_crop_num > 1 or height_crop_num > 1:
+                for crop_img in images_crop_raw:
+                    images_crop_list.append(
+                        self.image_transform(crop_img).to(self.dtype)
+                    )
+
+            # Calculate image tokens
+            num_queries = math.ceil((self.image_size // self.patch_size) / self.downsample_ratio)
+            num_queries_base = math.ceil((self.base_size // self.patch_size) / self.downsample_ratio)
+
+            tokenized_image = ([self.image_token_id] * num_queries_base + [self.image_token_id]) * num_queries_base
+            tokenized_image += [self.image_token_id]
+
+            if width_crop_num > 1 or height_crop_num > 1:
+                tokenized_image += ([self.image_token_id] * (num_queries * width_crop_num) + [self.image_token_id]) * (
+                    num_queries * height_crop_num)
+
+        else:  # crop_mode = False
+            crop_ratio = (1, 1)
+            images_spatial_crop.append([1, 1])
+
+            # For smaller base sizes, resize; for larger, pad
+            if self.base_size <= 640:
+                resized_image = image.resize((self.base_size, self.base_size), Image.LANCZOS)
+                images_list.append(self.image_transform(resized_image).to(self.dtype))
+            else:
+                global_view = ImageOps.pad(
+                    image, (self.base_size, self.base_size),
+                    color=tuple(int(x * 255) for x in self.image_transform.mean)
+                )
+                images_list.append(self.image_transform(global_view).to(self.dtype))
+
+            num_queries = math.ceil((self.base_size // self.patch_size) / self.downsample_ratio)
+            tokenized_image = ([self.image_token_id] * num_queries + [self.image_token_id]) * num_queries
+            tokenized_image += [self.image_token_id]
+
+        return images_list, images_crop_list, images_spatial_crop, tokenized_image, crop_ratio
+
+    def process_single_sample(self, messages: List[Dict]) -> Dict[str, Any]:
+            """
+            Process a single conversation into model inputs.
+            
+            Expected message format (Qwen2.5-VL native style):
+            [
+                {
+                    "role": "user",
+                    "content": [
+                        {"type": "image", "image": <PIL.Image or path or bytes>},
+                        {"type": "text", "text": "Describe this image."}
+                    ]
+                },
+                {
+                    "role": "assistant", 
+                    "content": [{"type": "text", "text": "This is a description..."}]
+                }
+            ]
+            
+            Also supports string content for backward compatibility.
+            """
+
+            # --- 1. Setup ---
+            tokenized_str = []
+            images_seq_mask = []
+            images_list, images_crop_list, images_spatial_crop = [], [], []
+
+            prompt_token_count = -1  # Index to start training
+            assistant_started = False
+
+            # Qwen2VL has NO bos_token, so we don't add one
+
+            for message in messages:
+                role = message["role"].lower()  # Normalize role to lowercase
+                content = message["content"]
+
+                # Check if this is the assistant's turn
+                if role == "assistant":
+                    if not assistant_started:
+                        # This is the split point. All tokens added *so far*
+                        # are part of the prompt.
+                        prompt_token_count = len(tokenized_str)
+                        assistant_started = True
+
+                # Process content based on format
+                if isinstance(content, list):
+                    # Qwen2.5-VL native format: content is a list of typed items
+                    content_parts = []
+                    
+                    for item in content:
+                        item_type = item.get("type", "")
+                        
+                        if item_type == "image":
+                            # Get image data from various possible keys
+                            image_data = item.get("image") or item.get("data")
+                            if image_data is not None:
+                                pil_image = self.deserialize_image(image_data)
+                                
+                                # Process the image through DeepSeek's encoder
+                                img_list, crop_list, spatial_crop, tok_img, _ = self.process_image(pil_image)
+                                
+                                images_list.extend(img_list)
+                                images_crop_list.extend(crop_list)
+                                images_spatial_crop.extend(spatial_crop)
+                                
+                                # Add image placeholder tokens
+                                tokenized_str.extend(tok_img)
+                                images_seq_mask.extend([True] * len(tok_img))
+                                
+                        elif item_type == "text":
+                            text = item.get("text", "")
+                            
+                            # For assistant, append EOS at the end of all text
+                            if role == "assistant" and item == content[-1]:
+                                if self.tokenizer.eos_token:
+                                    text = f"{text.strip()}{self.tokenizer.eos_token}"
+                            
+                            # Tokenize the text
+                            tokenized_text = text_encode(self.tokenizer, text, bos=False, eos=False)
+                            tokenized_str.extend(tokenized_text)
+                            images_seq_mask.extend([False] * len(tokenized_text))
+                            
+                else:
+                    # Legacy format: content is a string (backward compatibility)
+                    text_content = content
+                    
+                    # For assistant, append EOS token
+                    if role == "assistant" and self.tokenizer.eos_token:
+                        text_content = f"{text_content.strip()}{self.tokenizer.eos_token}"
+                    
+                    # Tokenize the text
+                    tokenized_text = text_encode(self.tokenizer, text_content, bos=False, eos=False)
+                    tokenized_str.extend(tokenized_text)
+                    images_seq_mask.extend([False] * len(tokenized_text))
+
+            # --- 2. Validation and Final Prep ---
+            # If we never found an assistant message, we're in a weird state
+            # (e.g., user-only prompt). We mask everything.
+            if not assistant_started:
+                print("Warning: No assistant message found in sample. Masking all tokens.")
+                prompt_token_count = len(tokenized_str)
+
+            # # DEBUG: Print after processing
+            # print(f"[DEBUG] tokenized_str length: {len(tokenized_str)}")
+            # print(f"[DEBUG] images_seq_mask length: {len(images_seq_mask)}, True count: {sum(images_seq_mask)}")
+            # print(f"[DEBUG] images_list length: {len(images_list)}")
+            # print(f"[DEBUG] images_crop_list length: {len(images_crop_list)}")
+            # print(f"[DEBUG] prompt_token_count: {prompt_token_count}")
+
+            # Prepare image tensors
+            images_ori = torch.stack(images_list, dim=0)
+            images_spatial_crop_tensor = torch.tensor(images_spatial_crop, dtype=torch.long)
+
+            if images_crop_list:
+                images_crop = torch.stack(images_crop_list, dim=0)
+            else:
+                images_crop = torch.zeros((1, 3, self.base_size, self.base_size), dtype=self.dtype)
+
+            return {
+                "input_ids": torch.tensor(tokenized_str, dtype=torch.long),
+                "images_seq_mask": torch.tensor(images_seq_mask, dtype=torch.bool),
+                "images_ori": images_ori,
+                "images_crop": images_crop,
+                "images_spatial_crop": images_spatial_crop_tensor,
+                "prompt_token_count": prompt_token_count, # This is now accurate
+            }
+
+    def __call__(self, features: List[Dict[str, Any]]) -> Dict[str, torch.Tensor]:
+        """
+        Collate batch of samples.
+        
+        Expected feature format:
+        {
+            "prompt": str,  # The user's question/instruction
+            "response": str,  # The assistant's response
+            "image": PIL.Image or bytes dict  # The image
+        }
+        
+        This will be converted to Qwen2.5-VL native conversation format:
+        [
+            {
+                "role": "user", 
+                "content": [
+                    {"type": "image", "image": <PIL.Image>},
+                    {"type": "text", "text": "<prompt>"}
+                ]
+            },
+            {
+                "role": "assistant", 
+                "content": [{"type": "text", "text": "<response>"}]
+            }
+        ]
+        """
+        batch_data = []
+
+        # Process each sample
+        for feature in features:
+            try:
+                # Get image from either 'image' or 'image_path' key (lazy loading support)
+                image_data = feature.get('image') or feature.get('image_path')
+                if image_data is None:
+                    raise ValueError("Sample missing both 'image' and 'image_path' keys")
+
+                # Use Qwen2.5-VL native message format
+                # content is a list of typed items: {"type": "image", ...} or {"type": "text", ...}
+                messages = [
+                    {
+                        "role": "user",
+                        "content": [
+                            {"type": "image", "image": image_data},
+                            {"type": "text", "text": feature['prompt']}
+                        ]
+                    },
+                    {
+                        "role": "assistant",
+                        "content": [
+                            {"type": "text", "text": feature["response"]}
+                        ]
+                    }
+                ]
+                
+                processed = self.process_single_sample(messages)
+                batch_data.append(processed)
+            except Exception as e:
+                print(f"Error processing sample: {e}")
+                continue
+
+        if not batch_data:
+            raise ValueError("No valid samples in batch")
+
+        # Extract lists
+        input_ids_list = [item['input_ids'] for item in batch_data]
+        images_seq_mask_list = [item['images_seq_mask'] for item in batch_data]
+        prompt_token_counts = [item['prompt_token_count'] for item in batch_data]
+
+        # Pad sequences using Qwen2VL's pad_token_id (151643 = <|endoftext|>)
+        input_ids = pad_sequence(input_ids_list, batch_first=True, padding_value=self.pad_token_id)
+        images_seq_mask = pad_sequence(images_seq_mask_list, batch_first=True, padding_value=False)
+
+        # Truncate to max_length if specified (prevents OOM on long sequences)
+        if self.max_length is not None and input_ids.shape[1] > self.max_length:
+            input_ids = input_ids[:, :self.max_length]
+            images_seq_mask = images_seq_mask[:, :self.max_length]
+            # Adjust prompt_token_counts if they exceed max_length
+            prompt_token_counts = [min(p, self.max_length) for p in prompt_token_counts]
+
+        # Create labels
+        labels = input_ids.clone()
+
+        # Mask padding tokens
+        labels[labels == self.pad_token_id] = -100
+
+        # Mask image tokens (model shouldn't predict these)
+        labels[images_seq_mask] = -100
+
+        # Mask user prompt tokens when train_on_responses_only=True (only train on assistant responses)
+        if self.train_on_responses_only:
+            for idx, prompt_count in enumerate(prompt_token_counts):
+                if prompt_count > 0:
+                    labels[idx, :prompt_count] = -100
+
+        # Create attention mask
+        attention_mask = (input_ids != self.pad_token_id).long()
+
+        images_batch = []
+        for item in batch_data:
+            images_batch.append((item['images_crop'], item['images_ori']))
+
+        images_spatial_crop = torch.cat([item['images_spatial_crop'] for item in batch_data], dim=0)
+
+        return {
+            "input_ids": input_ids,
+            "attention_mask": attention_mask,
+            "labels": labels,
+            "images": images_batch,
+            "images_seq_mask": images_seq_mask,
+            "images_spatial_crop": images_spatial_crop,
+        }
+

encoder.py

@@ -0,0 +1,1052 @@
+import torch.nn as nn
+import torch
+import torch.nn.functional as F
+import copy
+
+from contextlib import nullcontext
+import math
+from typing import Optional, Tuple
+# from megatron.model import LayerNorm
+
+from einops import rearrange
+from easydict import EasyDict as adict
+
+from typing import Optional, Tuple, Type
+from functools import partial
+
+class MlpProjector(nn.Module):
+    def __init__(self, cfg):
+        super().__init__()
+        self.cfg = cfg
+
+        if cfg.projector_type == "identity":
+            modules = nn.Identity()
+
+        elif cfg.projector_type == "linear":
+            modules = nn.Linear(cfg.input_dim, cfg.n_embed)
+
+        elif cfg.projector_type == "mlp_gelu":
+            mlp_depth = cfg.get("depth", 1)
+            modules = [nn.Linear(cfg.input_dim, cfg.n_embed)]
+            for _ in range(1, mlp_depth):
+                modules.append(nn.GELU())
+                modules.append(nn.Linear(cfg.n_embed, cfg.n_embed))
+            modules = nn.Sequential(*modules)
+        
+        elif cfg.projector_type == "normlayer_downsample_mlp_gelu":
+            mlp_depth = cfg.get("depth", 1)
+            mlp_ratio = cfg.get("mlp_ratio", 1)
+            modules = [
+                nn.LayerNorm(cfg.input_dim * cfg.downsample_ratio * cfg.downsample_ratio),
+                nn.Linear(cfg.input_dim * cfg.downsample_ratio * cfg.downsample_ratio, cfg.n_embed * mlp_ratio)
+            ]
+            for _ in range(1, mlp_depth - 1):
+                modules.append(nn.GELU())
+                modules.append(nn.Linear(cfg.n_embed * mlp_ratio, cfg.n_embed * mlp_ratio))
+            modules.append(nn.GELU())
+            modules.append(nn.Linear(cfg.n_embed * mlp_ratio, cfg.n_embed))
+            modules = nn.Sequential(*modules)
+        
+        elif cfg.projector_type == "downsample_mlp_gelu":
+            mlp_depth = cfg.get("depth", 1)
+            mlp_ratio = cfg.get("mlp_ratio", 1)
+            modules = [nn.Linear(cfg.input_dim * cfg.downsample_ratio * cfg.downsample_ratio, cfg.n_embed * mlp_ratio)]
+            for _ in range(1, mlp_depth - 1):
+                modules.append(nn.GELU())
+                modules.append(nn.Linear(cfg.n_embed * mlp_ratio, cfg.n_embed * mlp_ratio))
+            modules.append(nn.GELU())
+            modules.append(nn.Linear(cfg.n_embed * mlp_ratio, cfg.n_embed))
+            modules = nn.Sequential(*modules)
+
+        elif cfg.projector_type == "low_high_hybrid_split_mlp_gelu":
+            mlp_depth = cfg.get("depth", 1)
+            self.high_up_proj = nn.Linear(cfg.input_dim, cfg.n_embed // 2)
+            self.low_up_proj = nn.Linear(cfg.input_dim, cfg.n_embed // 2)
+
+            modules = []
+            for _ in range(1, mlp_depth):
+                modules.append(nn.GELU())
+                modules.append(nn.Linear(cfg.n_embed, cfg.n_embed))
+            modules = nn.Sequential(*modules)
+
+        elif cfg.projector_type == "hybrid_split_feature_mlp_gelu":
+            mlp_depth = cfg.get("depth", 1)
+            channel_div = cfg.get("channel_div", 0.5)
+            self.high_up_proj = nn.Linear(cfg.input_dim[0], int(cfg.n_embed * channel_div))
+            self.low_up_proj = nn.Linear(cfg.input_dim[1], cfg.n_embed - int(cfg.n_embed * channel_div))
+
+            modules = []
+            for _ in range(1, mlp_depth):
+                modules.append(nn.GELU())
+                modules.append(nn.Linear(cfg.n_embed, cfg.n_embed))
+            modules = nn.Sequential(*modules)
+
+        elif cfg.projector_type == "low_high_split_mlp_gelu":
+            mlp_depth = cfg.get("depth", 1)
+            modules = []
+            for _ in range(1, mlp_depth):
+                modules.append(nn.GELU())
+                modules.append(nn.Linear(cfg.n_embed // 2, cfg.n_embed // 2))
+            modules = nn.Sequential(*modules)
+            self.high_layers = nn.Sequential(*modules)
+            self.low_layers = copy.deepcopy(modules)
+
+        else:
+            raise ValueError(f"Unknown projector type: {cfg.projector_type}")
+
+        if cfg.get("token_pooling", False):
+            self.token_pooling_layer = nn.Linear(cfg.input_dim * 4, cfg.input_dim)
+
+        if cfg.get("conv_fusion_high_low_features", False):
+            self.fusion_layer = nn.Linear(cfg.input_dim, cfg.input_dim)
+        self.layers = modules
+
+    def forward(self, x):
+        if self.cfg.get("token_pooling", False):
+            batch_size, wxh, channels = x.shape
+            w = h = int(wxh**0.5)
+            x = x.view(batch_size, w, h, channels)
+            x = x.permute(0, 3, 1, 2)
+            # import ipdb; ipdb.set_trace()
+            patches = x.unfold(2, 2, 2).unfold(3, 2, 2)
+            batch_size, channels, h_patches, w_patches, _, _ = patches.size()
+            # 在通道维度上拼接
+            patches = patches.contiguous().view(batch_size, channels, h_patches * w_patches, -1)
+
+            # 通过线性层
+            patches = patches.permute(0, 2, 1, 3).contiguous()
+            patches = patches.view(batch_size, h_patches * w_patches, channels * 4)
+
+            x = self.token_pooling_layer(patches)
+        
+        if self.cfg.get("conv_fusion_high_low_features", False):
+            x = self.fusion_layer(x[:, 0]) + x[:, 1]
+
+        if self.cfg.projector_type == 'low_high_hybrid_split_mlp_gelu':
+            high_x, low_x = x[0], x[1]
+            high_x = self.high_up_proj(high_x)
+            low_x = self.low_up_proj(low_x)
+            x = torch.concat([high_x, low_x], dim=-1)
+        
+        if self.cfg.projector_type == 'hybrid_split_feature_mlp_gelu':
+            high_x = x[...,:self.cfg.input_dim[0]]
+            low_x = x[...,self.cfg.input_dim[0]:]
+            high_x = self.high_up_proj(high_x)
+            low_x = self.low_up_proj(low_x)
+            x = torch.concat([high_x, low_x], dim=-1)
+        
+        if self.cfg.projector_type == 'low_high_split_mlp_gelu':
+            high_x, low_x = x[0], x[1]
+            high_x = self.high_layers(high_x)
+            low_x = self.low_layers(low_x)
+            x = torch.concat([high_x, low_x], dim=-1)
+            return x
+        
+        if self.cfg.projector_type == 'downsample_mlp_gelu' or self.cfg.projector_type == 'normlayer_downsample_mlp_gelu':
+            bs, hw, input_dim = x.shape
+            h = w = int((hw) ** 0.5)
+
+            """compute padding"""
+            if h % self.cfg.downsample_ratio:
+                pad = self.cfg.downsample_ratio - h % self.cfg.downsample_ratio
+            else:
+                pad = 0
+            x = x.reshape(bs, h, w, input_dim)
+            if pad > 0:
+                x = F.pad(x, (0, 0, 0, pad, 0, pad), "constant", 0)
+
+            """4 to 1 concat"""
+            x = x.permute(0, 3, 1, 2)  # B, C, H, W
+            x = F.unfold(x, kernel_size=self.cfg.downsample_ratio, stride=self.cfg.downsample_ratio, padding=0) # B, C*4, HW // 4
+            x = x.permute(0, 2, 1)
+            
+        return self.layers(x)
+
+    @staticmethod
+    def get_flops_per_sample(cfg):
+        if cfg.projector_type == "linear":
+            fwd = 2 * cfg.input_dim * cfg.n_embed
+
+        elif "mlp_gelu" in cfg.projector_type :
+            mlp_depth = cfg.get("depth", 1)
+            downsample_ratio = cfg.get("downsample_ratio", 1)
+            input_dim = sum(cfg.input_dim) if isinstance(cfg.input_dim, list) else cfg.input_dim
+            input_dim = input_dim * downsample_ratio * downsample_ratio
+            fwd = 2 * input_dim * cfg.n_embed + (mlp_depth - 1) * 2 * cfg.n_embed * cfg.n_embed
+        else:
+            fwd = 0
+
+        return fwd * 3
+    
+
+#===================clip============================================================
+
+class LayerNormfp32(torch.nn.LayerNorm):
+    """Subclass torch's LayerNorm to handle fp16."""
+
+    def forward(self, x: torch.Tensor):
+        orig_type = x.dtype
+        ret = super().forward(x.type(torch.float32))
+        return ret.type(orig_type)
+
+
+def get_abs_pos(abs_pos, tgt_size):
+    # abs_pos: L, C
+    # tgt_size: M
+    # return: M, C
+
+    # print(tgt_size)
+    # print(abs_pos.shape)
+    # exit()
+    dim = abs_pos.size(-1)
+    # print(dim)
+    abs_pos_new = abs_pos.squeeze(0)
+    cls_token, old_pos_embed = abs_pos_new[:1], abs_pos_new[1:]
+
+
+
+    src_size = int(math.sqrt(abs_pos_new.shape[0] - 1))
+    tgt_size = int(math.sqrt(tgt_size))
+    dtype = abs_pos.dtype
+
+    if src_size != tgt_size:
+        old_pos_embed = old_pos_embed.view(1, src_size, src_size, dim).permute(0, 3, 1,
+                                                                                    2).contiguous()
+        old_pos_embed = old_pos_embed.to(torch.float32)
+        new_pos_embed = F.interpolate(
+            old_pos_embed,
+            size=(tgt_size, tgt_size),
+            mode='bicubic',
+            antialias=True,
+            align_corners=False,
+        ).to(dtype)
+        new_pos_embed = new_pos_embed.permute(0, 2, 3, 1)
+        new_pos_embed = new_pos_embed.view(tgt_size * tgt_size, dim)
+        vision_pos_embed = torch.cat([cls_token, new_pos_embed], dim=0)
+        vision_pos_embed = vision_pos_embed.view(1, tgt_size * tgt_size + 1, dim)
+        return vision_pos_embed
+    else:
+        return abs_pos
+
+@torch.jit.script
+def quick_gelu(x):
+    return x * torch.sigmoid(1.702 * x)
+
+
+
+class CLIPVisionEmbeddings(nn.Module):
+    def __init__(self, hidden_size=1024, image_size=224, patch_size=14, num_channels=3):
+        super().__init__()
+        self.embed_dim = hidden_size
+        self.image_size = image_size
+        self.patch_size = patch_size
+
+        self.class_embedding = torch.nn.Parameter(torch.randn(self.embed_dim))
+
+        self.patch_embedding = torch.nn.Conv2d(
+            in_channels=num_channels,
+            out_channels=self.embed_dim,
+            kernel_size=self.patch_size,
+            stride=self.patch_size,
+            bias=False,
+        )
+
+        self.num_patches = (self.image_size // self.patch_size) ** 2
+        self.num_positions = self.num_patches + 1
+        self.position_embedding = torch.nn.Embedding(self.num_positions, self.embed_dim)
+        self.register_buffer(
+            "position_ids", torch.arange(self.num_positions).expand((1, -1))
+        )
+
+    def forward(self, pixel_values, patch_embeds):
+        batch_size = pixel_values.shape[0]
+        # patch_embeds = self.patch_embedding(
+        #     pixel_values
+        # )  # shape = [*, width, grid, grid]
+
+
+        if patch_embeds is not None:
+            patch_embeds = patch_embeds
+            # print(patch_embeds.shape)
+        else:
+            patch_embeds = self.patch_embedding(pixel_values)  
+            # print(111111)
+        # shape = [*, width, grid, grid]
+        # patch_embeds = patch_embeds.flatten(2).transpose(1, 2)
+
+        patch_embeds = patch_embeds.flatten(2).transpose(1, 2)
+
+
+        class_embeds = self.class_embedding.expand(batch_size, 1, -1)
+        embeddings = torch.cat([class_embeds, patch_embeds], dim=1)
+
+        # x = torch.cat([cls_token, x], dim=1)
+        embeddings = embeddings + get_abs_pos(self.position_embedding(self.position_ids), embeddings.size(1))
+        # embeddings = embeddings + self.position_embedding(self.position_ids)
+        return embeddings
+
+
+class NoTPFeedForward(nn.Module):
+    def __init__(
+            self,
+            cfg,
+            dim: int,
+            hidden_dim: int,
+    ):
+        super().__init__()
+
+        self.fc1 = torch.nn.Linear(dim, hidden_dim, bias=True)
+        self.fc2 = torch.nn.Linear(hidden_dim, dim, bias=True)
+
+    def forward(self, x):
+        output = self.fc2(quick_gelu(self.fc1(x)))
+        return output
+
+
+
+
+class NoTPAttention(torch.nn.Module):
+    def __init__(self, cfg):
+        super().__init__()
+        self.num_heads = cfg.num_attention_heads
+        self.n_local_heads = cfg.num_attention_heads
+        self.head_dim = cfg.hidden_size // cfg.num_attention_heads
+        self.max_seq_len = cfg.seq_length
+        self.use_flash_attention = cfg.use_flash_attn
+
+        self.qkv_proj = torch.nn.Linear(cfg.hidden_size, cfg.hidden_size * 3, bias=True)
+        self.out_proj = torch.nn.Linear(cfg.hidden_size, cfg.hidden_size, bias=True)
+
+        # self.core_attention = CoreAttention(cfg, AttnType.self_attn)
+
+        self.attn_drop = cfg.attention_dropout
+
+    def forward(
+            self,
+            x: torch.Tensor,
+    ):
+        bsz, seqlen, _ = x.shape
+        xqkv = self.qkv_proj(x)
+        xqkv = xqkv.view(bsz, seqlen, 3, self.num_heads, self.head_dim)
+
+        if self.use_flash_attention:
+
+            xq, xk, xv = torch.split(xqkv, 1, dim=2)
+            xq = xq.squeeze(2)
+            xk = xk.squeeze(2)
+            xv = xv.squeeze(2)
+            # xq, xk, xv = xqkv[:, :, 0, ...], xqkv[:, :, 1, ...], xqkv[:, :, 2, ...]
+
+            # （B, num_head, S, head_size)
+            xq = xq.permute(0, 2, 1, 3)
+            xk = xk.permute(0, 2, 1, 3)
+            xv = xv.permute(0, 2, 1, 3)
+            # with torch.backends.cuda.sdp_kernel(enable_flash=True, enable_math=False, enable_mem_efficient=False):
+            output = torch.nn.functional.scaled_dot_product_attention(xq, xk, xv, attn_mask=None)
+            output = output.permute(0, 2, 1, 3).reshape(bsz, seqlen, -1)
+                # output = output.permute(0, 2, 1, 3).contiguous().view(bsz, seqlen, -1)
+        else:
+            # print(22222)
+            xq, xk, xv = torch.split(xqkv, 1, dim=2)
+            xq = xq.squeeze(2)
+            xk = xk.squeeze(2)
+            xv = xv.squeeze(2)
+            # xq, xk, xv = xqkv[:, :, 0, ...], xqkv[:, :, 1, ...], xqkv[:, :, 2, ...]
+
+            # （B, num_head, S, head_size)
+            xq = xq.permute(0, 2, 1, 3)
+            xk = xk.permute(0, 2, 1, 3)
+            xv = xv.permute(0, 2, 1, 3)
+            # with torch.backends.cuda.sdp_kernel(enable_flash=True, enable_math=False, enable_mem_efficient=False):
+            output = torch.nn.functional.scaled_dot_product_attention(xq, xk, xv, attn_mask=None)
+            output = output.permute(0, 2, 1, 3).reshape(bsz, seqlen, -1)
+            # output = output.permute(0, 2, 1, 3).contiguous().view(bsz, seqlen, -1)
+        output = self.out_proj(output)
+        return output
+
+class NoTPTransformerBlock(nn.Module):
+    def __init__(self, cfg, layer_id: int, multiple_of=256):
+        super().__init__()
+
+        self.n_heads = cfg.num_attention_heads
+        self.dim = cfg.hidden_size
+        self.head_dim = cfg.hidden_size // cfg.num_attention_heads
+        self.self_attn = NoTPAttention(cfg)
+        self.mlp = NoTPFeedForward(
+            cfg, dim=cfg.hidden_size, hidden_dim=cfg.ffn_hidden_size
+        )
+        self.layer_id = layer_id
+        self.layer_norm1 = torch.nn.LayerNorm(
+            cfg.hidden_size, eps=cfg.layernorm_epsilon
+        )
+        self.layer_norm2 = torch.nn.LayerNorm(
+            cfg.hidden_size, eps=cfg.layernorm_epsilon
+        )
+
+    def forward(self, x: torch.Tensor):
+        residual = self.self_attn.forward(self.layer_norm1(x))
+        h = x + residual
+        out = h + self.mlp.forward(self.layer_norm2(h))
+        return out
+
+
+class NoTPTransformer(nn.Module):
+    def __init__(self, cfg):
+        super().__init__()
+
+        self.cfg = cfg
+        # self.recompute_list = self.cfg.get("recompute_list", [])
+        self.num_layers = cfg.num_layers  # _get_num_layers(cfg)
+
+        self.layers = torch.nn.ModuleList()
+        for layer_id in range(self.num_layers):
+            self.layers.append(
+                NoTPTransformerBlock(
+                    cfg,
+                    layer_id + 1,
+                )
+            )
+
+    def forward(
+            self,
+            hidden_states,
+    ):
+
+        for lid, layer in enumerate(self.layers):
+            # if lid in self.recompute_list:
+            #     def custom(layer_id):
+            #         def custom_forward(*args, **kwargs):
+            #             x_ = self.layers[layer_id](*args, **kwargs)
+            #             return x_
+
+            #         return custom_forward
+
+            #     assert hidden_states.requires_grad == True, logger.warning(
+            #         "When using recalculation, the input must have grad fn"
+            #     )
+            #     hidden_states = tensor_parallel.checkpoint(
+            #         custom(lid),
+            #         False,
+            #         hidden_states.contiguous()
+            #     )
+            # else:
+            hidden_states = layer(hidden_states)
+
+        return hidden_states
+
+
+# from megatron.core.tensor_parallel.layers import non_tensor_paralleled, local_dp_reduce, local_dp_scatter
+
+class VitModel(nn.Module):
+    def __init__(
+            self,
+            cfg,
+            freeze_embed=False,
+            freeze_pre_norm=False
+    ) -> None:
+        super().__init__()
+
+        self.embeddings = CLIPVisionEmbeddings(hidden_size=cfg.hidden_size, image_size=cfg.image_size, patch_size=cfg.patch_size)
+
+        if freeze_embed:
+            for name, param in self.embeddings.named_parameters():
+                param.requires_grad = False
+
+        self.transformer = NoTPTransformer(cfg=cfg)
+
+        if cfg.get("fp32norm", False):
+            logger.info("Load fp32 layernorm for ViT.")
+            self.pre_layrnorm = LayerNormfp32(
+                cfg.hidden_size,
+                eps=cfg.get("pre_layernorm_epsilon", 1e-5),
+            )
+        else:
+            self.pre_layrnorm = torch.nn.LayerNorm(
+                cfg.hidden_size,
+                eps=cfg.get("pre_layernorm_epsilon", 1e-5),
+            )
+
+        # self.pre_layrnorm = RMSNorm(
+        #     cfg.hidden_size,
+        #     eps=cfg.get("pre_layernorm_epsilon", 1e-5),
+        #     sequence_parallel=False,
+        #     use_fp32=True,
+        #     use_optimus=True,
+        # )
+
+        if freeze_pre_norm:
+            for name, param in self.pre_layrnorm.named_parameters():
+                param.requires_grad = False
+
+        for p in self.parameters():
+            p.micro_dp = True
+
+    def set_input_tensor(self, input_tensor):
+        if not isinstance(input_tensor, list):
+            input_tensor = [input_tensor]
+        self.transformer.set_input_tensor(input_tensor[0])
+
+    def __str__(self) -> str:
+        return "open_clip"
+
+    def forward(
+            self,
+            x,
+            patch_embeds
+    ):
+        x = self.embeddings(x, patch_embeds)
+        hidden_states = self.pre_layrnorm(x)
+
+        # hidden_states, dis = local_dp_scatter(hidden_states)
+        output = self.transformer(hidden_states)
+
+        # output = local_dp_reduce(output, dis)
+
+        return output
+
+
+vit_model_cfg = adict(
+    num_layers=24,
+    hidden_size=1024,
+    num_heads = 16,
+    num_attention_heads=16,
+    ffn_hidden_size=4096,
+    seq_length=256,
+    max_position_embeddings=256,
+    use_flash_attn=False,
+    understand_projector_stride=2,
+    hidden_dropout = 0.0,
+    attention_dropout = 0.0,
+    no_persist_layer_norm = False,
+    layernorm_epsilon = 1e-5,
+    pre_layernorm_epsilon = 1e-5,
+    image_size = 224,
+    patch_size = 14,
+    recompute_list = []
+)
+
+def build_clip_l():
+    return VitModel(
+        cfg=vit_model_cfg,
+        freeze_embed=False,
+        freeze_pre_norm=False,
+    )
+
+
+
+
+
+#=========================Sam-Vary=================================
+
+
+def get_abs_pos_sam(abs_pos, tgt_size):
+
+    dtype = abs_pos.dtype
+
+    src_size = abs_pos.size(1)
+
+    if src_size != tgt_size:
+        old_pos_embed = abs_pos.permute(0, 3, 1, 2)
+        old_pos_embed = old_pos_embed.to(torch.float32)
+        new_pos_embed = F.interpolate(
+            old_pos_embed,
+            size=(tgt_size, tgt_size),
+            mode='bicubic',
+            antialias=True,
+            align_corners=False,
+        ).to(dtype)
+        new_pos_embed = new_pos_embed.permute(0, 2, 3, 1)
+        return new_pos_embed
+    else:
+        return abs_pos
+
+
+
+
+class MLPBlock(nn.Module):
+    def __init__(
+        self,
+        embedding_dim: int,
+        mlp_dim: int,
+        act: Type[nn.Module] = nn.GELU,
+    ) -> None:
+        super().__init__()
+        self.lin1 = nn.Linear(embedding_dim, mlp_dim)
+        self.lin2 = nn.Linear(mlp_dim, embedding_dim)
+        self.act = act()
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        return self.lin2(self.act(self.lin1(x)))
+
+
+# From https://github.com/facebookresearch/detectron2/blob/main/detectron2/layers/batch_norm.py # noqa
+# Itself from https://github.com/facebookresearch/ConvNeXt/blob/d1fa8f6fef0a165b27399986cc2bdacc92777e40/models/convnext.py#L119  # noqa
+class LayerNorm2d(nn.Module):
+    def __init__(self, num_channels: int, eps: float = 1e-6) -> None:
+        super().__init__()
+        self.weight = nn.Parameter(torch.ones(num_channels))
+        self.bias = nn.Parameter(torch.zeros(num_channels))
+        self.eps = eps
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        u = x.mean(1, keepdim=True)
+        s = (x - u).pow(2).mean(1, keepdim=True)
+        x = (x - u) / torch.sqrt(s + self.eps)
+        x = self.weight[:, None, None] * x + self.bias[:, None, None]
+        return x
+
+
+# This class and its supporting functions below lightly adapted from the ViTDet backbone available at: https://github.com/facebookresearch/detectron2/blob/main/detectron2/modeling/backbone/vit.py # noqa
+class ImageEncoderViT(nn.Module):
+    def __init__(
+        self,
+        img_size: int = 1024,
+        patch_size: int = 16,
+        in_chans: int = 3,
+        embed_dim: int = 768,
+        depth: int = 12,
+        num_heads: int = 12,
+        mlp_ratio: float = 4.0,
+        out_chans: int = 256,
+        qkv_bias: bool = True,
+        norm_layer: Type[nn.Module] = nn.LayerNorm,
+        act_layer: Type[nn.Module] = nn.GELU,
+        use_abs_pos: bool = True,
+        use_rel_pos: bool = False,
+        rel_pos_zero_init: bool = True,
+        window_size: int = 0,
+        global_attn_indexes: Tuple[int, ...] = (),
+    ) -> None:
+        """
+        Args:
+            img_size (int): Input image size.
+            patch_size (int): Patch size.
+            in_chans (int): Number of input image channels.
+            embed_dim (int): Patch embedding dimension.
+            depth (int): Depth of ViT.
+            num_heads (int): Number of attention heads in each ViT block.
+            mlp_ratio (float): Ratio of mlp hidden dim to embedding dim.
+            qkv_bias (bool): If True, add a learnable bias to query, key, value.
+            norm_layer (nn.Module): Normalization layer.
+            act_layer (nn.Module): Activation layer.
+            use_abs_pos (bool): If True, use absolute positional embeddings.
+            use_rel_pos (bool): If True, add relative positional embeddings to the attention map.
+            rel_pos_zero_init (bool): If True, zero initialize relative positional parameters.
+            window_size (int): Window size for window attention blocks.
+            global_attn_indexes (list): Indexes for blocks using global attention.
+        """
+        super().__init__()
+        self.img_size = img_size
+
+        self.patch_embed = PatchEmbed(
+            kernel_size=(patch_size, patch_size),
+            stride=(patch_size, patch_size),
+            in_chans=in_chans,
+            embed_dim=embed_dim,
+        )
+
+        self.pos_embed: Optional[nn.Parameter] = None
+        if use_abs_pos:
+            # Initialize absolute positional embedding with pretrain image size.
+            self.pos_embed = nn.Parameter(
+                torch.zeros(1, img_size // patch_size, img_size // patch_size, embed_dim)
+            )
+
+        self.blocks = nn.ModuleList()
+        for i in range(depth):
+            block = Block(
+                dim=embed_dim,
+                num_heads=num_heads,
+                mlp_ratio=mlp_ratio,
+                qkv_bias=qkv_bias,
+                norm_layer=norm_layer,
+                act_layer=act_layer,
+                use_rel_pos=use_rel_pos,
+                rel_pos_zero_init=rel_pos_zero_init,
+                window_size=window_size if i not in global_attn_indexes else 0,
+                input_size=(img_size // patch_size, img_size // patch_size),
+            )
+            self.blocks.append(block)
+
+        self.neck = nn.Sequential(
+            nn.Conv2d(
+                embed_dim,
+                out_chans,
+                kernel_size=1,
+                bias=False,
+            ),
+            LayerNorm2d(out_chans),
+            nn.Conv2d(
+                out_chans,
+                out_chans,
+                kernel_size=3,
+                padding=1,
+                bias=False,
+            ),
+            LayerNorm2d(out_chans),
+        )
+
+        self.net_2 = nn.Conv2d(256, 512, kernel_size=3, stride=2, padding=1, bias=False)
+        self.net_3 = nn.Conv2d(512, 1024, kernel_size=3, stride=2, padding=1, bias=False)
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        x = self.patch_embed(x)
+        if self.pos_embed is not None:
+            # x = x + self.pos_embed
+            x = x + get_abs_pos_sam(self.pos_embed, x.size(1))
+
+        for blk in self.blocks:
+            x = blk(x)
+
+        x = self.neck(x.permute(0, 3, 1, 2))
+        x2 = self.net_2(x)
+        x3 = self.net_3(x2.clone())
+
+        return x3
+
+
+class Block(nn.Module):
+    """Transformer blocks with support of window attention and residual propagation blocks"""
+
+    def __init__(
+        self,
+        dim: int,
+        num_heads: int,
+        mlp_ratio: float = 4.0,
+        qkv_bias: bool = True,
+        norm_layer: Type[nn.Module] = nn.LayerNorm,
+        act_layer: Type[nn.Module] = nn.GELU,
+        use_rel_pos: bool = False,
+        rel_pos_zero_init: bool = True,
+        window_size: int = 0,
+        input_size: Optional[Tuple[int, int]] = None,
+    ) -> None:
+        """
+        Args:
+            dim (int): Number of input channels.
+            num_heads (int): Number of attention heads in each ViT block.
+            mlp_ratio (float): Ratio of mlp hidden dim to embedding dim.
+            qkv_bias (bool): If True, add a learnable bias to query, key, value.
+            norm_layer (nn.Module): Normalization layer.
+            act_layer (nn.Module): Activation layer.
+            use_rel_pos (bool): If True, add relative positional embeddings to the attention map.
+            rel_pos_zero_init (bool): If True, zero initialize relative positional parameters.
+            window_size (int): Window size for window attention blocks. If it equals 0, then
+                use global attention.
+            input_size (tuple(int, int) or None): Input resolution for calculating the relative
+                positional parameter size.
+        """
+        super().__init__()
+        self.norm1 = norm_layer(dim)
+        self.attn = Attention(
+            dim,
+            num_heads=num_heads,
+            qkv_bias=qkv_bias,
+            use_rel_pos=use_rel_pos,
+            rel_pos_zero_init=rel_pos_zero_init,
+            input_size=input_size if window_size == 0 else (window_size, window_size),
+        )
+
+        self.norm2 = norm_layer(dim)
+        self.mlp = MLPBlock(embedding_dim=dim, mlp_dim=int(dim * mlp_ratio), act=act_layer)
+
+        self.window_size = window_size
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        shortcut = x
+        x = self.norm1(x)
+        # Window partition
+        if self.window_size > 0:
+            H, W = x.shape[1], x.shape[2]
+            x, pad_hw = window_partition(x, self.window_size)
+
+        x = self.attn(x)
+        # Reverse window partition
+        if self.window_size > 0:
+            x = window_unpartition(x, self.window_size, pad_hw, (H, W))
+
+        x = shortcut + x
+        x = x + self.mlp(self.norm2(x))
+
+        return x
+
+
+class Attention(nn.Module):
+    """Multi-head Attention block with relative position embeddings."""
+
+    def __init__(
+        self,
+        dim: int,
+        num_heads: int = 8,
+        qkv_bias: bool = True,
+        use_rel_pos: bool = False,
+        rel_pos_zero_init: bool = True,
+        input_size: Optional[Tuple[int, int]] = None,
+    ) -> None:
+        """
+        Args:
+            dim (int): Number of input channels.
+            num_heads (int): Number of attention heads.
+            qkv_bias (bool):  If True, add a learnable bias to query, key, value.
+            rel_pos (bool): If True, add relative positional embeddings to the attention map.
+            rel_pos_zero_init (bool): If True, zero initialize relative positional parameters.
+            input_size (tuple(int, int) or None): Input resolution for calculating the relative
+                positional parameter size.
+        """
+        super().__init__()
+        self.num_heads = num_heads
+        head_dim = dim // num_heads
+        self.scale = head_dim**-0.5
+
+        self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias)
+        self.proj = nn.Linear(dim, dim)
+
+        self.use_rel_pos = use_rel_pos
+        if self.use_rel_pos:
+            assert (
+                input_size is not None
+            ), "Input size must be provided if using relative positional encoding."
+            # initialize relative positional embeddings
+            self.rel_pos_h = nn.Parameter(torch.zeros(2 * input_size[0] - 1, head_dim))
+            self.rel_pos_w = nn.Parameter(torch.zeros(2 * input_size[1] - 1, head_dim))
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        B, H, W, _ = x.shape
+        # qkv with shape (3, B, nHead, H * W, C)
+        qkv = self.qkv(x).reshape(B, H * W, 3, self.num_heads, -1).permute(2, 0, 3, 1, 4)
+        # q, k, v with shape (B * nHead, H * W, C)
+        q, k, v = qkv.reshape(3, B * self.num_heads, H * W, -1).unbind(0)
+
+        rel_h, rel_w = None, None
+        if self.use_rel_pos:
+            rel_h, rel_w = add_decomposed_rel_pos(q, self.rel_pos_h, self.rel_pos_w, (H, W), (H, W))
+
+        q = q.view(B, self.num_heads, H * W, -1)
+        k = k.view(B, self.num_heads, H * W, -1)
+        v = v.view(B, self.num_heads, H * W, -1)
+
+        if self.use_rel_pos:
+            rel_h = rel_h.view(B, self.num_heads, rel_h.size(1), rel_h.size(2), rel_h.size(3))
+            rel_w = rel_w.view(B, self.num_heads, rel_w.size(1), rel_w.size(2), rel_w.size(3))
+            attn_bias = (rel_h + rel_w).view(B, self.num_heads, rel_h.size(2), rel_h.size(3) * rel_w.size(4))
+            x = torch.nn.functional.scaled_dot_product_attention(q, k, v, attn_mask=attn_bias)
+            # x = _attention_rel_h_rel_w(q, k, v, rel_h, rel_w)
+        else:
+            x = torch.nn.functional.scaled_dot_product_attention(q, k, v)
+
+        x = x.view(B, self.num_heads, H, W, -1).permute(0, 2, 3, 1, 4).reshape(B, H, W, -1)
+
+        x = self.proj(x)
+
+        return x
+
+
+def window_partition(x: torch.Tensor, window_size: int) -> Tuple[torch.Tensor, Tuple[int, int]]:
+    """
+    Partition into non-overlapping windows with padding if needed.
+    Args:
+        x (tensor): input tokens with [B, H, W, C].
+        window_size (int): window size.
+
+    Returns:
+        windows: windows after partition with [B * num_windows, window_size, window_size, C].
+        (Hp, Wp): padded height and width before partition
+    """
+    B, H, W, C = x.shape
+
+    pad_h = (window_size - H % window_size) % window_size
+    pad_w = (window_size - W % window_size) % window_size
+    if pad_h > 0 or pad_w > 0:
+        x = F.pad(x, (0, 0, 0, pad_w, 0, pad_h))
+    Hp, Wp = H + pad_h, W + pad_w
+
+    x = x.view(B, Hp // window_size, window_size, Wp // window_size, window_size, C)
+    windows = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(-1, window_size, window_size, C)
+    return windows, (Hp, Wp)
+
+
+def window_unpartition(
+    windows: torch.Tensor, window_size: int, pad_hw: Tuple[int, int], hw: Tuple[int, int]
+) -> torch.Tensor:
+    """
+    Window unpartition into original sequences and removing padding.
+    Args:
+        windows (tensor): input tokens with [B * num_windows, window_size, window_size, C].
+        window_size (int): window size.
+        pad_hw (Tuple): padded height and width (Hp, Wp).
+        hw (Tuple): original height and width (H, W) before padding.
+
+    Returns:
+        x: unpartitioned sequences with [B, H, W, C].
+    """
+    Hp, Wp = pad_hw
+    H, W = hw
+    B = windows.shape[0] // (Hp * Wp // window_size // window_size)
+    x = windows.view(B, Hp // window_size, Wp // window_size, window_size, window_size, -1)
+    x = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(B, Hp, Wp, -1)
+
+    if Hp > H or Wp > W:
+        x = x[:, :H, :W, :].contiguous()
+    return x
+
+
+def get_rel_pos(q_size: int, k_size: int, rel_pos: torch.Tensor) -> torch.Tensor:
+    """
+    Get relative positional embeddings according to the relative positions of
+        query and key sizes.
+    Args:
+        q_size (int): size of query q.
+        k_size (int): size of key k.
+        rel_pos (Tensor): relative position embeddings (L, C).
+
+    Returns:
+        Extracted positional embeddings according to relative positions.
+    """
+    max_rel_dist = int(2 * max(q_size, k_size) - 1)
+    # Interpolate rel pos if needed.
+    if rel_pos.shape[0] != max_rel_dist:
+        # Interpolate rel pos.
+        dtype = rel_pos.dtype
+        rel_pos = rel_pos.to(torch.float32)
+        rel_pos_resized = F.interpolate(
+            rel_pos.reshape(1, rel_pos.shape[0], -1).permute(0, 2, 1),
+            size=max_rel_dist,
+            mode="linear",
+        ).to(dtype)
+        rel_pos_resized = rel_pos_resized.reshape(-1, max_rel_dist).permute(1, 0)
+    else:
+        rel_pos_resized = rel_pos
+
+    # Scale the coords with short length if shapes for q and k are different.
+    q_coords = torch.arange(q_size, device=rel_pos.device)[:, None] * max(k_size / q_size, 1.0)
+    k_coords = torch.arange(k_size, device=rel_pos.device)[None, :] * max(q_size / k_size, 1.0)
+    relative_coords = (q_coords - k_coords) + (k_size - 1) * max(q_size / k_size, 1.0)
+
+    return rel_pos_resized[relative_coords.long()]
+
+
+def add_decomposed_rel_pos(
+    q: torch.Tensor,
+    rel_pos_h: torch.Tensor,
+    rel_pos_w: torch.Tensor,
+    q_size: Tuple[int, int],
+    k_size: Tuple[int, int],
+) -> torch.Tensor:
+    """
+    Calculate decomposed Relative Positional Embeddings from :paper:`mvitv2`.
+    https://github.com/facebookresearch/mvit/blob/19786631e330df9f3622e5402b4a419a263a2c80/mvit/models/attention.py   # noqa B950
+    Args:
+        q (Tensor): query q in the attention layer with shape (B, q_h * q_w, C).
+        rel_pos_h (Tensor): relative position embeddings (Lh, C) for height axis.
+        rel_pos_w (Tensor): relative position embeddings (Lw, C) for width axis.
+        q_size (Tuple): spatial sequence size of query q with (q_h, q_w).
+        k_size (Tuple): spatial sequence size of key k with (k_h, k_w).
+
+    Returns:
+        attn (Tensor): attention map with added relative positional embeddings.
+    """
+    q_h, q_w = q_size
+    k_h, k_w = k_size
+    Rh = get_rel_pos(q_h, k_h, rel_pos_h)
+    Rw = get_rel_pos(q_w, k_w, rel_pos_w)
+
+    B, _, dim = q.shape
+    r_q = q.reshape(B, q_h, q_w, dim)
+    rel_h = torch.einsum("bhwc,hkc->bhwk", r_q, Rh)
+    rel_w = torch.einsum("bhwc,wkc->bhwk", r_q, Rw)
+    rel_h = rel_h.unsqueeze(-1)
+    rel_w = rel_w.unsqueeze(-2)
+    rel_h = rel_h.reshape(B, q_h * q_w, k_h, 1)
+    rel_w = rel_w.reshape(B, q_h * q_w, 1, k_w)
+
+    return rel_h, rel_w
+
+
+class PatchEmbed(nn.Module):
+    """
+    Image to Patch Embedding.
+    """
+
+    def __init__(
+        self,
+        kernel_size: Tuple[int, int] = (16, 16),
+        stride: Tuple[int, int] = (16, 16),
+        padding: Tuple[int, int] = (0, 0),
+        in_chans: int = 3,
+        embed_dim: int = 768,
+    ) -> None:
+        """
+        Args:
+            kernel_size (Tuple): kernel size of the projection layer.
+            stride (Tuple): stride of the projection layer.
+            padding (Tuple): padding size of the projection layer.
+            in_chans (int): Number of input image channels.
+            embed_dim (int): Patch embedding dimension.
+        """
+        super().__init__()
+
+        self.proj = nn.Conv2d(
+            in_chans, embed_dim, kernel_size=kernel_size, stride=stride, padding=padding
+        )
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        x = self.proj(x)
+        # B C H W -> B H W C
+        x = x.permute(0, 2, 3, 1)
+        return x
+
+
+def build_sam_vit_b(checkpoint=None):
+    return _build_sam(
+        encoder_embed_dim=768,
+        encoder_depth=12,
+        encoder_num_heads=12,
+        encoder_global_attn_indexes=[2, 5, 8, 11],
+        checkpoint=checkpoint,
+    )
+
+def build_sam_fast_vit_b(checkpoint=None, compile_mode='max-autotune', dtype=torch.bfloat16):
+    image_encoder = build_sam_vit_b(checkpoint).eval().to(dtype)
+    # sam = _apply_eval_dtype_sam(sam, dtype)
+    image_encoder = torch.compile(image_encoder, mode=compile_mode)
+    return image_encoder
+
+
+def _build_sam(
+    encoder_embed_dim,
+    encoder_depth,
+    encoder_num_heads,
+    encoder_global_attn_indexes,
+    checkpoint=None,
+):
+    prompt_embed_dim = 256
+    image_size = 1024
+    vit_patch_size = 16
+    image_embedding_size = image_size // vit_patch_size
+    image_encoder=ImageEncoderViT(
+            depth=encoder_depth,
+            embed_dim=encoder_embed_dim,
+            img_size=image_size,
+            mlp_ratio=4,
+            norm_layer=partial(torch.nn.LayerNorm, eps=1e-6),
+            num_heads=encoder_num_heads,
+            patch_size=vit_patch_size,
+            qkv_bias=True,
+            use_rel_pos=True,
+            global_attn_indexes=encoder_global_attn_indexes,
+            window_size=14,
+            out_chans=prompt_embed_dim,
+        )
+    image_encoder.eval()
+    if checkpoint is not None:
+        # with open(checkpoint, "rb") as f:
+        state_dict = torch.load(checkpoint)
+        # print(state_dict.keys())
+        # for key in state_dict:
+        # image_encoder.load_state_dict({k[14:]: v for k, v in state_dict.items() if 'image_encoder' in k}, strict=False)
+        # ocr-anyting
+        # image_encoder.load_state_dict(state_dict, strict=True)
+        # tob
+        image_encoder.load_state_dict({k[30:]: v for k, v in state_dict.items() if 'vision_tower_high' in k}, strict=True)
+        print(checkpoint)
+    return image_encoder

model.py

@@ -0,0 +1,950 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from typing import Optional, List, Union, Tuple
+from transformers import Qwen2VLTextModel, Qwen2VLTextConfig, Qwen2VLPreTrainedModel, PretrainedConfig
+from transformers.models.llama.modeling_llama import LlamaAttention, LlamaRotaryEmbedding
+from transformers.generation.utils import GenerationMixin
+from transformers.modeling_utils import PreTrainedModel
+from transformers.modeling_outputs import ModelOutput
+from PIL import Image, ImageOps
+from encoder import build_sam_vit_b, build_clip_l, MlpProjector
+from addict import Dict as ADict
+import os
+import math
+from data import (
+    format_messages,
+    load_pil_images,
+    text_encode,
+    BasicImageTransform,
+    dynamic_preprocess,
+    re_match,
+    process_image_with_refs,
+    NoEOSTextStreamer,
+)
+from tqdm import tqdm
+from dataclasses import dataclass
+
+
+class DeepQwenVLConfig(PretrainedConfig):
+    """
+    Configuration class for DeepQwenVL model.
+    
+    This config wraps both the Qwen2VL text config and DeepSeek vision config.
+    When loading from a Qwen2-VL checkpoint, it will use the checkpoint's config
+    directly for the text model.
+    """
+    model_type = "deepqwen_vl"
+    
+    def __init__(
+        self,
+        deepseek_vision_hidden_size: int = 2048,
+        
+        # Projector settings
+        projector_type: str = "mlp",  # "vision_projector" or "mlp"
+        projector_input_dim: int = 2048,
+        projector_output_dim: int = None,
+        projector_hidden_dim: int = None,  # If None, uses projector_output_dim
+        
+        # Learnable vision tokens
+        image_newline_dim: int = None,  # If None, uses hidden_size
+        view_separator_dim: int = None,  # If None, uses hidden_size
+        
+        hidden_size: int = 1536,
+        intermediate_size: int = 8960,
+        num_hidden_layers: int = 28,
+        num_attention_heads: int = 12,
+        num_key_value_heads: int = 2,
+        hidden_act: str = "silu",
+        max_position_embeddings: int = 32768,
+        initializer_range: float = 0.02,
+        rms_norm_eps: float = 1e-6,
+        use_cache: bool = True,
+        tie_word_embeddings: bool = True,
+        rope_theta: float = 1000000.0,
+        attention_dropout: float = 0.0,
+        vocab_size: int = 151936,
+        
+        bos_token_id: int = 151643,
+        eos_token_id: int = 151645,
+        pad_token_id: int = 151643,
+        image_token_id: int = 151655,
+        video_token_id: int = 151656,
+        vision_start_token_id: int = 151652,
+        vision_end_token_id: int = 151653,
+        vision_token_id: int = 151654,
+        
+        rope_scaling: dict = None,
+        
+        **kwargs
+    ):
+        super().__init__(
+            bos_token_id=bos_token_id,
+            eos_token_id=eos_token_id,
+            pad_token_id=pad_token_id,
+            tie_word_embeddings=tie_word_embeddings,
+            **kwargs
+        )
+        
+        self.deepseek_vision_hidden_size = deepseek_vision_hidden_size
+        
+        # Projector settings
+        self.projector_type = projector_type
+        self.projector_input_dim = projector_input_dim
+        self.projector_output_dim = projector_output_dim if projector_output_dim else hidden_size
+        self.projector_hidden_dim = projector_hidden_dim if projector_hidden_dim else self.projector_output_dim
+        
+        # Learnable vision tokens
+        self.image_newline_dim = image_newline_dim if image_newline_dim else hidden_size
+        self.view_separator_dim = view_separator_dim if view_separator_dim else hidden_size
+        
+        # Text model settings
+        self.hidden_size = hidden_size
+        self.intermediate_size = intermediate_size
+        self.num_hidden_layers = num_hidden_layers
+        self.num_attention_heads = num_attention_heads
+        self.num_key_value_heads = num_key_value_heads
+        self.hidden_act = hidden_act
+        self.max_position_embeddings = max_position_embeddings
+        self.initializer_range = initializer_range
+        self.rms_norm_eps = rms_norm_eps
+        self.use_cache = use_cache
+        self.rope_theta = rope_theta
+        self.attention_dropout = attention_dropout
+        self.vocab_size = vocab_size
+        
+        # Special tokens
+        self.image_token_id = image_token_id
+        self.video_token_id = video_token_id
+        self.vision_start_token_id = vision_start_token_id
+        self.vision_end_token_id = vision_end_token_id
+        self.vision_token_id = vision_token_id
+        
+        # Rope scaling
+        if rope_scaling is None:
+            rope_scaling = {"type": "mrope", "mrope_section": [16, 24, 24]}
+        self.rope_scaling = rope_scaling
+    
+    def to_text_config(self) -> Qwen2VLTextConfig:
+        """Convert to Qwen2VLTextConfig for the text model."""
+        return Qwen2VLTextConfig(
+            hidden_size=self.hidden_size,
+            intermediate_size=self.intermediate_size,
+            num_hidden_layers=self.num_hidden_layers,
+            num_attention_heads=self.num_attention_heads,
+            num_key_value_heads=self.num_key_value_heads,
+            hidden_act=self.hidden_act,
+            max_position_embeddings=self.max_position_embeddings,
+            initializer_range=self.initializer_range,
+            rms_norm_eps=self.rms_norm_eps,
+            use_cache=self.use_cache,
+            tie_word_embeddings=self.tie_word_embeddings,
+            rope_theta=self.rope_theta,
+            attention_dropout=self.attention_dropout,
+            vocab_size=self.vocab_size,
+            bos_token_id=self.bos_token_id,
+            eos_token_id=self.eos_token_id,
+            pad_token_id=self.pad_token_id,
+            rope_scaling=self.rope_scaling,
+        )
+
+
+@dataclass
+class DeepQwenOutputWithPast(ModelOutput):
+    last_hidden_state: torch.FloatTensor = None
+    past_key_values: Optional[list[torch.FloatTensor]] = None
+    hidden_states: Optional[tuple[torch.FloatTensor]] = None
+    attentions: Optional[tuple[torch.FloatTensor]] = None
+
+@dataclass
+class DeepQwenCausalLMOutputWithPast(ModelOutput):
+    loss: Optional[torch.FloatTensor] = None
+    logits: Optional[torch.FloatTensor] = None
+    past_key_values: Optional[list[torch.FloatTensor]] = None
+    hidden_states: Optional[tuple[torch.FloatTensor]] = None
+    attentions: Optional[tuple[torch.FloatTensor]] = None
+
+
+class VisionProjector(nn.Module):
+    """
+    Vision projector with DeepSeek's pretrained layer + trainable adapter.
+
+    Architecture:
+        deepseek_proj: Linear(2048→1280) [FROZEN - loaded from DeepSeek checkpoint]
+        SiLU activation
+        norm: LayerNorm(1280) [TRAINABLE]
+        adapter: Linear(1280→1536) [TRAINABLE]
+
+    This preserves DeepSeek's learned vision-text alignment while adapting to Qwen's
+    embedding space. Total 2 layers like LLaVA's MLP projector.
+    """
+
+    def __init__(self, input_dim: int = 2048, hidden_dim: int = 1280, output_dim: int = 1536):
+        super().__init__()
+        # DeepSeek's original projection (will be frozen after loading weights)
+        self.deepseek_proj = nn.Linear(input_dim, hidden_dim)
+        # Adapter for Qwen (trainable)
+        self.norm = nn.LayerNorm(hidden_dim)
+        self.adapter = nn.Linear(hidden_dim, output_dim)
+        self._init_adapter_weights()
+
+    def _init_adapter_weights(self):
+        """Initialize adapter weights. deepseek_proj will be loaded from checkpoint."""
+        nn.init.ones_(self.norm.weight)
+        nn.init.zeros_(self.norm.bias)
+        nn.init.normal_(self.adapter.weight, mean=0.0, std=0.01)
+        nn.init.zeros_(self.adapter.bias)
+
+    def forward(self, x):
+        x = self.deepseek_proj(x)
+        x = F.silu(x)
+        x = self.norm(x)
+        x = self.adapter(x)
+        return x
+
+class DeepQwenVLPreTrainedModel(PreTrainedModel):
+    config_class = DeepQwenVLConfig
+    base_model_prefix = "model"
+    supports_gradient_checkpointing = True
+    _skip_keys_device_placement = "past_key_values"
+    _supports_flash_attn = True
+    _supports_sdpa = True
+    _supports_static_cache = True
+    _supports_attention_backend = True
+    
+    _keys_to_ignore_on_load_missing = [
+        "sam_model",
+        "vision_model", 
+        "projector",
+        "image_newline",
+        "view_separator",
+    ]
+    
+    def _init_weights(self, module):
+        """Initialize the weights."""
+        std = self.config.initializer_range if hasattr(self.config, 'initializer_range') else 0.02
+        if isinstance(module, nn.Linear):
+            module.weight.data.normal_(mean=0.0, std=std)
+            if module.bias is not None:
+                module.bias.data.zero_()
+        elif isinstance(module, nn.Embedding):
+            module.weight.data.normal_(mean=0.0, std=std)
+
+
+class DeepQwenVLModel(Qwen2VLTextModel):
+    """
+    DeepQwenVL Model that combines DeepSeek's vision encoders with Qwen2VL's text model.
+    
+    Accepts either:
+    - A DeepQwenVLConfig 
+    - A Qwen2VLTextConfig (for compatibility with from_pretrained from Qwen checkpoints)
+    - A generic PretrainedConfig (will extract necessary fields)
+    """
+    config_class = DeepQwenVLConfig
+    
+    def __init__(self, config):
+        if isinstance(config, DeepQwenVLConfig):
+            text_config = config.to_text_config()
+            output_hidden_size = config.projector_output_dim
+            vision_dim = config.deepseek_vision_hidden_size
+        elif isinstance(config, Qwen2VLTextConfig):
+            text_config = config
+            output_hidden_size = config.hidden_size
+            vision_dim = 2048
+        else:
+            text_config = config
+            output_hidden_size = getattr(config, 'hidden_size', 1536)
+            vision_dim = getattr(config, 'deepseek_vision_hidden_size', 2048)
+        
+        super(DeepQwenVLModel, self).__init__(text_config)
+        
+        self.config = config
+        self.output_hidden_size = output_hidden_size
+        
+        self.sam_model = build_sam_vit_b() 
+        self.vision_model = build_clip_l() 
+        
+        self.deepseek_vision_dim = vision_dim
+        self.deepseek_hidden_dim = 1280  # DeepSeek's projector output dimension
+        # New projector: DeepSeek layer (frozen) + adapter (trainable)
+        self.projector = VisionProjector(
+            input_dim=self.deepseek_vision_dim,      # 2048
+            hidden_dim=self.deepseek_hidden_dim,     # 1280 (DeepSeek's output)
+            output_dim=output_hidden_size            # 1536 (Qwen's hidden size)
+        )
+        
+        embed_std = 1 / torch.sqrt(torch.tensor(output_hidden_size, dtype=torch.float32))
+        self.image_newline = nn.Parameter(torch.randn(output_hidden_size) * embed_std)
+        self.view_separator = nn.Parameter(torch.randn(output_hidden_size) * embed_std)
+
+    def forward(
+        self,
+        input_ids: torch.LongTensor = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_values: Optional[List[torch.FloatTensor]] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        use_cache: Optional[bool] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        images: Optional[torch.FloatTensor] = None,
+        images_seq_mask: Optional[torch.FloatTensor] = None,
+        images_spatial_crop: Optional[torch.FloatTensor] = None,
+        return_dict: Optional[bool] = None,
+        cache_position: Optional[torch.LongTensor] = None,
+    ) -> Union[torch.Tensor, List[torch.Tensor]]:
+
+        if inputs_embeds is None:
+            inputs_embeds = self.get_input_embeddings()(input_ids)
+        
+        sam_model = getattr(self, 'sam_model', None)
+        vision_model = getattr(self, 'vision_model', None)
+
+        should_process_images = (
+            sam_model is not None 
+            and images is not None 
+            and images_seq_mask is not None
+            and (input_ids.shape[1] != 1 or self.training) 
+            and torch.sum(images[0][1]).item() != 0
+        )
+
+        if should_process_images:
+            idx = 0
+            for image, crop_shape in zip(images, images_spatial_crop):
+                images_in_this_batch = []
+                patches = image[0]
+                image_ori = image[1]
+
+                if torch.sum(patches).item() != 0:
+                    # Process local patches
+                    with torch.no_grad(): 
+                        local_features_1 = sam_model(patches)
+                        local_features_2 = vision_model(patches, local_features_1)
+                        local_features = torch.cat((local_features_2[:, 1:], local_features_1.flatten(2).permute(0, 2, 1)), dim=-1)
+                    local_features = local_features.detach()
+                    local_features = self.projector(local_features)
+
+                    # Process global image
+                    with torch.no_grad():
+                        global_features_1 = sam_model(image_ori)
+                        global_features_2 = vision_model(image_ori, global_features_1)
+                        global_features = torch.cat((global_features_2[:, 1:], global_features_1.flatten(2).permute(0, 2, 1)), dim=-1)
+                    global_features = global_features.detach()
+                    global_features = self.projector(global_features)
+
+                    # Reshape and add newline tokens
+                    _, hw, n_dim = global_features.shape
+                    h = w = int(hw ** 0.5)
+                    _2, hw2, n_dim2 = local_features.shape
+                    h2 = w2 = int(hw2 ** 0.5)
+                    width_crop_num, height_crop_num = crop_shape[0], crop_shape[1]
+
+                    global_features = global_features.view(h, w, n_dim)
+                    global_features = torch.cat(
+                        [global_features, self.image_newline[None, None, :].expand(h, 1, n_dim)], dim=1
+                    )
+                    global_features = global_features.view(-1, n_dim)
+
+                    local_features = local_features.view(
+                        height_crop_num, width_crop_num, h2, w2, n_dim2
+                    ).permute(0, 2, 1, 3, 4).reshape(height_crop_num*h2, width_crop_num*w2, n_dim2)
+                    local_features = torch.cat(
+                        [local_features, self.image_newline[None, None, :].expand(height_crop_num * h2, 1, n_dim2)], dim=1
+                    )
+                    local_features = local_features.view(-1, n_dim2)
+
+                    global_local_features = torch.cat([local_features, global_features, self.view_separator[None, :]], dim=0)
+                    images_in_this_batch.append(global_local_features)
+                else:
+                    # Global-only branch (small images)
+                    with torch.no_grad():
+                        global_features_1 = sam_model(image_ori)
+                        global_features_2 = vision_model(image_ori, global_features_1) 
+                        global_features = torch.cat((global_features_2[:, 1:], global_features_1.flatten(2).permute(0, 2, 1)), dim=-1)
+                    global_features = global_features.detach()
+                    global_features = self.projector(global_features)
+
+                    _, hw, n_dim = global_features.shape
+                    h = w = int(hw ** 0.5)
+                    global_features = global_features.view(h, w, n_dim)
+                    global_features = torch.cat(
+                        [global_features, self.image_newline[None, None, :].expand(h, 1, n_dim)], dim=1
+                    )
+                    global_features = global_features.view(-1, n_dim)
+                    global_local_features = torch.cat([global_features, self.view_separator[None, :]], dim=0)
+                    images_in_this_batch.append(global_local_features)
+
+                if images_in_this_batch:
+                    images_in_this_batch = torch.cat(images_in_this_batch, dim=0)
+                    inputs_embeds[idx].masked_scatter_(images_seq_mask[idx].unsqueeze(-1).cuda(), images_in_this_batch)
+                idx += 1
+
+        outputs = super().forward(
+            input_ids=None, attention_mask=attention_mask, past_key_values=past_key_values,
+            inputs_embeds=inputs_embeds, use_cache=use_cache, position_ids=position_ids,
+            output_attentions=output_attentions, output_hidden_states=output_hidden_states,
+            return_dict=return_dict, cache_position=cache_position
+        )
+
+        return DeepQwenOutputWithPast(
+            last_hidden_state=outputs.last_hidden_state,
+            past_key_values=outputs.past_key_values,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
+        ) if return_dict else outputs.to_tuple() 
+
+
+class DeepQwenVLForCausalLM(DeepQwenVLModel, GenerationMixin):
+    """
+    DeepQwenVL Model for causal language modeling with vision capabilities.
+    
+    Combines DeepSeek's vision encoders (SAM + CLIP) with Qwen2VL's text model.
+    """
+    config_class = DeepQwenVLConfig
+    _tied_weights_keys = ["lm_head.weight"]
+    
+    _keys_to_ignore_on_load_missing = [
+        # "sam_model",
+        # "vision_model", 
+        # "projector",
+        # "image_newline",
+        # "view_separator",
+    ]
+    
+    def __init__(self, config):
+        """
+        Initialize the model.
+
+        Args:
+            config: Can be DeepQwenVLConfig, Qwen2VLTextConfig, or a generic config
+                   from a Qwen2-VL checkpoint.
+        """
+        super().__init__(config)
+
+        hidden_size = getattr(config, 'hidden_size', 1536)
+        vocab_size = getattr(config, 'vocab_size', 151936)
+
+        self.lm_head = nn.Linear(hidden_size, vocab_size, bias=False)
+
+        self.post_init()
+    
+    def get_output_embeddings(self):
+        return getattr(self, 'lm_head', None)
+    
+    def set_output_embeddings(self, new_embeddings):
+        self.lm_head = new_embeddings
+
+    def forward(
+        self,
+        input_ids: torch.LongTensor = None,
+        labels: Optional[torch.LongTensor] = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_values: Optional[List[torch.FloatTensor]] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        use_cache: Optional[bool] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        images: Optional[torch.FloatTensor] = None,
+        images_seq_mask: Optional[torch.FloatTensor] = None,
+        images_spatial_crop: Optional[torch.FloatTensor] = None,
+        return_dict: Optional[bool] = None,
+        cache_position: Optional[torch.LongTensor] = None,
+    ) -> Union[torch.Tensor, List[torch.Tensor]]:
+
+        outputs = super().forward(
+            input_ids=input_ids, 
+            attention_mask=attention_mask, 
+            past_key_values=past_key_values,
+            inputs_embeds=inputs_embeds, 
+            use_cache=use_cache, 
+            position_ids = position_ids,
+            output_attentions=output_attentions, 
+            output_hidden_states=output_hidden_states, 
+            images=images,
+            images_seq_mask=images_seq_mask, 
+            images_spatial_crop=images_spatial_crop,
+            return_dict=True,
+            cache_position=cache_position,
+        )
+
+        hidden_states = outputs[0]
+        logits = self.lm_head(hidden_states)
+        logits = logits.float()
+
+        loss = None
+        if labels is not None:
+            loss = self.loss_function(logits=logits, labels=labels, vocab_size=self.config.vocab_size)
+
+        return DeepQwenCausalLMOutputWithPast(
+            loss=loss,
+            logits=logits,
+            past_key_values=outputs.past_key_values,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
+        )
+    
+    def prepare_inputs_for_generation(
+        self,
+        input_ids,
+        past_key_values=None,
+        attention_mask=None,
+        inputs_embeds=None,
+        cache_position=None,
+        position_ids=None,
+        images=None,
+        images_seq_mask=None,
+        images_spatial_crop=None,
+        **kwargs,
+    ):
+        model_inputs = super().prepare_inputs_for_generation(
+            input_ids,
+            past_key_values=past_key_values,
+            attention_mask=attention_mask,
+            inputs_embeds=inputs_embeds,
+            cache_position=cache_position,
+            position_ids=position_ids,
+            **kwargs,
+        )
+
+        model_inputs["images"] = images
+        model_inputs["images_seq_mask"] = images_seq_mask
+        model_inputs["images_spatial_crop"] = images_spatial_crop
+        model_inputs["position_ids"] = None
+
+        # Clear images after first forward pass (cache_position[0] != 0 means subsequent tokens)
+        if cache_position is not None and cache_position[0] != 0:
+            model_inputs["images"] = None
+            model_inputs["images_seq_mask"] = None
+            model_inputs["images_spatial_crop"] = None
+
+        return model_inputs
+    
+    def reinitialize_projector(self, vis_mlp=None, device=None, dtype=None):
+        """
+        Reinitialize the projector, image_newline, and view_separator.
+        Call this after from_pretrained when loading from a Qwen checkpoint.
+        """
+        if device is None:
+            for param in self.parameters():
+                if param.device.type != 'meta':
+                    device = param.device
+                    break
+            if device is None:
+                device = 'cpu'
+        if dtype is None:
+            dtype = torch.bfloat16
+            
+        input_dim = self.deepseek_vision_dim
+        output_dim = self.output_hidden_size
+        
+        if vis_mlp is not None:
+            self.projector = VisionProjector(input_dim=input_dim, output_dim=output_dim).to(device=device, dtype=dtype)
+            
+        else:
+            self.projector = nn.Linear(in_features=input_dim, out_features=output_dim).to(device=device, dtype=dtype)
+            nn.init.normal_(self.projector.weight, mean=0.0, std=0.01)
+            if self.projector.bias is not None:
+                nn.init.zeros_(self.projector.bias)
+        
+        embed_std = 1 / torch.sqrt(torch.tensor(output_dim, dtype=torch.float32))
+        self.image_newline = nn.Parameter(
+            torch.randn(output_dim, device=device, dtype=dtype) * embed_std.item()
+        )
+        self.view_separator = nn.Parameter(
+            torch.randn(output_dim, device=device, dtype=dtype) * embed_std.item()
+        )
+        
+        print(f"Projector reinitialized on {device} with dtype {dtype}")
+    
+    def load_pretrained_vision(self, pretrained_path: str):
+        try:
+            from safetensors import safe_open
+        except ImportError:
+            raise ImportError("Please install safetensors to load the pretrained vision model.")
+        
+        assert os.path.exists(pretrained_path), f"Pretrained path {pretrained_path} does not exist."
+
+        vision_weights = {}
+        with safe_open(f"{pretrained_path}/model-00001-of-000001.safetensors", framework="pt", device="cpu") as f:
+            for k in f.keys():
+                vision_weights[k] = f.get_tensor(k)
+        
+        prefixes = {
+            "sam_model": "model.sam_model.",
+            "vision_model": "model.vision_model.",
+        }
+
+        try:
+            for p in prefixes.keys():
+                state_dict = {}
+
+                for k, v in vision_weights.items():
+                    if k.startswith(prefixes[p]):
+                        new_key = k[len(prefixes[p]):]
+                        state_dict[new_key] = v
+
+                getattr(self, p).load_state_dict(state_dict, strict=False)
+
+            print("Pretrained vision model loaded successfully.")
+        except Exception as e:
+            print("Error loading pretrained vision model:", e)
+            raise e
+
+    def load_deepseek_projector(self, pretrained_path: str):
+        """
+        Load DeepSeek's projector weights into the deepseek_proj layer.
+
+        DeepSeek checkpoint has:
+            - projector.weight: shape (1280, 2048)
+            - projector.bias: shape (1280,)
+
+        These get loaded into self.projector.deepseek_proj
+        """
+        try:
+            from safetensors import safe_open
+        except ImportError:
+            raise ImportError("Please install safetensors to load DeepSeek projector.")
+
+        assert os.path.exists(pretrained_path), f"Pretrained path {pretrained_path} does not exist."
+
+        # Find safetensors file
+        safetensor_files = [f for f in os.listdir(pretrained_path) if f.endswith('.safetensors')]
+        if not safetensor_files:
+            raise FileNotFoundError(f"No safetensors files found in {pretrained_path}")
+
+        safetensor_path = os.path.join(pretrained_path, safetensor_files[0])
+
+        projector_weights = {}
+        with safe_open(safetensor_path, framework="pt", device="cpu") as f:
+            for k in f.keys():
+                if 'projector' in k:
+                    projector_weights[k] = f.get_tensor(k)
+
+        # Load into deepseek_proj
+        if 'projector.weight' in projector_weights:
+            self.projector.deepseek_proj.weight.data = projector_weights['projector.weight']
+            self.projector.deepseek_proj.bias.data = projector_weights['projector.bias']
+            print(f"Loaded DeepSeek projector weights: {self.projector.deepseek_proj.weight.shape}")
+            print(f"  Weight mean: {self.projector.deepseek_proj.weight.mean().item():.6f}")
+            print(f"  Weight std: {self.projector.deepseek_proj.weight.std().item():.6f}")
+        elif 'model.projector.weight' in projector_weights:
+            self.projector.deepseek_proj.weight.data = projector_weights['model.projector.weight']
+            self.projector.deepseek_proj.bias.data = projector_weights['model.projector.bias']
+            print(f"Loaded DeepSeek projector weights (model. prefix)")
+        else:
+            print(f"Warning: Could not find projector weights. Available keys: {list(projector_weights.keys())}")
+
+    def disable_torch_init(self):
+        """
+        Disable the redundant torch default initialization to accelerate model creation.
+        """
+        import torch
+        setattr(torch.nn.Linear, "reset_parameters", lambda self: None)
+        setattr(torch.nn.LayerNorm, "reset_parameters", lambda self: None)
+
+    def infer(
+        self, 
+        tokenizer, 
+        prompt='', 
+        image_file='', 
+        output_path = '', 
+        base_size=1024, 
+        image_size=640, 
+        crop_mode=True, 
+        test_compress=False, 
+        save_results=False, 
+        eval_mode=False
+    ):
+        self.disable_torch_init()
+        
+        os.makedirs(output_path, exist_ok=True)
+        os.makedirs(f'{output_path}/images', exist_ok=True)
+        conversation = [
+            {
+                "role": "user",
+                "content": [
+                    {
+                        "type": "image",
+                        "image": f"{image_file}",
+                    },
+                    {"type": "text", "text": f"{prompt}"},
+                ],
+            }
+        ]
+        
+        formatted_prompt = tokenizer.apply_chat_template(conversation, tokenize=False, add_generation_prompt=True)
+
+        patch_size = 16
+        downsample_ratio = 4
+        images = load_pil_images(conversation)
+
+        valid_img_tokens = 0
+        ratio = 1
+
+        image_draw = images[0].copy()
+
+        w,h = image_draw.size
+        ratio = 1 - ((max(w, h) - min(w, h)) / (max(w, h)))
+    
+
+        image_transform=BasicImageTransform(mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5), normalize=True)
+        images_seq_mask = []
+
+        image_token = '<|image_pad|>'
+        image_token_id = 151655
+        text_splits = formatted_prompt.split(image_token)
+
+        images_list, images_crop_list, images_seq_mask = [], [], []
+        tokenized_str = []
+        images_spatial_crop = []
+        for text_sep, image in zip(text_splits, images):
+
+            tokenized_sep = text_encode(tokenizer, text_sep, bos=False, eos=False)
+            tokenized_str += tokenized_sep
+            images_seq_mask += [False] * len(tokenized_sep)
+
+            if crop_mode:
+
+                if image.size[0] <= 640 and image.size[1] <= 640:
+                    crop_ratio = [1, 1]
+
+                else:
+                    if crop_mode:
+                        images_crop_raw, crop_ratio = dynamic_preprocess(image)
+                    else:
+                        crop_ratio = [1, 1]
+                
+                global_view = ImageOps.pad(image, (base_size, base_size),
+                                        color=tuple(int(x * 255) for x in image_transform.mean))
+                
+                if base_size == 1024:
+                    valid_img_tokens += int(256 * ratio)
+                elif base_size == 1280:
+                    valid_img_tokens += int(400 * ratio)
+                # elif base_size == 640:
+                    # valid_img_tokens += int(100 * ratio)
+                
+
+                images_list.append(image_transform(global_view).to(torch.bfloat16))
+
+                # global_view_tensor = image_transform(global_view).to(torch.bfloat16)
+
+                width_crop_num, height_crop_num = crop_ratio
+
+                images_spatial_crop.append([width_crop_num, height_crop_num])
+                
+                
+                if width_crop_num > 1 or height_crop_num > 1:
+                    """process the local views"""
+                    
+                    for i in range(len(images_crop_raw)):
+                        images_crop_list.append(image_transform(images_crop_raw[i]).to(torch.bfloat16))
+                
+                if image_size == 640:
+                    valid_img_tokens += len(images_crop_list) * 100
+
+                num_queries = math.ceil((image_size // patch_size) / downsample_ratio)
+                num_queries_base = math.ceil((base_size // patch_size) / downsample_ratio)
+
+                """add image tokens"""
+            
+                tokenized_image = ([image_token_id] * num_queries_base + [image_token_id]) * num_queries_base
+                tokenized_image += [image_token_id]
+                if width_crop_num > 1 or height_crop_num > 1:
+                    tokenized_image += ([image_token_id] * (num_queries * width_crop_num) + [image_token_id]) * (
+                                num_queries * height_crop_num)
+                tokenized_str += tokenized_image
+                images_seq_mask += [True] * len(tokenized_image)
+                # num_image_tokens.append(len(tokenized_image))
+
+            else:
+                """process the global view"""
+                if image_size <= 640:
+                    image = image.resize((image_size, image_size))
+                global_view = ImageOps.pad(image, (image_size, image_size),
+                                        color=tuple(int(x * 255) for x in image_transform.mean))
+                images_list.append(image_transform(global_view).to(torch.bfloat16))
+
+                if base_size == 1024:
+                    valid_img_tokens += int(256 * ratio)
+                elif base_size == 1280:
+                    valid_img_tokens += int(400 * ratio)
+                elif base_size == 640:
+                    valid_img_tokens += int(100 * 1)
+                elif base_size == 512:
+                    valid_img_tokens += int(64 * 1)
+
+                width_crop_num, height_crop_num = 1, 1
+
+                images_spatial_crop.append([width_crop_num, height_crop_num])
+
+
+                """add image tokens"""
+                num_queries = math.ceil((image_size // patch_size) / downsample_ratio)
+
+                tokenized_image = ([image_token_id] * num_queries + [image_token_id]) * num_queries
+                tokenized_image += [image_token_id]
+                # tokenized_image += ([self.image_token_id] * (num_queries * width_crop_num) + [self.image_token_id]) * (
+                #             num_queries * height_crop_num)
+                tokenized_str += tokenized_image
+                images_seq_mask += [True] * len(tokenized_image)
+                # num_image_tokens.append(len(tokenized_image))
+        
+        """process the last text split"""
+        tokenized_sep = text_encode(tokenizer, text_splits[-1], bos=False, eos=False)
+        tokenized_str += tokenized_sep
+        images_seq_mask += [False] * len(tokenized_sep)
+
+        # Qwen2VL has NO bos_token (bos_token_id is None)
+        # The chat template already handles proper formatting
+
+        input_ids = torch.LongTensor(tokenized_str)
+
+        images_seq_mask = torch.tensor(images_seq_mask, dtype=torch.bool)
+
+        if len(images_list) == 0:
+            images_ori = torch.zeros((1, 3, image_size, image_size))
+            images_spatial_crop = torch.zeros((1, 2), dtype=torch.long)
+            images_crop = torch.zeros((1, 3, base_size, base_size))
+
+        else:
+            images_ori = torch.stack(images_list, dim=0)
+            images_spatial_crop = torch.tensor(images_spatial_crop, dtype=torch.long)
+            if images_crop_list:
+                images_crop = torch.stack(images_crop_list, dim=0)
+            else:
+                images_crop = torch.zeros((1, 3, base_size, base_size))
+
+
+
+        if not eval_mode:
+            streamer = NoEOSTextStreamer(tokenizer, skip_prompt=True, skip_special_tokens=False)
+            with torch.autocast("cuda", dtype=torch.bfloat16):
+                with torch.no_grad():
+                    output_ids = self.generate(
+                        input_ids.unsqueeze(0).cuda(),
+                        images=[(images_crop.cuda(), images_ori.cuda())],
+                        images_seq_mask=images_seq_mask.unsqueeze(0).cuda(),
+                        images_spatial_crop=images_spatial_crop,
+                        temperature=0.5,
+                        eos_token_id=tokenizer.eos_token_id,
+                        streamer=streamer,
+                        max_new_tokens=8192,
+                        no_repeat_ngram_size=20,
+                        use_cache=True
+                    )
+        else:
+            with torch.autocast("cuda", dtype=torch.bfloat16):
+                with torch.no_grad():
+                    output_ids = self.generate(
+                        input_ids.unsqueeze(0).cuda(),
+                        images=[(images_crop.cuda(), images_ori.cuda())],
+                        images_seq_mask=images_seq_mask.unsqueeze(0).cuda(),
+                        images_spatial_crop=images_spatial_crop,
+                        temperature=0.5,
+                        eos_token_id=tokenizer.eos_token_id,
+                        max_new_tokens=8192,
+                        no_repeat_ngram_size=35,
+                        use_cache=True
+                    )
+
+        # Check if conversation has image
+        has_image = any(
+            (isinstance(item, dict) and item.get('type') == 'image')
+            for msg in conversation
+            for item in (msg.get('content', []) if isinstance(msg.get('content'), list) else [])
+        )
+        
+        if has_image and eval_mode:
+                outputs = tokenizer.decode(output_ids[0, input_ids.unsqueeze(0).cuda().shape[1]:], skip_special_tokens=False)
+                # Qwen2VL's EOS token is <|im_end|>
+                stop_str = tokenizer.eos_token or '<|im_end|>'
+                if outputs.endswith(stop_str):
+                    outputs = outputs[:-len(stop_str)]
+                outputs = outputs.strip()
+
+                return outputs
+        
+        if has_image and test_compress:
+            outputs = tokenizer.decode(output_ids[0, input_ids.unsqueeze(0).cuda().shape[1]:], skip_special_tokens=False)
+            pure_texts_outputs_token_length = len(text_encode(tokenizer, outputs, bos=False, eos=False))
+            print('='*50)
+            print('image size: ', (w, h))
+            print('valid image tokens: ', int(valid_img_tokens))
+            print('output texts tokens (valid): ', pure_texts_outputs_token_length)
+            print('compression ratio: ', round(pure_texts_outputs_token_length/valid_img_tokens, 2))
+            print('='*50)
+
+
+        if has_image and save_results:
+            outputs = tokenizer.decode(output_ids[0, input_ids.unsqueeze(0).cuda().shape[1]:], skip_special_tokens=False)
+            # Qwen2VL's EOS token
+            stop_str = tokenizer.eos_token or '<|im_end|>'
+
+            print('='*15 + 'save results:' + '='*15)
+            
+            if outputs.endswith(stop_str):
+                outputs = outputs[:-len(stop_str)]
+            outputs = outputs.strip()
+
+            matches_ref, matches_images, mathes_other = re_match(outputs)
+            result = process_image_with_refs(image_draw, matches_ref, output_path)
+
+            for idx, a_match_image in enumerate(tqdm(matches_images, desc="image")):
+                outputs = outputs.replace(a_match_image, '![](images/' + str(idx) + '.jpg)\n')
+            
+            for idx, a_match_other in enumerate(tqdm(mathes_other, desc="other")):
+                outputs = outputs.replace(a_match_other, '').replace('\\coloneqq', ':=').replace('\\eqqcolon', '=:')
+
+            with open(f'{output_path}/result.mmd', 'w', encoding = 'utf-8') as afile:
+                afile.write(outputs)
+
+            if 'line_type' in outputs:
+                import matplotlib.pyplot as plt
+                lines = eval(outputs)['Line']['line']
+
+                line_type = eval(outputs)['Line']['line_type']
+                endpoints = eval(outputs)['Line']['line_endpoint']
+
+                fig, ax = plt.subplots(figsize=(3,3), dpi=200)
+                ax.set_xlim(-15, 15)
+                ax.set_ylim(-15, 15)
+
+                for idx, line in enumerate(lines):
+                    try:
+                        p0 = eval(line.split(' -- ')[0])
+                        p1 = eval(line.split(' -- ')[-1])
+
+                        if line_type[idx] == '--':
+                            ax.plot([p0[0], p1[0]], [p0[1], p1[1]], linewidth=0.8, color='k')
+                        else:
+                            ax.plot([p0[0], p1[0]], [p0[1], p1[1]], linewidth = 0.8, color = 'k')
+
+                        ax.scatter(p0[0], p0[1], s=5, color = 'k')
+                        ax.scatter(p1[0], p1[1], s=5, color = 'k')
+                    except:
+                        pass
+
+                for endpoint in endpoints:
+
+                    label = endpoint.split(': ')[0]
+                    (x, y) = eval(endpoint.split(': ')[1])
+                    ax.annotate(label, (x, y), xytext=(1, 1), textcoords='offset points', 
+                                fontsize=5, fontweight='light')
+                
+
+                plt.savefig(f'{output_path}/geo.jpg')
+                plt.close()
+
+            result.save(f"{output_path}/result_with_boxes.jpg")
+
+
+## TODO
+
+# new training loop:
+## image -> vision encoder -> projection ->! txt_decoder -> embedding -> pool  
+#                                                                              => alignment(text_pooling, image_pooling)
+## text -> text encoder -> projection -> embedding -> pool                     
+
+## cant let projection layer output into text decoder

model.safetensors.index.json

@@ -0,0 +1,827 @@
+{
+  "metadata": {
+    "total_parameters": 1947057152,
+    "total_size": 6985491464
+  },
+  "weight_map": {
+    "embed_tokens.weight": "model-00001-of-00002.safetensors",
+    "image_newline": "model-00001-of-00002.safetensors",
+    "layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.0.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.0.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "layers.0.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.0.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.0.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.0.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.0.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.0.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.0.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.1.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.1.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "layers.1.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.1.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.1.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.1.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.1.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.1.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.1.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.10.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "layers.10.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.10.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.10.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "layers.10.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.10.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.10.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.10.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.10.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.10.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.10.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.11.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.11.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.11.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "layers.11.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.11.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.11.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.11.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.11.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.11.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.11.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.12.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "layers.12.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.12.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.12.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.12.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "layers.12.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.12.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.12.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.12.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.12.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.12.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.12.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "layers.13.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.13.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.13.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.13.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "layers.13.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.13.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.13.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.13.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.13.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.13.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.13.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.14.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "layers.14.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.14.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.14.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.14.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "layers.14.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.14.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.14.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.14.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.14.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.14.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.14.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.15.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "layers.15.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.15.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.15.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.15.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "layers.15.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.15.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.15.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.15.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.15.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.15.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.15.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.16.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "layers.16.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.16.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.16.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.16.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "layers.16.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.16.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.16.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.16.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.16.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.16.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.16.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.17.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "layers.17.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.17.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.17.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.17.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "layers.17.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.17.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.17.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.17.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.17.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.17.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.17.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.18.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "layers.18.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.18.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.18.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.18.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "layers.18.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.18.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.18.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.18.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.18.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.18.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.18.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.19.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "layers.19.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.19.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.19.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.19.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "layers.19.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.19.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.19.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.19.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.19.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.19.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.19.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.2.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.2.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "layers.2.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.2.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.2.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.2.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.2.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.2.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.2.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.20.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "layers.20.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.20.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.20.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.20.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "layers.20.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.20.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.20.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.20.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.20.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.20.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.20.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.21.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "layers.21.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "layers.21.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.21.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.21.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "layers.21.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.21.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.21.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.21.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.21.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.21.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.21.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.22.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "layers.22.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "layers.22.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "layers.22.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "layers.22.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "layers.22.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "layers.22.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "layers.22.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "layers.22.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "layers.22.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "layers.22.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "layers.22.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "layers.23.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "layers.23.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "layers.23.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "layers.23.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "layers.23.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "layers.23.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "layers.23.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "layers.23.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "layers.23.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "layers.23.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "layers.23.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "layers.23.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "layers.24.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "layers.24.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "layers.24.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "layers.24.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "layers.24.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "layers.24.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "layers.24.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "layers.24.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "layers.24.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "layers.24.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "layers.24.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "layers.24.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "layers.25.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "layers.25.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "layers.25.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "layers.25.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "layers.25.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "layers.25.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "layers.25.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "layers.25.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "layers.25.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "layers.25.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "layers.25.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "layers.25.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "layers.26.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "layers.26.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "layers.26.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "layers.26.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "layers.26.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "layers.26.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "layers.26.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "layers.26.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "layers.26.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "layers.26.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "layers.26.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "layers.26.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "layers.27.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "layers.27.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "layers.27.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
+    "layers.27.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
+    "layers.27.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "layers.27.self_attn.k_proj.bias": "model-00002-of-00002.safetensors",
+    "layers.27.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
+    "layers.27.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "layers.27.self_attn.q_proj.bias": "model-00002-of-00002.safetensors",
+    "layers.27.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
+    "layers.27.self_attn.v_proj.bias": "model-00002-of-00002.safetensors",
+    "layers.27.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
+    "layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.3.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.3.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "layers.3.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.3.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.3.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.3.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.3.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.3.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.4.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.4.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "layers.4.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.4.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.4.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.4.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.4.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.4.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.4.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.5.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.5.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "layers.5.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.5.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.5.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.5.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.5.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.5.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.5.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.6.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.6.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "layers.6.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.6.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.6.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.6.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.6.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.6.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.6.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.7.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.7.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "layers.7.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.7.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.7.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.7.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.7.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.7.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.8.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "layers.8.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.8.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.8.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "layers.8.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.8.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.8.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.8.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.8.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.8.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.8.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.9.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.9.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.9.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "layers.9.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.9.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.9.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.9.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.9.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
+    "layers.9.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
+    "layers.9.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
+    "norm.weight": "model-00002-of-00002.safetensors",
+    "projector.adapter.bias": "model-00002-of-00002.safetensors",
+    "projector.adapter.weight": "model-00002-of-00002.safetensors",
+    "projector.deepseek_proj.bias": "model-00002-of-00002.safetensors",
+    "projector.deepseek_proj.weight": "model-00002-of-00002.safetensors",
+    "projector.norm.bias": "model-00002-of-00002.safetensors",
+    "projector.norm.weight": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.0.attn.proj.bias": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.0.attn.proj.weight": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.0.attn.qkv.bias": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.0.attn.qkv.weight": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.0.attn.rel_pos_h": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.0.attn.rel_pos_w": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.0.mlp.lin1.bias": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.0.mlp.lin1.weight": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.0.mlp.lin2.bias": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.0.mlp.lin2.weight": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.0.norm1.bias": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.0.norm1.weight": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.0.norm2.bias": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.0.norm2.weight": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.1.attn.proj.bias": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.1.attn.proj.weight": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.1.attn.qkv.bias": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.1.attn.qkv.weight": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.1.attn.rel_pos_h": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.1.attn.rel_pos_w": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.1.mlp.lin1.bias": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.1.mlp.lin1.weight": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.1.mlp.lin2.bias": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.1.mlp.lin2.weight": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.1.norm1.bias": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.1.norm1.weight": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.1.norm2.bias": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.1.norm2.weight": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.10.attn.proj.bias": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.10.attn.proj.weight": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.10.attn.qkv.bias": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.10.attn.qkv.weight": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.10.attn.rel_pos_h": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.10.attn.rel_pos_w": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.10.mlp.lin1.bias": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.10.mlp.lin1.weight": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.10.mlp.lin2.bias": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.10.mlp.lin2.weight": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.10.norm1.bias": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.10.norm1.weight": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.10.norm2.bias": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.10.norm2.weight": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.11.attn.proj.bias": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.11.attn.proj.weight": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.11.attn.qkv.bias": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.11.attn.qkv.weight": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.11.attn.rel_pos_h": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.11.attn.rel_pos_w": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.11.mlp.lin1.bias": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.11.mlp.lin1.weight": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.11.mlp.lin2.bias": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.11.mlp.lin2.weight": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.11.norm1.bias": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.11.norm1.weight": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.11.norm2.bias": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.11.norm2.weight": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.2.attn.proj.bias": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.2.attn.proj.weight": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.2.attn.qkv.bias": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.2.attn.qkv.weight": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.2.attn.rel_pos_h": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.2.attn.rel_pos_w": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.2.mlp.lin1.bias": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.2.mlp.lin1.weight": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.2.mlp.lin2.bias": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.2.mlp.lin2.weight": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.2.norm1.bias": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.2.norm1.weight": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.2.norm2.bias": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.2.norm2.weight": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.3.attn.proj.bias": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.3.attn.proj.weight": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.3.attn.qkv.bias": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.3.attn.qkv.weight": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.3.attn.rel_pos_h": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.3.attn.rel_pos_w": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.3.mlp.lin1.bias": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.3.mlp.lin1.weight": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.3.mlp.lin2.bias": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.3.mlp.lin2.weight": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.3.norm1.bias": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.3.norm1.weight": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.3.norm2.bias": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.3.norm2.weight": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.4.attn.proj.bias": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.4.attn.proj.weight": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.4.attn.qkv.bias": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.4.attn.qkv.weight": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.4.attn.rel_pos_h": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.4.attn.rel_pos_w": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.4.mlp.lin1.bias": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.4.mlp.lin1.weight": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.4.mlp.lin2.bias": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.4.mlp.lin2.weight": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.4.norm1.bias": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.4.norm1.weight": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.4.norm2.bias": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.4.norm2.weight": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.5.attn.proj.bias": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.5.attn.proj.weight": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.5.attn.qkv.bias": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.5.attn.qkv.weight": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.5.attn.rel_pos_h": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.5.attn.rel_pos_w": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.5.mlp.lin1.bias": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.5.mlp.lin1.weight": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.5.mlp.lin2.bias": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.5.mlp.lin2.weight": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.5.norm1.bias": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.5.norm1.weight": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.5.norm2.bias": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.5.norm2.weight": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.6.attn.proj.bias": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.6.attn.proj.weight": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.6.attn.qkv.bias": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.6.attn.qkv.weight": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.6.attn.rel_pos_h": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.6.attn.rel_pos_w": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.6.mlp.lin1.bias": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.6.mlp.lin1.weight": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.6.mlp.lin2.bias": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.6.mlp.lin2.weight": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.6.norm1.bias": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.6.norm1.weight": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.6.norm2.bias": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.6.norm2.weight": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.7.attn.proj.bias": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.7.attn.proj.weight": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.7.attn.qkv.bias": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.7.attn.qkv.weight": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.7.attn.rel_pos_h": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.7.attn.rel_pos_w": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.7.mlp.lin1.bias": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.7.mlp.lin1.weight": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.7.mlp.lin2.bias": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.7.mlp.lin2.weight": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.7.norm1.bias": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.7.norm1.weight": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.7.norm2.bias": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.7.norm2.weight": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.8.attn.proj.bias": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.8.attn.proj.weight": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.8.attn.qkv.bias": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.8.attn.qkv.weight": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.8.attn.rel_pos_h": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.8.attn.rel_pos_w": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.8.mlp.lin1.bias": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.8.mlp.lin1.weight": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.8.mlp.lin2.bias": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.8.mlp.lin2.weight": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.8.norm1.bias": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.8.norm1.weight": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.8.norm2.bias": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.8.norm2.weight": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.9.attn.proj.bias": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.9.attn.proj.weight": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.9.attn.qkv.bias": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.9.attn.qkv.weight": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.9.attn.rel_pos_h": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.9.attn.rel_pos_w": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.9.mlp.lin1.bias": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.9.mlp.lin1.weight": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.9.mlp.lin2.bias": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.9.mlp.lin2.weight": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.9.norm1.bias": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.9.norm1.weight": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.9.norm2.bias": "model-00002-of-00002.safetensors",
+    "sam_model.blocks.9.norm2.weight": "model-00002-of-00002.safetensors",
+    "sam_model.neck.0.weight": "model-00002-of-00002.safetensors",
+    "sam_model.neck.1.bias": "model-00002-of-00002.safetensors",
+    "sam_model.neck.1.weight": "model-00002-of-00002.safetensors",
+    "sam_model.neck.2.weight": "model-00002-of-00002.safetensors",
+    "sam_model.neck.3.bias": "model-00002-of-00002.safetensors",
+    "sam_model.neck.3.weight": "model-00002-of-00002.safetensors",
+    "sam_model.net_2.weight": "model-00002-of-00002.safetensors",
+    "sam_model.net_3.weight": "model-00002-of-00002.safetensors",
+    "sam_model.patch_embed.proj.bias": "model-00002-of-00002.safetensors",
+    "sam_model.patch_embed.proj.weight": "model-00002-of-00002.safetensors",
+    "sam_model.pos_embed": "model-00002-of-00002.safetensors",
+    "view_separator": "model-00001-of-00002.safetensors",
+    "vision_model.embeddings.class_embedding": "model-00002-of-00002.safetensors",
+    "vision_model.embeddings.patch_embedding.weight": "model-00002-of-00002.safetensors",
+    "vision_model.embeddings.position_embedding.weight": "model-00002-of-00002.safetensors",
+    "vision_model.embeddings.position_ids": "model-00002-of-00002.safetensors",
+    "vision_model.pre_layrnorm.bias": "model-00002-of-00002.safetensors",
+    "vision_model.pre_layrnorm.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.0.layer_norm1.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.0.layer_norm1.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.0.layer_norm2.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.0.layer_norm2.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.0.mlp.fc1.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.0.mlp.fc1.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.0.mlp.fc2.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.0.mlp.fc2.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.0.self_attn.out_proj.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.0.self_attn.out_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.0.self_attn.qkv_proj.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.0.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.1.layer_norm1.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.1.layer_norm1.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.1.layer_norm2.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.1.layer_norm2.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.1.mlp.fc1.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.1.mlp.fc1.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.1.mlp.fc2.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.1.mlp.fc2.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.1.self_attn.out_proj.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.1.self_attn.out_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.1.self_attn.qkv_proj.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.1.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.10.layer_norm1.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.10.layer_norm1.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.10.layer_norm2.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.10.layer_norm2.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.10.mlp.fc1.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.10.mlp.fc1.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.10.mlp.fc2.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.10.mlp.fc2.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.10.self_attn.out_proj.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.10.self_attn.out_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.10.self_attn.qkv_proj.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.10.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.11.layer_norm1.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.11.layer_norm1.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.11.layer_norm2.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.11.layer_norm2.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.11.mlp.fc1.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.11.mlp.fc1.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.11.mlp.fc2.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.11.mlp.fc2.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.11.self_attn.out_proj.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.11.self_attn.out_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.11.self_attn.qkv_proj.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.11.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.12.layer_norm1.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.12.layer_norm1.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.12.layer_norm2.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.12.layer_norm2.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.12.mlp.fc1.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.12.mlp.fc1.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.12.mlp.fc2.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.12.mlp.fc2.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.12.self_attn.out_proj.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.12.self_attn.out_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.12.self_attn.qkv_proj.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.12.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.13.layer_norm1.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.13.layer_norm1.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.13.layer_norm2.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.13.layer_norm2.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.13.mlp.fc1.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.13.mlp.fc1.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.13.mlp.fc2.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.13.mlp.fc2.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.13.self_attn.out_proj.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.13.self_attn.out_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.13.self_attn.qkv_proj.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.13.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.14.layer_norm1.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.14.layer_norm1.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.14.layer_norm2.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.14.layer_norm2.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.14.mlp.fc1.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.14.mlp.fc1.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.14.mlp.fc2.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.14.mlp.fc2.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.14.self_attn.out_proj.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.14.self_attn.out_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.14.self_attn.qkv_proj.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.14.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.15.layer_norm1.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.15.layer_norm1.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.15.layer_norm2.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.15.layer_norm2.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.15.mlp.fc1.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.15.mlp.fc1.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.15.mlp.fc2.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.15.mlp.fc2.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.15.self_attn.out_proj.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.15.self_attn.out_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.15.self_attn.qkv_proj.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.15.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.16.layer_norm1.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.16.layer_norm1.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.16.layer_norm2.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.16.layer_norm2.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.16.mlp.fc1.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.16.mlp.fc1.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.16.mlp.fc2.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.16.mlp.fc2.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.16.self_attn.out_proj.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.16.self_attn.out_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.16.self_attn.qkv_proj.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.16.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.17.layer_norm1.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.17.layer_norm1.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.17.layer_norm2.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.17.layer_norm2.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.17.mlp.fc1.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.17.mlp.fc1.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.17.mlp.fc2.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.17.mlp.fc2.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.17.self_attn.out_proj.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.17.self_attn.out_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.17.self_attn.qkv_proj.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.17.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.18.layer_norm1.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.18.layer_norm1.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.18.layer_norm2.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.18.layer_norm2.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.18.mlp.fc1.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.18.mlp.fc1.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.18.mlp.fc2.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.18.mlp.fc2.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.18.self_attn.out_proj.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.18.self_attn.out_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.18.self_attn.qkv_proj.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.18.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.19.layer_norm1.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.19.layer_norm1.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.19.layer_norm2.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.19.layer_norm2.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.19.mlp.fc1.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.19.mlp.fc1.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.19.mlp.fc2.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.19.mlp.fc2.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.19.self_attn.out_proj.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.19.self_attn.out_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.19.self_attn.qkv_proj.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.19.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.2.layer_norm1.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.2.layer_norm1.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.2.layer_norm2.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.2.layer_norm2.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.2.mlp.fc1.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.2.mlp.fc1.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.2.mlp.fc2.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.2.mlp.fc2.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.2.self_attn.out_proj.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.2.self_attn.out_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.2.self_attn.qkv_proj.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.2.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.20.layer_norm1.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.20.layer_norm1.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.20.layer_norm2.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.20.layer_norm2.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.20.mlp.fc1.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.20.mlp.fc1.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.20.mlp.fc2.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.20.mlp.fc2.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.20.self_attn.out_proj.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.20.self_attn.out_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.20.self_attn.qkv_proj.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.20.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.21.layer_norm1.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.21.layer_norm1.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.21.layer_norm2.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.21.layer_norm2.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.21.mlp.fc1.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.21.mlp.fc1.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.21.mlp.fc2.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.21.mlp.fc2.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.21.self_attn.out_proj.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.21.self_attn.out_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.21.self_attn.qkv_proj.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.21.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.22.layer_norm1.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.22.layer_norm1.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.22.layer_norm2.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.22.layer_norm2.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.22.mlp.fc1.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.22.mlp.fc1.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.22.mlp.fc2.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.22.mlp.fc2.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.22.self_attn.out_proj.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.22.self_attn.out_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.22.self_attn.qkv_proj.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.22.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.23.layer_norm1.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.23.layer_norm1.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.23.layer_norm2.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.23.layer_norm2.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.23.mlp.fc1.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.23.mlp.fc1.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.23.mlp.fc2.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.23.mlp.fc2.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.23.self_attn.out_proj.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.23.self_attn.out_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.23.self_attn.qkv_proj.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.23.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.3.layer_norm1.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.3.layer_norm1.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.3.layer_norm2.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.3.layer_norm2.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.3.mlp.fc1.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.3.mlp.fc1.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.3.mlp.fc2.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.3.mlp.fc2.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.3.self_attn.out_proj.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.3.self_attn.out_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.3.self_attn.qkv_proj.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.3.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.4.layer_norm1.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.4.layer_norm1.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.4.layer_norm2.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.4.layer_norm2.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.4.mlp.fc1.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.4.mlp.fc1.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.4.mlp.fc2.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.4.mlp.fc2.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.4.self_attn.out_proj.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.4.self_attn.out_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.4.self_attn.qkv_proj.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.4.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.5.layer_norm1.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.5.layer_norm1.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.5.layer_norm2.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.5.layer_norm2.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.5.mlp.fc1.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.5.mlp.fc1.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.5.mlp.fc2.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.5.mlp.fc2.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.5.self_attn.out_proj.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.5.self_attn.out_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.5.self_attn.qkv_proj.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.5.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.6.layer_norm1.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.6.layer_norm1.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.6.layer_norm2.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.6.layer_norm2.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.6.mlp.fc1.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.6.mlp.fc1.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.6.mlp.fc2.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.6.mlp.fc2.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.6.self_attn.out_proj.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.6.self_attn.out_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.6.self_attn.qkv_proj.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.6.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.7.layer_norm1.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.7.layer_norm1.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.7.layer_norm2.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.7.layer_norm2.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.7.mlp.fc1.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.7.mlp.fc1.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.7.mlp.fc2.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.7.mlp.fc2.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.7.self_attn.out_proj.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.7.self_attn.out_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.7.self_attn.qkv_proj.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.7.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.8.layer_norm1.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.8.layer_norm1.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.8.layer_norm2.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.8.layer_norm2.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.8.mlp.fc1.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.8.mlp.fc1.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.8.mlp.fc2.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.8.mlp.fc2.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.8.self_attn.out_proj.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.8.self_attn.out_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.8.self_attn.qkv_proj.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.8.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.9.layer_norm1.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.9.layer_norm1.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.9.layer_norm2.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.9.layer_norm2.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.9.mlp.fc1.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.9.mlp.fc1.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.9.mlp.fc2.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.9.mlp.fc2.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.9.self_attn.out_proj.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.9.self_attn.out_proj.weight": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.9.self_attn.qkv_proj.bias": "model-00002-of-00002.safetensors",
+    "vision_model.transformer.layers.9.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors"
+  }
+}