first commit

.gitattributes

@@ -0,0 +1,36 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,95 @@
+---
+license: apache-2.0
+---
+
+# Model Summary
+## 25EMBAI-VLM-FM is a Vision-Language Foundation Model built by combining:
+### Vision Encoder: ViT-H/14 (OpenCLIP)
+### Language Model: Qwen-based LLM
+### Bridging Modules: Resampler + Projector (image → LLM embedding space)
+
+It takes an image, encodes it into patch tokens, compresses them into a fixed-length set of visual tokens, projects them into the language model’s hidden space, and then performs multimodal reasoning conditioned on a text prompt.
+
+## Architecture Flow
+Image → ViT-H/14 → Resampler → Projector → Qwen LLM → Text Output
+LLM Input Format
[Batch, K_image_tokens + T_text_tokens, D_hidden]
+
+## Training Summary
+### Pre-training (Stage 1 & 2)
+Hardware: 8 × H100 80GB
+Stage 1 (3.6h):
Freeze ViT + LLM → Train Resampler + Projector
+Stage 2 (5.4h):
Unfreeze all → Train end-to-end
+Data: ~2M image–caption pairs (BLIP3 style)
+### Instruction Fine-tuning
+~2M images + ~200M text tokens
+~20 multimodal tasks: VQA, OCR, captioning, commands
+max_length: 1024
+effective batch size: ~64
+
+
+# Usage
+
+## Install
+pip install torch transformers pillow
+
+## Inference Example
+from transformers import AutoModel, AutoTokenizer, AutoImageProcessor 
+
+import torch 
+
+from PIL import Image 
+
+model_path = "YOUR_HF_USERNAME/25EMBAI-VLM-FM"
+
+dtype = torch.bfloat16
+
+### Load model
+model = AutoModel.from_pretrained(
+    model_path,
+    trust_remote_code=True,
+).to(device="cuda", dtype=dtype)
+
+### Load tokenizer
+tokenizer = AutoTokenizer.from_pretrained(model_path)
+
+### Load image processor from model assets
+image_processor = AutoImageProcessor.from_pretrained(
+    model_path,
+    trust_remote_code=True,
+)
+
+model.eval()
+
+### Load image
+img = Image.open("sample.png").convert("RGB")
+
+### Transform image → visual embeddings
+pixel = image_processor(img, return_tensors="pt")["pixel_values"].to(
+    dtype=dtype, device="cuda"
+)
+
+### Prompt
+prompt = "please describe this image."
+
+### Multimodal generation
+output = model.generate_text(
+    images=pixel,
+    prompt=prompt,
+    max_new_tokens=512,
+    do_sample=True,
+    top_p=0.9,
+    temperature=0.7,
+)
+
+print(output)
+# Limitations & Biases
+This model is an early-stage prototype.
It will be updated and reorganized in future releases.
+Because it was trained on web-scale multimodal data:
+It may reflect social biases and stereotypes
+It may hallucinate, invent facts, or produce unverifiable content
+It may perform suboptimally on:
+Non-English languages
+Specialized and domain-specific tasks
+Safety-critical contexts
+This model is not recommended for medical, legal, or safety-critical use without additional validation, guardrails, or fine-tuning.
+Users should apply external filtering, grounding, and safety alignment before deployment.

VLM_prototype/.gitattributes

@@ -0,0 +1,2 @@
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

VLM_prototype/added_tokens.json

@@ -0,0 +1,28 @@
+{
+  "</think>": 151668,
+  "</tool_call>": 151658,
+  "</tool_response>": 151666,
+  "<think>": 151667,
+  "<tool_call>": 151657,
+  "<tool_response>": 151665,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

VLM_prototype/chat_template.jinja

@@ -0,0 +1,61 @@
+{%- if tools %}
+    {{- '<|im_start|>system\n' }}
+    {%- if messages[0].role == 'system' %}
+        {{- messages[0].content + '\n\n' }}
+    {%- endif %}
+    {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
+    {%- for tool in tools %}
+        {{- "\n" }}
+        {{- tool | tojson }}
+    {%- endfor %}
+    {{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
+{%- else %}
+    {%- if messages[0].role == 'system' %}
+        {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
+    {%- endif %}
+{%- endif %}
+{%- for message in messages %}
+    {%- if message.content is string %}
+        {%- set content = message.content %}
+    {%- else %}
+        {%- set content = '' %}
+    {%- endif %}
+    {%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
+        {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
+    {%- elif message.role == "assistant" %}
+        {{- '<|im_start|>' + message.role + '\n' + content }}
+        {%- if message.tool_calls %}
+            {%- for tool_call in message.tool_calls %}
+                {%- if (loop.first and content) or (not loop.first) %}
+                    {{- '\n' }}
+                {%- endif %}
+                {%- if tool_call.function %}
+                    {%- set tool_call = tool_call.function %}
+                {%- endif %}
+                {{- '<tool_call>\n{"name": "' }}
+                {{- tool_call.name }}
+                {{- '", "arguments": ' }}
+                {%- if tool_call.arguments is string %}
+                    {{- tool_call.arguments }}
+                {%- else %}
+                    {{- tool_call.arguments | tojson }}
+                {%- endif %}
+                {{- '}\n</tool_call>' }}
+            {%- endfor %}
+        {%- endif %}
+        {{- '<|im_end|>\n' }}
+    {%- elif message.role == "tool" %}
+        {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
+            {{- '<|im_start|>user' }}
+        {%- endif %}
+        {{- '\n<tool_response>\n' }}
+        {{- content }}
+        {{- '\n</tool_response>' }}
+        {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
+            {{- '<|im_end|>\n' }}
+        {%- endif %}
+    {%- endif %}
+{%- endfor %}
+{%- if add_generation_prompt %}
+    {{- '<|im_start|>assistant\n' }}
+{%- endif %}

VLM_prototype/config.json

@@ -0,0 +1,168 @@
+{
+  "_class_name": "VLMModel",
+  "architectures": [
+    "VLMModel"
+  ],
+  "model_type": "vision-language",
+  "version": 1,
+  "vision_model": "ViT-H-14-378-quickgelu",
+  "vision_pretrained": "dfn5b",
+  "vision_width": 1280,
+  "lm_model": "Qwen/Qwen3-4B-Instruct-2507",
+  "lm_hidden_size": 2560,
+  "n_vis_tokens": 196,
+  "auto_map": {
+    "AutoConfig": "model.VLMConfig",
+    "AutoModel": "model.VLMModel"
+  },
+  "pad_token_id": 151643,
+  "eos_token_id": 151645,
+  "model_args": {
+    "vision_model": "ViT-H-14-378-quickgelu",
+    "vision_pretrained": "dfn5b",
+    "lm_model": "Qwen/Qwen3-4B-Instruct-2507",
+    "vision_freeze": true,
+    "projector_freeze": true,
+    "llm_freeze": true,
+    "n_vis_tokens": 196,
+    "checkpoint_path": null,
+    "use_chat_template": true
+  },
+  "lm_config": {
+    "vocab_size": 151936,
+    "max_position_embeddings": 262144,
+    "hidden_size": 2560,
+    "intermediate_size": 9728,
+    "num_hidden_layers": 36,
+    "num_attention_heads": 32,
+    "use_sliding_window": false,
+    "sliding_window": null,
+    "max_window_layers": 36,
+    "num_key_value_heads": 8,
+    "head_dim": 128,
+    "hidden_act": "silu",
+    "initializer_range": 0.02,
+    "rms_norm_eps": 1e-06,
+    "use_cache": false,
+    "rope_theta": 5000000,
+    "rope_scaling": null,
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "layer_types": [
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention"
+    ],
+    "return_dict": true,
+    "output_hidden_states": false,
+    "torchscript": false,
+    "dtype": "float32",
+    "pruned_heads": {},
+    "tie_word_embeddings": true,
+    "chunk_size_feed_forward": 0,
+    "is_encoder_decoder": false,
+    "is_decoder": false,
+    "cross_attention_hidden_size": null,
+    "add_cross_attention": false,
+    "tie_encoder_decoder": false,
+    "architectures": [
+      "Qwen3ForCausalLM"
+    ],
+    "finetuning_task": null,
+    "id2label": {
+      "0": "LABEL_0",
+      "1": "LABEL_1"
+    },
+    "label2id": {
+      "LABEL_0": 0,
+      "LABEL_1": 1
+    },
+    "task_specific_params": null,
+    "problem_type": null,
+    "tokenizer_class": null,
+    "prefix": null,
+    "bos_token_id": 151643,
+    "pad_token_id": null,
+    "eos_token_id": 151645,
+    "sep_token_id": null,
+    "decoder_start_token_id": null,
+    "max_length": 20,
+    "min_length": 0,
+    "do_sample": false,
+    "early_stopping": false,
+    "num_beams": 1,
+    "temperature": 1.0,
+    "top_k": 50,
+    "top_p": 1.0,
+    "typical_p": 1.0,
+    "repetition_penalty": 1.0,
+    "length_penalty": 1.0,
+    "no_repeat_ngram_size": 0,
+    "encoder_no_repeat_ngram_size": 0,
+    "bad_words_ids": null,
+    "num_return_sequences": 1,
+    "output_scores": false,
+    "return_dict_in_generate": false,
+    "forced_bos_token_id": null,
+    "forced_eos_token_id": null,
+    "remove_invalid_values": false,
+    "exponential_decay_length_penalty": null,
+    "suppress_tokens": null,
+    "begin_suppress_tokens": null,
+    "num_beam_groups": 1,
+    "diversity_penalty": 0.0,
+    "_name_or_path": "Qwen/Qwen3-4B-Instruct-2507",
+    "transformers_version": "4.57.1",
+    "model_type": "qwen3",
+    "tf_legacy_loss": false,
+    "use_bfloat16": false,
+    "output_attentions": false
+  },
+  "resampler_config": {
+    "K": 196,
+    "n_layers": 2,
+    "use_pos": true,
+    "q_grid": 14,
+    "adaptive_kv_pos": true,
+    "use_q_self_attn": true
+  },
+  "projector_config": {
+    "type": "Projector",
+    "in_features": 1280,
+    "out_features": 2560,
+    "num_layers": 2
+  }
+}

VLM_prototype/model.py

@@ -0,0 +1,792 @@
+# model.py
+from __future__ import annotations
+from dataclasses import dataclass, asdict
+from typing import Any, Dict, Optional, List
+import json, os
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import torchvision.transforms as T
+
+from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer, PretrainedConfig
+from transformers.image_processing_utils import BaseImageProcessor
+
+import open_clip
+from PIL import Image
+from safetensors.torch import save_file as _save_sf, load_file as _load_sf
+
+from utils import *
+from resampler import VisualResampler
+
+class VLMConfig(PretrainedConfig):
+    model_type = "vision-language"
+
+    def __init__(
+        self,
+        vision_model: str = "ViT-H-14-378-quickgelu",
+        vision_pretrained: str = "dfn5b",
+        lm_model: str = "Qwen/Qwen3-4B-Instruct-2507",
+        n_vis_tokens: int = 196,
+        use_chat_template: bool = True,
+        dtype: str = torch.bfloat16,
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+        self.vision_model = vision_model
+        self.vision_pretrained = vision_pretrained
+        self.lm_model = lm_model
+        self.n_vis_tokens = n_vis_tokens
+        self.use_chat_template = use_chat_template
+
+# ------------------------------- Args -------------------------------
+
+@dataclass
+class ModelArgs:
+    vision_model: str = "ViT-H-14-378-quickgelu"
+    vision_pretrained: str = "dfn5b"
+    # ✅ 기본값을 Qwen3 계열 Instruct로 설정 (원하면 CLI에서 덮어쓰기)
+    lm_model: str = "Qwen/Qwen3-4B-Instruct-2507"
+    vision_freeze: bool = False
+    projector_freeze: bool = False
+    llm_freeze: bool = False
+    n_vis_tokens: int = 196
+    checkpoint_path: Optional[str] = None
+    use_chat_template: bool = True
+
+# ------------------------------- Vision wrappers -------------------------------
+
+class OpenCLIPVisionOnly(nn.Module):
+    def __init__(self, clip_model):
+        super().__init__()
+        self.visual = clip_model.visual
+        self._feat_dim = self._infer_feat_dim()
+
+    def _infer_feat_dim(self) -> int:
+        v = self.visual
+        candidates = [
+            getattr(v, "width", None),
+            getattr(getattr(v, "ln_post", None), "normalized_shape", None),
+            getattr(getattr(v, "conv1", None), "out_channels", None),
+            getattr(v, "embed_dim", None),
+        ]
+        for cand in candidates:
+            if cand is None:
+                continue
+            if isinstance(cand, (list, tuple)):
+                return int(cand[0])
+            return int(cand)
+        # 안전 폴백
+        return 768
+
+    @property
+    def feat_dim(self) -> int:
+        return self._feat_dim
+
+    def tokens_or_global(self, pixel_values: torch.Tensor) -> torch.Tensor:
+        v = self.visual
+        # 1) 패치 임베딩
+        x = v.conv1(pixel_values)                                   # [B, C, H/ps, W/ps]
+        x = x.reshape(x.shape[0], x.shape[1], -1).permute(0, 2, 1)  # [B, HW, C]
+
+        # 2) CLS 붙이기
+        cls = v.class_embedding.to(x.dtype)
+        cls = cls.unsqueeze(0).expand(x.size(0), 1, -1)             # [B,1,C]
+        x = torch.cat([cls, x], dim=1)                              # [B, 1+HW, C]
+
+        # 3) 위치 임베딩(해상도 보간)
+        pe = getattr(v, "positional_embedding", None)
+        if pe is not None:
+            if pe.dim() == 2:
+                pe = pe.unsqueeze(0)                                # [1, N, C]
+            if pe.shape[1] != x.shape[1]:
+                cls_pe, patch_pe = pe[:, :1, :], pe[:, 1:, :]
+                s0 = int((patch_pe.shape[1]) ** 0.5)
+                s1 = int((x.shape[1] - 1) ** 0.5)
+                patch_pe = patch_pe.reshape(1, s0, s0, -1).permute(0, 3, 1, 2)
+                patch_pe = F.interpolate(patch_pe, size=(s1, s1), mode="bicubic", align_corners=False)
+                patch_pe = patch_pe.permute(0, 2, 3, 1).reshape(1, s1 * s1, -1)
+                pe = torch.cat([cls_pe, patch_pe], dim=1)
+            x = x + pe.to(dtype=x.dtype, device=x.device)
+
+        x = v.ln_pre(x)
+        for blk in v.transformer.resblocks:
+            x = blk(x)
+        x = v.ln_post(x)                                            # [B, 1+T, C]
+
+        # CLS 제외 패치 토큰 반환
+        return x[:, 1:, :]                                          # [B, T, C]
+
+# ------------------------------- Adapters -------------------------------
+
+class Projector(nn.Module):
+    def __init__(self, d_in: int, d_out: int):
+        super().__init__()
+        self.net = nn.Sequential(
+            nn.LayerNorm(d_in),
+            nn.Linear(d_in, d_out),
+            nn.GELU(),
+            nn.Linear(d_out, d_out),
+            nn.LayerNorm(d_out),
+        )
+    def forward(self, x):
+        return self.net(x)
+
+# ------------------------------- VLM -------------------------------
+
+class VLMModel(nn.Module):
+    def __init__(self, margs: ModelArgs, device: Optional[torch.device] = None):
+        super().__init__()
+        self.margs = margs
+        self.device_ = device or torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+        # --- Vision ---
+        clip, _, _ = open_clip.create_model_and_transforms(
+            margs.vision_model, pretrained=margs.vision_pretrained, device=self.device_
+        )
+        self.vision = OpenCLIPVisionOnly(clip).to(self.device_)
+        self.vision_width = int(self.vision.feat_dim)
+
+        # export용 메타
+        self.vision_model_name = margs.vision_model
+        self.vision_pretrained_tag = margs.vision_pretrained
+
+        if margs.vision_freeze:
+            for p in self.vision.parameters():
+                p.requires_grad_(False)
+
+        # --- Tokens pipeline
+        lm_cfg = AutoConfig.from_pretrained(margs.lm_model, trust_remote_code=True)
+        lm_hidden = lm_cfg.hidden_size
+
+        n_vis_tokens = margs.n_vis_tokens
+        self.resampler = VisualResampler(
+            C=self.vision_width,
+            K=n_vis_tokens,
+            n_heads=8,
+            n_layers=2,
+            kv_dim=self.vision_width,
+            use_pos=True,
+            q_grid=int(n_vis_tokens ** 0.5) if int(n_vis_tokens ** 0.5) ** 2 == n_vis_tokens else None,
+            adaptive_kv_pos=True,
+            dropout=0.0,
+            use_q_self_attn=True
+        ).to(self.device_)
+
+        self.projector = Projector(d_in=self.vision_width, d_out=lm_hidden).to(self.device_)
+        if margs.projector_freeze:
+            for p in self.projector.parameters():
+                p.requires_grad_(False)
+     
+        # --- LM (Qwen3) --- 
+        self.lm = AutoModelForCausalLM.from_pretrained(
+            margs.lm_model, trust_remote_code=True
+        )
+        self.tokenizer = AutoTokenizer.from_pretrained(
+            margs.lm_model, use_fast=True, trust_remote_code=True
+        )
+
+        self.lm.to(self.device_)
+        if self.tokenizer.pad_token is None:
+            self.tokenizer.pad_token = self.tokenizer.eos_token
+        self.lm_hidden = self.lm.config.hidden_size
+        
+        if hasattr(self.lm, "config"):
+            self.lm.config.use_cache = False
+        
+        if margs.llm_freeze:
+            for p in self.lm.parameters():
+                p.requires_grad_(False)
+    
+        if margs.checkpoint_path:
+            target_path, kind = _resolve_ckpt_target_from_path(margs.checkpoint_path)
+            if not os.path.exists(target_path):
+                raise FileNotFoundError(f"[ckpt] not found: {target_path} (kind={kind})")
+            rank0_print(f"[ckpt] target={target_path} ({kind})")
+
+            # ▶ 단일 파일/샤드 디렉터리/루트 인덱스 모두 지원
+            state = _hf_load_state_dict_any(target_path, map_location=self.device_)
+
+            # {'state_dict': ...} 래핑 해제
+            if isinstance(state, dict) and "state_dict" in state and isinstance(state["state_dict"], dict):
+                state = state["state_dict"]
+
+            # DDP 저장본 접두어 제거
+            if any(isinstance(k, str) and k.startswith("module.") for k in state.keys()):
+                state = {k.replace("module.", "", 1): v for k, v in state.items()}
+
+            # ▶ 현재 모델(self)에 주입 (버그 fix: self.model X)
+            missing, unexpected = self.load_state_dict(state, strict=False)
+            rank0_print(f"[load_state_dict] missing:{len(missing)} unexpected:{len(unexpected)}")
+            if missing:
+                rank0_print("  - missing (first 10): " + ", ".join(missing[:10]))
+            if unexpected:
+                rank0_print("  - unexpected (first 10): " + ", ".join(unexpected[:10]))
+
+
+    @property
+    def device(self):
+        return self.device_
+    
+    @property
+    def tok_emb(self):
+        return self.lm.get_input_embeddings()
+
+    # ---- Vision encode (patch tokens 우선) ----
+    def encode_patches(self, pixel_values: torch.Tensor) -> torch.Tensor:
+        """
+        Returns: patch tokens [B, T, C] (CLS 제외). 불가 시 [B, 1, C] 글로벌 토큰.
+        """
+        return self.vision.tokens_or_global(pixel_values)
+
+    def gradient_checkpointing_enable(self, gradient_checkpointing_kwargs=None):
+        if hasattr(self.lm, "gradient_checkpointing_enable"):
+            if gradient_checkpointing_kwargs is not None:
+                self.lm.gradient_checkpointing_enable(
+                    gradient_checkpointing_kwargs=gradient_checkpointing_kwargs
+                )
+            else:
+                self.lm.gradient_checkpointing_enable()
+        if hasattr(self.lm, "config"):
+            self.lm.config.use_cache = False
+        if hasattr(self.lm, "enable_input_require_grads"):
+            try:
+                self.lm.enable_input_require_grads()
+            except Exception:
+                pass
+
+    def gradient_checkpointing_disable(self):
+        if hasattr(self.lm, "gradient_checkpointing_disable"):
+            self.lm.gradient_checkpointing_disable()
+
+    # ---- Visual-first concat (no splice, no markers) ----
+    def _concat_visual_first(
+        self,
+        text_embeds: torch.Tensor,        # [B, T, D]
+        attention_mask: torch.Tensor,     # [B, T]
+        proj_feats: Optional[torch.Tensor],  # [B, K, D] or None
+    ):
+        if proj_feats is None:
+            return text_embeds, attention_mask
+        B, T, D = text_embeds.shape
+        K = proj_feats.size(1)
+        new_embeds = torch.cat([proj_feats, text_embeds], dim=1)  # [B, K+T, D]
+        pad_ones = torch.ones(B, K, dtype=attention_mask.dtype, device=attention_mask.device)
+        new_attn  = torch.cat([pad_ones, attention_mask], dim=1)  # [B, K+T]
+        return new_embeds, new_attn
+        
+    # ---- Forward ----
+    def forward(
+        self,
+        images: Optional[torch.Tensor] = None,
+        input_ids: Optional[torch.Tensor] = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        labels: Optional[torch.Tensor] = None,
+    ) -> Dict[str, Any]:
+        # input_ids 없으면 labels에서 복구
+        if input_ids is None:
+            if labels is None:
+                raise ValueError("Either input_ids or labels must be provided.")
+            pad_id = self.tokenizer.pad_token_id
+            input_ids = labels.clone()
+            input_ids[input_ids == -100] = pad_id
+            if attention_mask is None:
+                attention_mask = (input_ids != pad_id).long()
+
+        vis_tokens_lm = None
+        if images is not None:
+            patches   = self.encode_patches(images)     # [B,Tv,Cv]
+            patches_K = self.resampler(patches)         # [B,K,Cv]
+            vis_tokens_lm = self.projector(patches_K)   # [B,K,Dlm]
+            lm_dtype = next(self.lm.parameters()).dtype
+            if vis_tokens_lm.dtype != lm_dtype:
+                vis_tokens_lm = vis_tokens_lm.to(lm_dtype)
+
+        # 텍스트 임베딩 및 마스크 준비
+        text_emb = self.tok_emb(input_ids)  # [B,T,D]
+        attn_in  = attention_mask if attention_mask is not None else torch.ones_like(input_ids)
+
+        # 🔁 splice 제거 → 항상 "비전 토큰 앞 concat"
+        inputs_embeds, attn = self._concat_visual_first(
+            text_embeds=text_emb,
+            attention_mask=attn_in,
+            proj_feats=vis_tokens_lm,   # 이미지 없으면 None
+        )
+
+        # 라벨 재구성
+        new_labels = None
+        if labels is not None:
+            if vis_tokens_lm is None:
+                new_labels = labels
+            else:
+                B, K, _ = vis_tokens_lm.shape
+                pad_mask = torch.full((B, K), -100, dtype=labels.dtype, device=labels.device)
+                new_labels = torch.cat([pad_mask, labels], dim=1)  # [B, K+T]
+
+        out = self.lm(
+            inputs_embeds=inputs_embeds if vis_tokens_lm is not None else None,
+            attention_mask=attn if vis_tokens_lm is not None else attention_mask,
+            input_ids=None if vis_tokens_lm is not None else input_ids,
+            labels=new_labels if vis_tokens_lm is not None else labels,
+            output_hidden_states=False,
+            use_cache=False,
+        )
+        return {"loss": out.loss}
+
+    # ---- Inference helper (Qwen chat template) ----
+    @torch.no_grad()
+    def generate_text(
+        self,
+        images: Optional[torch.Tensor] = None,
+        prompt: Optional[str] = None,
+        max_new_tokens: int = 128,
+        do_sample: bool = True,
+        top_p: float = 0.9,
+        temperature: float = 0.7,
+        **gen_kw,
+    ) -> str:
+        self.eval()
+        device = self.device
+
+        system_prompt = "You are a helpful assistant."
+        user_text = (prompt or "").strip()
+
+        if getattr(self.margs, "use_chat_template", True):
+            messages = []
+            if system_prompt.strip():
+                messages.append({"role": "system", "content": system_prompt.strip()})
+            messages.append({"role": "user", "content": user_text})
+
+            input_ids = self.tokenizer.apply_chat_template(
+                messages,
+                tokenize=True,
+                add_generation_prompt=True,
+                return_tensors="pt",
+            )
+            attention_mask = torch.ones_like(input_ids)
+        else:
+            # (fallback) 단순 문자열
+            text = f"System: {system_prompt}\nUser: {user_text}\nAssistant: "
+            enc = self.tokenizer(text, return_tensors="pt", add_special_tokens=False)
+            input_ids = enc.input_ids
+            attention_mask = enc.attention_mask
+
+        input_ids = input_ids.to(device)
+        attention_mask = attention_mask.to(device)
+
+        if images is not None:
+            # 비전 임베딩
+            with torch.autocast(device_type="cuda", dtype=torch.bfloat16):
+                vision_feats = self.encode_patches(images.to(device, non_blocking=True))
+            patches_K = self.resampler(vision_feats)                # [1,K,Cv]
+            projected_feats = self.projector(patches_K)             # [1,K,Dlm]
+            lm_dtype = next(self.lm.parameters()).dtype
+            if projected_feats.dtype != lm_dtype:
+                projected_feats = projected_feats.to(lm_dtype)
+
+            # 텍스트 임베딩
+            text_embeds = self.lm.get_input_embeddings()(input_ids) # [1,T,D]
+            inputs_embeds, attn = self._concat_visual_first(
+                text_embeds=text_embeds,
+                attention_mask=attention_mask,
+                proj_feats=projected_feats,
+            )
+            gen_out = self.lm.generate(
+                inputs_embeds=inputs_embeds,
+                attention_mask=attn,
+                max_new_tokens=max_new_tokens,
+                do_sample=gen_kw.get("do_sample", do_sample),
+                temperature=gen_kw.get("temperature", temperature),
+                top_p=gen_kw.get("top_p", top_p),
+                eos_token_id=self.tokenizer.eos_token_id,
+                pad_token_id=self.tokenizer.pad_token_id,
+                use_cache=True
+            )
+            seq = gen_out[0]
+            decoded = self.tokenizer.decode(seq, skip_special_tokens=True)
+            return decoded.strip()
+        else:
+            gen_out = self.lm.generate(
+                input_ids=input_ids,
+                attention_mask=attention_mask,
+                max_new_tokens=max_new_tokens,
+                do_sample=gen_kw.get("do_sample", do_sample),
+                temperature=gen_kw.get("temperature", temperature),
+                top_p=gen_kw.get("top_p", top_p),
+                eos_token_id=self.tokenizer.eos_token_id,
+                pad_token_id=self.tokenizer.pad_token_id,
+                use_cache=True
+            )
+            seq = gen_out[0]
+            decoded = self.tokenizer.decode(seq[input_ids.size(1):], skip_special_tokens=True)
+            return decoded.strip()
+
+    @staticmethod
+    def _break_shared_tensors(sd: Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor]:
+        by_ptr: Dict[tuple, list] = {}
+        for k, v in sd.items():
+            if not isinstance(v, torch.Tensor):
+                continue
+
+            if v.device.type != "meta":
+                # 새 API 우선 사용, 없으면 옛날 storage()로 폴백
+                if hasattr(v, "untyped_storage"):
+                    storage = v.untyped_storage()
+                else:
+                    storage = v.storage()
+                ptr = storage.data_ptr()
+            else:
+                # meta tensor는 storage가 없으니 그냥 data_ptr
+                ptr = v.data_ptr()
+
+            by_ptr.setdefault((ptr, v.dtype, tuple(v.size())), []).append(k)
+
+        sd = dict(sd)  # shallow copy
+        for (_ptr, _dtype, _shape), keys in by_ptr.items():
+            if len(keys) <= 1:
+                continue
+            master = keys[0]
+            for k in keys[1:]:
+                sd[k] = sd[k].clone()
+        return sd
+
+    @staticmethod
+    def _retie_lm_head_if_possible(model: "VLMModel"):
+        try:
+            emb = model.lm.get_input_embeddings().weight
+        except Exception:
+            emb = None
+        candidates = [
+            "lm.lm_head.weight",
+            "lm.get_output_embeddings.weight",
+        ]
+        for name in candidates:
+            head = model
+            try:
+                for part in name.split("."):
+                    head = getattr(head, part)
+                if isinstance(head, torch.nn.Parameter) and emb is not None and head.shape == emb.shape:
+                    with torch.no_grad():
+                        head.set_(emb)
+                    break
+            except Exception:
+                continue
+
+    # ---------------- config I/O ----------------
+    def _export_vlm_config(self) -> Dict[str, Any]:
+        """
+        HuggingFace 호환용 config.json 생성기.
+
+        - model_type: "vision-language"  (커스텀 타입)
+        - architectures: ["VLMModel"]   (AutoModel가 참고)
+        - auto_map:
+            * AutoConfig -> model.VLMConfig
+            * AutoModel  -> model.VLMModel
+        """
+        # 1) 기본 VLM 메타
+        cfg: Dict[str, Any] = {
+            "_class_name": self.__class__.__name__,   # "VLMModel"
+            "architectures": [self.__class__.__name__],
+            "model_type": "vision-language",
+            "version": 1,
+
+            # --- 간단 메타 ---
+            "vision_model": getattr(self, "vision_model_name", None) or "ViT-B-16-quickgelu",
+            "vision_pretrained": getattr(self, "vision_pretrained_tag", None) or "metaclip_400m",
+            "vision_width": int(getattr(self, "vision_width", 0)),
+
+            "lm_model": getattr(self.lm, "name_or_path", None) or self.margs.lm_model,
+            "lm_hidden_size": int(getattr(self, "lm_hidden", 0)),
+
+            "n_vis_tokens": getattr(self.resampler, "K", None) or self.margs.n_vis_tokens
+        }
+
+        # 1-1) remote code용 매핑 정보
+        cfg["auto_map"] = {
+            # model.py 모듈에서 VLMConfig / VLMModel을 import
+            "AutoConfig": "model.VLMConfig",
+            "AutoModel": "model.VLMModel",
+        }
+
+        # 1-2) 토크나이저 관련 메타 (있으면)
+        try:
+            tok = getattr(self, "tokenizer", None)
+            if tok is not None:
+                if tok.pad_token_id is not None:
+                    cfg["pad_token_id"] = int(tok.pad_token_id)
+                if tok.eos_token_id is not None:
+                    cfg["eos_token_id"] = int(tok.eos_token_id)
+                if tok.bos_token_id is not None:
+                    cfg["bos_token_id"] = int(tok.bos_token_id)
+        except Exception:
+            pass
+
+        # 2) 원본 ModelArgs 저장
+        margs = getattr(self, "margs", None)
+        if margs is not None:
+            try:
+                cfg["model_args"] = asdict(margs)
+            except Exception:
+                try:
+                    cfg["model_args"] = dict(margs.__dict__)
+                except Exception:
+                    pass
+
+        # 3) LLM(HF) config 전체도 같이 저장
+        if hasattr(self.lm, "config") and self.lm.config is not None:
+            try:
+                cfg["lm_config"] = self.lm.config.to_dict()
+            except Exception:
+                pass
+
+        # 4) Resampler 관련 하이퍼파라미터
+        res = getattr(self, "resampler", None)
+        if res is not None:
+            res_cfg = {}
+            for attr in [
+                "K", "n_heads", "n_layers", "kv_dim",
+                "use_pos", "q_grid", "adaptive_kv_pos",
+                "dropout", "use_q_self_attn",
+            ]:
+                if hasattr(res, attr):
+                    v = getattr(res, attr)
+                    if isinstance(v, torch.Tensor):
+                        v = v.item() if v.numel() == 1 else list(v.shape)
+                    res_cfg[attr] = v
+            cfg["resampler_config"] = res_cfg
+
+        # 5) Projector 구조 요약
+        proj = getattr(self, "projector", None)
+        if proj is not None:
+            proj_cfg = {
+                "type": proj.__class__.__name__,
+            }
+            if hasattr(proj, "net") and isinstance(proj.net, nn.Sequential):
+                in_dim = None
+                out_dim = None
+                for m in proj.net:
+                    if isinstance(m, nn.Linear):
+                        if in_dim is None:
+                            in_dim = m.in_features
+                        out_dim = m.out_features
+                if in_dim is not None:
+                    proj_cfg["in_features"] = int(in_dim)
+                if out_dim is not None:
+                    proj_cfg["out_features"] = int(out_dim)
+                proj_cfg["num_layers"] = sum(
+                    1 for m in proj.net if isinstance(m, nn.Linear)
+                )
+            cfg["projector_config"] = proj_cfg
+
+        return cfg
+
+    def save_pretrained(
+            self,
+            save_directory: str,
+            safe: bool = True,
+            **kwargs,   # HF가 넘기는 기타 인자 무시용
+        ) -> None:
+            os.makedirs(save_directory, exist_ok=True)
+
+            # 1) state_dict 추출 및 공유 스토리지 해제
+            sd = self.state_dict()
+            sd = self._break_shared_tensors(sd)
+
+            # 2) 저장 (safetensors 권장)
+            if safe:
+                _save_sf(sd, os.path.join(save_directory, "model.safetensors"))
+            else:
+                torch.save(sd, os.path.join(save_directory, "pytorch_model.bin"))
+
+            # 3) 구성/토크나이저
+            with open(os.path.join(save_directory, "config.json"), "w", encoding="utf-8") as f:
+                json.dump(self._export_vlm_config(), f, ensure_ascii=False, indent=2)
+            try:
+                if getattr(self, "tokenizer", None) is not None:
+                    self.tokenizer.save_pretrained(save_directory)
+            except Exception:
+                pass
+
+    @classmethod
+    def from_pretrained(
+        cls,
+        pretrained_model_name_or_path: str,
+        *model_args,
+        config: Optional[Any] = None,
+        **kwargs,
+    ):
+        """
+        AutoModel.from_pretrained(..., trust_remote_code=True)가 호출하는 진입점.
+        pretrained_model_name_or_path: save_pretrained로 저장된 디렉토리 경로
+        """
+        load_directory = pretrained_model_name_or_path
+
+        # 1) config.json 로드
+        cfg_path = os.path.join(load_directory, "config.json")
+        if not os.path.exists(cfg_path):
+            raise FileNotFoundError(f"config.json not found in {load_directory}")
+        with open(cfg_path, "r", encoding="utf-8") as f:
+            cfg = json.load(f)
+
+        # 2) ModelArgs 복원 (config 안의 model_args가 있으면 우선 사용)
+        model_args_cfg = cfg.get("model_args", {})
+        margs = ModelArgs(
+            vision_model     = model_args_cfg.get("vision_model",     cfg.get("vision_model", "ViT-B-16-quickgelu")),
+            vision_pretrained= model_args_cfg.get("vision_pretrained",cfg.get("vision_pretrained", "metaclip_400m")),
+            lm_model         = model_args_cfg.get("lm_model",         cfg.get("lm_model", "state-spaces/mamba-370m-hf")),
+            n_vis_tokens     = int(model_args_cfg.get("n_vis_tokens", cfg.get("n_vis_tokens", 196))),
+            use_chat_template= model_args_cfg.get("use_chat_template", True),
+        )
+
+        model = cls(margs)
+
+        # 5) 가중치 로드 (CPU로 먼저 불러오고 이후 .to(device))
+        wt_sf = os.path.join(load_directory, "model.safetensors")
+        wt_pt = os.path.join(load_directory, "pytorch_model.bin")
+        if os.path.exists(wt_sf):
+            sd = _load_sf(wt_sf, device="cpu")
+        elif os.path.exists(wt_pt):
+            sd = torch.load(wt_pt, map_location="cpu")
+        else:
+            raise FileNotFoundError(
+                f"No weight file found in {load_directory} "
+                f"(expected model.safetensors or pytorch_model.bin)"
+            )
+
+        # 6) DDP prefix 제거
+        if any(k.startswith("module.") for k in sd):
+            sd = {k.replace("module.", "", 1): v for k, v in sd.items()}
+
+        missing, unexpected = model.load_state_dict(sd, strict=False)
+        print(f"[from_pretrained] missing={len(missing)} unexpected={len(unexpected)}")
+        if missing:
+            print("  - missing (first 10):", missing[:10])
+        if unexpected:
+            print("  - unexpected (first 10):", unexpected[:10])
+
+        # 7) 로드 후 tie 복구(메모리 절감)
+        try:
+            cls._retie_lm_head_if_possible(model)
+        except Exception:
+            pass
+        
+        return model
+
+    @classmethod
+    def register_for_auto_class(cls, auto_class=None):
+        """
+        transformers.AutoModel.from_pretrained(..., trust_remote_code=True)가
+        커스텀 모델을 AutoModel 레지스트리에 등록하려고 호출하는 메서드.
+        """
+        return
+
+class OpenCLIPImageProcessor(BaseImageProcessor):
+    model_input_names = ["pixel_values"]
+    image_processor_type = "open_clip"
+
+    def __init__(
+        self,
+        vision_model: str = "ViT-H-14-378-quickgelu",
+        vision_pretrained: str = "dfn5b",
+        is_train: bool = False,
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+
+        self.vision_model = vision_model
+        self.vision_pretrained = vision_pretrained
+        self.is_train = is_train
+
+        # HF AutoImageProcessor용 remote code 매핑
+        self.auto_map = {"AutoImageProcessor": "model.OpenCLIPImageProcessor"}
+
+        # 실제 torchvision transform은 lazy하게 생성 (처음 호출 시)
+        self._transform = None
+
+    @classmethod
+    def register_for_auto_class(cls, auto_class=None):
+        # AutoImageProcessor.from_pretrained(..., trust_remote_code=True) 호출 시 사용될 수 있는 훅
+        # 우리는 별도 레지스트리 안 써도 되니 no-op
+        return
+
+    def _ensure_transform(self):
+        if self._transform is not None:
+            return
+
+        clip_model, preprocess_train, preprocess_val = open_clip.create_model_and_transforms(
+            self.vision_model,
+            pretrained=self.vision_pretrained,
+            device="cpu",  # 전처리용이니 CPU로 충분
+        )
+        del clip_model  # 모델은 필요 없음
+
+        self._transform = preprocess_train if self.is_train else preprocess_val
+
+    def to_dict(self) -> Dict[str, Any]:
+        # BaseImageProcessor → dict 직렬화 시 호출
+        config = super().to_dict()
+        config.update(
+            {
+                "vision_model": self.vision_model,
+                "vision_pretrained": self.vision_pretrained,
+                "is_train": self.is_train,
+                "auto_map": self.auto_map,
+                "image_processor_type": self.image_processor_type,
+            }
+        )
+        return config
+
+    def __call__(
+        self,
+        images,
+        return_tensors: Optional[str] = "pt",
+        **kwargs,
+    ) -> Dict[str, Any]:
+        """
+        images: PIL.Image 또는 그 리스트
+        return: {"pixel_values": Tensor[B, 3, H, W]}
+        """
+        self._ensure_transform()
+
+        if not isinstance(images, (list, tuple)):
+            images = [images]
+
+        proc = []
+        for img in images:
+            if not isinstance(img, Image.Image):
+                raise TypeError(f"Expected PIL.Image, but got {type(img)}")
+            proc.append(self._transform(img))
+        pixel_values = torch.stack(proc, dim=0)  # [B,3,H,W]
+
+        # HF 스타일에선 대부분 "pt" 텐서면 충분
+        return {"pixel_values": pixel_values}
+    
+# ------------------------------- Builder -------------------------------
+
+def build_model(
+    vision_model: str,
+    vision_pretrained: str,
+    lm_model: str,
+    vision_freeze: bool,
+    projector_freeze: bool,
+    llm_freeze: bool,
+    device: Optional[torch.device] = None,
+    checkpoint_path: Optional[str] = None,
+    n_vis_tokens: int = 196,
+    use_chat_template: bool = True,
+) -> VLMModel:    
+    margs = ModelArgs(
+        vision_model=vision_model,
+        vision_pretrained=vision_pretrained,
+        lm_model=lm_model,
+        checkpoint_path=checkpoint_path,
+        vision_freeze=vision_freeze,
+        projector_freeze=projector_freeze,
+        llm_freeze=llm_freeze,
+        n_vis_tokens=n_vis_tokens,
+        use_chat_template=use_chat_template,
+    )
+    model = VLMModel(margs, device)
+    model.to(device or model.device)
+    return model
+

VLM_prototype/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:db36e5ec2007edfa7df1427237f8097ada6772f26535179e44c7d7b6d6fe8e1c
+size 20426846152

VLM_prototype/preprocessor_config.json

@@ -0,0 +1,10 @@
+{
+  "_transform": null,
+  "auto_map": {
+    "AutoImageProcessor": "model.OpenCLIPImageProcessor"
+  },
+  "image_processor_type": "open_clip",
+  "is_train": false,
+  "vision_model": "ViT-H-14-378-quickgelu",
+  "vision_pretrained": "dfn5b"
+}

VLM_prototype/special_tokens_map.json

@@ -0,0 +1,31 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

VLM_prototype/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aeb13307a71acd8fe81861d94ad54ab689df773318809eed3cbe794b4492dae4
+size 11422654

VLM_prototype/tokenizer_config.json

@@ -0,0 +1,239 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151665": {
+      "content": "<tool_response>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151666": {
+      "content": "</tool_response>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151667": {
+      "content": "<think>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151668": {
+      "content": "</think>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 1010000,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}