Upload folder using huggingface_hub

added_tokens.json

@@ -0,0 +1,7 @@
+{
+  "<im_col>": 100281,
+  "<im_end>": 100279,
+  "<im_patch>": 100280,
+  "<im_start>": 100278,
+  "<|image|>": 100282
+}

chat_template.jinja

@@ -0,0 +1,4 @@
+{% for message in messages %}{{'<|im_start|>' + message['role'] + '
+' + message['content'] + '<|im_end|>' + '
+'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant
+' }}{% endif %}

config.json

@@ -0,0 +1,115 @@
+{
+  "architectures": [
+    "MolmoOlmo3ForConditionalGeneration"
+  ],
+  "auto_map": {
+    "AutoConfig": "configuration_molmo_olmo3.MolmoOlmo3Config",
+    "AutoModelForImageTextToText": "modeling_molmo_olmo3.MolmoOlmo3ForConditionalGeneration",
+    "AutoImageProcessor": "image_processing_molmo_olmo3.MolmoOlmo3ImageProcessor",
+    "AutoProcessor": "processing_molmo_olmo3.MolmoOlmo3Processor"
+  },
+  "bos_token_id": 100257,
+  "dtype": "float32",
+  "eos_token_id": 100257,
+  "image_col_token_id": 100281,
+  "image_end_token_id": 100279,
+  "image_prompt_token_id": 100282,
+  "image_start_token_id": 100278,
+  "image_token_id": 100280,
+  "include_cls_token": true,
+  "lm_head_vocab_size": 100352,
+  "model_type": "molmo_olmo3",
+  "projector_hidden_act": "silu",
+  "projector_intermediate_size": 11008,
+  "text_config": {
+    "architectures": [
+      "Olmo3ForCausalLM"
+    ],
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "dtype": "bfloat16",
+    "eos_token_id": 100257,
+    "hidden_act": "silu",
+    "hidden_size": 4096,
+    "initializer_range": 0.02,
+    "intermediate_size": 11008,
+    "layer_types": [
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention",
+      "full_attention"
+    ],
+    "max_position_embeddings": 65536,
+    "model_type": "olmo3",
+    "num_attention_heads": 32,
+    "num_hidden_layers": 32,
+    "num_key_value_heads": 32,
+    "pad_token_id": 100277,
+    "rms_norm_eps": 1e-06,
+    "rope_scaling": {
+      "attention_factor": 1.2079441541679836,
+      "beta_fast": 32,
+      "beta_slow": 1,
+      "factor": 8.0,
+      "original_max_position_embeddings": 8192,
+      "rope_type": "yarn"
+    },
+    "rope_theta": 500000,
+    "sliding_window": 4096,
+    "use_cache": true,
+    "vocab_size": 100480
+  },
+  "text_hidden_size": 4096,
+  "tie_word_embeddings": false,
+  "transformers_version": "4.57.1",
+  "vision_config": {
+    "attention_dropout": 0.0,
+    "hidden_act": "gelu",
+    "hidden_size": 768,
+    "image_size": 224,
+    "initializer_factor": 1.0,
+    "initializer_range": 0.02,
+    "intermediate_size": 3072,
+    "layer_norm_eps": 1e-05,
+    "model_type": "clip_vision_model",
+    "num_attention_heads": 12,
+    "num_channels": 3,
+    "num_hidden_layers": 12,
+    "patch_size": 16,
+    "projection_dim": 512
+  },
+  "vision_feature_layer": -1,
+  "vision_final_norm_attr": "norm",
+  "vision_hidden_size": 768,
+  "vision_strip_final_norm": false,
+  "vision_tower_name_or_path": "amitha/clip-vit-b16-datacomp-medium",
+  "vision_trust_remote_code": false
+}

configuration_molmo_olmo3.py

@@ -0,0 +1,109 @@
+# coding=utf-8
+"""Configuration for the Molmo-v1 (CLIP vision) VLM in HuggingFace format.
+
+LLaVA-style composition:
+  - vision_tower : a HuggingFace vision encoder (CLIPVisionModel), referenced by
+                   `vision_tower_name_or_path`; its weights are NOT stored in this
+                   checkpoint (loaded from the referenced repo at load time).
+  - multi_modal_projector : SwiGLU image projector + a separate CLS Linear projector.
+  - language_model : a native transformers `Olmo3ForCausalLM` (text_config).
+
+The vision encoder architecture is stored in `vision_config` (metadata only) so the
+module can be constructed without network access; the trained vision *weights* live
+in the referenced `vision_tower_name_or_path` repo.
+"""
+
+from transformers.configuration_utils import PretrainedConfig
+from transformers.models.auto import CONFIG_MAPPING, AutoConfig
+from transformers.models.olmo3.configuration_olmo3 import Olmo3Config
+
+
+class MolmoOlmo3Config(PretrainedConfig):
+    model_type = "molmo_olmo3"
+    sub_configs = {"text_config": Olmo3Config, "vision_config": AutoConfig}
+
+    def __init__(
+        self,
+        text_config=None,
+        vision_config=None,
+        vision_tower_name_or_path="amitha/clip-vit-b16-datacomp-1b-medium-subset",
+        vision_trust_remote_code=False,
+        vision_feature_layer=-1,  # Molmo vit_layers=[-1] -> last block output (pre-final-norm)
+        # DINOv3 applies a final LayerNorm to last_hidden_state that Molmo discards; strip it
+        # so the tower output is the pre-norm last-block features Molmo's connector consumes.
+        vision_strip_final_norm=False,
+        vision_final_norm_attr="norm",
+        projector_intermediate_size=11008,
+        projector_hidden_act="silu",
+        include_cls_token=True,
+        # lm_head covers the real vocab; ids >= lm_head_vocab_size are image
+        # placeholder tokens (never generation targets) and are masked to -inf.
+        lm_head_vocab_size=100352,
+        # Molmo's get_tokenizer pads to vocab_size=100278 (no padding tokens), so the
+        # 5 image special tokens land at 100278..100282 and index wte.embedding directly.
+        image_token_id=100280,        # <im_patch>
+        image_start_token_id=100278,  # <im_start>
+        image_end_token_id=100279,    # <im_end>
+        image_col_token_id=100281,    # <im_col>
+        image_prompt_token_id=100282, # <|image|>
+        bos_token_id=100257,
+        eos_token_id=100257,
+        pad_token_id=None,
+        tie_word_embeddings=False,
+        **kwargs,
+    ):
+        # --- text config (native Olmo3) ---
+        if text_config is None:
+            text_config = {}
+        if isinstance(text_config, dict):
+            text_config = Olmo3Config(**text_config)
+        self.text_config = text_config
+
+        # --- vision config (architecture metadata for the referenced encoder) ---
+        if vision_config is None:
+            # Default to the CLIP ViT-B/16 (224) vision tower used by this VLM family.
+            vision_config = CONFIG_MAPPING["clip_vision_model"](
+                hidden_size=768,
+                intermediate_size=3072,
+                num_hidden_layers=12,
+                num_attention_heads=12,
+                num_channels=3,
+                image_size=224,
+                patch_size=16,
+                hidden_act="quick_gelu",
+                layer_norm_eps=1e-5,
+            )
+        elif isinstance(vision_config, dict):
+            vision_model_type = vision_config.get("model_type", "clip_vision_model")
+            vision_config = CONFIG_MAPPING[vision_model_type](**vision_config)
+        self.vision_config = vision_config
+
+        self.vision_tower_name_or_path = vision_tower_name_or_path
+        self.vision_trust_remote_code = vision_trust_remote_code
+        self.vision_feature_layer = vision_feature_layer
+        self.vision_strip_final_norm = vision_strip_final_norm
+        self.vision_final_norm_attr = vision_final_norm_attr
+        self.vision_hidden_size = getattr(vision_config, "hidden_size", 768)
+        self.text_hidden_size = self.text_config.hidden_size
+
+        self.projector_intermediate_size = projector_intermediate_size
+        self.projector_hidden_act = projector_hidden_act
+        self.include_cls_token = include_cls_token
+
+        self.lm_head_vocab_size = lm_head_vocab_size
+        self.image_token_id = image_token_id
+        self.image_start_token_id = image_start_token_id
+        self.image_end_token_id = image_end_token_id
+        self.image_col_token_id = image_col_token_id
+        self.image_prompt_token_id = image_prompt_token_id
+
+        super().__init__(
+            bos_token_id=bos_token_id,
+            eos_token_id=eos_token_id,
+            pad_token_id=pad_token_id,
+            tie_word_embeddings=tie_word_embeddings,
+            **kwargs,
+        )
+
+
+__all__ = ["MolmoOlmo3Config"]

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "bos_token_id": 100257,
+  "eos_token_id": 100257,
+  "pad_token_id": 100257,
+  "transformers_version": "4.57.1"
+}

image_processing_molmo_olmo3.py

@@ -0,0 +1,92 @@
+# coding=utf-8
+"""Image processor for the Molmo-v1 (CLIP vision) VLM.
+
+Replicates Molmo's `hf_resize_and_center_crop` + OpenAI-CLIP normalization exactly
+(PIL BICUBIC, shortest-edge resize, center crop, /255, normalize) so the resulting
+(3, 224, 224) array is bit-identical to the Molmo preprocessor. The HF CLIPVisionModel
+performs the Conv2d patchification internally, matching Molmo's un-patchify+Conv2d path.
+"""
+
+from typing import Dict, List, Optional, Union
+
+import numpy as np
+import PIL.Image
+import torch
+
+from transformers.image_processing_utils import BaseImageProcessor, BatchFeature
+from transformers.image_utils import ImageInput
+
+
+OPENAI_CLIP_MEAN = (0.48145466, 0.4578275, 0.40821073)
+OPENAI_CLIP_STD = (0.26862954, 0.26130258, 0.27577711)
+
+
+def _to_uint8_rgb(image) -> np.ndarray:
+    if isinstance(image, PIL.Image.Image):
+        return np.array(image.convert("RGB"))
+    arr = np.asarray(image)
+    if arr.dtype != np.uint8:
+        # assume already in [0,255] if float
+        arr = arr.astype(np.uint8)
+    if arr.ndim == 2:
+        arr = np.stack([arr] * 3, axis=-1)
+    return arr[:, :, :3]
+
+
+def hf_resize_and_center_crop(image: np.ndarray, output_size) -> np.ndarray:
+    """Exactly mirrors olmo/data/model_preprocessor.py:hf_resize_and_center_crop."""
+    desired_h, desired_w = output_size
+    height, width = image.shape[:2]
+    scale = max(desired_h / height, desired_w / width)
+    new_h = int(height * scale)
+    new_w = int(width * scale)
+
+    pil_image = PIL.Image.fromarray(image)
+    pil_image = pil_image.resize((new_w, new_h), PIL.Image.BICUBIC)
+
+    top = (new_h - desired_h) // 2
+    left = (new_w - desired_w) // 2
+    pil_image = pil_image.crop((left, top, left + desired_w, top + desired_h))
+
+    return np.array(pil_image).astype(np.float32) / 255.0
+
+
+class MolmoOlmo3ImageProcessor(BaseImageProcessor):
+    model_input_names = ["pixel_values"]
+
+    def __init__(
+        self,
+        image_size: int = 224,
+        image_mean=OPENAI_CLIP_MEAN,
+        image_std=OPENAI_CLIP_STD,
+        **kwargs,
+    ):
+        super().__init__(**kwargs)
+        self.image_size = image_size
+        self.image_mean = list(image_mean)
+        self.image_std = list(image_std)
+
+    def preprocess_one(self, image) -> np.ndarray:
+        arr = _to_uint8_rgb(image)
+        resized = hf_resize_and_center_crop(arr, (self.image_size, self.image_size))  # (H,W,3) in [0,1]
+        resized = resized - np.array(self.image_mean, dtype=np.float32)[None, None, :]
+        resized = resized / np.array(self.image_std, dtype=np.float32)[None, None, :]
+        # HWC -> CHW
+        return np.transpose(resized, (2, 0, 1))
+
+    def preprocess(
+        self,
+        images: Union[ImageInput, List[ImageInput]],
+        return_tensors: Optional[str] = "pt",
+        **kwargs,
+    ) -> BatchFeature:
+        if not isinstance(images, (list, tuple)):
+            images = [images]
+        pixel_values = np.stack([self.preprocess_one(im) for im in images], axis=0)  # (n, 3, H, W)
+        data = {"pixel_values": pixel_values}
+        if return_tensors == "pt":
+            data = {"pixel_values": torch.from_numpy(pixel_values)}
+        return BatchFeature(data=data, tensor_type=None)
+
+
+__all__ = ["MolmoOlmo3ImageProcessor"]

model-00001-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2cad7f999b323ea993cf3fa3a2831739ca8a3f2c717762bc4d02728e945c4648
+size 4964718424

model-00002-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:11d12d32c86599a8b80f968b49ae74422c18aa72ce4f57dd31fc5ad3875d8369
+size 4857405840

model-00003-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aa1faec702b61fbeada74ff49597659e205caa6eab5e2b8c12181cc94bb5ac89
+size 4857405904

model-00004-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:110f3c8e62bd293022396152b35a4812f87fe04f569074c51c6d0235d76ddb6e
+size 4857405904

model-00005-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6102fd06f1961c650427cf208446df21426ee55ff439ddc06a8fbd13392b8cb8
+size 4857405904

model-00006-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:11bf1fe86d5e55c426d8713e0a11eb448ac30297e7187533e2e898e34d072810
+size 3418675336

model-00007-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8f5d1ff492b1e20a8a430e3ebf37818482af15aa632c73829fd77fa08d20a854
+size 1989489064

model.safetensors.index.json

@@ -0,0 +1,566 @@
+{
+  "metadata": {
+    "total_parameters": 7450608128,
+    "total_size": 29802432512
+  },
+  "weight_map": {
+    "language_model.lm_head.weight": "model-00007-of-00007.safetensors",
+    "language_model.model.embed_tokens.weight": "model-00001-of-00007.safetensors",
+    "language_model.model.layers.0.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
+    "language_model.model.layers.0.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
+    "language_model.model.layers.0.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
+    "language_model.model.layers.0.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
+    "language_model.model.layers.0.post_feedforward_layernorm.weight": "model-00001-of-00007.safetensors",
+    "language_model.model.layers.0.self_attn.k_norm.weight": "model-00001-of-00007.safetensors",
+    "language_model.model.layers.0.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "language_model.model.layers.0.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "language_model.model.layers.0.self_attn.q_norm.weight": "model-00001-of-00007.safetensors",
+    "language_model.model.layers.0.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "language_model.model.layers.0.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "language_model.model.layers.1.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
+    "language_model.model.layers.1.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
+    "language_model.model.layers.1.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
+    "language_model.model.layers.1.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
+    "language_model.model.layers.1.post_feedforward_layernorm.weight": "model-00001-of-00007.safetensors",
+    "language_model.model.layers.1.self_attn.k_norm.weight": "model-00001-of-00007.safetensors",
+    "language_model.model.layers.1.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "language_model.model.layers.1.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "language_model.model.layers.1.self_attn.q_norm.weight": "model-00001-of-00007.safetensors",
+    "language_model.model.layers.1.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "language_model.model.layers.1.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "language_model.model.layers.10.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "language_model.model.layers.10.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "language_model.model.layers.10.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "language_model.model.layers.10.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "language_model.model.layers.10.post_feedforward_layernorm.weight": "model-00003-of-00007.safetensors",
+    "language_model.model.layers.10.self_attn.k_norm.weight": "model-00003-of-00007.safetensors",
+    "language_model.model.layers.10.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "language_model.model.layers.10.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "language_model.model.layers.10.self_attn.q_norm.weight": "model-00003-of-00007.safetensors",
+    "language_model.model.layers.10.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "language_model.model.layers.10.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "language_model.model.layers.11.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "language_model.model.layers.11.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "language_model.model.layers.11.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "language_model.model.layers.11.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "language_model.model.layers.11.post_feedforward_layernorm.weight": "model-00003-of-00007.safetensors",
+    "language_model.model.layers.11.self_attn.k_norm.weight": "model-00003-of-00007.safetensors",
+    "language_model.model.layers.11.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "language_model.model.layers.11.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "language_model.model.layers.11.self_attn.q_norm.weight": "model-00003-of-00007.safetensors",
+    "language_model.model.layers.11.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "language_model.model.layers.11.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "language_model.model.layers.12.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "language_model.model.layers.12.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "language_model.model.layers.12.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "language_model.model.layers.12.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "language_model.model.layers.12.post_feedforward_layernorm.weight": "model-00003-of-00007.safetensors",
+    "language_model.model.layers.12.self_attn.k_norm.weight": "model-00003-of-00007.safetensors",
+    "language_model.model.layers.12.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "language_model.model.layers.12.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "language_model.model.layers.12.self_attn.q_norm.weight": "model-00003-of-00007.safetensors",
+    "language_model.model.layers.12.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "language_model.model.layers.12.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "language_model.model.layers.13.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "language_model.model.layers.13.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "language_model.model.layers.13.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "language_model.model.layers.13.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "language_model.model.layers.13.post_feedforward_layernorm.weight": "model-00003-of-00007.safetensors",
+    "language_model.model.layers.13.self_attn.k_norm.weight": "model-00003-of-00007.safetensors",
+    "language_model.model.layers.13.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "language_model.model.layers.13.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "language_model.model.layers.13.self_attn.q_norm.weight": "model-00003-of-00007.safetensors",
+    "language_model.model.layers.13.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "language_model.model.layers.13.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "language_model.model.layers.14.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "language_model.model.layers.14.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "language_model.model.layers.14.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "language_model.model.layers.14.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "language_model.model.layers.14.post_feedforward_layernorm.weight": "model-00003-of-00007.safetensors",
+    "language_model.model.layers.14.self_attn.k_norm.weight": "model-00003-of-00007.safetensors",
+    "language_model.model.layers.14.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "language_model.model.layers.14.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "language_model.model.layers.14.self_attn.q_norm.weight": "model-00003-of-00007.safetensors",
+    "language_model.model.layers.14.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "language_model.model.layers.14.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "language_model.model.layers.15.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "language_model.model.layers.15.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
+    "language_model.model.layers.15.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
+    "language_model.model.layers.15.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "language_model.model.layers.15.post_feedforward_layernorm.weight": "model-00004-of-00007.safetensors",
+    "language_model.model.layers.15.self_attn.k_norm.weight": "model-00003-of-00007.safetensors",
+    "language_model.model.layers.15.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
+    "language_model.model.layers.15.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "language_model.model.layers.15.self_attn.q_norm.weight": "model-00003-of-00007.safetensors",
+    "language_model.model.layers.15.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
+    "language_model.model.layers.15.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
+    "language_model.model.layers.16.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "language_model.model.layers.16.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "language_model.model.layers.16.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "language_model.model.layers.16.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "language_model.model.layers.16.post_feedforward_layernorm.weight": "model-00004-of-00007.safetensors",
+    "language_model.model.layers.16.self_attn.k_norm.weight": "model-00004-of-00007.safetensors",
+    "language_model.model.layers.16.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "language_model.model.layers.16.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "language_model.model.layers.16.self_attn.q_norm.weight": "model-00004-of-00007.safetensors",
+    "language_model.model.layers.16.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "language_model.model.layers.16.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "language_model.model.layers.17.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "language_model.model.layers.17.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "language_model.model.layers.17.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "language_model.model.layers.17.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "language_model.model.layers.17.post_feedforward_layernorm.weight": "model-00004-of-00007.safetensors",
+    "language_model.model.layers.17.self_attn.k_norm.weight": "model-00004-of-00007.safetensors",
+    "language_model.model.layers.17.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "language_model.model.layers.17.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "language_model.model.layers.17.self_attn.q_norm.weight": "model-00004-of-00007.safetensors",
+    "language_model.model.layers.17.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "language_model.model.layers.17.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "language_model.model.layers.18.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "language_model.model.layers.18.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "language_model.model.layers.18.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "language_model.model.layers.18.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "language_model.model.layers.18.post_feedforward_layernorm.weight": "model-00004-of-00007.safetensors",
+    "language_model.model.layers.18.self_attn.k_norm.weight": "model-00004-of-00007.safetensors",
+    "language_model.model.layers.18.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "language_model.model.layers.18.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "language_model.model.layers.18.self_attn.q_norm.weight": "model-00004-of-00007.safetensors",
+    "language_model.model.layers.18.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "language_model.model.layers.18.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "language_model.model.layers.19.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "language_model.model.layers.19.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "language_model.model.layers.19.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "language_model.model.layers.19.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "language_model.model.layers.19.post_feedforward_layernorm.weight": "model-00004-of-00007.safetensors",
+    "language_model.model.layers.19.self_attn.k_norm.weight": "model-00004-of-00007.safetensors",
+    "language_model.model.layers.19.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "language_model.model.layers.19.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "language_model.model.layers.19.self_attn.q_norm.weight": "model-00004-of-00007.safetensors",
+    "language_model.model.layers.19.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "language_model.model.layers.19.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "language_model.model.layers.2.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
+    "language_model.model.layers.2.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
+    "language_model.model.layers.2.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
+    "language_model.model.layers.2.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
+    "language_model.model.layers.2.post_feedforward_layernorm.weight": "model-00001-of-00007.safetensors",
+    "language_model.model.layers.2.self_attn.k_norm.weight": "model-00001-of-00007.safetensors",
+    "language_model.model.layers.2.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "language_model.model.layers.2.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "language_model.model.layers.2.self_attn.q_norm.weight": "model-00001-of-00007.safetensors",
+    "language_model.model.layers.2.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "language_model.model.layers.2.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "language_model.model.layers.20.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "language_model.model.layers.20.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "language_model.model.layers.20.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "language_model.model.layers.20.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "language_model.model.layers.20.post_feedforward_layernorm.weight": "model-00004-of-00007.safetensors",
+    "language_model.model.layers.20.self_attn.k_norm.weight": "model-00004-of-00007.safetensors",
+    "language_model.model.layers.20.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "language_model.model.layers.20.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "language_model.model.layers.20.self_attn.q_norm.weight": "model-00004-of-00007.safetensors",
+    "language_model.model.layers.20.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "language_model.model.layers.20.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "language_model.model.layers.21.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "language_model.model.layers.21.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
+    "language_model.model.layers.21.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
+    "language_model.model.layers.21.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "language_model.model.layers.21.post_feedforward_layernorm.weight": "model-00005-of-00007.safetensors",
+    "language_model.model.layers.21.self_attn.k_norm.weight": "model-00004-of-00007.safetensors",
+    "language_model.model.layers.21.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
+    "language_model.model.layers.21.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "language_model.model.layers.21.self_attn.q_norm.weight": "model-00004-of-00007.safetensors",
+    "language_model.model.layers.21.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
+    "language_model.model.layers.21.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
+    "language_model.model.layers.22.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "language_model.model.layers.22.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "language_model.model.layers.22.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "language_model.model.layers.22.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "language_model.model.layers.22.post_feedforward_layernorm.weight": "model-00005-of-00007.safetensors",
+    "language_model.model.layers.22.self_attn.k_norm.weight": "model-00005-of-00007.safetensors",
+    "language_model.model.layers.22.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "language_model.model.layers.22.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "language_model.model.layers.22.self_attn.q_norm.weight": "model-00005-of-00007.safetensors",
+    "language_model.model.layers.22.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "language_model.model.layers.22.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "language_model.model.layers.23.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "language_model.model.layers.23.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "language_model.model.layers.23.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "language_model.model.layers.23.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "language_model.model.layers.23.post_feedforward_layernorm.weight": "model-00005-of-00007.safetensors",
+    "language_model.model.layers.23.self_attn.k_norm.weight": "model-00005-of-00007.safetensors",
+    "language_model.model.layers.23.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "language_model.model.layers.23.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "language_model.model.layers.23.self_attn.q_norm.weight": "model-00005-of-00007.safetensors",
+    "language_model.model.layers.23.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "language_model.model.layers.23.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "language_model.model.layers.24.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "language_model.model.layers.24.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "language_model.model.layers.24.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "language_model.model.layers.24.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "language_model.model.layers.24.post_feedforward_layernorm.weight": "model-00005-of-00007.safetensors",
+    "language_model.model.layers.24.self_attn.k_norm.weight": "model-00005-of-00007.safetensors",
+    "language_model.model.layers.24.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "language_model.model.layers.24.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "language_model.model.layers.24.self_attn.q_norm.weight": "model-00005-of-00007.safetensors",
+    "language_model.model.layers.24.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "language_model.model.layers.24.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "language_model.model.layers.25.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "language_model.model.layers.25.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "language_model.model.layers.25.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "language_model.model.layers.25.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "language_model.model.layers.25.post_feedforward_layernorm.weight": "model-00005-of-00007.safetensors",
+    "language_model.model.layers.25.self_attn.k_norm.weight": "model-00005-of-00007.safetensors",
+    "language_model.model.layers.25.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "language_model.model.layers.25.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "language_model.model.layers.25.self_attn.q_norm.weight": "model-00005-of-00007.safetensors",
+    "language_model.model.layers.25.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "language_model.model.layers.25.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "language_model.model.layers.26.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "language_model.model.layers.26.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "language_model.model.layers.26.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "language_model.model.layers.26.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "language_model.model.layers.26.post_feedforward_layernorm.weight": "model-00005-of-00007.safetensors",
+    "language_model.model.layers.26.self_attn.k_norm.weight": "model-00005-of-00007.safetensors",
+    "language_model.model.layers.26.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "language_model.model.layers.26.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "language_model.model.layers.26.self_attn.q_norm.weight": "model-00005-of-00007.safetensors",
+    "language_model.model.layers.26.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "language_model.model.layers.26.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "language_model.model.layers.27.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "language_model.model.layers.27.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
+    "language_model.model.layers.27.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
+    "language_model.model.layers.27.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "language_model.model.layers.27.post_feedforward_layernorm.weight": "model-00006-of-00007.safetensors",
+    "language_model.model.layers.27.self_attn.k_norm.weight": "model-00005-of-00007.safetensors",
+    "language_model.model.layers.27.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
+    "language_model.model.layers.27.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "language_model.model.layers.27.self_attn.q_norm.weight": "model-00005-of-00007.safetensors",
+    "language_model.model.layers.27.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
+    "language_model.model.layers.27.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
+    "language_model.model.layers.28.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "language_model.model.layers.28.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "language_model.model.layers.28.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "language_model.model.layers.28.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "language_model.model.layers.28.post_feedforward_layernorm.weight": "model-00006-of-00007.safetensors",
+    "language_model.model.layers.28.self_attn.k_norm.weight": "model-00006-of-00007.safetensors",
+    "language_model.model.layers.28.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "language_model.model.layers.28.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "language_model.model.layers.28.self_attn.q_norm.weight": "model-00006-of-00007.safetensors",
+    "language_model.model.layers.28.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "language_model.model.layers.28.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "language_model.model.layers.29.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "language_model.model.layers.29.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "language_model.model.layers.29.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "language_model.model.layers.29.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "language_model.model.layers.29.post_feedforward_layernorm.weight": "model-00006-of-00007.safetensors",
+    "language_model.model.layers.29.self_attn.k_norm.weight": "model-00006-of-00007.safetensors",
+    "language_model.model.layers.29.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "language_model.model.layers.29.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "language_model.model.layers.29.self_attn.q_norm.weight": "model-00006-of-00007.safetensors",
+    "language_model.model.layers.29.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "language_model.model.layers.29.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "language_model.model.layers.3.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "language_model.model.layers.3.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
+    "language_model.model.layers.3.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
+    "language_model.model.layers.3.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "language_model.model.layers.3.post_feedforward_layernorm.weight": "model-00002-of-00007.safetensors",
+    "language_model.model.layers.3.self_attn.k_norm.weight": "model-00001-of-00007.safetensors",
+    "language_model.model.layers.3.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
+    "language_model.model.layers.3.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "language_model.model.layers.3.self_attn.q_norm.weight": "model-00001-of-00007.safetensors",
+    "language_model.model.layers.3.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
+    "language_model.model.layers.3.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
+    "language_model.model.layers.30.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "language_model.model.layers.30.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "language_model.model.layers.30.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "language_model.model.layers.30.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "language_model.model.layers.30.post_feedforward_layernorm.weight": "model-00006-of-00007.safetensors",
+    "language_model.model.layers.30.self_attn.k_norm.weight": "model-00006-of-00007.safetensors",
+    "language_model.model.layers.30.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "language_model.model.layers.30.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "language_model.model.layers.30.self_attn.q_norm.weight": "model-00006-of-00007.safetensors",
+    "language_model.model.layers.30.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "language_model.model.layers.30.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "language_model.model.layers.31.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "language_model.model.layers.31.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
+    "language_model.model.layers.31.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
+    "language_model.model.layers.31.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "language_model.model.layers.31.post_feedforward_layernorm.weight": "model-00006-of-00007.safetensors",
+    "language_model.model.layers.31.self_attn.k_norm.weight": "model-00006-of-00007.safetensors",
+    "language_model.model.layers.31.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
+    "language_model.model.layers.31.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "language_model.model.layers.31.self_attn.q_norm.weight": "model-00006-of-00007.safetensors",
+    "language_model.model.layers.31.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
+    "language_model.model.layers.31.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
+    "language_model.model.layers.4.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "language_model.model.layers.4.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "language_model.model.layers.4.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "language_model.model.layers.4.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "language_model.model.layers.4.post_feedforward_layernorm.weight": "model-00002-of-00007.safetensors",
+    "language_model.model.layers.4.self_attn.k_norm.weight": "model-00002-of-00007.safetensors",
+    "language_model.model.layers.4.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "language_model.model.layers.4.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "language_model.model.layers.4.self_attn.q_norm.weight": "model-00002-of-00007.safetensors",
+    "language_model.model.layers.4.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "language_model.model.layers.4.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "language_model.model.layers.5.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "language_model.model.layers.5.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "language_model.model.layers.5.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "language_model.model.layers.5.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "language_model.model.layers.5.post_feedforward_layernorm.weight": "model-00002-of-00007.safetensors",
+    "language_model.model.layers.5.self_attn.k_norm.weight": "model-00002-of-00007.safetensors",
+    "language_model.model.layers.5.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "language_model.model.layers.5.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "language_model.model.layers.5.self_attn.q_norm.weight": "model-00002-of-00007.safetensors",
+    "language_model.model.layers.5.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "language_model.model.layers.5.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "language_model.model.layers.6.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "language_model.model.layers.6.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "language_model.model.layers.6.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "language_model.model.layers.6.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "language_model.model.layers.6.post_feedforward_layernorm.weight": "model-00002-of-00007.safetensors",
+    "language_model.model.layers.6.self_attn.k_norm.weight": "model-00002-of-00007.safetensors",
+    "language_model.model.layers.6.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "language_model.model.layers.6.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "language_model.model.layers.6.self_attn.q_norm.weight": "model-00002-of-00007.safetensors",
+    "language_model.model.layers.6.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "language_model.model.layers.6.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "language_model.model.layers.7.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "language_model.model.layers.7.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "language_model.model.layers.7.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "language_model.model.layers.7.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "language_model.model.layers.7.post_feedforward_layernorm.weight": "model-00002-of-00007.safetensors",
+    "language_model.model.layers.7.self_attn.k_norm.weight": "model-00002-of-00007.safetensors",
+    "language_model.model.layers.7.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "language_model.model.layers.7.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "language_model.model.layers.7.self_attn.q_norm.weight": "model-00002-of-00007.safetensors",
+    "language_model.model.layers.7.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "language_model.model.layers.7.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "language_model.model.layers.8.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "language_model.model.layers.8.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "language_model.model.layers.8.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "language_model.model.layers.8.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "language_model.model.layers.8.post_feedforward_layernorm.weight": "model-00002-of-00007.safetensors",
+    "language_model.model.layers.8.self_attn.k_norm.weight": "model-00002-of-00007.safetensors",
+    "language_model.model.layers.8.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "language_model.model.layers.8.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "language_model.model.layers.8.self_attn.q_norm.weight": "model-00002-of-00007.safetensors",
+    "language_model.model.layers.8.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "language_model.model.layers.8.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "language_model.model.layers.9.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "language_model.model.layers.9.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
+    "language_model.model.layers.9.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
+    "language_model.model.layers.9.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "language_model.model.layers.9.post_feedforward_layernorm.weight": "model-00003-of-00007.safetensors",
+    "language_model.model.layers.9.self_attn.k_norm.weight": "model-00002-of-00007.safetensors",
+    "language_model.model.layers.9.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
+    "language_model.model.layers.9.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "language_model.model.layers.9.self_attn.q_norm.weight": "model-00002-of-00007.safetensors",
+    "language_model.model.layers.9.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
+    "language_model.model.layers.9.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
+    "language_model.model.norm.weight": "model-00006-of-00007.safetensors",
+    "multi_modal_projector.cls_projector.weight": "model-00001-of-00007.safetensors",
+    "multi_modal_projector.image_projector.w1.weight": "model-00001-of-00007.safetensors",
+    "multi_modal_projector.image_projector.w2.weight": "model-00001-of-00007.safetensors",
+    "multi_modal_projector.image_projector.w3.weight": "model-00001-of-00007.safetensors",
+    "vision_tower.vision_model.embeddings.class_embedding": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.embeddings.patch_embedding.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.embeddings.position_embedding.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.0.layer_norm1.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.0.layer_norm1.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.0.layer_norm2.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.0.layer_norm2.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.0.mlp.fc1.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.0.mlp.fc1.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.0.mlp.fc2.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.0.mlp.fc2.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.0.self_attn.k_proj.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.0.self_attn.k_proj.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.0.self_attn.out_proj.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.0.self_attn.out_proj.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.0.self_attn.q_proj.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.0.self_attn.q_proj.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.0.self_attn.v_proj.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.0.self_attn.v_proj.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.1.layer_norm1.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.1.layer_norm1.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.1.layer_norm2.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.1.layer_norm2.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.1.mlp.fc1.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.1.mlp.fc1.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.1.mlp.fc2.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.1.mlp.fc2.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.1.self_attn.k_proj.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.1.self_attn.k_proj.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.1.self_attn.out_proj.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.1.self_attn.out_proj.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.1.self_attn.q_proj.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.1.self_attn.q_proj.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.1.self_attn.v_proj.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.1.self_attn.v_proj.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.10.layer_norm1.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.10.layer_norm1.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.10.layer_norm2.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.10.layer_norm2.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.10.mlp.fc1.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.10.mlp.fc1.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.10.mlp.fc2.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.10.mlp.fc2.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.10.self_attn.k_proj.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.10.self_attn.k_proj.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.10.self_attn.out_proj.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.10.self_attn.out_proj.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.10.self_attn.q_proj.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.10.self_attn.q_proj.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.10.self_attn.v_proj.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.10.self_attn.v_proj.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.11.layer_norm1.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.11.layer_norm1.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.11.layer_norm2.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.11.layer_norm2.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.11.mlp.fc1.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.11.mlp.fc1.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.11.mlp.fc2.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.11.mlp.fc2.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.11.self_attn.k_proj.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.11.self_attn.k_proj.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.11.self_attn.out_proj.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.11.self_attn.out_proj.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.11.self_attn.q_proj.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.11.self_attn.q_proj.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.11.self_attn.v_proj.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.11.self_attn.v_proj.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.2.layer_norm1.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.2.layer_norm1.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.2.layer_norm2.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.2.layer_norm2.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.2.mlp.fc1.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.2.mlp.fc1.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.2.mlp.fc2.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.2.mlp.fc2.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.2.self_attn.k_proj.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.2.self_attn.k_proj.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.2.self_attn.out_proj.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.2.self_attn.out_proj.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.2.self_attn.q_proj.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.2.self_attn.q_proj.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.2.self_attn.v_proj.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.2.self_attn.v_proj.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.3.layer_norm1.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.3.layer_norm1.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.3.layer_norm2.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.3.layer_norm2.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.3.mlp.fc1.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.3.mlp.fc1.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.3.mlp.fc2.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.3.mlp.fc2.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.3.self_attn.k_proj.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.3.self_attn.k_proj.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.3.self_attn.out_proj.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.3.self_attn.out_proj.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.3.self_attn.q_proj.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.3.self_attn.q_proj.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.3.self_attn.v_proj.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.3.self_attn.v_proj.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.4.layer_norm1.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.4.layer_norm1.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.4.layer_norm2.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.4.layer_norm2.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.4.mlp.fc1.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.4.mlp.fc1.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.4.mlp.fc2.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.4.mlp.fc2.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.4.self_attn.k_proj.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.4.self_attn.k_proj.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.4.self_attn.out_proj.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.4.self_attn.out_proj.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.4.self_attn.q_proj.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.4.self_attn.q_proj.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.4.self_attn.v_proj.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.4.self_attn.v_proj.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.5.layer_norm1.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.5.layer_norm1.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.5.layer_norm2.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.5.layer_norm2.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.5.mlp.fc1.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.5.mlp.fc1.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.5.mlp.fc2.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.5.mlp.fc2.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.5.self_attn.k_proj.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.5.self_attn.k_proj.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.5.self_attn.out_proj.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.5.self_attn.out_proj.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.5.self_attn.q_proj.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.5.self_attn.q_proj.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.5.self_attn.v_proj.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.5.self_attn.v_proj.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.6.layer_norm1.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.6.layer_norm1.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.6.layer_norm2.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.6.layer_norm2.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.6.mlp.fc1.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.6.mlp.fc1.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.6.mlp.fc2.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.6.mlp.fc2.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.6.self_attn.k_proj.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.6.self_attn.k_proj.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.6.self_attn.out_proj.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.6.self_attn.out_proj.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.6.self_attn.q_proj.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.6.self_attn.q_proj.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.6.self_attn.v_proj.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.6.self_attn.v_proj.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.7.layer_norm1.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.7.layer_norm1.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.7.layer_norm2.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.7.layer_norm2.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.7.mlp.fc1.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.7.mlp.fc1.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.7.mlp.fc2.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.7.mlp.fc2.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.7.self_attn.k_proj.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.7.self_attn.k_proj.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.7.self_attn.out_proj.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.7.self_attn.out_proj.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.7.self_attn.q_proj.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.7.self_attn.q_proj.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.7.self_attn.v_proj.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.7.self_attn.v_proj.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.8.layer_norm1.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.8.layer_norm1.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.8.layer_norm2.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.8.layer_norm2.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.8.mlp.fc1.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.8.mlp.fc1.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.8.mlp.fc2.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.8.mlp.fc2.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.8.self_attn.k_proj.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.8.self_attn.k_proj.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.8.self_attn.out_proj.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.8.self_attn.out_proj.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.8.self_attn.q_proj.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.8.self_attn.q_proj.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.8.self_attn.v_proj.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.8.self_attn.v_proj.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.9.layer_norm1.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.9.layer_norm1.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.9.layer_norm2.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.9.layer_norm2.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.9.mlp.fc1.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.9.mlp.fc1.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.9.mlp.fc2.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.9.mlp.fc2.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.9.self_attn.k_proj.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.9.self_attn.k_proj.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.9.self_attn.out_proj.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.9.self_attn.out_proj.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.9.self_attn.q_proj.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.9.self_attn.q_proj.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.9.self_attn.v_proj.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.encoder.layers.9.self_attn.v_proj.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.post_layernorm.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.post_layernorm.weight": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.pre_layrnorm.bias": "model-00007-of-00007.safetensors",
+    "vision_tower.vision_model.pre_layrnorm.weight": "model-00007-of-00007.safetensors"
+  }
+}

modeling_molmo_olmo3.py

@@ -0,0 +1,291 @@
+# coding=utf-8
+"""Modeling code for the Molmo-v1 (CLIP vision) VLM in HuggingFace format.
+
+LLaVA-style wrapper that reproduces Molmo-v1 inference exactly:
+  vision_tower (referenced HF CLIPVisionModel, frozen)
+    -> multi_modal_projector (SwiGLU image projector + CLS Linear projector)
+    -> additive insertion at <im_patch> positions
+    -> language_model (native transformers Olmo3ForCausalLM)
+
+The vision tower weights are NOT stored in this checkpoint; they are loaded from
+`config.vision_tower_name_or_path` at load time (see `from_pretrained`).
+"""
+
+from typing import List, Optional, Tuple, Union
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+from transformers.activations import ACT2FN
+from transformers.modeling_outputs import CausalLMOutputWithPast
+from transformers.modeling_utils import PreTrainedModel
+from transformers.generation import GenerationMixin
+from transformers.models.auto import AutoModel
+from transformers.models.clip.modeling_clip import CLIPVisionModel
+from transformers.models.olmo3.modeling_olmo3 import Olmo3ForCausalLM
+
+try:
+    from .configuration_molmo_olmo3 import MolmoOlmo3Config
+except ImportError:  # allow direct (non-package) import from converter/validation scripts
+    from configuration_molmo_olmo3 import MolmoOlmo3Config
+
+
+class MolmoImageProjectorMLP(nn.Module):
+    """SwiGLU projector matching Molmo `ImageProjectorMLP`: w2(act(w1(x)) * w3(x))."""
+
+    def __init__(self, in_dim: int, hidden_dim: int, out_dim: int, hidden_act: str = "silu"):
+        super().__init__()
+        self.w1 = nn.Linear(in_dim, hidden_dim, bias=False)   # gate
+        self.w2 = nn.Linear(hidden_dim, out_dim, bias=False)  # down
+        self.w3 = nn.Linear(in_dim, hidden_dim, bias=False)   # up
+        self.act = ACT2FN[hidden_act]
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        return self.w2(self.act(self.w1(x)) * self.w3(x))
+
+
+class MolmoMultiModalProjector(nn.Module):
+    def __init__(self, config: MolmoOlmo3Config):
+        super().__init__()
+        self.image_projector = MolmoImageProjectorMLP(
+            config.vision_hidden_size,
+            config.projector_intermediate_size,
+            config.text_hidden_size,
+            config.projector_hidden_act,
+        )
+        self.include_cls_token = config.include_cls_token
+        if self.include_cls_token:
+            self.cls_projector = nn.Linear(
+                config.vision_hidden_size, config.text_hidden_size, bias=False
+            )
+
+    def forward(self, patch_features: torch.Tensor, cls_features: Optional[torch.Tensor]) -> torch.Tensor:
+        """patch_features: (..., num_patches, vision_hidden); cls_features: (..., vision_hidden).
+
+        Returns (..., [1+]num_patches, text_hidden) with the projected CLS prepended.
+        """
+        projected = self.image_projector(patch_features)
+        if self.include_cls_token and cls_features is not None:
+            cls_projected = self.cls_projector(cls_features).unsqueeze(-2)
+            projected = torch.cat([cls_projected, projected], dim=-2)
+        return projected
+
+
+class MolmoOlmo3PreTrainedModel(PreTrainedModel):
+    config_class = MolmoOlmo3Config
+    base_model_prefix = "model"
+    supports_gradient_checkpointing = True
+    _no_split_modules = ["Olmo3DecoderLayer", "CLIPEncoderLayer"]
+    _supports_flash_attn_2 = True
+    _supports_sdpa = True
+    _supports_cache_class = True
+    # The vision tower is referenced (not stored): drop on save, ignore-missing on load.
+    _keys_to_ignore_on_save = [r"^vision_tower\."]
+    _keys_to_ignore_on_load_missing = [r"^vision_tower\."]
+    _keys_to_ignore_on_load_unexpected = [r"^vision_tower\."]
+
+
+class MolmoOlmo3ForConditionalGeneration(MolmoOlmo3PreTrainedModel, GenerationMixin):
+    def __init__(self, config: MolmoOlmo3Config):
+        super().__init__(config)
+        self.config = config
+        # Vision tower is referenced (not stored): built lazily from
+        # config.vision_tower_name_or_path via load_vision_tower_weights() (called by
+        # from_pretrained). Building via from_pretrained fetches code+arch+weights together,
+        # which is required for trust_remote_code towers like DINOv3.
+        self.vision_tower = None
+        self.multi_modal_projector = MolmoMultiModalProjector(config)
+        self.language_model = Olmo3ForCausalLM(config.text_config)
+        self.lm_head_vocab_size = config.lm_head_vocab_size
+        self.post_init()
+
+    # ---- weight tying / embedding plumbing ----
+    def get_input_embeddings(self):
+        return self.language_model.get_input_embeddings()
+
+    def set_input_embeddings(self, value):
+        self.language_model.set_input_embeddings(value)
+
+    def get_output_embeddings(self):
+        return self.language_model.get_output_embeddings()
+
+    # ---- vision-by-reference loading ----
+    @staticmethod
+    def _is_clip_vision(config) -> bool:
+        return getattr(config.vision_config, "model_type", "") in ("clip_vision_model", "clip")
+
+    def load_vision_tower_weights(self):
+        """Instantiate the frozen vision encoder from the referenced HF repo (code+weights)."""
+        name = self.config.vision_tower_name_or_path
+        if self._is_clip_vision(self.config):
+            # CLIPVisionModel.from_pretrained extracts vision weights from a full CLIP repo.
+            tower = CLIPVisionModel.from_pretrained(name)
+        else:
+            # DINOv3 etc.: from_pretrained fetches the trust_remote_code modeling + weights.
+            tower = AutoModel.from_pretrained(
+                name, trust_remote_code=self.config.vision_trust_remote_code
+            )
+        # Molmo discards the vision encoder's final norm (uses the pre-norm last block).
+        # DINOv3's last_hidden_state has a final LayerNorm; strip it for bit-identity.
+        if getattr(self.config, "vision_strip_final_norm", False):
+            attr = getattr(self.config, "vision_final_norm_attr", "norm")
+            if hasattr(tower, attr):
+                setattr(tower, attr, nn.Identity())
+        ref_param = next(self.language_model.parameters())
+        self.vision_tower = tower.to(device=ref_param.device, dtype=ref_param.dtype).eval()
+
+    @classmethod
+    def from_pretrained(cls, *args, **kwargs):
+        model = super().from_pretrained(*args, **kwargs)
+        model.load_vision_tower_weights()
+        model.vision_tower.eval()
+        return model
+
+    # ---- vision feature path (matches Molmo encode_image + projector) ----
+    def get_image_features(self, pixel_values: torch.Tensor) -> torch.Tensor:
+        """pixel_values: (batch, num_images, 3, H, W) or (batch, 3, H, W).
+
+        Returns (batch, num_images, features_per_image, text_hidden).
+        features_per_image = 197 (1 CLS + 196 patches) for the CLIP-B/16 224 tower.
+        """
+        if pixel_values.dim() == 4:
+            pixel_values = pixel_values.unsqueeze(1)
+        b, n = pixel_values.shape[:2]
+        flat = pixel_values.flatten(0, 1).to(self.vision_tower.dtype)
+        # Molmo (vit_layers=[-1]) uses the LAST transformer block output BEFORE any final norm.
+        # For CLIP, hidden_states[-1] == last_hidden_state; for DINOv3, last_hidden_state has an
+        # extra final LayerNorm applied, so we must use hidden_states[vision_feature_layer] (pre-norm).
+        vision_out = self.vision_tower(pixel_values=flat, output_hidden_states=True)
+        feats = vision_out.hidden_states[self.config.vision_feature_layer]  # (b*n, 197, vision_hidden)
+
+        cls_feats = feats[:, 0]
+        patch_feats = feats[:, 1:]
+        projected = self.multi_modal_projector(patch_feats, cls_feats)  # (b*n, 197, text_hidden)
+        return projected.view(b, n, projected.shape[-2], projected.shape[-1])
+
+    def _merge_image_features(self, inputs_embeds, image_features, image_input_idx):
+        """Additive scatter of image features at image_input_idx (matches Molmo)."""
+        batch_size = inputs_embeds.shape[0]
+        image_features = image_features.reshape(batch_size, -1, image_features.shape[-1])
+        image_input_idx = image_input_idx.reshape(batch_size, -1)
+        valid = image_input_idx >= 0
+        batch_idx = torch.arange(batch_size, device=inputs_embeds.device)[:, None]
+        batch_idx = batch_idx.expand(-1, image_input_idx.shape[1])
+        image_features = image_features.to(inputs_embeds.device, inputs_embeds.dtype)
+        inputs_embeds[batch_idx[valid], image_input_idx[valid]] += image_features[valid]
+        return inputs_embeds
+
+    def forward(
+        self,
+        input_ids: torch.LongTensor = None,
+        pixel_values: Optional[torch.FloatTensor] = None,
+        image_input_idx: Optional[torch.LongTensor] = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_values=None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        labels: Optional[torch.LongTensor] = None,
+        use_cache: Optional[bool] = None,
+        cache_position: Optional[torch.LongTensor] = None,
+        logits_to_keep: Union[int, torch.Tensor] = 0,
+        **kwargs,
+    ) -> CausalLMOutputWithPast:
+        if inputs_embeds is None:
+            # clamp -1 placeholders (loss-mask sentinels) to 0 before embedding (matches Molmo)
+            safe_ids = input_ids * (input_ids != -1).to(input_ids.dtype)
+            inputs_embeds = self.get_input_embeddings()(safe_ids)
+            if pixel_values is not None:
+                image_features = self.get_image_features(pixel_values)
+                inputs_embeds = self._merge_image_features(
+                    inputs_embeds, image_features, image_input_idx
+                )
+
+        # position_ids = clamp(cumsum(attn_mask)-1, 0) when not provided (matches Molmo).
+        # Compute on BOTH prefill and cached decode (slice to the current query length): some
+        # transformers versions call forward during generation with position_ids AND
+        # cache_position both None, which would otherwise leave RoPE positions undefined.
+        if position_ids is None and attention_mask is not None:
+            position_ids = torch.clamp(
+                torch.cumsum(attention_mask.to(torch.int32), dim=-1) - 1, min=0
+            )
+            query_len = inputs_embeds.shape[1] if inputs_embeds is not None else input_ids.shape[1]
+            if position_ids.shape[1] != query_len:
+                position_ids = position_ids[:, -query_len:]
+
+        outputs = self.language_model(
+            input_ids=None,
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            past_key_values=past_key_values,
+            inputs_embeds=inputs_embeds,
+            use_cache=use_cache,
+            cache_position=cache_position,
+            logits_to_keep=logits_to_keep,
+            **kwargs,
+        )
+        logits = outputs.logits
+        # Mask the 128 additional (image placeholder) vocab ids so generation matches
+        # Molmo's lm_head (which only spans the first `lm_head_vocab_size` tokens).
+        if logits.shape[-1] > self.lm_head_vocab_size:
+            logits = logits.clone()
+            logits[..., self.lm_head_vocab_size:] = float("-inf")
+
+        loss = None
+        if labels is not None:
+            loss = self.language_model.loss_function(
+                logits=logits, labels=labels, vocab_size=logits.shape[-1]
+            )
+
+        return CausalLMOutputWithPast(
+            loss=loss,
+            logits=logits,
+            past_key_values=outputs.past_key_values,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
+        )
+
+    def prepare_inputs_for_generation(
+        self,
+        input_ids,
+        past_key_values=None,
+        attention_mask=None,
+        inputs_embeds=None,
+        pixel_values=None,
+        image_input_idx=None,
+        cache_position=None,
+        **kwargs,
+    ):
+        model_inputs = self.language_model.prepare_inputs_for_generation(
+            input_ids,
+            past_key_values=past_key_values,
+            attention_mask=attention_mask,
+            inputs_embeds=inputs_embeds,
+            cache_position=cache_position,
+            **kwargs,
+        )
+        # Feed images only on the prefill step (no cached context yet). Detect prefill
+        # robustly: cache_position may be None when this runs under some transformers
+        # versions, so do NOT rely solely on `cache_position[0] == 0` — that guard silently
+        # drops the image and yields a text-only hallucination. Fall back to inspecting the
+        # KV cache length.
+        cache_len = 0
+        if past_key_values is not None and hasattr(past_key_values, "get_seq_length"):
+            try:
+                cache_len = past_key_values.get_seq_length()
+            except Exception:
+                cache_len = 0
+        is_prefill = (
+            (cache_position is not None and int(cache_position[0]) == 0)
+            or (cache_position is None and past_key_values is None)
+            or (cache_position is None and cache_len == 0)
+        )
+        if is_prefill and pixel_values is not None:
+            model_inputs["pixel_values"] = pixel_values
+            model_inputs["image_input_idx"] = image_input_idx
+        # We always re-embed inside forward; let it own the embedding lookup.
+        model_inputs.pop("inputs_embeds", None)
+        return model_inputs
+
+
+__all__ = ["MolmoOlmo3ForConditionalGeneration", "MolmoOlmo3PreTrainedModel"]

preprocessor_config.json

@@ -0,0 +1,19 @@
+{
+  "auto_map": {
+    "AutoImageProcessor": "image_processing_molmo_olmo3.MolmoOlmo3ImageProcessor",
+    "AutoProcessor": "processing_molmo_olmo3.MolmoOlmo3Processor"
+  },
+  "image_mean": [
+    0.48145466,
+    0.4578275,
+    0.40821073
+  ],
+  "image_processor_type": "MolmoOlmo3ImageProcessor",
+  "image_size": 224,
+  "image_std": [
+    0.26862954,
+    0.26130258,
+    0.27577711
+  ],
+  "processor_class": "MolmoOlmo3Processor"
+}

processing_molmo_olmo3.py

@@ -0,0 +1,201 @@
+# coding=utf-8
+"""Processor for the Molmo-v1 (CLIP vision) VLM.
+
+Reproduces the Molmo preprocessor token layout exactly for this VLM's config
+(crop_mode=resize, max_crops=1, image_pooling_2d=none, include_cls_token=true):
+
+  per image block (213 tokens; 197 <im_patch>):
+    [<im_start>] [<im_patch>(CLS)] then 14x([<im_patch>*14][<im_col>]) [<im_end>]
+
+  full sequence: [BOS] + <pre-image text> + image_block + <post-image text>
+  image_input_idx: the 197 <im_patch> positions (CLS first, then 196 row-major),
+                   each +1 for the prepended BOS.
+"""
+
+from typing import List, Optional, Union
+
+import numpy as np
+import torch
+
+from transformers.processing_utils import ProcessorMixin
+from transformers.feature_extraction_utils import BatchFeature
+
+
+class MolmoOlmo3Processor(ProcessorMixin):
+    attributes = ["image_processor", "tokenizer"]
+    image_processor_class = "AutoImageProcessor"
+    tokenizer_class = "AutoTokenizer"
+
+    # token-id constants (dolma2 base 100278; specials appended at 100278..100282)
+    IMAGE_PROMPT_TOKEN_ID = 100282   # <|image|>
+    IMAGE_START_TOKEN_ID = 100278    # <im_start>
+    IMAGE_END_TOKEN_ID = 100279      # <im_end>
+    IMAGE_PATCH_TOKEN_ID = 100280    # <im_patch>
+    IMAGE_COL_TOKEN_ID = 100281      # <im_col>
+    BOS_TOKEN_ID = 100257
+
+    # The only styles these models were trained on (system_prompt_kind='demo_or_style').
+    # long_caption/user_qa/synthetic_qa saw the "{style}:" prefix only ~10% of the time
+    # (no prefix the other ~90%); transcript was always prefixed.
+    KNOWN_STYLES = ("long_caption", "transcript", "user_qa", "synthetic_qa")
+
+    def __init__(
+        self,
+        image_processor=None,
+        tokenizer=None,
+        image_token_length_w: int = 14,
+        image_token_length_h: int = 14,
+        include_cls_token: bool = True,
+        use_col_tokens: bool = True,
+        always_start_with_space: bool = True,
+        **kwargs,
+    ):
+        self.image_token_length_w = image_token_length_w
+        self.image_token_length_h = image_token_length_h
+        self.include_cls_token = include_cls_token
+        self.use_col_tokens = use_col_tokens
+        self.always_start_with_space = always_start_with_space
+        super().__init__(image_processor, tokenizer, **kwargs)
+
+    def format_prompt(self, question: str, style=None) -> str:
+        """Reproduce Molmo's DataFormatter (system_prompt='demo_or_style', message_format='none').
+
+        Usage:
+          - VQA / instruction (most common): `text="your question"`, `style=None`
+            -> " your question". This matches ~90% of training (no prefix), so leaving
+            style unset is usually best.
+          - Captioning: `text=""`, `style=None` -> a bare " " prompt; or
+            `text="", style="long_caption"` / `style="transcript"` to request that mode
+            explicitly. (Training produced captions/transcripts from an empty user turn.)
+          - Steer output mode: pass `style` in {long_caption, transcript, user_qa,
+            synthetic_qa} -> "{style}: ...". Note long_caption/user_qa/synthetic_qa only
+            saw the prefix ~10% of the time in training; transcript was always prefixed.
+
+        always_start_with_space -> a single leading space is always prepended.
+        """
+        if style is not None and style not in self.KNOWN_STYLES:
+            import warnings
+            warnings.warn(
+                f"style={style!r} was not used to train these models; the model may ignore "
+                f"or mishandle it. Known styles: {self.KNOWN_STYLES}. Use style=None for the "
+                f"default (no-prefix) behavior the model saw ~90% of the time."
+            )
+        prefix = "" if not style else f"{style}:"
+        if prefix and question:
+            text = prefix + " " + question
+        elif prefix:
+            text = prefix
+        else:
+            text = question
+        if self.always_start_with_space:
+            text = " " + text
+        return text
+
+    def _image_block(self) -> np.ndarray:
+        """The 213-token image block for a single resized crop."""
+        per_row = np.full((self.image_token_length_w,), self.IMAGE_PATCH_TOKEN_ID, dtype=np.int32)
+        if self.use_col_tokens:
+            per_row = np.concatenate([per_row, [self.IMAGE_COL_TOKEN_ID]], 0)
+        extra = np.tile(per_row, [self.image_token_length_h])
+        joint = [[self.IMAGE_START_TOKEN_ID]]
+        if self.include_cls_token:
+            joint.append([self.IMAGE_PATCH_TOKEN_ID])
+        joint += [extra, [self.IMAGE_END_TOKEN_ID]]
+        return np.concatenate(joint, 0).astype(np.int32)
+
+    def _image_input_idx(self, image_block: np.ndarray) -> np.ndarray:
+        """Positions of <im_patch> within the block, (1, features_per_image)."""
+        tokens_per_image = self.image_token_length_w * self.image_token_length_h
+        features_per_image = tokens_per_image + (1 if self.include_cls_token else 0)
+        idx = np.nonzero(image_block == self.IMAGE_PATCH_TOKEN_ID)[0].astype(np.int32)
+        return idx.reshape(1, features_per_image)
+
+    def __call__(
+        self,
+        text: Union[str, List[str]],
+        images=None,
+        style=None,
+        apply_prompt_format: bool = True,
+        return_tensors: Optional[str] = "pt",
+        **kwargs,
+    ) -> BatchFeature:
+        """Tokenize text + splice image features.
+
+        By default (apply_prompt_format=True) the text is wrapped with the training-time
+        formatting (leading space + optional "{style}: " prefix) and the image is placed
+        first (Molmo inserts the image at the start when no <|image|> marker is present).
+        Pass apply_prompt_format=False to feed pre-formatted text, or include an explicit
+        <|image|> marker to control image placement.
+        """
+        if isinstance(text, (list, tuple)):
+            if len(text) != 1:
+                raise NotImplementedError("MolmoOlmo3Processor supports a single prompt at a time.")
+            text = text[0]
+        if images is not None and not isinstance(images, (list, tuple)):
+            images = [images]
+
+        if apply_prompt_format and self.IMAGE_PROMPT_TOKEN_ID not in \
+                self.tokenizer.encode(text, add_special_tokens=False):
+            text = self.format_prompt(text, style=style)
+
+        tokens = np.array(self.tokenizer.encode(text, add_special_tokens=False), dtype=np.int32)
+
+        if not images:
+            input_ids = np.pad(tokens, [[1, 0]], constant_values=self.BOS_TOKEN_ID)
+            return self._finalize({"input_tokens": input_ids}, None, None, return_tensors)
+
+        marker_pos = np.argwhere(tokens == self.IMAGE_PROMPT_TOKEN_ID)
+        # No marker -> image first (token_ix=-1, matching Molmo's no-marker behavior).
+        image_idx = marker_pos[:, 0] if len(marker_pos) else np.array([-1] * len(images))
+        assert len(image_idx) == len(images), "number of <|image|> markers must match images"
+
+        block = self._image_block()
+        patch_idx = self._image_input_idx(block)
+        all_pixel = self.image_processor(images, return_tensors=None)["pixel_values"]  # (n,3,H,W)
+
+        out_tokens, all_image_idx = [], []
+        for ix in range(len(images)):
+            token_ix = image_idx[ix]
+            if token_ix == -1:
+                start, token_ix = 0, 0
+            else:
+                start = 0 if ix == 0 else image_idx[ix - 1] + 1
+            all_image_idx.append(patch_idx + token_ix)
+            out_tokens.append(tokens[start:token_ix])
+            out_tokens.append(block)
+        end = (image_idx[-1] + 1) if image_idx[-1] != -1 else 0
+        out_tokens.append(tokens[end:])
+
+        input_ids = np.concatenate(out_tokens, 0)
+        image_input_idx = np.concatenate(all_image_idx, 0)
+
+        # prepend BOS; shift image_input_idx by +1 (matches Molmo inference path)
+        input_ids = np.pad(input_ids, [[1, 0]], constant_values=self.BOS_TOKEN_ID)
+        image_input_idx = np.where(image_input_idx < 0, image_input_idx, image_input_idx + 1)
+
+        return self._finalize(
+            {"input_tokens": input_ids, "image_input_idx": image_input_idx[None]},
+            all_pixel, image_input_idx[None], return_tensors,
+        )
+
+    def _finalize(self, out, pixel_values, image_input_idx, return_tensors):
+        input_ids = out["input_tokens"].astype(np.int64)[None]  # (1, seq)
+        attention_mask = np.ones_like(input_ids)
+        data = {"input_ids": input_ids, "attention_mask": attention_mask}
+        if pixel_values is not None:
+            data["pixel_values"] = pixel_values[None]  # (1, n_images, 3, H, W)
+            data["image_input_idx"] = image_input_idx  # (1, n_images, features_per_image)
+        if return_tensors == "pt":
+            data = {k: torch.as_tensor(v) for k, v in data.items()}
+            if "pixel_values" in data:
+                data["pixel_values"] = data["pixel_values"].to(torch.float32)
+        return BatchFeature(data=data, tensor_type=None)
+
+    def batch_decode(self, *args, **kwargs):
+        return self.tokenizer.batch_decode(*args, **kwargs)
+
+    def decode(self, *args, **kwargs):
+        return self.tokenizer.decode(*args, **kwargs)
+
+
+__all__ = ["MolmoOlmo3Processor"]

processor_config.json

@@ -0,0 +1,11 @@
+{
+  "always_start_with_space": true,
+  "auto_map": {
+    "AutoProcessor": "processing_molmo_olmo3.MolmoOlmo3Processor"
+  },
+  "image_token_length_h": 14,
+  "image_token_length_w": 14,
+  "include_cls_token": true,
+  "processor_class": "MolmoOlmo3Processor",
+  "use_col_tokens": true
+}

special_tokens_map.json

@@ -0,0 +1,37 @@
+{
+  "additional_special_tokens": [
+    "<im_start>",
+    "<im_end>",
+    "<im_patch>",
+    "<im_col>",
+    "<|image|>"
+  ],
+  "bos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|pad|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,240 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "100256": {
+      "content": "<|extra_id_0|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100257": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100258": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100259": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100260": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100261": {
+      "content": "|||PHONE_NUMBER|||",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100262": {
+      "content": "|||EMAIL_ADDRESS|||",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100263": {
+      "content": "|||IP_ADDRESS|||",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100264": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100265": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100266": {
+      "content": "<|extra_id_1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100267": {
+      "content": "<|extra_id_2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100268": {
+      "content": "<|extra_id_3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100269": {
+      "content": "<|extra_id_4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100270": {
+      "content": "<|extra_id_5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100271": {
+      "content": "<|extra_id_6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100272": {
+      "content": "<|extra_id_7|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100273": {
+      "content": "<|extra_id_8|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100274": {
+      "content": "<|extra_id_9|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100275": {
+      "content": "<|extra_id_10|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "100276": {
+      "content": "<|endofprompt|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100277": {
+      "content": "<|pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100278": {
+      "content": "<im_start>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100279": {
+      "content": "<im_end>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100280": {
+      "content": "<im_patch>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100281": {
+      "content": "<im_col>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100282": {
+      "content": "<|image|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<im_start>",
+    "<im_end>",
+    "<im_patch>",
+    "<im_col>",
+    "<|image|>"
+  ],
+  "auto_map": {
+    "AutoProcessor": "processing_molmo_olmo3.MolmoOlmo3Processor"
+  },
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "extra_special_tokens": {},
+  "model_max_length": 8192,
+  "pad_token": "<|pad|>",
+  "processor_class": "MolmoOlmo3Processor",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}