Upload GPTQ-INT4 quantized model

README.md

@@ -0,0 +1,41 @@
+---
+license: apache-2.0
+base_model: AIDC-AI/Ovis2-4B
+tags:
+  - quantized
+  - gptq
+  - vision-language-model
+  - vlm
+---
+
+# AIDC-AI__Ovis2-4B__gptq_int4
+
+This is a **GPTQ** (4-bit) quantized version of [AIDC-AI/Ovis2-4B](https://huggingface.co/AIDC-AI/Ovis2-4B).
+
+## Quantization Details
+
+- **Method**: GPTQ
+- **Bits**: 4
+- **Base model**: AIDC-AI/Ovis2-4B
+- **Group size**: 128
+
+## Usage
+
+```python
+from transformers import AutoProcessor, AutoModelForImageTextToText
+import torch
+
+model = AutoModelForImageTextToText.from_pretrained(
+    "{REPO_ID}",
+    torch_dtype=torch.float16,
+    device_map="auto",
+    trust_remote_code=True,
+)
+processor = AutoProcessor.from_pretrained("{REPO_ID}", trust_remote_code=True)
+```
+
+Replace `{REPO_ID}` with the repo ID of this model.
+
+## Original Model
+
+See [AIDC-AI/Ovis2-4B](https://huggingface.co/AIDC-AI/Ovis2-4B) for the original FP16 model.

config.json

@@ -9,6 +9,8 @@
   "conversation_formatter_class": "QwenConversationFormatter",
   "disable_tie_weight": false,
   "dtype": "bfloat16",
+  "eos_token": "<|im_end|>",
+  "eos_token_id": 151645,
   "hidden_size": 2048,
   "llm_attn_implementation": "flash_attention_2",
   "llm_config": {
@@ -111,7 +113,55 @@
   },
   "model_type": "ovis",
   "multimodal_max_length": 32768,
+  "pad_token_id": 151643,
+  "quantization_config": {
+    "bits": 4,
+    "checkpoint_format": "gptq",
+    "desc_act": false,
+    "format": "gptq",
+    "group_size": 128,
+    "lm_head": false,
+    "meta": {
+      "act_group_aware": true,
+      "auto_forward_data_parallel": true,
+      "damp_auto_increment": 0.01,
+      "damp_percent": 0.05,
+      "failsafe": {
+        "smooth": null,
+        "strategy": "rtn",
+        "threshold": "0.5%"
+      },
+      "gc_mode": "interval",
+      "gptaq": null,
+      "hessian": {
+        "chunk_bytes": null,
+        "chunk_size": null,
+        "staging_dtype": "float32"
+      },
+      "mock_quantization": false,
+      "mse": 0.0,
+      "offload_to_disk": true,
+      "offload_to_disk_path": "./gptqmodel_offload/lzyvwgbq-jskkldgv/",
+      "pack_impl": "cpu",
+      "quantizer": [
+        "gptqmodel:5.8.0"
+      ],
+      "static_groups": false,
+      "true_sequential": true,
+      "uri": "https://github.com/modelcloud/gptqmodel",
+      "vram_strategy": "exclusive",
+      "wait_for_submodule_finalizers": false
+    },
+    "pack_dtype": "int32",
+    "quant_method": "gptq",
+    "sym": true
+  },
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
   "transformers_version": "5.3.0",
+  "use_cache": false,
   "visual_tokenizer_config": {
     "_attn_implementation_autoset": true,
     "_name_or_path": "",

configuration_ovis.py

@@ -11,7 +11,7 @@ IMAGE_TOKEN = "<image>"
 IMAGE_ATOM_ID = -300
 IMAGE_INDICATOR_IDS = [-301, -302, -303, -304, -305]
 
-AutoConfig.register("aimv2", AIMv2Config, exist_ok=True)
+AutoConfig.register("aimv2", AIMv2Config)
 AutoModel.register(AIMv2Config, AIMv2Model)
 
 # ----------------------------------------------------------------------
@@ -61,7 +61,7 @@ class Aimv2VisualTokenizerConfig(BaseVisualTokenizerConfig):
             self.backbone_kwargs['num_hidden_layers'] = self.depths[0]
 
 
-AutoConfig.register("aimv2_visual_tokenizer", Aimv2VisualTokenizerConfig, exist_ok=True)
+AutoConfig.register("aimv2_visual_tokenizer", Aimv2VisualTokenizerConfig)
 
 
 # ----------------------------------------------------------------------
@@ -141,7 +141,7 @@ class ConversationFormatter(ABC):
 
 
 class QwenConversationFormatter(ConversationFormatter):
-    support_tokenizer_types = ['QWenTokenizer', 'Qwen2TokenizerFast', 'Qwen2Tokenizer']
+    support_tokenizer_types = ['QWenTokenizer', 'Qwen2TokenizerFast']
 
     def __init__(self, tokenizer):
         super().__init__(tokenizer)

generation_config.json

@@ -0,0 +1,14 @@
+{
+  "bos_token_id": 151643,
+  "do_sample": true,
+  "eos_token_id": [
+    151645,
+    151643
+  ],
+  "multimodal_max_length": 32768,
+  "repetition_penalty": 1.05,
+  "temperature": 0.7,
+  "top_k": 20,
+  "top_p": 0.8,
+  "transformers_version": "5.3.0"
+}

model-00001-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1289d202add65fd0192f43e8e322f173e340d5d106c57812483a910c0cc91939
+size 4293719116

model-00002-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c8c1546900294fd4e694c2d65e9e58755f88585a60b4218e7bd8d73c75537436
+size 212071848

modeling_ovis.py

@@ -0,0 +1,590 @@
+# Copyright (C) 2025 AIDC-AI
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import os
+import importlib.metadata
+
+from packaging import version
+from importlib import import_module
+from typing import List, Callable, Union, Optional, Dict
+
+import PIL.Image
+import torch
+from torch import Tensor
+from torch.nn import init
+from torch.nn.functional import softmax, gumbel_softmax, pad
+from transformers.utils import is_flash_attn_2_available
+from transformers import PreTrainedModel, AutoModel, AutoTokenizer, AutoModelForCausalLM, AutoImageProcessor
+from transformers.generation.utils import GenerateOutput
+
+from .configuration_ovis import BaseVisualTokenizerConfig, Aimv2VisualTokenizerConfig
+from .configuration_ovis import OvisConfig, ConversationFormatter
+from .configuration_ovis import IGNORE_ID, IMAGE_ATOM_ID, IMAGE_INDICATOR_IDS, IMAGE_TOKEN_ID
+
+# ----------------------------------------------------------------------
+#                            Visual Tokenizer
+# ----------------------------------------------------------------------
+class BaseVisualTokenizer(PreTrainedModel):
+    base_model_prefix = "backbone"
+    main_input_name = None
+    _image_processor_class = None
+    _image_processor_kwargs = {}
+    _backbone_class = None
+    _backbone_name_or_path = None
+
+    def __init__(self, config: BaseVisualTokenizerConfig, *inputs, **kwargs):
+        super().__init__(config, *inputs, **kwargs)
+        self.image_processor = AutoImageProcessor.from_pretrained(kwargs['image_processor_name_or_path'])
+        self.backbone = AutoModel.from_config(self.config.backbone_config)
+        head_dim = self.config.vocab_size - len(IMAGE_INDICATOR_IDS)  # reserved tokens for IMAGE_INDICATORS
+        self.head = torch.nn.Sequential(
+            torch.nn.Linear(
+                self.backbone.config.hidden_size * self.config.hidden_stride * self.config.hidden_stride, head_dim,
+                bias=False
+            ),
+            torch.nn.LayerNorm(head_dim)
+        )
+
+        assert all((self.image_processor.do_resize,
+                    not getattr(self.image_processor, 'do_center_crop', False),
+                    self.image_processor.do_rescale,
+                    self.image_processor.do_normalize
+                    )), f"image_processor `{self.image_processor}` is not supported currently"
+
+    def get_backbone(self):
+        return self.backbone
+
+    def get_image_processor(self):
+        return self.image_processor
+
+    def mock_input(self):
+        height, width = self.get_image_size()
+        return torch.zeros(1, 3, height, width), self.construct_image_placeholders((1, 1))
+
+    def get_head(self):
+        return self.head
+
+    def get_image_size(self):
+        raise NotImplementedError
+
+    @staticmethod
+    def construct_image_placeholders(grid):
+        image_placeholders = [IMAGE_INDICATOR_IDS[0], IMAGE_ATOM_ID, IMAGE_INDICATOR_IDS[1]]
+        if grid[0] * grid[1] > 1:
+            for r in range(grid[0]):
+                for c in range(grid[1]):
+                    image_placeholders.append(IMAGE_ATOM_ID)
+                    if c < grid[1] - 1:
+                        image_placeholders.append(IMAGE_INDICATOR_IDS[2])
+                if r < grid[0] - 1:
+                    image_placeholders.append(IMAGE_INDICATOR_IDS[3])
+        image_placeholders.append(IMAGE_INDICATOR_IDS[4])
+        return image_placeholders
+
+    def preprocess_image(self, image: PIL.Image.Image, max_partition=9, covering_threshold=0.9, convert_to_rgb=True):
+        def _preprocess(img: PIL.Image.Image, side):
+            # first resize and preprocess
+            w, h = img.size
+            if w == h:
+                new_width = new_height = side
+            elif w > h:
+                new_width = side
+                new_height = int(h / w * new_width)
+            else:
+                new_height = side
+                new_width = int(w / h * new_height)
+            new_size = dict(height=new_height, width=new_width)
+            pixel_values = self.image_processor.preprocess(img, size=new_size, return_tensors='pt')['pixel_values']
+
+            # then pad to square
+            square_values = torch.zeros([1, 3, side, side], dtype=pixel_values.dtype, device=pixel_values.device)
+            new_height, new_width = pixel_values.shape[2:]
+            if new_height == new_width:
+                square_values[:, :, :, :] = pixel_values
+            elif new_height > new_width:
+                from_index = (side - new_width) // 2
+                square_values[:, :, :, from_index:from_index + new_width] = pixel_values
+            else:
+                from_index = (side - new_height) // 2
+                square_values[:, :, from_index:from_index + new_height, :] = pixel_values
+
+            return square_values
+
+        def _partition(img, grid):
+            w, h = img.size
+            row_height = h // grid[0]
+            col_width = w // grid[1]
+
+            partition = []
+            for row in range(grid[0]):
+                for col in range(grid[1]):
+                    left = col * col_width
+                    upper = row * row_height
+                    right = w if col == grid[1] - 1 else (col + 1) * col_width
+                    lower = h if row == grid[0] - 1 else (row + 1) * row_height
+                    partition.append((left, upper, right, lower))
+
+            return partition
+
+        def _covering_area(left, upper, right, lower, side):
+            w = right - left
+            h = lower - upper
+            w, h = max(w, h), min(w, h)
+            if w > side:
+                h = h / w * side
+                w = side
+            return w * h
+
+        def _get_best_grid(img, side):
+            img_area = img.size[0] * img.size[1]
+
+            candidate_grids = []
+            for i in range(1, max_partition + 1):
+                for j in range(1, max_partition + 1):
+                    if i * j <= max_partition:
+                        candidate_grids.append((i, j))
+
+            all_grids = []
+            good_grids = []
+            for grid in candidate_grids:
+                partition = _partition(img, grid)
+                covering_ratio = sum([_covering_area(*p, side) for p in partition]) / img_area
+                assert covering_ratio <= 1.0
+                all_grids.append((grid, covering_ratio))
+                if covering_ratio > covering_threshold:
+                    good_grids.append((grid, covering_ratio))
+
+            if len(good_grids) > 0:
+                # pick the good partition with minimum #sub_images and break the tie using covering_ratio
+                return sorted(good_grids, key=lambda x: (x[0][0] * x[0][1], -x[1]))[0][0]
+            else:
+                # pick the partition with maximum covering_ratio and break the tie using #sub_images
+                return sorted(all_grids, key=lambda x: (-x[1], x[0][0] * x[0][1]))[0][0]
+
+        if convert_to_rgb and image.mode != 'RGB':
+            image = image.convert('RGB')
+
+        sides = self.get_image_size()
+        if sides[0] != sides[1]:
+            raise ValueError('get_image_size() returns non-square size')
+        side = sides[0]
+        grid = _get_best_grid(image, side)
+        partition = _partition(image, grid)
+        crops = [image.crop(p) for p in partition]
+        if len(crops) > 1:
+            crops.insert(0, image)
+        pixel_values = torch.cat([_preprocess(crop, side) for crop in crops], dim=0)
+        image_placeholders = self.construct_image_placeholders(grid)
+        return pixel_values, image_placeholders
+
+    def tokenize(self, logits):
+        def st_argmax(y_soft, dim):  # straight-through softmax
+            index = y_soft.max(dim, keepdim=True)[1]
+            y_hard = torch.zeros_like(y_soft, memory_format=torch.legacy_contiguous_format).scatter_(dim, index, 1.0)
+            ret = y_hard - y_soft.detach() + y_soft
+            return ret
+
+        if self.config.tokenize_function == 'softmax':
+            tokens = softmax(logits, dim=-1)
+        elif self.config.tokenize_function == 'gumbel_argmax':
+            tokens = gumbel_softmax(logits, tau=self.config.tau, hard=True)
+        elif self.config.tokenize_function == 'st_argmax':
+            tokens = st_argmax(logits, dim=-1)
+        else:
+            raise ValueError(
+                f'Invalid `max_type`, expected softmax or gumbel_argmax or st_argmax, but got {self.config.tokenize_function}')
+        return tokens
+
+    def encode(self, pixel_values):
+        output = self.backbone(pixel_values, output_hidden_states=True, return_dict=True)
+        features = output.hidden_states[-1]
+        if self.config.drop_cls_token:
+            features = features[:, 1:, :]
+
+        # merge number of `hidden_stride * hidden_stride` hidden states together to reduce token sequence length
+        # e.g., for hidden_stride=2, this leads to a token length reduction: 1024 -> 256 for aimv2
+        if self.config.hidden_stride > 1:
+            n, l, d = features.shape  # this `d` maybe different from the above `d
+            sqrt_l = int(l ** 0.5)
+            assert sqrt_l ** 2 == l, "The token sequence length should be a perfect square."
+            features = features.reshape(n, sqrt_l, sqrt_l, d)
+            pl = (self.config.hidden_stride - (sqrt_l % self.config.hidden_stride)) % self.config.hidden_stride
+            features = pad(features, (0, 0, 0, pl, 0, pl), "constant", 0)
+            sqrt_l += pl
+            features = features.reshape(n, sqrt_l // self.config.hidden_stride, self.config.hidden_stride,
+                                        sqrt_l // self.config.hidden_stride, self.config.hidden_stride, d)
+            features = features.permute(0, 1, 3, 2, 4, 5)  # [n, sqrt_l/hs, sqrt_l/hs, hs, hs, d]
+            features = features.flatten(3)  # [n, sqrt_l/hs, sqrt_l/hs, hs*hs*d]
+            features = features.reshape(
+                n, -1, self.config.hidden_stride * self.config.hidden_stride * d)
+
+        return features
+
+    def forward(self, pixel_values) -> torch.Tensor:  # [BatchSize, ImageShape] -> [BatchSize, #Token, VocabSize]
+        features = self.encode(pixel_values)
+        logits = self.head(features)
+        tokens = self.tokenize(logits)
+        # tokens' shape is [BatchSize, #Token, VocabSize-5], so padding with [BatchSize, #Token, 5], after
+        # which, tokens' shape should become [BatchSize, #Token, VocabSize]
+        batch_size, token_len, _ = tokens.shape
+        padding_tensor = torch.zeros(size=(batch_size, token_len, len(IMAGE_INDICATOR_IDS)),
+                                     dtype=tokens.dtype,
+                                     device=tokens.device,
+                                     layout=tokens.layout,
+                                     requires_grad=False)
+        tokens = torch.cat((tokens, padding_tensor), dim=2)
+        return tokens
+
+
+class Aimv2VisualTokenizer(BaseVisualTokenizer):
+    config_class = Aimv2VisualTokenizerConfig
+    supports_gradient_checkpointing = True
+    _no_split_modules = ["AIMv2ViTPreprocessor", "AIMv2Block"]
+    _image_processor_kwargs = dict(do_center_crop=False)
+
+    def get_image_size(self):
+        height = self.image_processor.crop_size["height"]
+        width = self.image_processor.crop_size["width"]
+        return height, width
+
+
+AutoModel.register(Aimv2VisualTokenizerConfig, Aimv2VisualTokenizer)
+
+
+# ----------------------------------------------------------------------
+#                                  Ovis
+# ----------------------------------------------------------------------
+class VisualEmbedding(torch.nn.Embedding):
+    def forward(self, visual_tokens: Tensor) -> Tensor:
+        if visual_tokens.dtype in [torch.int8, torch.int16, torch.int32, torch.int64, torch.long]:
+            return super().forward(visual_tokens)
+        return torch.matmul(visual_tokens, self.weight)
+
+    def reset_parameters(self, mean=0., std=1.) -> None:
+        init.normal_(self.weight, mean=mean, std=std)
+        self._fill_padding_idx_with_zero()
+
+
+class OvisPreTrainedModel(PreTrainedModel):
+    config_class = OvisConfig
+    base_model_prefix = "ovis"
+
+
+class Ovis(OvisPreTrainedModel):
+
+    def __init__(self, config: OvisConfig, *inputs, **kwargs):
+        super().__init__(config, *inputs, **kwargs)
+        attn_kwargs = dict()
+        if self.config.llm_attn_implementation:
+            if self.config.llm_attn_implementation == "flash_attention_2":
+                assert (is_flash_attn_2_available() and
+                        version.parse(importlib.metadata.version("flash_attn")) >= version.parse("2.6.3")), \
+                    "Using `flash_attention_2` requires having `flash_attn>=2.6.3` installed."
+            attn_kwargs["attn_implementation"] = self.config.llm_attn_implementation
+        self.llm = AutoModelForCausalLM.from_config(self.config.llm_config, **attn_kwargs)
+        assert self.config.hidden_size == self.llm.config.hidden_size, "hidden size mismatch"
+        self.text_tokenizer = AutoTokenizer.from_pretrained(self.config.name_or_path)
+        self.visual_tokenizer = AutoModel.from_config(self.config.visual_tokenizer_config,
+                                                      image_processor_name_or_path=self.config.name_or_path)
+        self.vte = VisualEmbedding(
+            self.config.visual_tokenizer_config.vocab_size,
+            self.config.hidden_size,
+            device=self.visual_tokenizer.device,
+            dtype=self.visual_tokenizer.dtype
+        )
+
+        def _merge_modules(modules_list: tuple):
+            merged_modules = []
+            for modules in modules_list:
+                merged_modules.extend(modules if modules else [])
+            return merged_modules
+
+        self._no_split_modules = _merge_modules((self.llm._no_split_modules, self.visual_tokenizer._no_split_modules))
+        self._skip_keys_device_placement = self.llm._skip_keys_device_placement
+        self._keep_in_fp32_modules = _merge_modules(
+            (self.llm._keep_in_fp32_modules, self.visual_tokenizer._keep_in_fp32_modules))
+        self.is_parallelizable = all((self.llm.is_parallelizable, self.visual_tokenizer.is_parallelizable))
+        self.supports_gradient_checkpointing = True
+        self._supports_flash_attn_2 = True
+
+    def get_text_tokenizer(self):
+        return self.text_tokenizer
+
+    def get_visual_tokenizer(self):
+        return self.visual_tokenizer
+
+    def tie_weights(self):
+        if not self.config.disable_tie_weight:
+            self.get_llm().tie_weights()
+
+    def get_llm(self):
+        return self.llm
+
+    def get_vte(self):
+        return self.vte
+
+    def get_wte(self):
+        return self.llm.get_input_embeddings()
+
+    def get_conversation_formatter(self) -> ConversationFormatter:
+        if getattr(self, 'conversation_formatter', None) is None:
+            self.conversation_formatter = getattr(import_module(".configuration_ovis", __package__),
+                                                  self.config.conversation_formatter_class)(self.text_tokenizer)
+        return self.conversation_formatter
+
+    def forward(
+        self,
+        input_ids: torch.Tensor,
+        attention_mask: torch.Tensor,
+        labels: Optional[torch.Tensor],
+        pixel_values: List[Optional[torch.Tensor]],
+        **kwargs
+    ):
+        # assert self.training, "`forward` can only be used in training. For inference, use `generate`."
+        _, inputs_embeds, labels, attention_mask = self.merge_multimodal(
+            text_input_ids=input_ids,
+            text_attention_masks=attention_mask,
+            text_labels=labels,
+            pixel_values=pixel_values
+        )
+        return self.llm(inputs_embeds=inputs_embeds, labels=labels, attention_mask=attention_mask, **kwargs)
+
+    def merge_multimodal(
+        self,
+        text_input_ids: torch.Tensor,
+        text_attention_masks: torch.Tensor,
+        text_labels: Optional[torch.Tensor],
+        pixel_values: List[Optional[torch.Tensor]],
+        left_padding: bool = False
+    ):
+        input_device = text_input_ids.device
+        visual_vocab_szie = self.get_visual_tokenizer().config.vocab_size
+        visual_indicator_embeds = self.get_vte()(
+            torch.tensor(
+                list(range(visual_vocab_szie - 5, visual_vocab_szie)),
+                dtype=torch.long,
+                device=self.get_visual_tokenizer().device
+            )
+        ).to(device=input_device)
+
+        if self.training:
+            # When training, to be compatible with deepspeed zero, each sample has to include pixel_value tensor.
+            # For text-only sample, one can simply use a full zero tensor as pixel_value, which will be ignored
+            # (see below in this function); so, the gradient will not be affected.
+            num_images = [x.shape[0] for x in pixel_values]
+            visual_tokens = self.visual_tokenizer(torch.cat([x for x in pixel_values], dim=0))
+            visual_embeds = torch.split(self.get_vte()(visual_tokens).to(dtype=self.dtype, device=input_device),
+                                        split_size_or_sections=num_images, dim=0)
+            visual_input_ids = torch.split(torch.argmax(visual_tokens, dim=-1).to(device=input_device),
+                                           split_size_or_sections=num_images, dim=0)
+            visual_labels = [torch.full(x.shape, IGNORE_ID, dtype=torch.long, device=input_device) for x in
+                             visual_input_ids]
+        else:
+            # When inference, sample can include only text with `None` pixel_value
+            num_images = [x.shape[0] if x is not None else 0 for x in pixel_values]
+            if sum(num_images) > 0:
+                visual_tokens = self.visual_tokenizer(torch.cat([x for x in pixel_values if x is not None], dim=0))
+                visual_embeds = torch.split(self.get_vte()(visual_tokens).to(dtype=self.dtype, device=input_device),
+                                            split_size_or_sections=num_images, dim=0)
+                visual_input_ids = torch.split(torch.argmax(visual_tokens, dim=-1).to(device=input_device),
+                                               split_size_or_sections=num_images, dim=0)
+                visual_labels = [torch.full(x.shape, IGNORE_ID, dtype=torch.long, device=input_device) for x in
+                                 visual_input_ids]
+            else:
+                # just placeholders
+                visual_embeds = [None] * len(num_images)
+                visual_input_ids = [None] * len(num_images)
+                visual_labels = [None] * len(num_images)
+            # just placeholders
+            if text_labels is None:
+                text_labels = torch.full(text_input_ids.shape, IGNORE_ID, dtype=torch.long, device=input_device)
+
+        input_embeds = []
+        attention_masks = []
+        labels = []
+        for text_input_id, text_label, text_attention_mask, visual_embed, visual_input_id, visual_label in zip(
+                text_input_ids, text_labels, text_attention_masks, visual_embeds, visual_input_ids, visual_labels
+        ):
+            placeholder_token_mask = torch.lt(text_input_id, 0)
+            text_embed = self.get_wte()(torch.masked_fill(text_input_id, placeholder_token_mask, 0))
+            for i, indicator_id in enumerate(IMAGE_INDICATOR_IDS):
+                text_embed[text_input_id == indicator_id] = visual_indicator_embeds[i]
+            image_atom_positions = torch.where(torch.eq(text_input_id, IMAGE_ATOM_ID))[0].tolist()
+            if len(image_atom_positions) > 0:
+                input_embed_parts = []
+                attention_mask_parts = []
+                label_parts = []
+                prev_image_atom_position = -1
+                for index, image_atom_position in enumerate(image_atom_positions):
+                    input_embed_parts.append(
+                        text_embed[prev_image_atom_position + 1:image_atom_position, :])
+                    label_parts.append(
+                        text_label[prev_image_atom_position + 1:image_atom_position])
+                    attention_mask_parts.append(
+                        text_attention_mask[prev_image_atom_position + 1:image_atom_position])
+                    input_embed_parts.append(visual_embed[index])
+                    attention_mask_parts.append(
+                        torch.ones_like(visual_label[index], dtype=torch.bool))
+                    label_parts.append(visual_label[index])
+                    prev_image_atom_position = image_atom_position
+                if prev_image_atom_position + 1 < text_input_id.shape[0]:
+                    input_embed_parts.append(
+                        text_embed[prev_image_atom_position + 1:, :])
+                    attention_mask_parts.append(
+                        text_attention_mask[prev_image_atom_position + 1:])
+                    label_parts.append(
+                        text_label[prev_image_atom_position + 1:])
+                input_embed = torch.cat(input_embed_parts, dim=0)
+                attention_mask = torch.cat(attention_mask_parts, dim=0)
+                label = torch.cat(label_parts, dim=0)
+            else:
+                input_embed = text_embed
+                attention_mask = text_attention_mask
+                label = text_label
+                if self.training:
+                    # Make visual_embed & visual_indicator_embeds involved in the backward graph,
+                    # to be compatible with deepspeed zero and ddp.
+                    input_embed += torch.sum(visual_embed * 0.0) + torch.sum(visual_indicator_embeds * 0.0)
+            input_embeds.append(input_embed)
+            attention_masks.append(attention_mask)
+            labels.append(label)
+
+        if self.training:  # padding to self.config.multimodal_max_length for increased training speed
+            padding_size = max(0, self.config.multimodal_max_length - len(input_embeds[0]))
+            input_embeds[0] = torch.nn.ConstantPad2d((0, 0, 0, padding_size), 0.0)(input_embeds[0])
+            attention_masks[0] = torch.nn.ConstantPad1d((0, padding_size), False)(attention_masks[0])
+            labels[0] = torch.nn.ConstantPad1d((0, padding_size), IGNORE_ID)(labels[0])
+        batch_input_embeds = self.pad_truncate_sequence(input_embeds, batch_first=True, padding_value=0.0, left_padding=left_padding)
+        batch_attention_mask = self.pad_truncate_sequence(attention_masks, batch_first=True, padding_value=False, left_padding=left_padding)
+        batch_labels = self.pad_truncate_sequence(labels, batch_first=True, padding_value=IGNORE_ID, left_padding=left_padding)
+
+        return visual_input_ids, batch_input_embeds, batch_labels, batch_attention_mask
+
+    def pad_truncate_sequence(self, sequences: List[torch.Tensor], batch_first: bool = True, padding_value: float = 0.0, left_padding: bool = False) -> torch.Tensor:
+        if not left_padding:
+            pad_sequence = torch.nn.utils.rnn.pad_sequence(sequences, batch_first=batch_first, padding_value=padding_value)
+            return pad_sequence[:,:self.config.multimodal_max_length]
+        else:
+            pad_sequence = torch.nn.utils.rnn.pad_sequence([i.flip(dims=[0]) for i in sequences],batch_first=True, padding_value=padding_value).flip(dims=[1])
+            return pad_sequence[:,-self.config.multimodal_max_length:]
+
+    def preprocess_inputs(
+        self,
+        text_or_conversations: Union[List[Dict], str],
+        images: Optional[List[PIL.Image.Image]],
+        max_partition=9,
+        generation_preface='',
+        return_labels=False,
+        propagate_exception=True,
+        frame_selector=None,
+        frame_selector_kwargs=None
+    ):
+        # convert text to conversations
+        if isinstance(text_or_conversations, str):
+            conversations = [{
+                "from": "human",
+                "value": text_or_conversations
+            }]
+        elif isinstance(text_or_conversations, list):
+            conversations = text_or_conversations
+        else:
+            raise ValueError(f'Invalid type of `text_or_conversations`, expected `List[Dict]` or `str`,'
+                             f' but got {type(text_or_conversations)}')
+
+        if frame_selector is not None:
+            frame_selector_kwargs = frame_selector_kwargs or {}
+            conversations, images = frame_selector(conversations=conversations, frames=images, **frame_selector_kwargs)
+
+        # format conversations
+        prompt, raw_input_ids, raw_labels = self.get_conversation_formatter().format(
+            conversations, generation_preface=generation_preface)
+
+        # place image placeholders
+        input_ids = []
+        labels = []
+        pixel_values = []
+        invalidate_label = False
+        image_token_indices = [i for i, v in enumerate(raw_input_ids) if v == IMAGE_TOKEN_ID]
+        last_image_token_index = -1
+        for i in range(len(image_token_indices)):
+            head = 0 if i == 0 else image_token_indices[i - 1] + 1
+            tail = image_token_indices[i]
+            last_image_token_index = tail
+            input_ids.extend(raw_input_ids[head:tail])
+            labels.extend(raw_labels[head:tail])
+            try:
+                image = images[i]
+                raw_pixel_values, image_placeholders = self.visual_tokenizer.preprocess_image(
+                    image, max_partition=max_partition)
+            except Exception as e:
+                if propagate_exception:
+                    raise e
+                logging.exception(e)
+                invalidate_label = True
+                raw_pixel_values, image_placeholders = self.visual_tokenizer.mock_input()
+            input_ids.extend(image_placeholders)
+            labels.extend([IGNORE_ID] * len(image_placeholders))
+            pixel_values.append(raw_pixel_values)
+        input_ids.extend(raw_input_ids[last_image_token_index + 1:])
+        labels.extend(raw_labels[last_image_token_index + 1:])
+
+        # return tensors
+        input_ids = torch.tensor(input_ids, dtype=torch.long)
+        labels = torch.tensor([IGNORE_ID] * len(labels) if invalidate_label else labels, dtype=torch.long)
+        pixel_values = torch.cat(pixel_values, dim=0) if len(pixel_values) > 0 else None
+
+        if return_labels:
+            return prompt, input_ids, pixel_values, labels
+        else:
+            return prompt, input_ids, pixel_values
+
+    def save_pretrained(
+        self,
+        save_directory: Union[str, os.PathLike],
+        is_main_process: bool = True,
+        state_dict: Optional[dict] = None,
+        save_function: Callable = torch.save,
+        push_to_hub: bool = False,
+        max_shard_size: Union[int, str] = "5GB",
+        safe_serialization: bool = True,
+        variant: Optional[str] = None,
+        token: Optional[Union[str, bool]] = None,
+        save_peft_format: bool = True,
+        **kwargs
+    ):
+        super().save_pretrained(save_directory,
+                                is_main_process=is_main_process,
+                                state_dict=state_dict,
+                                save_function=save_function,
+                                safe_serialization=safe_serialization)
+        self.get_text_tokenizer().save_pretrained(save_directory)
+        self.get_visual_tokenizer().get_image_processor().save_pretrained(save_directory)
+
+    def generate(
+        self,
+        inputs: Optional[torch.Tensor] = None,
+        **kwargs
+    ) -> Union[GenerateOutput, torch.LongTensor]:
+        _, inputs_embeds, labels, attention_mask = self.merge_multimodal(
+            text_input_ids=inputs,
+            text_attention_masks=kwargs.pop('attention_mask'),
+            text_labels=None,
+            pixel_values=kwargs.pop('pixel_values'),
+            left_padding=True
+        )
+        inputs_embeds = inputs_embeds.detach()
+        torch.cuda.empty_cache()
+
+        return self.llm.generate(inputs=None, inputs_embeds=inputs_embeds, attention_mask=attention_mask, **kwargs)

preprocessor_config.json

@@ -0,0 +1,28 @@
+{
+  "crop_size": {
+    "height": 448,
+    "width": 448
+  },
+  "data_format": "channels_first",
+  "do_center_crop": false,
+  "do_convert_rgb": true,
+  "do_normalize": true,
+  "do_rescale": true,
+  "do_resize": true,
+  "image_mean": [
+    0.48145466,
+    0.4578275,
+    0.40821073
+  ],
+  "image_processor_type": "CLIPImageProcessorFast",
+  "image_std": [
+    0.26862954,
+    0.26130258,
+    0.27577711
+  ],
+  "resample": 3,
+  "rescale_factor": 0.00392156862745098,
+  "size": {
+    "shortest_edge": 448
+  }
+}

quant_log.csv

@@ -0,0 +1,253 @@
+layer,module,loss,samples,damp,time
+0,self_attn.k_proj,0.0000926561,0.05000,1.479
+0,self_attn.q_proj,0.0004441526,0.05000,1.486
+0,self_attn.v_proj,0.0000118854,0.05000,1.526
+0,self_attn.o_proj,0.0000509747,0.05000,0.488
+0,mlp.gate_proj,0.0005754575,0.05000,0.925
+0,mlp.up_proj,0.0004449969,0.05000,0.925
+0,mlp.down_proj,0.0000614704,0.05000,2.638
+1,self_attn.v_proj,0.0000047415,0.05000,1.379
+1,self_attn.q_proj,0.0000868213,0.05000,1.386
+1,self_attn.k_proj,0.0000241810,0.05000,1.393
+1,self_attn.o_proj,0.0000163117,0.05000,0.436
+1,mlp.gate_proj,0.0385717081,0.05000,0.929
+1,mlp.up_proj,0.0290021339,0.05000,0.933
+1,mlp.down_proj,0.0000056345,0.05000,2.672
+2,self_attn.q_proj,0.0001292669,0.05000,1.324
+2,self_attn.v_proj,0.0000073232,0.05000,1.331
+2,self_attn.k_proj,0.0000284607,0.05000,1.363
+2,self_attn.o_proj,0.0000094392,0.05000,0.446
+2,mlp.gate_proj,0.0082810673,0.05000,0.926
+2,mlp.up_proj,0.0080136446,0.05000,0.929
+2,mlp.down_proj,0.0269350808,0.05000,2.572
+3,self_attn.k_proj,0.0001170079,0.05000,1.427
+3,self_attn.q_proj,0.0005366353,0.05000,1.437
+3,self_attn.v_proj,0.0000215521,0.05000,1.439
+3,self_attn.o_proj,0.0000148144,0.05000,0.429
+3,mlp.gate_proj,0.0101124701,0.05000,0.956
+3,mlp.up_proj,0.0087271334,0.05000,0.963
+3,mlp.down_proj,0.0001437532,0.05000,2.713
+4,self_attn.q_proj,0.0004495623,0.05000,1.299
+4,self_attn.v_proj,0.0000305555,0.05000,1.323
+4,self_attn.k_proj,0.0000884943,0.05000,1.351
+4,self_attn.o_proj,0.0000298310,0.05000,0.433
+4,mlp.gate_proj,0.0096624994,0.05000,0.973
+4,mlp.up_proj,0.0069845970,0.05000,0.986
+4,mlp.down_proj,0.0000694115,0.05000,2.615
+5,self_attn.q_proj,0.0009599595,0.05000,1.503
+5,self_attn.v_proj,0.0000786844,0.05000,1.520
+5,self_attn.k_proj,0.0001727102,0.05000,1.529
+5,self_attn.o_proj,0.0000174932,0.05000,0.429
+5,mlp.gate_proj,0.0131947269,0.05000,0.968
+5,mlp.up_proj,0.0111081641,0.05000,0.971
+5,mlp.down_proj,0.0001038428,0.05000,2.544
+6,self_attn.q_proj,0.0006202141,0.05000,1.369
+6,self_attn.k_proj,0.0000994150,0.05000,1.376
+6,self_attn.v_proj,0.0000700387,0.05000,1.398
+6,self_attn.o_proj,0.0000451209,0.05000,0.440
+6,mlp.up_proj,0.0139774917,0.05000,0.940
+6,mlp.gate_proj,0.0164863941,0.05000,0.951
+6,mlp.down_proj,0.0001430597,0.05000,2.530
+7,self_attn.q_proj,0.0007890122,0.05000,1.466
+7,self_attn.v_proj,0.0000788149,0.05000,1.467
+7,self_attn.k_proj,0.0001257278,0.05000,1.466
+7,self_attn.o_proj,0.0000292612,0.05000,0.455
+7,mlp.gate_proj,0.0158752722,0.05000,0.954
+7,mlp.up_proj,0.0129439407,0.05000,0.957
+7,mlp.down_proj,0.0001964390,0.05000,2.723
+8,self_attn.q_proj,0.0007963218,0.05000,1.280
+8,self_attn.k_proj,0.0001210402,0.05000,1.289
+8,self_attn.v_proj,0.0000847841,0.05000,1.323
+8,self_attn.o_proj,0.0000442622,0.05000,0.432
+8,mlp.gate_proj,0.0170453784,0.05000,0.913
+8,mlp.up_proj,0.0121827184,0.05000,0.917
+8,mlp.down_proj,0.0002522487,0.05000,2.543
+9,self_attn.k_proj,0.0001248283,0.05000,1.415
+9,self_attn.v_proj,0.0000729903,0.05000,1.415
+9,self_attn.q_proj,0.0007056552,0.05000,1.434
+9,self_attn.o_proj,0.0000737344,0.05000,0.427
+9,mlp.up_proj,0.0105305310,0.05000,0.946
+9,mlp.gate_proj,0.0156497282,0.05000,0.945
+9,mlp.down_proj,0.0002750355,0.05000,2.662
+10,self_attn.q_proj,0.0010030885,0.05000,1.415
+10,self_attn.v_proj,0.0001205626,0.05000,1.425
+10,self_attn.k_proj,0.0001500157,0.05000,1.433
+10,self_attn.o_proj,0.0000719801,0.05000,0.423
+10,mlp.up_proj,0.0074459834,0.05000,0.876
+10,mlp.gate_proj,0.0113039666,0.05000,0.877
+10,mlp.down_proj,0.0003157377,0.05000,2.603
+11,self_attn.q_proj,0.0007359214,0.05000,1.301
+11,self_attn.v_proj,0.0000977395,0.05000,1.325
+11,self_attn.k_proj,0.0001095498,0.05000,1.329
+11,self_attn.o_proj,0.0001212573,0.05000,0.425
+11,mlp.up_proj,0.0042120424,0.05000,0.937
+11,mlp.gate_proj,0.0058166186,0.05000,0.943
+11,mlp.down_proj,0.0003497882,0.05000,2.562
+12,self_attn.k_proj,0.0001183058,0.05000,1.428
+12,self_attn.v_proj,0.0000878876,0.05000,1.438
+12,self_attn.q_proj,0.0007691479,0.05000,1.445
+12,self_attn.o_proj,0.0001109989,0.05000,0.416
+12,mlp.gate_proj,0.0063900089,0.05000,0.951
+12,mlp.up_proj,0.0045523345,0.05000,0.961
+12,mlp.down_proj,0.0003277593,0.05000,2.648
+13,self_attn.q_proj,0.0009233296,0.05000,1.406
+13,self_attn.v_proj,0.0000616283,0.05000,1.438
+13,self_attn.k_proj,0.0001702990,0.05000,1.441
+13,self_attn.o_proj,0.0000684895,0.05000,0.456
+13,mlp.gate_proj,0.0034981644,0.05000,0.951
+13,mlp.up_proj,0.0033732841,0.05000,0.956
+13,mlp.down_proj,0.0002857845,0.05000,2.568
+14,self_attn.k_proj,0.0001107448,0.05000,1.253
+14,self_attn.q_proj,0.0007179622,0.05000,1.262
+14,self_attn.v_proj,0.0000722210,0.05000,1.286
+14,self_attn.o_proj,0.0001577273,0.05000,0.435
+14,mlp.up_proj,0.0032660126,0.05000,0.936
+14,mlp.gate_proj,0.0033751456,0.05000,0.940
+14,mlp.down_proj,0.0002750711,0.05000,2.512
+15,self_attn.q_proj,0.0007260779,0.05000,1.240
+15,self_attn.k_proj,0.0001140344,0.05000,1.259
+15,self_attn.v_proj,0.0000717924,0.05000,1.279
+15,self_attn.o_proj,0.0001466886,0.05000,0.435
+15,mlp.gate_proj,0.0027777215,0.05000,0.956
+15,mlp.up_proj,0.0029199004,0.05000,0.969
+15,mlp.down_proj,0.0002450036,0.05000,2.557
+16,self_attn.k_proj,0.0001236907,0.05000,1.357
+16,self_attn.q_proj,0.0007415169,0.05000,1.370
+16,self_attn.v_proj,0.0000728653,0.05000,1.394
+16,self_attn.o_proj,0.0001548824,0.05000,0.427
+16,mlp.up_proj,0.0029338877,0.05000,0.930
+16,mlp.gate_proj,0.0029728621,0.05000,0.936
+16,mlp.down_proj,0.0002227558,0.05000,2.520
+17,self_attn.q_proj,0.0011978788,0.05000,1.411
+17,self_attn.k_proj,0.0002049842,0.05000,1.442
+17,self_attn.v_proj,0.0001110038,0.05000,1.449
+17,self_attn.o_proj,0.0001096319,0.05000,0.456
+17,mlp.gate_proj,0.0025995242,0.05000,0.938
+17,mlp.up_proj,0.0026509292,0.05000,0.941
+17,mlp.down_proj,0.0002133363,0.05000,2.545
+18,self_attn.q_proj,0.0007654526,0.05000,1.373
+18,self_attn.v_proj,0.0000810104,0.05000,1.386
+18,self_attn.k_proj,0.0001118958,0.05000,1.415
+18,self_attn.o_proj,0.0001209933,0.05000,0.437
+18,mlp.gate_proj,0.0026745959,0.05000,0.988
+18,mlp.up_proj,0.0026052926,0.05000,0.993
+18,mlp.down_proj,0.0001979110,0.05000,2.526
+19,self_attn.q_proj,0.0008727050,0.05000,1.484
+19,self_attn.v_proj,0.0000879996,0.05000,1.511
+19,self_attn.k_proj,0.0001525676,0.05000,1.517
+19,self_attn.o_proj,0.0001613215,0.05000,0.445
+19,mlp.gate_proj,0.0025457988,0.05000,0.981
+19,mlp.up_proj,0.0026727604,0.05000,0.990
+19,mlp.down_proj,0.0001955881,0.05000,2.512
+20,self_attn.q_proj,0.0013551767,0.05000,1.482
+20,self_attn.v_proj,0.0001676866,0.05000,1.505
+20,self_attn.k_proj,0.0001561945,0.05000,1.517
+20,self_attn.o_proj,0.0001528362,0.05000,0.437
+20,mlp.gate_proj,0.0026263935,0.05000,0.962
+20,mlp.up_proj,0.0027138740,0.05000,0.977
+20,mlp.down_proj,0.0002239240,0.05000,2.573
+21,self_attn.v_proj,0.0001106188,0.05000,1.444
+21,self_attn.k_proj,0.0001505467,0.05000,1.450
+21,self_attn.q_proj,0.0009562430,0.05000,1.456
+21,self_attn.o_proj,0.0001582452,0.05000,0.433
+21,mlp.gate_proj,0.0027915318,0.05000,0.984
+21,mlp.up_proj,0.0026419421,0.05000,0.997
+21,mlp.down_proj,0.0002030738,0.05000,2.577
+22,self_attn.q_proj,0.0011346004,0.05000,1.464
+22,self_attn.v_proj,0.0001487743,0.05000,1.496
+22,self_attn.k_proj,0.0001729702,0.05000,1.499
+22,self_attn.o_proj,0.0001293588,0.05000,0.436
+22,mlp.gate_proj,0.0027813923,0.05000,0.973
+22,mlp.up_proj,0.0027358021,0.05000,0.979
+22,mlp.down_proj,0.0002278643,0.05000,2.698
+23,self_attn.q_proj,0.0011565852,0.05000,1.424
+23,self_attn.k_proj,0.0001676210,0.05000,1.443
+23,self_attn.v_proj,0.0000908016,0.05000,1.451
+23,self_attn.o_proj,0.0001457047,0.05000,0.434
+23,mlp.gate_proj,0.0028181392,0.05000,0.982
+23,mlp.up_proj,0.0029544076,0.05000,0.990
+23,mlp.down_proj,0.0002631970,0.05000,2.507
+24,self_attn.k_proj,0.0001165944,0.05000,1.458
+24,self_attn.v_proj,0.0001046166,0.05000,1.466
+24,self_attn.q_proj,0.0008574808,0.05000,1.481
+24,self_attn.o_proj,0.0001873872,0.05000,0.441
+24,mlp.up_proj,0.0029398206,0.05000,0.966
+24,mlp.gate_proj,0.0028506532,0.05000,0.975
+24,mlp.down_proj,0.0002885577,0.05000,2.585
+25,self_attn.q_proj,0.0012196636,0.05000,1.466
+25,self_attn.k_proj,0.0001167471,0.05000,1.479
+25,self_attn.v_proj,0.0001735481,0.05000,1.490
+25,self_attn.o_proj,0.0001489266,0.05000,0.451
+25,mlp.gate_proj,0.0032135869,0.05000,0.950
+25,mlp.up_proj,0.0032790664,0.05000,0.953
+25,mlp.down_proj,0.0003838621,0.05000,2.757
+26,self_attn.v_proj,0.0001645723,0.05000,1.358
+26,self_attn.q_proj,0.0009637461,0.05000,1.371
+26,self_attn.k_proj,0.0001212483,0.05000,1.376
+26,self_attn.o_proj,0.0002090202,0.05000,0.427
+26,mlp.up_proj,0.0038802780,0.05000,0.938
+26,mlp.gate_proj,0.0036459990,0.05000,0.942
+26,mlp.down_proj,0.0005390834,0.05000,2.532
+27,self_attn.k_proj,0.0001402346,0.05000,1.335
+27,self_attn.v_proj,0.0002450933,0.05000,1.359
+27,self_attn.q_proj,0.0016638996,0.05000,1.369
+27,self_attn.o_proj,0.0003133276,0.05000,0.422
+27,mlp.up_proj,0.0040160986,0.05000,0.946
+27,mlp.gate_proj,0.0039604067,0.05000,0.953
+27,mlp.down_proj,0.0006700770,0.05000,2.505
+28,self_attn.v_proj,0.0002193391,0.05000,1.531
+28,self_attn.q_proj,0.0011876772,0.05000,1.540
+28,self_attn.k_proj,0.0001271059,0.05000,1.549
+28,self_attn.o_proj,0.0002864604,0.05000,0.440
+28,mlp.up_proj,0.0045377837,0.05000,0.981
+28,mlp.gate_proj,0.0045106539,0.05000,0.991
+28,mlp.down_proj,0.0007840230,0.05000,2.636
+29,self_attn.q_proj,0.0011711047,0.05000,1.465
+29,self_attn.k_proj,0.0001328981,0.05000,1.484
+29,self_attn.v_proj,0.0002036485,0.05000,1.487
+29,self_attn.o_proj,0.0002445588,0.05000,0.433
+29,mlp.gate_proj,0.0053584442,0.05000,0.863
+29,mlp.up_proj,0.0054983478,0.05000,0.867
+29,mlp.down_proj,0.0010751190,0.05000,2.710
+30,self_attn.q_proj,0.0016243049,0.05000,1.236
+30,self_attn.k_proj,0.0001383619,0.05000,1.274
+30,self_attn.v_proj,0.0004685805,0.05000,1.281
+30,self_attn.o_proj,0.0002138880,0.05000,0.422
+30,mlp.up_proj,0.0074140683,0.05000,0.931
+30,mlp.gate_proj,0.0067897284,0.05000,0.931
+30,mlp.down_proj,0.0253920822,0.05000,2.541
+31,self_attn.q_proj,0.0018998680,0.05000,1.392
+31,self_attn.k_proj,0.0001888821,0.05000,1.397
+31,self_attn.v_proj,0.0005093985,0.05000,1.399
+31,self_attn.o_proj,0.0004122503,0.05000,0.442
+31,mlp.gate_proj,0.0078921620,0.05000,1.006
+31,mlp.up_proj,0.0090096026,0.05000,1.026
+31,mlp.down_proj,0.0036010571,0.05000,2.663
+32,self_attn.v_proj,0.0013319228,0.05000,1.439
+32,self_attn.k_proj,0.0002597295,0.05000,1.439
+32,self_attn.q_proj,0.0029558921,0.05000,1.455
+32,self_attn.o_proj,0.0003896669,0.05000,0.489
+32,mlp.gate_proj,0.0077764657,0.05000,0.945
+32,mlp.up_proj,0.0088612360,0.05000,0.960
+32,mlp.down_proj,0.0033728618,0.05000,2.690
+33,self_attn.q_proj,0.0028402272,0.05000,1.341
+33,self_attn.k_proj,0.0002332372,0.05000,1.346
+33,self_attn.v_proj,0.0027896581,0.05000,1.362
+33,self_attn.o_proj,0.0004359603,0.05000,0.474
+33,mlp.gate_proj,0.0072181967,0.05000,0.921
+33,mlp.up_proj,0.0090274869,0.05000,0.927
+33,mlp.down_proj,0.0057725686,0.05000,2.693
+34,self_attn.q_proj,0.0011860414,0.05000,1.332
+34,self_attn.v_proj,0.0003992968,0.05000,1.340
+34,self_attn.k_proj,0.0001209296,0.05000,1.356
+34,self_attn.o_proj,0.0005546022,0.05000,0.431
+34,mlp.gate_proj,0.0077909217,0.05000,1.025
+34,mlp.up_proj,0.0090878079,0.05000,1.049
+34,mlp.down_proj,0.0041094714,0.05000,2.748
+35,self_attn.q_proj,0.0010943316,0.05000,1.456
+35,self_attn.v_proj,0.0003437671,0.05000,1.478
+35,self_attn.k_proj,0.0001138098,0.05000,1.483
+35,self_attn.o_proj,0.0005312666,0.05000,0.440
+35,mlp.up_proj,0.0124718128,0.05000,0.965
+35,mlp.gate_proj,0.0117675475,0.05000,0.969
+35,mlp.down_proj,0.0163082102,0.05000,2.571

quant_meta.json

@@ -0,0 +1,9 @@
+{
+  "model_id": "AIDC-AI/Ovis2-4B",
+  "family": "ovis2",
+  "method": "gptq",
+  "bits": 4,
+  "group_size": 128,
+  "quant_time_s": 231.2,
+  "n_calib": 32
+}

quantize_config.json

@@ -0,0 +1,42 @@
+{
+  "bits": 4,
+  "group_size": 128,
+  "desc_act": false,
+  "lm_head": false,
+  "quant_method": "gptq",
+  "checkpoint_format": "gptq",
+  "pack_dtype": "int32",
+  "meta": {
+    "quantizer": [
+      "gptqmodel:5.8.0"
+    ],
+    "uri": "https://github.com/modelcloud/gptqmodel",
+    "damp_percent": 0.05,
+    "damp_auto_increment": 0.01,
+    "static_groups": false,
+    "true_sequential": true,
+    "mse": 0.0,
+    "gptaq": null,
+    "act_group_aware": true,
+    "failsafe": {
+      "strategy": "rtn",
+      "threshold": "0.5%",
+      "smooth": null
+    },
+    "offload_to_disk": true,
+    "offload_to_disk_path": "./gptqmodel_offload/lzyvwgbq-jskkldgv/",
+    "pack_impl": "cpu",
+    "mock_quantization": false,
+    "gc_mode": "interval",
+    "wait_for_submodule_finalizers": false,
+    "auto_forward_data_parallel": true,
+    "hessian": {
+      "chunk_size": null,
+      "chunk_bytes": null,
+      "staging_dtype": "float32"
+    },
+    "vram_strategy": "exclusive"
+  },
+  "sym": true,
+  "format": "gptq"
+}