Spaces:
Paused
Paused
File size: 4,926 Bytes
9a03d09 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 | # Copyright 2025 The Intel and The HuggingFace Inc. teams. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from typing import TYPE_CHECKING
from ...utils import is_auto_round_available, logging
from ..base import DiffusersQuantizer
if TYPE_CHECKING:
from ...models.modeling_utils import ModelMixin
logger = logging.get_logger(__name__)
class AutoRoundQuantizer(DiffusersQuantizer):
r"""
Diffusers Quantizer for AutoRound (https://github.com/intel/auto-round).
AutoRound is a weight-only quantization method that uses sign gradient descent to jointly optimize rounding values
and min-max ranges for weights. It supports W4A16 (4-bit weight, 16-bit activation) quantization for efficient
inference.
This quantizer only supports loading pre-quantized AutoRound models. On-the-fly quantization (calibration) is not
supported through this interface.
"""
# AutoRound requires data calibration — we only support loading pre-quantized checkpoints.
requires_calibration = True
required_packages = ["auto_round"]
def __init__(self, quantization_config, **kwargs):
super().__init__(quantization_config, **kwargs)
def validate_environment(self, *args, **kwargs):
"""
Validates that the auto-round library (>= 0.5) is installed and captures the device_map for later use during
model conversion.
"""
self.device_map = kwargs.get("device_map", None)
if not is_auto_round_available():
raise ImportError(
"Loading an AutoRound quantized model requires the auto-round library "
"(`pip install 'auto-round>=0.13.0'`)"
)
if not self.pre_quantized:
raise ValueError(
"AutoRound quantizer in diffusers only supports loading pre-quantized models. "
"To quantize a model from scratch, use the AutoRound CLI or Python API "
"(https://github.com/intel/auto-round) directly, then load the result with Diffusers."
)
def _process_model_before_weight_loading(
self,
model: "ModelMixin",
device_map,
keep_in_fp32_modules: list[str] = [],
**kwargs,
):
"""
Replaces target nn.Linear layers with AutoRound's quantized QuantLinear layers before weights are loaded from
the checkpoint.
Uses `auto_round.inference.convert_model.convert_hf_model` which:
- Inspects the model architecture and the quantization config (bits, group_size, sym, backend).
- Replaces eligible nn.Linear modules with the appropriate QuantLinear variant (the packed-weight layer that
stores qweight, scales, qzeros).
- Returns the converted model and a set of used backend names.
`infer_target_device` resolves the device_map into a single target device string that AutoRound uses to select
the correct kernel backend (e.g. "cuda", "cpu").
"""
from auto_round.inference.convert_model import convert_hf_model, infer_target_device
target_device = infer_target_device(self.device_map)
model, used_backends = convert_hf_model(model, target_device)
self.used_backends = used_backends
def _process_model_after_weight_loading(self, model, **kwargs):
"""
Finalizes the model after all quantized weights (qweight, scales, qzeros, etc.) have been loaded into the
QuantLinear layers.
Uses `auto_round.inference.convert_model.post_init` which:
- Performs backend-specific finalization (e.g. repacking weights into the kernel's expected memory layout,
moving buffers to the correct device).
- Freezes quantized parameters (requires_grad=False).
- Prepares the model for inference.
"""
from auto_round.inference.convert_model import post_init
post_init(model, self.used_backends)
return model
@property
def is_trainable(self) -> bool:
"""AutoRound W4A16 pre-quantized models do not support training."""
return False
@property
def is_serializable(self):
"""AutoRound quantized models can be serialized (the quantization config may be
updated by the backend, e.g. for GPTQ/AWQ-compatible formats)."""
return True
@property
def is_compileable(self) -> bool:
return True
|