File size: 551 Bytes
005efa2 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 | """Weight-only block-wise quantization of the fp16 Core ML model (int4 or int8)."""
import sys
import coremltools as ct
import coremltools.optimize.coreml as cto
src, bits, out = sys.argv[1], sys.argv[2], sys.argv[3]
dtype = {"4": "int4", "8": "int8"}[bits]
cfg = cto.OptimizationConfig(
global_config=cto.OpLinearQuantizerConfig(
mode="linear_symmetric", dtype=dtype, granularity="per_block", block_size=32
)
)
m = ct.models.MLModel(src, skip_model_load=True)
q = cto.linear_quantize_weights(m, cfg)
q.save(out)
print("saved", out)
|