File size: 3,071 Bytes
45446c9 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 | """Weight compression of the EmbeddingGemma 2 Core ML multifunction packages.
The public coremltools PTQ API refuses multifunction models, so this loads the
whole program into pymil, runs the compression pass over every function,
deduplicates consts across functions (weight sharing) and exports it again as
a multifunction package, as `save_multifunction` does.
usage: quantize.py <fp16 dir> <out dir> [mode]
mode: int8 (linear symmetric, per channel; default) | int4 | palN (k-means, N bits)
"""
import shutil
import sys
from pathlib import Path
import coremltools as ct
import coremltools.optimize as cto
from coremltools.converters.mil.converter import mil_convert
from coremltools.converters.mil.frontend.milproto.load import load as milproto_load
from coremltools.converters.mil.mil.passes.pass_registry import PASS_REGISTRY
from coremltools.converters.mil.mil.passes.graph_pass import PassOption
from coremltools.models.utils import _try_get_weights_dir_path
src, out = Path(sys.argv[1]), Path(sys.argv[2])
mode = sys.argv[3] if len(sys.argv) > 3 else "int8"
out.mkdir(parents=True, exist_ok=True)
if mode == "int8":
op = cto.coreml.OpLinearQuantizerConfig(
mode="linear_symmetric", dtype="int8", granularity="per_channel",
weight_threshold=2048)
pass_name = "compression::linear_quantize_weights"
elif mode == "int4":
op = cto.coreml.OpLinearQuantizerConfig(
mode="linear_symmetric", dtype="int4", granularity="per_block",
block_size=32, weight_threshold=2048)
pass_name = "compression::linear_quantize_weights"
else:
op = cto.coreml.OpPalettizerConfig(
mode="kmeans", nbits=int(mode[3:]), granularity="per_grouped_channel",
group_size=16, weight_threshold=2048)
pass_name = "compression::palettize_weights"
config = cto.coreml.OptimizationConfig(global_config=op)
for name in ("EmbeddingGemma2Text", "EmbeddingGemma2Vision"):
path = str(src / f"{name}.mlpackage")
spec = ct.utils.load_spec(path)
print(name, "spec", spec.specificationVersion, "functions",
[f.name for f in spec.description.functions], flush=True)
prog = milproto_load(spec, spec.specificationVersion, _try_get_weights_dir_path(path))
p = PASS_REGISTRY[pass_name]
p.set_options([PassOption("config", config), PassOption("joint_compression", False)])
p.apply(prog)
PASS_REGISTRY["common::const_deduplication"]._deduplicate_const_across_functions(prog)
prog.skip_all_passes = True
prog.export_as_multifunction = len(spec.description.functions) > 0
if prog.export_as_multifunction:
prog.default_function_name = spec.description.defaultFunctionName
model = mil_convert(
prog, convert_to="mlprogram", convert_from="milinternal",
specification_version=spec.specificationVersion,
compute_units=ct.ComputeUnit.CPU_ONLY,
model_description=spec.description, skip_model_load=True)
dest = out / f"{name}.mlpackage"
if dest.exists():
shutil.rmtree(dest)
model.save(str(dest))
print("saved", dest, flush=True)
|