Download scripts/quantize.py from lorelaiassistant/embeddinggemma-2-coreml-int8: direct link, hf CLI and curl.
- Browser
- Download file 3.07 kB
-
https://huggingface.co/lorelaiassistant/embeddinggemma-2-coreml-int8/resolve/main/scripts/quantize.py
- Command line
-
hf download hf://lorelaiassistant/embeddinggemma-2-coreml-int8/scripts/quantize.py
-
curl -L -o quantize.py https://huggingface.co/lorelaiassistant/embeddinggemma-2-coreml-int8/resolve/main/scripts/quantize.py
3.07 kB
| """Weight compression of the EmbeddingGemma 2 Core ML multifunction packages. | |
| The public coremltools PTQ API refuses multifunction models, so this loads the | |
| whole program into pymil, runs the compression pass over every function, | |
| deduplicates consts across functions (weight sharing) and exports it again as | |
| a multifunction package, as `save_multifunction` does. | |
| usage: quantize.py <fp16 dir> <out dir> [mode] | |
| mode: int8 (linear symmetric, per channel; default) | int4 | palN (k-means, N bits) | |
| """ | |
| import shutil | |
| import sys | |
| from pathlib import Path | |
| import coremltools as ct | |
| import coremltools.optimize as cto | |
| from coremltools.converters.mil.converter import mil_convert | |
| from coremltools.converters.mil.frontend.milproto.load import load as milproto_load | |
| from coremltools.converters.mil.mil.passes.pass_registry import PASS_REGISTRY | |
| from coremltools.converters.mil.mil.passes.graph_pass import PassOption | |
| from coremltools.models.utils import _try_get_weights_dir_path | |
| src, out = Path(sys.argv[1]), Path(sys.argv[2]) | |
| mode = sys.argv[3] if len(sys.argv) > 3 else "int8" | |
| out.mkdir(parents=True, exist_ok=True) | |
| if mode == "int8": | |
| op = cto.coreml.OpLinearQuantizerConfig( | |
| mode="linear_symmetric", dtype="int8", granularity="per_channel", | |
| weight_threshold=2048) | |
| pass_name = "compression::linear_quantize_weights" | |
| elif mode == "int4": | |
| op = cto.coreml.OpLinearQuantizerConfig( | |
| mode="linear_symmetric", dtype="int4", granularity="per_block", | |
| block_size=32, weight_threshold=2048) | |
| pass_name = "compression::linear_quantize_weights" | |
| else: | |
| op = cto.coreml.OpPalettizerConfig( | |
| mode="kmeans", nbits=int(mode[3:]), granularity="per_grouped_channel", | |
| group_size=16, weight_threshold=2048) | |
| pass_name = "compression::palettize_weights" | |
| config = cto.coreml.OptimizationConfig(global_config=op) | |
| for name in ("EmbeddingGemma2Text", "EmbeddingGemma2Vision"): | |
| path = str(src / f"{name}.mlpackage") | |
| spec = ct.utils.load_spec(path) | |
| print(name, "spec", spec.specificationVersion, "functions", | |
| [f.name for f in spec.description.functions], flush=True) | |
| prog = milproto_load(spec, spec.specificationVersion, _try_get_weights_dir_path(path)) | |
| p = PASS_REGISTRY[pass_name] | |
| p.set_options([PassOption("config", config), PassOption("joint_compression", False)]) | |
| p.apply(prog) | |
| PASS_REGISTRY["common::const_deduplication"]._deduplicate_const_across_functions(prog) | |
| prog.skip_all_passes = True | |
| prog.export_as_multifunction = len(spec.description.functions) > 0 | |
| if prog.export_as_multifunction: | |
| prog.default_function_name = spec.description.defaultFunctionName | |
| model = mil_convert( | |
| prog, convert_to="mlprogram", convert_from="milinternal", | |
| specification_version=spec.specificationVersion, | |
| compute_units=ct.ComputeUnit.CPU_ONLY, | |
| model_description=spec.description, skip_model_load=True) | |
| dest = out / f"{name}.mlpackage" | |
| if dest.exists(): | |
| shutil.rmtree(dest) | |
| model.save(str(dest)) | |
| print("saved", dest, flush=True) | |