muscriptor-onnx / tools /quantize_int8.py
fpp-ableton's picture
Add MuScriptor small + medium ONNX (INT8), tools, notice
bedfc1d verified
Raw History Blame Contribute Delete
850 Bytes
#!/usr/bin/env python3
"""INT8 dynamic weight quantization of an exported MuScriptor lm.onnx (LM head kept fp32).
Usage: python quantize_int8.py out/small/lm.onnx out/small/lm-int8.onnx --card 1393
"""
import argparse
import onnx
from onnxruntime.quantization import QuantType, quantize_dynamic
ap = argparse.ArgumentParser()
ap.add_argument("src")
ap.add_argument("dst")
ap.add_argument("--card", type=int, required=True, help="vocabulary size of the model (1393 small, 1395 medium)")
a = ap.parse_args()
m = onnx.load(a.src)
inits = {i.name: i for i in m.graph.initializer}
head = [n.name for n in m.graph.node if n.op_type == "MatMul" and any(i in inits and list(inits[i].dims)[-1] == a.card for i in n.input)]
quantize_dynamic(a.src, a.dst, weight_type=QuantType.QInt8, op_types_to_quantize=["MatMul"], nodes_to_exclude=head, per_channel=True)