File size: 850 Bytes
bedfc1d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
#!/usr/bin/env python3
"""INT8 dynamic weight quantization of an exported MuScriptor lm.onnx (LM head kept fp32).

Usage: python quantize_int8.py out/small/lm.onnx out/small/lm-int8.onnx --card 1393
"""
import argparse
import onnx
from onnxruntime.quantization import QuantType, quantize_dynamic

ap = argparse.ArgumentParser()
ap.add_argument("src")
ap.add_argument("dst")
ap.add_argument("--card", type=int, required=True, help="vocabulary size of the model (1393 small, 1395 medium)")
a = ap.parse_args()
m = onnx.load(a.src)
inits = {i.name: i for i in m.graph.initializer}
head = [n.name for n in m.graph.node if n.op_type == "MatMul" and any(i in inits and list(inits[i].dims)[-1] == a.card for i in n.input)]
quantize_dynamic(a.src, a.dst, weight_type=QuantType.QInt8, op_types_to_quantize=["MatMul"], nodes_to_exclude=head, per_channel=True)