#!/usr/bin/env python3 """INT8 dynamic weight quantization of an exported MuScriptor lm.onnx (LM head kept fp32). Usage: python quantize_int8.py out/small/lm.onnx out/small/lm-int8.onnx --card 1393 """ import argparse import onnx from onnxruntime.quantization import QuantType, quantize_dynamic ap = argparse.ArgumentParser() ap.add_argument("src") ap.add_argument("dst") ap.add_argument("--card", type=int, required=True, help="vocabulary size of the model (1393 small, 1395 medium)") a = ap.parse_args() m = onnx.load(a.src) inits = {i.name: i for i in m.graph.initializer} head = [n.name for n in m.graph.node if n.op_type == "MatMul" and any(i in inits and list(inits[i].dims)[-1] == a.card for i in n.input)] quantize_dynamic(a.src, a.dst, weight_type=QuantType.QInt8, op_types_to_quantize=["MatMul"], nodes_to_exclude=head, per_channel=True)