python/optimize_onnx.py

"""
Produce a compact fp16 variant of the exported ONNX (and best-effort onnxslim).

The 804 MB fp32 export carries ~425 MB of constant-folded fp32 tensors (Swin attention masks /
positional-embedding tables). fp16 (keep_io_types=True so I/O stay float32) ~halves the file and is
native/fast on WebGPU & DirectML. CPU runs fp16 too (via casts), just slower than fp32.

Note: onnxslim's shape-inference pass trips protobuf's 2 GB serialization limit on the fp32 model,
so we convert to fp16 first (smaller) and only then try to slim.
"""
import time
from pathlib import Path

import onnx

ROOT = Path(__file__).resolve().parent.parent
SRC = ROOT / "models" / "mvanet_box_segmenter.onnx"
FP16 = ROOT / "models" / "mvanet_box_segmenter_fp16.onnx"
FP16_SLIM = ROOT / "models" / "mvanet_box_segmenter_fp16_slim.onnx"


def mb(p: Path) -> float:
    return p.stat().st_size / 1e6


def main() -> None:
    print(f"[src ] {SRC.name}  {mb(SRC):.1f} MB")

    # fp16 (input/output kept float32 so the JS harness is unchanged)
    import numpy as np
    from onnx import TensorProto, numpy_helper
    from onnxconverter_common import float16

    def fix_resize_scale_inputs(model) -> int:
        """Resize/Upsample roi & scales inputs must be float32 even when X is fp16 (ONNX spec).
        The fp16 converter wrongly casts the Constants feeding them; cast those back to fp32."""
        inits = {t.name: t for t in model.graph.initializer}
        consts = {n.output[0]: n for n in model.graph.node if n.op_type == "Constant"}
        targets = set()
        for n in model.graph.node:
            if n.op_type in ("Resize", "Upsample"):
                for idx in (1, 2):  # roi, scales
                    if idx < len(n.input) and n.input[idx]:
                        targets.add(n.input[idx])
        fixed = 0
        for name in targets:
            if name in inits and inits[name].data_type == TensorProto.FLOAT16:
                arr = numpy_helper.to_array(inits[name]).astype(np.float32)
                inits[name].CopyFrom(numpy_helper.from_array(arr, name)); fixed += 1
            elif name in consts:
                for a in consts[name].attribute:
                    if a.name == "value" and a.t.data_type == TensorProto.FLOAT16:
                        arr = numpy_helper.to_array(a.t).astype(np.float32)
                        a.t.CopyFrom(numpy_helper.from_array(arr, a.t.name)); fixed += 1
        return fixed

    from onnx import helper

    def wrap_fp32_io(model) -> None:
        """Make external input/output float32 around an all-fp16 interior, via explicit Cast nodes.
        (keep_io_types=True left the fp32 input feeding dtype-agnostic reshape branches, which then
        clashed with cast fp16 branches at a Concat — so we go uniform fp16 + manual I/O casts.)"""
        g = model.graph
        # INPUT: rename the fp16 graph input value, add a fp32 input that Casts into it.
        gi = g.input[0]
        old = gi.name
        internal = old + "_f16"
        for n in g.node:
            n.input[:] = [internal if x == old else x for x in n.input]
        shape = [d.dim_value for d in gi.type.tensor_type.shape.dim]
        g.input.remove(gi)
        g.input.insert(0, helper.make_tensor_value_info(old, TensorProto.FLOAT, shape))
        g.node.insert(0, helper.make_node("Cast", [old], [internal], to=TensorProto.FLOAT16,
                                           name="cast_input_to_fp16"))
        # OUTPUT: rename the fp16 graph output, Cast it back to fp32.
        go = g.output[0]
        oold = go.name
        ointernal = oold + "_f16"
        for n in g.node:
            n.output[:] = [ointernal if x == oold else x for x in n.output]
            n.input[:] = [ointernal if x == oold else x for x in n.input]
        oshape = [d.dim_value for d in go.type.tensor_type.shape.dim]
        g.output.remove(go)
        g.output.insert(0, helper.make_tensor_value_info(oold, TensorProto.FLOAT, oshape))
        g.node.append(helper.make_node("Cast", [ointernal], [oold], to=TensorProto.FLOAT,
                                        name="cast_output_to_fp32"))

    # Shape inference must be ON: otherwise the converter can't see branch types (e.g. the global-view
    # Resize branch) and leaves a mixed fp16/fp32 Concat in SplitMultiView. keep_io_types=True keeps
    # the model's input/output float32 so the Node harness is unchanged.
    # onnxconverter_common leaves several mixed fp16/fp32 boundaries (Concat in SplitMultiView, the
    # WindowSDPA scale Div, ...) that onnx.checker accepts but ORT rejects. General fix: infer types,
    # then Cast any fp32 input of a float-compute op to fp16 so every such op is uniformly fp16.
    FLOAT_OPS = {"Add", "Sub", "Mul", "Div", "Concat", "Where", "Sum", "Min", "Max", "Pow",
                 "MatMul", "Gemm", "PRelu", "Sqrt", "Reciprocal", "Neg", "Softmax", "Erf", "Sigmoid"}

    def reconcile_float_types(model) -> int:
        """Forward-propagate per-tensor dtypes ourselves (onnx shape-inference under-types this graph),
        then Cast any fp32 input of a float-compute op to fp16 so ORT's strict type check passes."""
        from onnx import TensorProto, helper
        FP16, FP32, INT64, BOOL = (TensorProto.FLOAT16, TensorProto.FLOAT,
                                   TensorProto.INT64, TensorProto.BOOL)
        INT_OUT = {"Shape", "Size", "NonZero", "ArgMax", "ArgMin"}
        BOOL_OUT = {"Equal", "Greater", "Less", "GreaterOrEqual", "LessOrEqual",
                    "And", "Or", "Not", "Xor", "IsNaN", "IsInf"}
        dt = {t.name: t.data_type for t in model.graph.initializer}
        for v in model.graph.input:
            dt[v.name] = v.type.tensor_type.elem_type

        def out_dtype(n):
            op = n.op_type
            if op == "Constant":
                for a in n.attribute:
                    if a.name == "value":
                        return a.t.data_type
                    if a.name in ("value_float", "value_floats"):
                        return FP32
                    if a.name in ("value_int", "value_ints"):
                        return INT64
                return None
            if op == "ConstantOfShape":
                for a in n.attribute:
                    if a.name == "value":
                        return a.t.data_type
                return FP32
            if op in ("Cast", "CastLike"):
                return next((a.i for a in n.attribute if a.name == "to"), None)
            if op in INT_OUT:
                return INT64
            if op in BOOL_OUT:
                return BOOL
            return dt.get(n.input[0]) if n.input else None

        for _ in range(3):  # a few passes in case nodes aren't perfectly topo-sorted
            for n in model.graph.node:
                d = out_dtype(n)
                if d is not None:
                    for o in n.output:
                        if o:
                            dt[o] = d

        out_nodes, c = [], 0
        for n in model.graph.node:
            if n.op_type in FLOAT_OPS:
                for idx, i in enumerate(list(n.input)):
                    if i and dt.get(i) == FP32:
                        co = f"{i}__f16_{c}"
                        out_nodes.append(helper.make_node("Cast", [i], [co], to=FP16, name=f"recon_{c}"))
                        n.input[idx] = co
                        dt[co] = FP16
                        c += 1
            out_nodes.append(n)
        model.graph.ClearField("node")
        model.graph.node.extend(out_nodes)
        return c

    # Pipeline order matters:
    #   convert fp16  ->  fix Resize scales  ->  onnxslim (folds 3377 Constants, dedupes)
    #   ->  reconcile float types  ->  clear stale value_info  ->  save.
    # reconcile + value_info-clear MUST be the LAST steps: onnxslim eliminates "redundant" casts
    # (undoing reconcile) and the original fp32 value_info conflicts with the fp16 tensors (ORT rejects).
    tmp1 = FP16.with_name("_fp16_pre.onnx")
    m = onnx.load(str(SRC))
    t = time.time()
    m16 = float16.convert_float_to_float16(m, keep_io_types=True, disable_shape_infer=False)
    fix_resize_scale_inputs(m16)
    onnx.save(m16, str(tmp1))

    from onnxslim import slim
    slim(str(tmp1), str(FP16))                       # slim in place into the final path

    ms = onnx.load(str(FP16))
    n_fixed = fix_resize_scale_inputs(ms)            # re-assert in case slim refolded
    n_recon = reconcile_float_types(ms)
    ms.graph.ClearField("value_info")               # drop stale fp32 annotations; ORT re-infers
    onnx.checker.check_model(ms)
    onnx.save(ms, str(FP16))
    tmp1.unlink(missing_ok=True)
    print(f"[fp16] slim+reconcile: resize-fixes={n_fixed} reconcile-casts={n_recon}")
    print(f"[fp16] {FP16.name}  {mb(FP16):.1f} MB  ({time.time()-t:.1f}s)")


if __name__ == "__main__":
    main()