tinycast/quant.py

"""Post-training INT8 quantization (fake-quant) for TinyCast.

Measures the GIFT-Eval accuracy of an INT8-deployed model by simulating INT8 arithmetic in
floating point. Per-output-channel symmetric INT8 weights on every Linear and Conv1d; optional
per-tensor dynamic INT8 activations. RMSNorm, the SiLU gate, the min-max (de)normalization,
and the rFFT period detector stay in full precision: they are off the convolutional mixing path
and run as fp/LUT ops on the target runtime.

Modes (env ``TINYCAST_INT8``):
  ``w8``    per-channel INT8 weights, fp activations. The ~145 KB weight footprint; isolates the
            weight-quantization error.
  ``w8a8``  + per-tensor dynamic INT8 activations (scale from each tensor's own range, so no
            calibration set is needed; an optimistic but faithful estimate of full INT8 compute).

The weight quant is applied in-place to the parameter tensors, so it is correct regardless of
whether a module is invoked via ``__call__`` or functionally (the separable pointwise conv is run
as ``F.linear(weight.squeeze(-1))``). Activation quant (``w8a8``) uses a forward-pre-hook on every
Linear/Conv1d to quantize inputs, plus a forward-hook on Conv1d to quantize the depthwise output
that feeds the functional pointwise, covering every activation site on the mixing path.
"""
from __future__ import annotations

import torch
import torch.nn as nn

_QMIN, _QMAX = -128, 127  # int8 symmetric (zero-point 0)


@torch.no_grad()
def _fq_weight_per_outchannel(w: torch.Tensor) -> torch.Tensor:
    """Symmetric per-output-channel (axis 0) int8 fake-quant of a weight tensor.

    Linear weight is (out, in); Conv1d weight is (out, in/groups, k). Axis 0 is the output
    channel in both, so a per-axis-0 scale is the standard per-channel weight scheme.
    """
    red = tuple(d for d in range(w.dim()) if d != 0)
    amax = w.abs().amax(dim=red, keepdim=True).clamp_(min=1e-12)
    scale = amax / _QMAX
    return (torch.round(w / scale).clamp_(_QMIN, _QMAX) * scale).to(w.dtype)


def _fq_act_dynamic(x: torch.Tensor) -> torch.Tensor:
    """Per-tensor symmetric int8 fake-quant with a dynamic (this-tensor) scale."""
    if not torch.is_floating_point(x):
        return x
    amax = x.detach().abs().amax().clamp(min=1e-12)
    scale = amax / _QMAX
    return torch.round(x / scale).clamp(_QMIN, _QMAX) * scale


def _pre_hook(_mod, inp):
    if not inp:
        return None
    return (_fq_act_dynamic(inp[0]),) + tuple(inp[1:])


def _post_hook(_mod, _inp, out):
    return _fq_act_dynamic(out)


def quantize_int8_(model: nn.Module, mode: str = "w8") -> nn.Module:
    """In-place INT8 fake-quant of ``model``. ``mode`` in {"w8", "w8a8"}. Returns ``model``."""
    mode = mode.strip().lower()
    if mode not in ("w8", "w8a8"):
        raise ValueError(f"unknown INT8 mode {mode!r} (expected 'w8' or 'w8a8')")
    n_w = 0
    with torch.no_grad():
        for m in model.modules():
            if isinstance(m, (nn.Linear, nn.Conv1d)):
                m.weight.data.copy_(_fq_weight_per_outchannel(m.weight.data))
                n_w += 1
                if mode == "w8a8":
                    m.register_forward_pre_hook(_pre_hook)
                    if isinstance(m, nn.Conv1d):
                        m.register_forward_hook(_post_hook)
    print(
        f"[quant] INT8 {mode}: fake-quantized {n_w} Linear/Conv1d weight tensors"
        + (" + per-tensor dynamic activation quant" if mode == "w8a8" else ""),
        flush=True,
    )
    return model