"""Weight-only quantisation of the Linear layers (precision variants int8 / int4 of the same checkpoint). int8: per-output-channel symmetric int8 (scale fp16) -> `W = q * s[:, None]` int4: asymmetric 4-bit (scale + zero point per group of 128 along the input dim, fp16), two nibbles per byte; the first trunk layers (most sensitive, see tests) can be kept int8 via `int8_layers` Kept in higher precision (not quantised): embeddings, norms, the decision readout, the audio projector, the audio subsampler / output projection and the vision patch embed / merger (small, numerically sensitive). The quantised file keeps every non-quantised tensor as in the bf16 checkpoint and stores `.weight` as int8/uint8 + `.scale` (+ `.qcfg`). Runtime: `QLinear` dequantises on the fly (GPU: fused into a bf16 matmul per call; CPU: dequantised once into fp32 or an int8 dynamic matmul). python -m julia2.quant --src --dst /int8 --bits 8 """ import argparse import json from pathlib import Path import torch import torch.nn as nn import torch.nn.functional as F # tensors whose Linear weights are quantised: trunk layers (incl. attention / linear-attention / mlp), vision blocks, audio conformer layers QUANT_PREFIXES = ("trunk.language_model.layers.", "trunk.visual.blocks.", "audio.encoder.layers.") SKIP_SUBSTR = ("norm", "A_log", "dt_bias", "per_dim_scale", "conv1d", "_min", "_max") def quantize_tensor(w, bits, group=128): """w [out, in] float -> (q int8|uint8-packed, scale fp16, meta).""" w = w.float() if bits == 8: s = w.abs().amax(1).clamp_min(1e-8) / 127.0 q = torch.round(w / s[:, None]).clamp(-127, 127).to(torch.int8) return q, s.to(torch.float16), {"bits": 8} o, i = w.shape g = group if i % group == 0 else i wg = w.view(o, i // g, g) mn, mx = wg.amin(2), wg.amax(2) s = ((mx - mn) / 15.0).clamp_min(1e-8); z = torch.round(-mn / s).clamp(0, 15) # asymmetric: q = round(w/s) + z in 0..15 q = (torch.round(wg / s[:, :, None]) + z[:, :, None]).clamp(0, 15).to(torch.uint8).view(o, i) packed = (q[:, 0::2] | (q[:, 1::2] << 4)).to(torch.uint8) # two nibbles per byte return packed, torch.stack([s, z], -1).to(torch.float16), {"bits": 4, "group": g} # scale [o, i/g, 2] = (s, z) def dequantize(q, s, meta, dtype=torch.float32): if meta["bits"] == 8: return q.to(dtype) * s.to(dtype)[:, None] o = q.shape[0]; g = meta["group"] lo = (q & 0x0F); hi = (q >> 4) full = torch.stack([lo, hi], -1).view(o, -1, g).to(dtype) # [o, in/g, g] in 0..15 sz = s.to(dtype) return ((full - sz[:, :, 1:2]) * sz[:, :, 0:1]).view(o, -1) class QLinear(nn.Module): """Drop-in for nn.Linear holding quantised weights. GPU: dequantise per call into the activation dtype (bandwidth-bound matmuls; the dequant is fused by the graph capture). CPU: dequantised once to the compute dtype (bf16 on AMX/AVX512-BF16 CPUs, else fp32) and cached -> same speed as the bf16 variant with 1/2 (int8) or 1/4 (int4) of the weight bytes on disk / in RAM before the cache fills. Optional (JULIA2_CPU_INT8_DYNAMIC=1): int8 weights x dynamically int8-quantised activations through the fbgemm/onednn kernel — measured not faster than bf16 AMX on Sapphire Rapids and costs text agreement (.95), so off by default.""" def __init__(self, q, s, meta, bias, out_features, in_features): super().__init__() self.register_buffer("q", q); self.register_buffer("s", s) self.bias = None if bias is None else nn.Parameter(bias, requires_grad=False) self.meta, self.out_features, self.in_features = meta, out_features, in_features self._w = None; self._packed = None def _cpu_int8(self, x): if self._packed is None: import warnings with warnings.catch_warnings(): warnings.simplefilter("ignore") wq = torch._make_per_channel_quantized_tensor(self.q, self.s.float(), torch.zeros(self.q.shape[0], dtype=torch.long), 0) self._packed = torch.ops.quantized.linear_prepack(wq, None if self.bias is None else self.bias.float()) y = torch.ops.quantized.linear_dynamic(x.float(), self._packed, reduce_range=False) return y.to(x.dtype) def forward(self, x): if x.device.type == "cpu": if self.meta["bits"] == 8 and CPU_INT8_DYNAMIC and _cpu_int8_ok(): return self._cpu_int8(x) # opt-in: per-tensor dynamic activation quantisation costs agreement (.95 on text) and is not faster than bf16 AMX here if self._w is None or self._w.dtype != x.dtype: self._w = dequantize(self.q, self.s, self.meta, x.dtype) return F.linear(x, self._w, None if self.bias is None else self.bias.to(x.dtype)) w = dequantize(self.q, self.s, self.meta, x.dtype) return F.linear(x, w, None if self.bias is None else self.bias.to(x.dtype)) _CPU_INT8 = None CPU_INT8_DYNAMIC = bool(int(__import__("os").environ.get("JULIA2_CPU_INT8_DYNAMIC", "0"))) def _cpu_int8_ok(): global _CPU_INT8 if _CPU_INT8 is None: try: import warnings with warnings.catch_warnings(): warnings.simplefilter("ignore") torch.backends.quantized.engine = "x86" if "x86" in torch.backends.quantized.supported_engines else torch.backends.quantized.engine w = torch._make_per_channel_quantized_tensor(torch.zeros(4, 8, dtype=torch.int8), torch.ones(4), torch.zeros(4, dtype=torch.long), 0) torch.ops.quantized.linear_dynamic(torch.zeros(2, 8), torch.ops.quantized.linear_prepack(w, None), reduce_range=False) _CPU_INT8 = True except Exception: _CPU_INT8 = False return _CPU_INT8 def swap_linears(module, state, prefix): """Replace nn.Linear children whose quantised tensors exist in `state` (keys relative to `prefix`).""" for name, child in list(module.named_children()): full = f"{prefix}{name}" if isinstance(child, nn.Linear) and f"{full}.q" in state: q, s, meta = state[f"{full}.q"], state[f"{full}.s"], json.loads(state[f"{full}.qcfg"]) setattr(module, name, QLinear(q, s, meta, child.bias.data if child.bias is not None else None, child.out_features, child.in_features)) else: swap_linears(child, state, full + ".") def quantize_file(src, dst, bits, group=128, int8_layers=()): """int8_layers: trunk layer indices kept at int8 inside an int4 file (mixed precision).""" from safetensors import safe_open from safetensors.torch import save_file f = safe_open(str(Path(src) / "model.safetensors"), "pt") out, n_q, n_bytes = {}, 0, 0 for k in f.keys(): t = f.get_tensor(k) if k.endswith(".weight") and t.dim() == 2 and any(k.startswith(p) for p in QUANT_PREFIXES) and not any(s in k for s in SKIP_SUBSTR): b = bits if bits == 4 and k.startswith("trunk.language_model.layers.") and int(k.split(".")[3]) in int8_layers: b = 8 q, s, meta = quantize_tensor(t, b, group) base = k[: -len(".weight")] out[base + ".q"], out[base + ".s"] = q.contiguous(), s.contiguous() out[base + ".qcfg"] = torch.tensor(list(json.dumps(meta).encode()), dtype=torch.uint8) n_q += 1; n_bytes += q.numel() * q.element_size() + s.numel() * 2 else: out[k] = t.contiguous() Path(dst).mkdir(parents=True, exist_ok=True) save_file(out, str(Path(dst) / "model.safetensors"), metadata={"precision": f"int{bits}", "group": str(group), "int8_layers": ",".join(map(str, int8_layers)), "format": "julia2-wq-v1"}) return n_q, n_bytes if __name__ == "__main__": ap = argparse.ArgumentParser(); ap.add_argument("--src", required=True); ap.add_argument("--dst", required=True); ap.add_argument("--bits", type=int, default=8); ap.add_argument("--group", type=int, default=128) ap.add_argument("--int8_layers", default="", help="comma-separated trunk layers kept int8 in an int4 file") a = ap.parse_args() n, b = quantize_file(a.src, a.dst, a.bits, a.group, tuple(int(x) for x in a.int8_layers.split(",") if x)) print(f"quantised {n} linear weights -> {a.dst} ({b/1e9:.2f} GB of quantised tensors)")