"""Weight-only quantisation of the Linear layers (precision variants int8 / int4 of the same checkpoint). int8: per-output-channel symmetric int8 (scale fp16) -> `W = q * s[:, None]` int4: asymmetric 4-bit (scale + zero point per group of 128 along the input dim, fp16), two nibbles per byte; the first trunk layers (most sensitive, see tests) can be kept int8 via `int8_layers` Kept in higher precision (not quantised): embeddings, norms, the decision readout, the audio projector, the audio subsampler / output projection and the vision patch embed / merger (small, numerically sensitive). The quantised file keeps every non-quantised tensor as in the bf16 checkpoint and stores `.weight` as int8/uint8 + `.scale` (+ `.qcfg`). Runtime: `QLinear` dequantises on the fly (GPU: fused into a bf16 matmul per call; CPU: dequantised once into fp32 or an int8 dynamic matmul). python -m julia2.quant --src --dst /int8 --bits 8 """ import argparse import json from pathlib import Path import torch import torch.nn as nn import torch.nn.functional as F # tensors whose Linear weights are quantised: trunk layers (incl. attention / linear-attention / mlp), vision blocks, audio conformer layers QUANT_PREFIXES = ("trunk.language_model.layers.", "trunk.visual.blocks.", "audio.encoder.layers.", "audio.proj.") SKIP_SUBSTR = ("norm", "A_log", "dt_bias", "per_dim_scale", ".conv1d", "_min", "_max") # ".conv1d": the audio lconv1d.linear_* layers are Linear (J42) def quantize_tensor(w, bits, group=128): """w [out, in] float -> (q int8|uint8-packed, scale fp16, meta).""" w = w.float() if bits == 8: s = w.abs().amax(1).clamp_min(1e-8) / 127.0 q = torch.round(w / s[:, None]).clamp(-127, 127).to(torch.int8) return q, s.to(torch.float16), {"bits": 8} o, i = w.shape g = group if i % group == 0 else i wg = w.view(o, i // g, g) mn, mx = wg.amin(2), wg.amax(2) if bits == 2: s = ((mx - mn) / 3.0).clamp_min(1e-8); z = torch.round(-mn / s).clamp(0, 3) q = (torch.round(wg / s[:, :, None]) + z[:, :, None]).clamp(0, 3).to(torch.uint8).view(o, i) packed = (q[:, 0::4] | (q[:, 1::4] << 2) | (q[:, 2::4] << 4) | (q[:, 3::4] << 6)).to(torch.uint8) # four values per byte return packed, torch.stack([s, z], -1).to(torch.float16), {"bits": 2, "group": g} s = ((mx - mn) / 15.0).clamp_min(1e-8); z = torch.round(-mn / s).clamp(0, 15) # asymmetric: q = round(w/s) + z in 0..15 q = (torch.round(wg / s[:, :, None]) + z[:, :, None]).clamp(0, 15).to(torch.uint8).view(o, i) packed = (q[:, 0::2] | (q[:, 1::2] << 4)).to(torch.uint8) # two nibbles per byte return packed, torch.stack([s, z], -1).to(torch.float16), {"bits": 4, "group": g} # scale [o, i/g, 2] = (s, z) def dequantize(q, s, meta, dtype=torch.float32): if meta["bits"] == 8: return q.to(dtype) * s.to(dtype)[:, None] o = q.shape[0]; g = meta["group"] if meta["bits"] == 2: full = torch.stack([q & 3, (q >> 2) & 3, (q >> 4) & 3, q >> 6], -1).view(o, -1, g).to(dtype) # [o, in/g, g] in 0..3 else: full = torch.stack([q & 0x0F, q >> 4], -1).view(o, -1, g).to(dtype) # [o, in/g, g] in 0..15 sz = s.to(dtype) return ((full - sz[:, :, 1:2]) * sz[:, :, 0:1]).view(o, -1) class QLinear(nn.Module): """Drop-in for nn.Linear holding quantised weights. GPU: dequantise per call into the activation dtype (bandwidth-bound matmuls; the dequant is fused by the graph capture). CPU: dequantised once to the compute dtype (bf16 on AMX/AVX512-BF16 CPUs, else fp32) and cached -> same speed as the bf16 variant with 1/2 (int8) or 1/4 (int4) of the weight bytes on disk / in RAM before the cache fills. Optional (JULIA2_CPU_INT8_DYNAMIC=1): int8 weights x dynamically int8-quantised activations through the fbgemm/onednn kernel — measured not faster than bf16 AMX on Sapphire Rapids and costs text agreement (.95), so off by default.""" def __init__(self, q, s, meta, bias, out_features, in_features): super().__init__() self.register_buffer("q", q); self.register_buffer("s", s) self.bias = None if bias is None else nn.Parameter(bias, requires_grad=False) self.meta, self.out_features, self.in_features = meta, out_features, in_features self._w = None; self._packed = None if TINY4_MAX_ROWS and meta["bits"] == 4 and q.is_cuda: self._pack_tiny4() # at load: packing lazily inside a compiled CUDA-graph region (audio hop) gives a buffer that the next replay overwrites def _cpu_int8(self, x): if self._packed is None: import warnings with warnings.catch_warnings(): warnings.simplefilter("ignore") wq = torch._make_per_channel_quantized_tensor(self.q, self.s.float(), torch.zeros(self.q.shape[0], dtype=torch.long), 0) self._packed = torch.ops.quantized.linear_prepack(wq, None if self.bias is None else self.bias.float()) y = torch.ops.quantized.linear_dynamic(x.float(), self._packed, reduce_range=False) return y.to(x.dtype) def forward(self, x): if x.device.type == "cpu": if self.meta["bits"] == 8 and CPU_INT8_DYNAMIC and _cpu_int8_ok(): return self._cpu_int8(x) # opt-in: per-tensor dynamic activation quantisation costs agreement (.95 on text) and is not faster than bf16 AMX here if CPU_KEEP_QUANT: # low RAM: dequantise per call, keep only the quantised weights resident w = (_compiled_dequantize() if COMPILED_DEQUANT else dequantize)(self.q, self.s, self.meta, x.dtype) return F.linear(x, w, None if self.bias is None else self.bias.to(x.dtype)) if self._w is None or self._w.dtype != x.dtype: self._w = dequantize(self.q, self.s, self.meta, x.dtype) return F.linear(x, self._w, None if self.bias is None else self.bias.to(x.dtype)) if self._w is not None and self._w.dtype == x.dtype: # GPU tower cache (TOWER_CACHE) return F.linear(x, self._w, None if self.bias is None else self.bias.to(x.dtype)) rows = x.numel() // x.shape[-1] if TINY4_MAX_ROWS and self.meta["bits"] == 4 and rows <= TINY4_MAX_ROWS and x.dtype == torch.bfloat16: y = self._tiny4(x.reshape(rows, -1)).view(*x.shape[:-1], self.out_features) return y if self.bias is None else y + self.bias.to(y.dtype) w = (_compiled_dequantize() if COMPILED_DEQUANT else dequantize)(self.q, self.s, self.meta, x.dtype) return F.linear(x, w, None if self.bias is None else self.bias.to(x.dtype)) def _tiny4(self, x): """torch's int4 weight-only kernel: reads the 4-bit weights directly. Fast for few rows (question branches). Same quantised values as `dequantize`: w = (q - z) * s = (q - 8) * s + (8 - z) * s.""" if self._packed is None: self._pack_tiny4() return torch.ops.aten._weight_int4pack_mm(x.contiguous(), self._packed, self.meta["group"], self._sz) def _pack_tiny4(self): o = self.q.shape[0] q = torch.stack([self.q & 0x0F, self.q >> 4], -1).view(o, -1).to(torch.int32) self._packed = torch.ops.aten._convert_weight_to_int4pack((q[:, ::2] << 4 | q[:, 1::2]).to(torch.uint8), 8) sc, z = self.s[..., 0].float(), self.s[..., 1].float() self._sz = torch.stack([sc, (8 - z) * sc], -1).transpose(0, 1).contiguous().to(torch.bfloat16) _CPU_INT8 = None # CPU: 1 keeps the weights quantised in RAM and dequantises per call (the release caches a full-precision copy). CPU_KEEP_QUANT = bool(int(__import__("os").environ.get("JULIA2_CPU_KEEP_QUANT", "0"))) # J2: compile the dequantise step into one fused kernel per weight shape (3x faster than the eager ops on an L40S). COMPILED_DEQUANT = bool(int(__import__("os").environ.get("JULIA2_COMPILED_DEQUANT", "0"))) _CDEQ = None def _compiled_dequantize(): global _CDEQ if _CDEQ is None: import torch._dynamo torch._dynamo.config.cache_size_limit = max(torch._dynamo.config.cache_size_limit, 64) # one entry per weight shape and bit width _CDEQ = torch.compile(dequantize, dynamic=False) return _CDEQ # int4 layers on CUDA: calls with at most this many rows use the int4 kernel (J2); 0 keeps the release path. TINY4_MAX_ROWS = int(__import__("os").environ.get("JULIA2_TINY4_MAX_ROWS", "0")) # J23: CUDA: keep a bf16 copy of the audio tower weights (+0.4 GB GPU memory, no per-call dequantise). TOWER_CACHE = bool(int(__import__("os").environ.get("JULIA2_TOWER_CACHE", "0"))) CPU_INT8_DYNAMIC = bool(int(__import__("os").environ.get("JULIA2_CPU_INT8_DYNAMIC", "0"))) def _cpu_int8_ok(): global _CPU_INT8 if _CPU_INT8 is None: try: import warnings with warnings.catch_warnings(): warnings.simplefilter("ignore") torch.backends.quantized.engine = "x86" if "x86" in torch.backends.quantized.supported_engines else torch.backends.quantized.engine w = torch._make_per_channel_quantized_tensor(torch.zeros(4, 8, dtype=torch.int8), torch.ones(4), torch.zeros(4, dtype=torch.long), 0) torch.ops.quantized.linear_dynamic(torch.zeros(2, 8), torch.ops.quantized.linear_prepack(w, None), reduce_range=False) _CPU_INT8 = True except Exception: _CPU_INT8 = False return _CPU_INT8 class QEmbedding(nn.Module): """Drop-in for nn.Embedding with quantised rows. Only the looked-up rows are dequantised, so the cost per token is one row. int8: one fp16 scale per row. int4: asymmetric groups along the hidden dim, as in `quantize_tensor`.""" def __init__(self, q, s, meta, dtype, hot=None): super().__init__() self.register_buffer("q", q); self.register_buffer("s", s) self.meta, self.dtype = meta, dtype self.hot = hot is not None # frequent tokens with more bits: hot = (map [V] int32: hot index, or -1 - cold index; q, s, meta) if self.hot: self.register_buffer("hmap", hot[0]); self.register_buffer("hq", hot[1]); self.register_buffer("hs", hot[2]); self.hmeta = hot[3] def forward(self, ids): flat = ids.reshape(-1) if not self.hot: return dequantize(self.q[flat], self.s[flat], self.meta, self.dtype).view(*ids.shape, -1) h = self.hmap[flat].long() xh = dequantize(self.hq[h.clamp_min(0)], self.hs[h.clamp_min(0)], self.hmeta, self.dtype) cold = (-1 - h).clamp_min(0) if self.hmeta.get("compact") else flat # compact: q/s hold only the cold rows x = dequantize(self.q[cold], self.s[cold], self.meta, self.dtype) return torch.where((h >= 0)[:, None], xh, x).view(*ids.shape, -1) def cache_dequantised(module, dtype): """Store the dequantised weights of every QLinear under `module` (GPU tower cache).""" for m in module.modules(): if isinstance(m, QLinear): m._w = dequantize(m.q, m.s, m.meta, dtype) def swap_linears(module, state, prefix): """Replace nn.Linear children whose quantised tensors exist in `state` (keys relative to `prefix`).""" for name, child in list(module.named_children()): full = f"{prefix}{name}" if isinstance(child, nn.Embedding) and f"{full}.q" in state: hot = (state[f"{full}.hot.map"], state[f"{full}.hot.q"], state[f"{full}.hot.s"], json.loads(state[f"{full}.hot.qcfg"])) if f"{full}.hot.q" in state else None setattr(module, name, QEmbedding(state[f"{full}.q"], state[f"{full}.s"], json.loads(state[f"{full}.qcfg"]), child.weight.dtype, hot)) elif isinstance(child, nn.Linear) and f"{full}.q" in state: q, s, meta = state[f"{full}.q"], state[f"{full}.s"], json.loads(state[f"{full}.qcfg"]) setattr(module, name, QLinear(q, s, meta, child.bias.data if child.bias is not None else None, child.out_features, child.in_features)) else: swap_linears(child, state, full + ".") EMBED = "trunk.language_model.embed_tokens.weight" def quantize_file(src, dst, bits, group=128, int8_layers=(), embed_bits=0, embed_group=64): """int8_layers: trunk layer indices kept at int8 inside an int4 file (mixed precision). embed_bits: 8 or 4 quantises the input embedding table (a lookup table only: Julia-2 has no LM head); 0 keeps bf16. bits=16 keeps every Linear in bf16 (to test the embedding alone).""" from safetensors import safe_open from safetensors.torch import save_file f = safe_open(str(Path(src) / "model.safetensors"), "pt") out, n_q, n_bytes = {}, 0, 0 for k in f.keys(): t = f.get_tensor(k) is_embed = k == EMBED and embed_bits if is_embed or (bits < 16 and k.endswith(".weight") and t.dim() == 2 and any(k.startswith(p) for p in QUANT_PREFIXES) and not any(s in k for s in SKIP_SUBSTR)): b = embed_bits if is_embed else bits if bits == 4 and k.startswith("trunk.language_model.layers.") and int(k.split(".")[3]) in int8_layers: b = 8 q, s, meta = quantize_tensor(t, b, embed_group if is_embed else group) base = k[: -len(".weight")] out[base + ".q"], out[base + ".s"] = q.contiguous(), s.contiguous() out[base + ".qcfg"] = torch.tensor(list(json.dumps(meta).encode()), dtype=torch.uint8) n_q += 1; n_bytes += q.numel() * q.element_size() + s.numel() * 2 else: out[k] = t.contiguous() Path(dst).mkdir(parents=True, exist_ok=True) save_file(out, str(Path(dst) / "model.safetensors"), metadata={"precision": f"int{bits}", "group": str(group), "int8_layers": ",".join(map(str, int8_layers)), "embed_bits": str(embed_bits), "format": "julia2-wq-v1"}) return n_q, n_bytes if __name__ == "__main__": ap = argparse.ArgumentParser(); ap.add_argument("--src", required=True); ap.add_argument("--dst", required=True); ap.add_argument("--bits", type=int, default=8); ap.add_argument("--group", type=int, default=128) ap.add_argument("--int8_layers", default="", help="comma-separated trunk layers kept int8 in an int4 file") ap.add_argument("--embed_bits", type=int, default=0, help="8 or 4: quantise the input embedding table too") ap.add_argument("--embed_group", type=int, default=64) a = ap.parse_args() n, b = quantize_file(a.src, a.dst, a.bits, a.group, tuple(int(x) for x in a.int8_layers.split(",") if x), a.embed_bits, a.embed_group) print(f"quantised {n} linear weights -> {a.dst} ({b/1e9:.2f} GB of quantised tensors)")