Julia-2 / runtime /julia2 /quant.py
Julia Klee
Julia-2: julia2_audio_v1 (text = base_v2 byte-identical, + pruned audio tower, stem p9), int8/int4 variants, lean runtime 0.1.0 (audio, LiveSession, modalities), card with speech, live calls and variants
1d1c6db
Raw History Blame Contribute Delete
8.4 kB
"""Weight-only quantisation of the Linear layers (precision variants int8 / int4 of the same checkpoint).
int8: per-output-channel symmetric int8 (scale fp16) -> `W = q * s[:, None]`
int4: asymmetric 4-bit (scale + zero point per group of 128 along the input dim, fp16), two nibbles per byte;
the first trunk layers (most sensitive, see tests) can be kept int8 via `int8_layers`
Kept in higher precision (not quantised): embeddings, norms, the decision readout, the audio projector, the audio
subsampler / output projection and the vision patch embed / merger (small, numerically sensitive). The quantised file keeps
every non-quantised tensor as in the bf16 checkpoint and stores `<name>.weight` as int8/uint8 + `<name>.scale` (+ `.qcfg`).
Runtime: `QLinear` dequantises on the fly (GPU: fused into a bf16 matmul per call; CPU: dequantised once into fp32 or an
int8 dynamic matmul).
python -m julia2.quant --src <repo> --dst <repo>/int8 --bits 8
"""
import argparse
import json
from pathlib import Path
import torch
import torch.nn as nn
import torch.nn.functional as F
# tensors whose Linear weights are quantised: trunk layers (incl. attention / linear-attention / mlp), vision blocks, audio conformer layers
QUANT_PREFIXES = ("trunk.language_model.layers.", "trunk.visual.blocks.", "audio.encoder.layers.")
SKIP_SUBSTR = ("norm", "A_log", "dt_bias", "per_dim_scale", "conv1d", "_min", "_max")
def quantize_tensor(w, bits, group=128):
"""w [out, in] float -> (q int8|uint8-packed, scale fp16, meta)."""
w = w.float()
if bits == 8:
s = w.abs().amax(1).clamp_min(1e-8) / 127.0
q = torch.round(w / s[:, None]).clamp(-127, 127).to(torch.int8)
return q, s.to(torch.float16), {"bits": 8}
o, i = w.shape
g = group if i % group == 0 else i
wg = w.view(o, i // g, g)
mn, mx = wg.amin(2), wg.amax(2)
s = ((mx - mn) / 15.0).clamp_min(1e-8); z = torch.round(-mn / s).clamp(0, 15) # asymmetric: q = round(w/s) + z in 0..15
q = (torch.round(wg / s[:, :, None]) + z[:, :, None]).clamp(0, 15).to(torch.uint8).view(o, i)
packed = (q[:, 0::2] | (q[:, 1::2] << 4)).to(torch.uint8) # two nibbles per byte
return packed, torch.stack([s, z], -1).to(torch.float16), {"bits": 4, "group": g} # scale [o, i/g, 2] = (s, z)
def dequantize(q, s, meta, dtype=torch.float32):
if meta["bits"] == 8:
return q.to(dtype) * s.to(dtype)[:, None]
o = q.shape[0]; g = meta["group"]
lo = (q & 0x0F); hi = (q >> 4)
full = torch.stack([lo, hi], -1).view(o, -1, g).to(dtype) # [o, in/g, g] in 0..15
sz = s.to(dtype)
return ((full - sz[:, :, 1:2]) * sz[:, :, 0:1]).view(o, -1)
class QLinear(nn.Module):
"""Drop-in for nn.Linear holding quantised weights.
GPU: dequantise per call into the activation dtype (bandwidth-bound matmuls; the dequant is fused by the graph capture).
CPU: dequantised once to the compute dtype (bf16 on AMX/AVX512-BF16 CPUs, else fp32) and cached -> same speed as the
bf16 variant with 1/2 (int8) or 1/4 (int4) of the weight bytes on disk / in RAM before the cache fills. Optional
(JULIA2_CPU_INT8_DYNAMIC=1): int8 weights x dynamically int8-quantised activations through the fbgemm/onednn kernel —
measured not faster than bf16 AMX on Sapphire Rapids and costs text agreement (.95), so off by default."""
def __init__(self, q, s, meta, bias, out_features, in_features):
super().__init__()
self.register_buffer("q", q); self.register_buffer("s", s)
self.bias = None if bias is None else nn.Parameter(bias, requires_grad=False)
self.meta, self.out_features, self.in_features = meta, out_features, in_features
self._w = None; self._packed = None
def _cpu_int8(self, x):
if self._packed is None:
import warnings
with warnings.catch_warnings():
warnings.simplefilter("ignore")
wq = torch._make_per_channel_quantized_tensor(self.q, self.s.float(), torch.zeros(self.q.shape[0], dtype=torch.long), 0)
self._packed = torch.ops.quantized.linear_prepack(wq, None if self.bias is None else self.bias.float())
y = torch.ops.quantized.linear_dynamic(x.float(), self._packed, reduce_range=False)
return y.to(x.dtype)
def forward(self, x):
if x.device.type == "cpu":
if self.meta["bits"] == 8 and CPU_INT8_DYNAMIC and _cpu_int8_ok():
return self._cpu_int8(x) # opt-in: per-tensor dynamic activation quantisation costs agreement (.95 on text) and is not faster than bf16 AMX here
if self._w is None or self._w.dtype != x.dtype:
self._w = dequantize(self.q, self.s, self.meta, x.dtype)
return F.linear(x, self._w, None if self.bias is None else self.bias.to(x.dtype))
w = dequantize(self.q, self.s, self.meta, x.dtype)
return F.linear(x, w, None if self.bias is None else self.bias.to(x.dtype))
_CPU_INT8 = None
CPU_INT8_DYNAMIC = bool(int(__import__("os").environ.get("JULIA2_CPU_INT8_DYNAMIC", "0")))
def _cpu_int8_ok():
global _CPU_INT8
if _CPU_INT8 is None:
try:
import warnings
with warnings.catch_warnings():
warnings.simplefilter("ignore")
torch.backends.quantized.engine = "x86" if "x86" in torch.backends.quantized.supported_engines else torch.backends.quantized.engine
w = torch._make_per_channel_quantized_tensor(torch.zeros(4, 8, dtype=torch.int8), torch.ones(4), torch.zeros(4, dtype=torch.long), 0)
torch.ops.quantized.linear_dynamic(torch.zeros(2, 8), torch.ops.quantized.linear_prepack(w, None), reduce_range=False)
_CPU_INT8 = True
except Exception:
_CPU_INT8 = False
return _CPU_INT8
def swap_linears(module, state, prefix):
"""Replace nn.Linear children whose quantised tensors exist in `state` (keys relative to `prefix`)."""
for name, child in list(module.named_children()):
full = f"{prefix}{name}"
if isinstance(child, nn.Linear) and f"{full}.q" in state:
q, s, meta = state[f"{full}.q"], state[f"{full}.s"], json.loads(state[f"{full}.qcfg"])
setattr(module, name, QLinear(q, s, meta, child.bias.data if child.bias is not None else None, child.out_features, child.in_features))
else:
swap_linears(child, state, full + ".")
def quantize_file(src, dst, bits, group=128, int8_layers=()):
"""int8_layers: trunk layer indices kept at int8 inside an int4 file (mixed precision)."""
from safetensors import safe_open
from safetensors.torch import save_file
f = safe_open(str(Path(src) / "model.safetensors"), "pt")
out, n_q, n_bytes = {}, 0, 0
for k in f.keys():
t = f.get_tensor(k)
if k.endswith(".weight") and t.dim() == 2 and any(k.startswith(p) for p in QUANT_PREFIXES) and not any(s in k for s in SKIP_SUBSTR):
b = bits
if bits == 4 and k.startswith("trunk.language_model.layers.") and int(k.split(".")[3]) in int8_layers:
b = 8
q, s, meta = quantize_tensor(t, b, group)
base = k[: -len(".weight")]
out[base + ".q"], out[base + ".s"] = q.contiguous(), s.contiguous()
out[base + ".qcfg"] = torch.tensor(list(json.dumps(meta).encode()), dtype=torch.uint8)
n_q += 1; n_bytes += q.numel() * q.element_size() + s.numel() * 2
else:
out[k] = t.contiguous()
Path(dst).mkdir(parents=True, exist_ok=True)
save_file(out, str(Path(dst) / "model.safetensors"), metadata={"precision": f"int{bits}", "group": str(group), "int8_layers": ",".join(map(str, int8_layers)), "format": "julia2-wq-v1"})
return n_q, n_bytes
if __name__ == "__main__":
ap = argparse.ArgumentParser(); ap.add_argument("--src", required=True); ap.add_argument("--dst", required=True); ap.add_argument("--bits", type=int, default=8); ap.add_argument("--group", type=int, default=128)
ap.add_argument("--int8_layers", default="", help="comma-separated trunk layers kept int8 in an int4 file")
a = ap.parse_args()
n, b = quantize_file(a.src, a.dst, a.bits, a.group, tuple(int(x) for x in a.int8_layers.split(",") if x))
print(f"quantised {n} linear weights -> {a.dst} ({b/1e9:.2f} GB of quantised tensors)")