Julia Klee
Julia-2: julia2_audio_v1 (text = base_v2 byte-identical, + pruned audio tower, stem p9), int8/int4 variants, lean runtime 0.1.0 (audio, LiveSession, modalities), card with speech, live calls and variants
1d1c6db Download runtime/julia2/quant.py from SupersonicLabs/Julia-2: direct link, hf CLI and curl.
- Browser
- Download file 8.4 kB
-
https://huggingface.co/SupersonicLabs/Julia-2/resolve/main/runtime/julia2/quant.py
- Command line
-
hf download hf://SupersonicLabs/Julia-2/runtime/julia2/quant.py
-
curl -L -o quant.py https://huggingface.co/SupersonicLabs/Julia-2/resolve/main/runtime/julia2/quant.py
8.4 kB
| """Weight-only quantisation of the Linear layers (precision variants int8 / int4 of the same checkpoint). | |
| int8: per-output-channel symmetric int8 (scale fp16) -> `W = q * s[:, None]` | |
| int4: asymmetric 4-bit (scale + zero point per group of 128 along the input dim, fp16), two nibbles per byte; | |
| the first trunk layers (most sensitive, see tests) can be kept int8 via `int8_layers` | |
| Kept in higher precision (not quantised): embeddings, norms, the decision readout, the audio projector, the audio | |
| subsampler / output projection and the vision patch embed / merger (small, numerically sensitive). The quantised file keeps | |
| every non-quantised tensor as in the bf16 checkpoint and stores `<name>.weight` as int8/uint8 + `<name>.scale` (+ `.qcfg`). | |
| Runtime: `QLinear` dequantises on the fly (GPU: fused into a bf16 matmul per call; CPU: dequantised once into fp32 or an | |
| int8 dynamic matmul). | |
| python -m julia2.quant --src <repo> --dst <repo>/int8 --bits 8 | |
| """ | |
| import argparse | |
| import json | |
| from pathlib import Path | |
| import torch | |
| import torch.nn as nn | |
| import torch.nn.functional as F | |
| # tensors whose Linear weights are quantised: trunk layers (incl. attention / linear-attention / mlp), vision blocks, audio conformer layers | |
| QUANT_PREFIXES = ("trunk.language_model.layers.", "trunk.visual.blocks.", "audio.encoder.layers.") | |
| SKIP_SUBSTR = ("norm", "A_log", "dt_bias", "per_dim_scale", "conv1d", "_min", "_max") | |
| def quantize_tensor(w, bits, group=128): | |
| """w [out, in] float -> (q int8|uint8-packed, scale fp16, meta).""" | |
| w = w.float() | |
| if bits == 8: | |
| s = w.abs().amax(1).clamp_min(1e-8) / 127.0 | |
| q = torch.round(w / s[:, None]).clamp(-127, 127).to(torch.int8) | |
| return q, s.to(torch.float16), {"bits": 8} | |
| o, i = w.shape | |
| g = group if i % group == 0 else i | |
| wg = w.view(o, i // g, g) | |
| mn, mx = wg.amin(2), wg.amax(2) | |
| s = ((mx - mn) / 15.0).clamp_min(1e-8); z = torch.round(-mn / s).clamp(0, 15) # asymmetric: q = round(w/s) + z in 0..15 | |
| q = (torch.round(wg / s[:, :, None]) + z[:, :, None]).clamp(0, 15).to(torch.uint8).view(o, i) | |
| packed = (q[:, 0::2] | (q[:, 1::2] << 4)).to(torch.uint8) # two nibbles per byte | |
| return packed, torch.stack([s, z], -1).to(torch.float16), {"bits": 4, "group": g} # scale [o, i/g, 2] = (s, z) | |
| def dequantize(q, s, meta, dtype=torch.float32): | |
| if meta["bits"] == 8: | |
| return q.to(dtype) * s.to(dtype)[:, None] | |
| o = q.shape[0]; g = meta["group"] | |
| lo = (q & 0x0F); hi = (q >> 4) | |
| full = torch.stack([lo, hi], -1).view(o, -1, g).to(dtype) # [o, in/g, g] in 0..15 | |
| sz = s.to(dtype) | |
| return ((full - sz[:, :, 1:2]) * sz[:, :, 0:1]).view(o, -1) | |
| class QLinear(nn.Module): | |
| """Drop-in for nn.Linear holding quantised weights. | |
| GPU: dequantise per call into the activation dtype (bandwidth-bound matmuls; the dequant is fused by the graph capture). | |
| CPU: dequantised once to the compute dtype (bf16 on AMX/AVX512-BF16 CPUs, else fp32) and cached -> same speed as the | |
| bf16 variant with 1/2 (int8) or 1/4 (int4) of the weight bytes on disk / in RAM before the cache fills. Optional | |
| (JULIA2_CPU_INT8_DYNAMIC=1): int8 weights x dynamically int8-quantised activations through the fbgemm/onednn kernel — | |
| measured not faster than bf16 AMX on Sapphire Rapids and costs text agreement (.95), so off by default.""" | |
| def __init__(self, q, s, meta, bias, out_features, in_features): | |
| super().__init__() | |
| self.register_buffer("q", q); self.register_buffer("s", s) | |
| self.bias = None if bias is None else nn.Parameter(bias, requires_grad=False) | |
| self.meta, self.out_features, self.in_features = meta, out_features, in_features | |
| self._w = None; self._packed = None | |
| def _cpu_int8(self, x): | |
| if self._packed is None: | |
| import warnings | |
| with warnings.catch_warnings(): | |
| warnings.simplefilter("ignore") | |
| wq = torch._make_per_channel_quantized_tensor(self.q, self.s.float(), torch.zeros(self.q.shape[0], dtype=torch.long), 0) | |
| self._packed = torch.ops.quantized.linear_prepack(wq, None if self.bias is None else self.bias.float()) | |
| y = torch.ops.quantized.linear_dynamic(x.float(), self._packed, reduce_range=False) | |
| return y.to(x.dtype) | |
| def forward(self, x): | |
| if x.device.type == "cpu": | |
| if self.meta["bits"] == 8 and CPU_INT8_DYNAMIC and _cpu_int8_ok(): | |
| return self._cpu_int8(x) # opt-in: per-tensor dynamic activation quantisation costs agreement (.95 on text) and is not faster than bf16 AMX here | |
| if self._w is None or self._w.dtype != x.dtype: | |
| self._w = dequantize(self.q, self.s, self.meta, x.dtype) | |
| return F.linear(x, self._w, None if self.bias is None else self.bias.to(x.dtype)) | |
| w = dequantize(self.q, self.s, self.meta, x.dtype) | |
| return F.linear(x, w, None if self.bias is None else self.bias.to(x.dtype)) | |
| _CPU_INT8 = None | |
| CPU_INT8_DYNAMIC = bool(int(__import__("os").environ.get("JULIA2_CPU_INT8_DYNAMIC", "0"))) | |
| def _cpu_int8_ok(): | |
| global _CPU_INT8 | |
| if _CPU_INT8 is None: | |
| try: | |
| import warnings | |
| with warnings.catch_warnings(): | |
| warnings.simplefilter("ignore") | |
| torch.backends.quantized.engine = "x86" if "x86" in torch.backends.quantized.supported_engines else torch.backends.quantized.engine | |
| w = torch._make_per_channel_quantized_tensor(torch.zeros(4, 8, dtype=torch.int8), torch.ones(4), torch.zeros(4, dtype=torch.long), 0) | |
| torch.ops.quantized.linear_dynamic(torch.zeros(2, 8), torch.ops.quantized.linear_prepack(w, None), reduce_range=False) | |
| _CPU_INT8 = True | |
| except Exception: | |
| _CPU_INT8 = False | |
| return _CPU_INT8 | |
| def swap_linears(module, state, prefix): | |
| """Replace nn.Linear children whose quantised tensors exist in `state` (keys relative to `prefix`).""" | |
| for name, child in list(module.named_children()): | |
| full = f"{prefix}{name}" | |
| if isinstance(child, nn.Linear) and f"{full}.q" in state: | |
| q, s, meta = state[f"{full}.q"], state[f"{full}.s"], json.loads(state[f"{full}.qcfg"]) | |
| setattr(module, name, QLinear(q, s, meta, child.bias.data if child.bias is not None else None, child.out_features, child.in_features)) | |
| else: | |
| swap_linears(child, state, full + ".") | |
| def quantize_file(src, dst, bits, group=128, int8_layers=()): | |
| """int8_layers: trunk layer indices kept at int8 inside an int4 file (mixed precision).""" | |
| from safetensors import safe_open | |
| from safetensors.torch import save_file | |
| f = safe_open(str(Path(src) / "model.safetensors"), "pt") | |
| out, n_q, n_bytes = {}, 0, 0 | |
| for k in f.keys(): | |
| t = f.get_tensor(k) | |
| if k.endswith(".weight") and t.dim() == 2 and any(k.startswith(p) for p in QUANT_PREFIXES) and not any(s in k for s in SKIP_SUBSTR): | |
| b = bits | |
| if bits == 4 and k.startswith("trunk.language_model.layers.") and int(k.split(".")[3]) in int8_layers: | |
| b = 8 | |
| q, s, meta = quantize_tensor(t, b, group) | |
| base = k[: -len(".weight")] | |
| out[base + ".q"], out[base + ".s"] = q.contiguous(), s.contiguous() | |
| out[base + ".qcfg"] = torch.tensor(list(json.dumps(meta).encode()), dtype=torch.uint8) | |
| n_q += 1; n_bytes += q.numel() * q.element_size() + s.numel() * 2 | |
| else: | |
| out[k] = t.contiguous() | |
| Path(dst).mkdir(parents=True, exist_ok=True) | |
| save_file(out, str(Path(dst) / "model.safetensors"), metadata={"precision": f"int{bits}", "group": str(group), "int8_layers": ",".join(map(str, int8_layers)), "format": "julia2-wq-v1"}) | |
| return n_q, n_bytes | |
| if __name__ == "__main__": | |
| ap = argparse.ArgumentParser(); ap.add_argument("--src", required=True); ap.add_argument("--dst", required=True); ap.add_argument("--bits", type=int, default=8); ap.add_argument("--group", type=int, default=128) | |
| ap.add_argument("--int8_layers", default="", help="comma-separated trunk layers kept int8 in an int4 file") | |
| a = ap.parse_args() | |
| n, b = quantize_file(a.src, a.dst, a.bits, a.group, tuple(int(x) for x in a.int8_layers.split(",") if x)) | |
| print(f"quantised {n} linear weights -> {a.dst} ({b/1e9:.2f} GB of quantised tensors)") | |