Download runtime/julia2/quant.py from SupersonicLabs/Julia-2-Flash: direct link, hf CLI and curl.
- Browser
- Download file 14.8 kB
-
https://huggingface.co/SupersonicLabs/Julia-2-Flash/resolve/main/runtime/julia2/quant.py
- Command line
-
hf download hf://SupersonicLabs/Julia-2-Flash/runtime/julia2/quant.py
-
curl -L -o quant.py https://huggingface.co/SupersonicLabs/Julia-2-Flash/resolve/main/runtime/julia2/quant.py
14.8 kB
| """Weight-only quantisation of the Linear layers (precision variants int8 / int4 of the same checkpoint). | |
| int8: per-output-channel symmetric int8 (scale fp16) -> `W = q * s[:, None]` | |
| int4: asymmetric 4-bit (scale + zero point per group of 128 along the input dim, fp16), two nibbles per byte; | |
| the first trunk layers (most sensitive, see tests) can be kept int8 via `int8_layers` | |
| Kept in higher precision (not quantised): embeddings, norms, the decision readout, the audio projector, the audio | |
| subsampler / output projection and the vision patch embed / merger (small, numerically sensitive). The quantised file keeps | |
| every non-quantised tensor as in the bf16 checkpoint and stores `<name>.weight` as int8/uint8 + `<name>.scale` (+ `.qcfg`). | |
| Runtime: `QLinear` dequantises on the fly (GPU: fused into a bf16 matmul per call; CPU: dequantised once into fp32 or an | |
| int8 dynamic matmul). | |
| python -m julia2.quant --src <repo> --dst <repo>/int8 --bits 8 | |
| """ | |
| import argparse | |
| import json | |
| from pathlib import Path | |
| import torch | |
| import torch.nn as nn | |
| import torch.nn.functional as F | |
| # tensors whose Linear weights are quantised: trunk layers (incl. attention / linear-attention / mlp), vision blocks, audio conformer layers | |
| QUANT_PREFIXES = ("trunk.language_model.layers.", "trunk.visual.blocks.", "audio.encoder.layers.", "audio.proj.") | |
| SKIP_SUBSTR = ("norm", "A_log", "dt_bias", "per_dim_scale", ".conv1d", "_min", "_max") # ".conv1d": the audio lconv1d.linear_* layers are Linear (J42) | |
| def quantize_tensor(w, bits, group=128): | |
| """w [out, in] float -> (q int8|uint8-packed, scale fp16, meta).""" | |
| w = w.float() | |
| if bits == 8: | |
| s = w.abs().amax(1).clamp_min(1e-8) / 127.0 | |
| q = torch.round(w / s[:, None]).clamp(-127, 127).to(torch.int8) | |
| return q, s.to(torch.float16), {"bits": 8} | |
| o, i = w.shape | |
| g = group if i % group == 0 else i | |
| wg = w.view(o, i // g, g) | |
| mn, mx = wg.amin(2), wg.amax(2) | |
| if bits == 2: | |
| s = ((mx - mn) / 3.0).clamp_min(1e-8); z = torch.round(-mn / s).clamp(0, 3) | |
| q = (torch.round(wg / s[:, :, None]) + z[:, :, None]).clamp(0, 3).to(torch.uint8).view(o, i) | |
| packed = (q[:, 0::4] | (q[:, 1::4] << 2) | (q[:, 2::4] << 4) | (q[:, 3::4] << 6)).to(torch.uint8) # four values per byte | |
| return packed, torch.stack([s, z], -1).to(torch.float16), {"bits": 2, "group": g} | |
| s = ((mx - mn) / 15.0).clamp_min(1e-8); z = torch.round(-mn / s).clamp(0, 15) # asymmetric: q = round(w/s) + z in 0..15 | |
| q = (torch.round(wg / s[:, :, None]) + z[:, :, None]).clamp(0, 15).to(torch.uint8).view(o, i) | |
| packed = (q[:, 0::2] | (q[:, 1::2] << 4)).to(torch.uint8) # two nibbles per byte | |
| return packed, torch.stack([s, z], -1).to(torch.float16), {"bits": 4, "group": g} # scale [o, i/g, 2] = (s, z) | |
| def dequantize(q, s, meta, dtype=torch.float32): | |
| if meta["bits"] == 8: | |
| return q.to(dtype) * s.to(dtype)[:, None] | |
| o = q.shape[0]; g = meta["group"] | |
| if meta["bits"] == 2: | |
| full = torch.stack([q & 3, (q >> 2) & 3, (q >> 4) & 3, q >> 6], -1).view(o, -1, g).to(dtype) # [o, in/g, g] in 0..3 | |
| else: | |
| full = torch.stack([q & 0x0F, q >> 4], -1).view(o, -1, g).to(dtype) # [o, in/g, g] in 0..15 | |
| sz = s.to(dtype) | |
| return ((full - sz[:, :, 1:2]) * sz[:, :, 0:1]).view(o, -1) | |
| class QLinear(nn.Module): | |
| """Drop-in for nn.Linear holding quantised weights. | |
| GPU: dequantise per call into the activation dtype (bandwidth-bound matmuls; the dequant is fused by the graph capture). | |
| CPU: dequantised once to the compute dtype (bf16 on AMX/AVX512-BF16 CPUs, else fp32) and cached -> same speed as the | |
| bf16 variant with 1/2 (int8) or 1/4 (int4) of the weight bytes on disk / in RAM before the cache fills. Optional | |
| (JULIA2_CPU_INT8_DYNAMIC=1): int8 weights x dynamically int8-quantised activations through the fbgemm/onednn kernel — | |
| measured not faster than bf16 AMX on Sapphire Rapids and costs text agreement (.95), so off by default.""" | |
| def __init__(self, q, s, meta, bias, out_features, in_features): | |
| super().__init__() | |
| self.register_buffer("q", q); self.register_buffer("s", s) | |
| self.bias = None if bias is None else nn.Parameter(bias, requires_grad=False) | |
| self.meta, self.out_features, self.in_features = meta, out_features, in_features | |
| self._w = None; self._packed = None | |
| if TINY4_MAX_ROWS and meta["bits"] == 4 and q.is_cuda: | |
| self._pack_tiny4() # at load: packing lazily inside a compiled CUDA-graph region (audio hop) gives a buffer that the next replay overwrites | |
| def _cpu_int8(self, x): | |
| if self._packed is None: | |
| import warnings | |
| with warnings.catch_warnings(): | |
| warnings.simplefilter("ignore") | |
| wq = torch._make_per_channel_quantized_tensor(self.q, self.s.float(), torch.zeros(self.q.shape[0], dtype=torch.long), 0) | |
| self._packed = torch.ops.quantized.linear_prepack(wq, None if self.bias is None else self.bias.float()) | |
| y = torch.ops.quantized.linear_dynamic(x.float(), self._packed, reduce_range=False) | |
| return y.to(x.dtype) | |
| def forward(self, x): | |
| if x.device.type == "cpu": | |
| if self.meta["bits"] == 8 and CPU_INT8_DYNAMIC and _cpu_int8_ok(): | |
| return self._cpu_int8(x) # opt-in: per-tensor dynamic activation quantisation costs agreement (.95 on text) and is not faster than bf16 AMX here | |
| if CPU_KEEP_QUANT: # low RAM: dequantise per call, keep only the quantised weights resident | |
| w = (_compiled_dequantize() if COMPILED_DEQUANT else dequantize)(self.q, self.s, self.meta, x.dtype) | |
| return F.linear(x, w, None if self.bias is None else self.bias.to(x.dtype)) | |
| if self._w is None or self._w.dtype != x.dtype: | |
| self._w = dequantize(self.q, self.s, self.meta, x.dtype) | |
| return F.linear(x, self._w, None if self.bias is None else self.bias.to(x.dtype)) | |
| if self._w is not None and self._w.dtype == x.dtype: # GPU tower cache (TOWER_CACHE) | |
| return F.linear(x, self._w, None if self.bias is None else self.bias.to(x.dtype)) | |
| rows = x.numel() // x.shape[-1] | |
| if TINY4_MAX_ROWS and self.meta["bits"] == 4 and rows <= TINY4_MAX_ROWS and x.dtype == torch.bfloat16: | |
| y = self._tiny4(x.reshape(rows, -1)).view(*x.shape[:-1], self.out_features) | |
| return y if self.bias is None else y + self.bias.to(y.dtype) | |
| w = (_compiled_dequantize() if COMPILED_DEQUANT else dequantize)(self.q, self.s, self.meta, x.dtype) | |
| return F.linear(x, w, None if self.bias is None else self.bias.to(x.dtype)) | |
| def _tiny4(self, x): | |
| """torch's int4 weight-only kernel: reads the 4-bit weights directly. Fast for few rows (question branches). | |
| Same quantised values as `dequantize`: w = (q - z) * s = (q - 8) * s + (8 - z) * s.""" | |
| if self._packed is None: | |
| self._pack_tiny4() | |
| return torch.ops.aten._weight_int4pack_mm(x.contiguous(), self._packed, self.meta["group"], self._sz) | |
| def _pack_tiny4(self): | |
| o = self.q.shape[0] | |
| q = torch.stack([self.q & 0x0F, self.q >> 4], -1).view(o, -1).to(torch.int32) | |
| self._packed = torch.ops.aten._convert_weight_to_int4pack((q[:, ::2] << 4 | q[:, 1::2]).to(torch.uint8), 8) | |
| sc, z = self.s[..., 0].float(), self.s[..., 1].float() | |
| self._sz = torch.stack([sc, (8 - z) * sc], -1).transpose(0, 1).contiguous().to(torch.bfloat16) | |
| _CPU_INT8 = None | |
| # CPU: 1 keeps the weights quantised in RAM and dequantises per call (the release caches a full-precision copy). | |
| CPU_KEEP_QUANT = bool(int(__import__("os").environ.get("JULIA2_CPU_KEEP_QUANT", "0"))) | |
| # J2: compile the dequantise step into one fused kernel per weight shape (3x faster than the eager ops on an L40S). | |
| COMPILED_DEQUANT = bool(int(__import__("os").environ.get("JULIA2_COMPILED_DEQUANT", "0"))) | |
| _CDEQ = None | |
| def _compiled_dequantize(): | |
| global _CDEQ | |
| if _CDEQ is None: | |
| import torch._dynamo | |
| torch._dynamo.config.cache_size_limit = max(torch._dynamo.config.cache_size_limit, 64) # one entry per weight shape and bit width | |
| _CDEQ = torch.compile(dequantize, dynamic=False) | |
| return _CDEQ | |
| # int4 layers on CUDA: calls with at most this many rows use the int4 kernel (J2); 0 keeps the release path. | |
| TINY4_MAX_ROWS = int(__import__("os").environ.get("JULIA2_TINY4_MAX_ROWS", "0")) | |
| # J23: CUDA: keep a bf16 copy of the audio tower weights (+0.4 GB GPU memory, no per-call dequantise). | |
| TOWER_CACHE = bool(int(__import__("os").environ.get("JULIA2_TOWER_CACHE", "0"))) | |
| CPU_INT8_DYNAMIC = bool(int(__import__("os").environ.get("JULIA2_CPU_INT8_DYNAMIC", "0"))) | |
| def _cpu_int8_ok(): | |
| global _CPU_INT8 | |
| if _CPU_INT8 is None: | |
| try: | |
| import warnings | |
| with warnings.catch_warnings(): | |
| warnings.simplefilter("ignore") | |
| torch.backends.quantized.engine = "x86" if "x86" in torch.backends.quantized.supported_engines else torch.backends.quantized.engine | |
| w = torch._make_per_channel_quantized_tensor(torch.zeros(4, 8, dtype=torch.int8), torch.ones(4), torch.zeros(4, dtype=torch.long), 0) | |
| torch.ops.quantized.linear_dynamic(torch.zeros(2, 8), torch.ops.quantized.linear_prepack(w, None), reduce_range=False) | |
| _CPU_INT8 = True | |
| except Exception: | |
| _CPU_INT8 = False | |
| return _CPU_INT8 | |
| class QEmbedding(nn.Module): | |
| """Drop-in for nn.Embedding with quantised rows. Only the looked-up rows are dequantised, so the cost per token is | |
| one row. int8: one fp16 scale per row. int4: asymmetric groups along the hidden dim, as in `quantize_tensor`.""" | |
| def __init__(self, q, s, meta, dtype, hot=None): | |
| super().__init__() | |
| self.register_buffer("q", q); self.register_buffer("s", s) | |
| self.meta, self.dtype = meta, dtype | |
| self.hot = hot is not None # frequent tokens with more bits: hot = (map [V] int32: hot index, or -1 - cold index; q, s, meta) | |
| if self.hot: | |
| self.register_buffer("hmap", hot[0]); self.register_buffer("hq", hot[1]); self.register_buffer("hs", hot[2]); self.hmeta = hot[3] | |
| def forward(self, ids): | |
| flat = ids.reshape(-1) | |
| if not self.hot: | |
| return dequantize(self.q[flat], self.s[flat], self.meta, self.dtype).view(*ids.shape, -1) | |
| h = self.hmap[flat].long() | |
| xh = dequantize(self.hq[h.clamp_min(0)], self.hs[h.clamp_min(0)], self.hmeta, self.dtype) | |
| cold = (-1 - h).clamp_min(0) if self.hmeta.get("compact") else flat # compact: q/s hold only the cold rows | |
| x = dequantize(self.q[cold], self.s[cold], self.meta, self.dtype) | |
| return torch.where((h >= 0)[:, None], xh, x).view(*ids.shape, -1) | |
| def cache_dequantised(module, dtype): | |
| """Store the dequantised weights of every QLinear under `module` (GPU tower cache).""" | |
| for m in module.modules(): | |
| if isinstance(m, QLinear): | |
| m._w = dequantize(m.q, m.s, m.meta, dtype) | |
| def swap_linears(module, state, prefix): | |
| """Replace nn.Linear children whose quantised tensors exist in `state` (keys relative to `prefix`).""" | |
| for name, child in list(module.named_children()): | |
| full = f"{prefix}{name}" | |
| if isinstance(child, nn.Embedding) and f"{full}.q" in state: | |
| hot = (state[f"{full}.hot.map"], state[f"{full}.hot.q"], state[f"{full}.hot.s"], json.loads(state[f"{full}.hot.qcfg"])) if f"{full}.hot.q" in state else None | |
| setattr(module, name, QEmbedding(state[f"{full}.q"], state[f"{full}.s"], json.loads(state[f"{full}.qcfg"]), child.weight.dtype, hot)) | |
| elif isinstance(child, nn.Linear) and f"{full}.q" in state: | |
| q, s, meta = state[f"{full}.q"], state[f"{full}.s"], json.loads(state[f"{full}.qcfg"]) | |
| setattr(module, name, QLinear(q, s, meta, child.bias.data if child.bias is not None else None, child.out_features, child.in_features)) | |
| else: | |
| swap_linears(child, state, full + ".") | |
| EMBED = "trunk.language_model.embed_tokens.weight" | |
| def quantize_file(src, dst, bits, group=128, int8_layers=(), embed_bits=0, embed_group=64): | |
| """int8_layers: trunk layer indices kept at int8 inside an int4 file (mixed precision). | |
| embed_bits: 8 or 4 quantises the input embedding table (a lookup table only: Julia-2 has no LM head); 0 keeps bf16. | |
| bits=16 keeps every Linear in bf16 (to test the embedding alone).""" | |
| from safetensors import safe_open | |
| from safetensors.torch import save_file | |
| f = safe_open(str(Path(src) / "model.safetensors"), "pt") | |
| out, n_q, n_bytes = {}, 0, 0 | |
| for k in f.keys(): | |
| t = f.get_tensor(k) | |
| is_embed = k == EMBED and embed_bits | |
| if is_embed or (bits < 16 and k.endswith(".weight") and t.dim() == 2 and any(k.startswith(p) for p in QUANT_PREFIXES) and not any(s in k for s in SKIP_SUBSTR)): | |
| b = embed_bits if is_embed else bits | |
| if bits == 4 and k.startswith("trunk.language_model.layers.") and int(k.split(".")[3]) in int8_layers: | |
| b = 8 | |
| q, s, meta = quantize_tensor(t, b, embed_group if is_embed else group) | |
| base = k[: -len(".weight")] | |
| out[base + ".q"], out[base + ".s"] = q.contiguous(), s.contiguous() | |
| out[base + ".qcfg"] = torch.tensor(list(json.dumps(meta).encode()), dtype=torch.uint8) | |
| n_q += 1; n_bytes += q.numel() * q.element_size() + s.numel() * 2 | |
| else: | |
| out[k] = t.contiguous() | |
| Path(dst).mkdir(parents=True, exist_ok=True) | |
| save_file(out, str(Path(dst) / "model.safetensors"), metadata={"precision": f"int{bits}", "group": str(group), "int8_layers": ",".join(map(str, int8_layers)), | |
| "embed_bits": str(embed_bits), "format": "julia2-wq-v1"}) | |
| return n_q, n_bytes | |
| if __name__ == "__main__": | |
| ap = argparse.ArgumentParser(); ap.add_argument("--src", required=True); ap.add_argument("--dst", required=True); ap.add_argument("--bits", type=int, default=8); ap.add_argument("--group", type=int, default=128) | |
| ap.add_argument("--int8_layers", default="", help="comma-separated trunk layers kept int8 in an int4 file") | |
| ap.add_argument("--embed_bits", type=int, default=0, help="8 or 4: quantise the input embedding table too") | |
| ap.add_argument("--embed_group", type=int, default=64) | |
| a = ap.parse_args() | |
| n, b = quantize_file(a.src, a.dst, a.bits, a.group, tuple(int(x) for x in a.int8_layers.split(",") if x), a.embed_bits, a.embed_group) | |
| print(f"quantised {n} linear weights -> {a.dst} ({b/1e9:.2f} GB of quantised tensors)") | |