Julia-2-Flash / runtime /tools /make_variants.py
vianahnz's picture
Julia 2 Flash
b61b4c4 verified
Raw History Blame Contribute Delete
1.76 kB
"""One pass over the fused bf16 file -> int8/ and int4/ (mixed: trunk layers 0-11 int8, rest int4 g128) variants."""
import json, sys, hashlib
from pathlib import Path
import torch
HERE = Path(__file__).resolve().parent; sys.path.insert(0, str(HERE.parent))
from julia2.quant import quantize_tensor, QUANT_PREFIXES, SKIP_SUBSTR
from safetensors import safe_open
from safetensors.torch import save_file
repo = Path(sys.argv[1]); int8_layers = set(range(12))
f = safe_open(str(repo / "model.safetensors"), "pt"); o8, o4 = {}, {}
for k in f.keys():
t = f.get_tensor(k)
if k.endswith(".weight") and t.dim() == 2 and any(k.startswith(p) for p in QUANT_PREFIXES) and not any(s in k for s in SKIP_SUBSTR):
base = k[:-7]
q, s, meta = quantize_tensor(t, 8); o8[base + ".q"], o8[base + ".s"], o8[base + ".qcfg"] = q, s, torch.tensor(list(json.dumps(meta).encode()), dtype=torch.uint8)
b = 8 if (k.startswith("trunk.language_model.layers.") and int(k.split(".")[3]) in int8_layers) else 4
q, s, meta = quantize_tensor(t, b, 128); o4[base + ".q"], o4[base + ".s"], o4[base + ".qcfg"] = q, s, torch.tensor(list(json.dumps(meta).encode()), dtype=torch.uint8)
else:
o8[k] = o4[k] = t.contiguous()
for name, out, md in (("int8", o8, {"precision": "int8", "scheme": "per-channel symmetric int8, fp16 scales"}), ("int4", o4, {"precision": "int4", "scheme": "asymmetric 4-bit g128 (fp16 scale+zero); trunk layers 0-11 int8", "int8_layers": "0-11"})):
(repo / name).mkdir(exist_ok=True); p = repo / name / "model.safetensors"
save_file({k: v.contiguous() for k, v in out.items()}, str(p), metadata={**md, "format": "julia2-wq-v1"})
print(name, p.stat().st_size / 1e9, "GB", hashlib.sha256(p.read_bytes()).hexdigest())