clairic-flash-4bit / convert.py
lucataco's picture
Clairic Flash, MLX 4-bit, with runtime, server and conversion script
ae97c5d verified
Raw History Blame Contribute Delete
8.71 kB
"""Convert a Cloudflare Clairic checkpoint (PyTorch `Qwen3_5ForConditionalGeneration` + typed head) to MLX.
python convert.py <src_checkpoint> <dst_dir> [--bits 4] [--group-size 64] [--mode affine]
[--keep in_proj_a,in_proj_b] [--high <regex> --high-bits 8] [--dtype bfloat16]
What it does:
- renames the backbone tensors to the `mlx_vlm.models.qwen3_5` layout with mlx-vlm's own `sanitize` (conv1d
axis move, patch-embed transpose, +1 on the zero-centred RMSNorm weights) and drops the unused `mtp.*` draft
head (the config sets `mtp_num_hidden_layers` to 0 and the runtime never calls it);
- stores the RMSNorm weights as exact fp32 `1 + w` (the runtime evaluates these norms in fp32);
- quantizes the language model's linear layers, token embedding and vocabulary head (`--bits 0` keeps
everything in `--dtype`); the vision tower, norms, conv1d, `A_log` / `dt_bias` and any `--keep` module-path
suffixes are never quantized;
- copies the typed head (`clair_head.safetensors`, fp32, unchanged), tokenizer, configs and licence.
The folder is not a stock mlx-lm / mlx-vlm chat model: answers come from the typed head and from
grammar-constrained decoding driven by the runtime shipped next to the weights.
"""
from __future__ import annotations
import argparse
import json
import re
import shutil
import time
from pathlib import Path
import mlx.core as mx
import mlx.nn as nn
from mlx.utils import tree_flatten
from mlx_vlm.models.qwen3_5 import Model, ModelConfig
SHARD_BYTES = 5 << 30
COPY = [
"LICENSE", "chat_template.jinja", "clair_head.safetensors", "clair_head_config.json", "generation_config.json",
"preprocessor_config.json", "tokenizer.json", "tokenizer_config.json",
]
NORM_SUFFIXES = (
".input_layernorm.weight", ".post_attention_layernorm.weight", "model.norm.weight", ".q_norm.weight", ".k_norm.weight",
)
def main() -> None:
ap = argparse.ArgumentParser()
ap.add_argument("src")
ap.add_argument("dst")
ap.add_argument("--bits", type=int, default=4, help="0 = no quantization")
ap.add_argument("--group-size", type=int, default=64)
ap.add_argument("--mode", default="affine")
ap.add_argument("--keep", default="in_proj_a,in_proj_b", help="comma-separated module-path suffixes to leave unquantized")
ap.add_argument("--high", default="", help="regular expression of module paths quantized with --high-bits instead")
ap.add_argument("--high-bits", type=int, default=8)
ap.add_argument("--dtype", default="bfloat16", choices=["bfloat16", "float32"])
args = ap.parse_args()
src, dst = Path(args.src), Path(args.dst)
dst.mkdir(parents=True, exist_ok=True)
dtype = getattr(mx, args.dtype)
keep = tuple(s for s in args.keep.split(",") if s)
config = json.loads((src / "config.json").read_text())
started = time.time()
# ---- load (lazily), rename and sanitize ----------------------------------------------------------
raw: dict[str, mx.array] = {}
for shard in sorted(src.glob("model*.safetensors")):
raw.update(mx.load(str(shard)))
dropped = [key for key in raw if key.startswith("mtp.")]
if "lm_head.weight" not in raw or not any(key.startswith("model.visual.") for key in raw):
raise ValueError("Expected a full Qwen3_5ForConditionalGeneration checkpoint (lm_head and vision tower)")
# Copies: mlx-vlm's sanitize shifts the norm weights in place (`value += 1.0`).
originals = {key: mx.array(value) for key, value in raw.items()
if key.startswith("model.language_model.") and (key.endswith(NORM_SUFFIXES) or "conv1d.weight" in key)}
model = Model(ModelConfig.from_dict(config))
weights = model.sanitize(raw)
weights = model.vision_tower.sanitize(weights)
# Zero-centred RMSNorm weights: the reference computes (1 + w) in fp32. mlx-vlm's sanitize adds the 1 in
# bf16, which rounds weights above 2 to 1/64. Store the exact fp32 sum instead (a few hundred small vectors).
norm_keys = set()
for key, value in originals.items():
new_key = "language_model.model." + key.removeprefix("model.language_model.")
new = weights[new_key]
if "conv1d.weight" in key:
assert new.shape == (value.shape[0], value.shape[2], value.shape[1]), (key, new.shape)
continue
exact = value.astype(mx.float32) + 1.0
# sanitize must have applied the shift itself (within bf16 rounding); otherwise its detection changed.
assert mx.abs(new.astype(mx.float32) - exact).max().item() <= 0.04, f"norm weight {key} was not shifted by +1"
weights[new_key] = exact
norm_keys.add(new_key)
print(f"sanitized {len(weights)} tensors ({len(dropped)} mtp tensors dropped); "
f"{len(norm_keys)} RMSNorm weights stored as exact fp32 (1 + w)")
def cast(key: str, value: mx.array) -> mx.array:
if not mx.issubdtype(value.dtype, mx.floating) or key in norm_keys:
return value
return value.astype(dtype)
weights = {key: cast(key, value) for key, value in weights.items()}
model.load_weights(list(weights.items()), strict=True)
# ---- quantize ------------------------------------------------------------------------------------
quantized: list[str] = []
skipped: list[str] = []
raised: dict[str, dict] = {}
high = re.compile(args.high) if args.high else None
if args.bits:
def predicate(path: str, module: nn.Module) -> bool | dict:
if not hasattr(module, "to_quantized"):
return False
if path.startswith("vision_tower") or path.endswith(keep or ("\0",)):
skipped.append(path)
return False
if module.weight.shape[-1] % args.group_size:
skipped.append(path)
return False
quantized.append(path)
settings = {"bits": args.bits, "group_size": args.group_size, "mode": args.mode}
if high is not None and high.search(path):
settings["bits"] = args.high_bits
raised[path] = settings
return settings
nn.quantize(model, class_predicate=predicate)
# The standard MLX layout: global settings, then per-module entries: `false` for language-model modules
# left unquantized, and the settings of modules quantized with a different bit width.
config["quantization"] = {"group_size": args.group_size, "bits": args.bits, "mode": args.mode,
**{path: False for path in sorted(skipped) if path.startswith("language_model")},
**{path: raised[path] for path in sorted(raised)}}
config["quantization_config"] = config["quantization"]
else:
config.pop("quantization", None)
config.pop("quantization_config", None)
# ---- save shards (evaluating one shard at a time) -------------------------------------------------
for old in dst.glob("model*.safetensors*"):
old.unlink()
params = dict(tree_flatten(model.parameters()))
shards: list[dict[str, mx.array]] = [{}]
size = 0
for key in sorted(params):
nbytes = params[key].nbytes
if size and size + nbytes > SHARD_BYTES:
shards.append({})
size = 0
shards[-1][key] = params[key]
size += nbytes
index = {"metadata": {"format": "mlx", "total_size": sum(v.nbytes for v in params.values()),
"total_parameters": sum(v.size for v in params.values())}, "weight_map": {}}
for number, shard in enumerate(shards, 1):
name = f"model-{number:05d}-of-{len(shards):05d}.safetensors"
mx.eval(*shard.values())
mx.save_safetensors(str(dst / name), shard, metadata={"format": "mlx"})
for key in shard:
index["weight_map"][key] = name
print(f" {name}: {len(shard)} tensors, {sum(v.nbytes for v in shard.values()) / 1e9:.2f} GB", flush=True)
for key in shard: # release
params[key] = None
shard.clear()
mx.clear_cache()
(dst / "model.safetensors.index.json").write_text(json.dumps(index, indent=2))
(dst / "config.json").write_text(json.dumps(config, indent=2))
for name in COPY:
shutil.copy2(src / name, dst / name)
total = sum(f.stat().st_size for f in dst.glob("model-*.safetensors"))
print(f"wrote {dst} in {time.time() - started:.0f}s: {total / 1e9:.2f} GB of backbone weights, "
f"{len(quantized)} modules quantized ({len(raised)} of them at {args.high_bits} bits), "
f"{len(skipped)} quantizable modules kept in {args.dtype}")
if __name__ == "__main__":
main()