"""Convert a Cloudflare Clairic checkpoint (PyTorch `Qwen3_5ForConditionalGeneration` + typed head) to MLX. python convert.py [--bits 4] [--group-size 64] [--mode affine] [--keep in_proj_a,in_proj_b] [--high --high-bits 8] [--dtype bfloat16] What it does: - renames the backbone tensors to the `mlx_vlm.models.qwen3_5` layout with mlx-vlm's own `sanitize` (conv1d axis move, patch-embed transpose, +1 on the zero-centred RMSNorm weights) and drops the unused `mtp.*` draft head (the config sets `mtp_num_hidden_layers` to 0 and the runtime never calls it); - stores the RMSNorm weights as exact fp32 `1 + w` (the runtime evaluates these norms in fp32); - quantizes the language model's linear layers, token embedding and vocabulary head (`--bits 0` keeps everything in `--dtype`); the vision tower, norms, conv1d, `A_log` / `dt_bias` and any `--keep` module-path suffixes are never quantized; - copies the typed head (`clair_head.safetensors`, fp32, unchanged), tokenizer, configs and licence. The folder is not a stock mlx-lm / mlx-vlm chat model: answers come from the typed head and from grammar-constrained decoding driven by the runtime shipped next to the weights. """ from __future__ import annotations import argparse import json import re import shutil import time from pathlib import Path import mlx.core as mx import mlx.nn as nn from mlx.utils import tree_flatten from mlx_vlm.models.qwen3_5 import Model, ModelConfig SHARD_BYTES = 5 << 30 COPY = [ "LICENSE", "chat_template.jinja", "clair_head.safetensors", "clair_head_config.json", "generation_config.json", "preprocessor_config.json", "tokenizer.json", "tokenizer_config.json", ] NORM_SUFFIXES = ( ".input_layernorm.weight", ".post_attention_layernorm.weight", "model.norm.weight", ".q_norm.weight", ".k_norm.weight", ) def main() -> None: ap = argparse.ArgumentParser() ap.add_argument("src") ap.add_argument("dst") ap.add_argument("--bits", type=int, default=4, help="0 = no quantization") ap.add_argument("--group-size", type=int, default=64) ap.add_argument("--mode", default="affine") ap.add_argument("--keep", default="in_proj_a,in_proj_b", help="comma-separated module-path suffixes to leave unquantized") ap.add_argument("--high", default="", help="regular expression of module paths quantized with --high-bits instead") ap.add_argument("--high-bits", type=int, default=8) ap.add_argument("--dtype", default="bfloat16", choices=["bfloat16", "float32"]) args = ap.parse_args() src, dst = Path(args.src), Path(args.dst) dst.mkdir(parents=True, exist_ok=True) dtype = getattr(mx, args.dtype) keep = tuple(s for s in args.keep.split(",") if s) config = json.loads((src / "config.json").read_text()) started = time.time() # ---- load (lazily), rename and sanitize ---------------------------------------------------------- raw: dict[str, mx.array] = {} for shard in sorted(src.glob("model*.safetensors")): raw.update(mx.load(str(shard))) dropped = [key for key in raw if key.startswith("mtp.")] if "lm_head.weight" not in raw or not any(key.startswith("model.visual.") for key in raw): raise ValueError("Expected a full Qwen3_5ForConditionalGeneration checkpoint (lm_head and vision tower)") # Copies: mlx-vlm's sanitize shifts the norm weights in place (`value += 1.0`). originals = {key: mx.array(value) for key, value in raw.items() if key.startswith("model.language_model.") and (key.endswith(NORM_SUFFIXES) or "conv1d.weight" in key)} model = Model(ModelConfig.from_dict(config)) weights = model.sanitize(raw) weights = model.vision_tower.sanitize(weights) # Zero-centred RMSNorm weights: the reference computes (1 + w) in fp32. mlx-vlm's sanitize adds the 1 in # bf16, which rounds weights above 2 to 1/64. Store the exact fp32 sum instead (a few hundred small vectors). norm_keys = set() for key, value in originals.items(): new_key = "language_model.model." + key.removeprefix("model.language_model.") new = weights[new_key] if "conv1d.weight" in key: assert new.shape == (value.shape[0], value.shape[2], value.shape[1]), (key, new.shape) continue exact = value.astype(mx.float32) + 1.0 # sanitize must have applied the shift itself (within bf16 rounding); otherwise its detection changed. assert mx.abs(new.astype(mx.float32) - exact).max().item() <= 0.04, f"norm weight {key} was not shifted by +1" weights[new_key] = exact norm_keys.add(new_key) print(f"sanitized {len(weights)} tensors ({len(dropped)} mtp tensors dropped); " f"{len(norm_keys)} RMSNorm weights stored as exact fp32 (1 + w)") def cast(key: str, value: mx.array) -> mx.array: if not mx.issubdtype(value.dtype, mx.floating) or key in norm_keys: return value return value.astype(dtype) weights = {key: cast(key, value) for key, value in weights.items()} model.load_weights(list(weights.items()), strict=True) # ---- quantize ------------------------------------------------------------------------------------ quantized: list[str] = [] skipped: list[str] = [] raised: dict[str, dict] = {} high = re.compile(args.high) if args.high else None if args.bits: def predicate(path: str, module: nn.Module) -> bool | dict: if not hasattr(module, "to_quantized"): return False if path.startswith("vision_tower") or path.endswith(keep or ("\0",)): skipped.append(path) return False if module.weight.shape[-1] % args.group_size: skipped.append(path) return False quantized.append(path) settings = {"bits": args.bits, "group_size": args.group_size, "mode": args.mode} if high is not None and high.search(path): settings["bits"] = args.high_bits raised[path] = settings return settings nn.quantize(model, class_predicate=predicate) # The standard MLX layout: global settings, then per-module entries: `false` for language-model modules # left unquantized, and the settings of modules quantized with a different bit width. config["quantization"] = {"group_size": args.group_size, "bits": args.bits, "mode": args.mode, **{path: False for path in sorted(skipped) if path.startswith("language_model")}, **{path: raised[path] for path in sorted(raised)}} config["quantization_config"] = config["quantization"] else: config.pop("quantization", None) config.pop("quantization_config", None) # ---- save shards (evaluating one shard at a time) ------------------------------------------------- for old in dst.glob("model*.safetensors*"): old.unlink() params = dict(tree_flatten(model.parameters())) shards: list[dict[str, mx.array]] = [{}] size = 0 for key in sorted(params): nbytes = params[key].nbytes if size and size + nbytes > SHARD_BYTES: shards.append({}) size = 0 shards[-1][key] = params[key] size += nbytes index = {"metadata": {"format": "mlx", "total_size": sum(v.nbytes for v in params.values()), "total_parameters": sum(v.size for v in params.values())}, "weight_map": {}} for number, shard in enumerate(shards, 1): name = f"model-{number:05d}-of-{len(shards):05d}.safetensors" mx.eval(*shard.values()) mx.save_safetensors(str(dst / name), shard, metadata={"format": "mlx"}) for key in shard: index["weight_map"][key] = name print(f" {name}: {len(shard)} tensors, {sum(v.nbytes for v in shard.values()) / 1e9:.2f} GB", flush=True) for key in shard: # release params[key] = None shard.clear() mx.clear_cache() (dst / "model.safetensors.index.json").write_text(json.dumps(index, indent=2)) (dst / "config.json").write_text(json.dumps(config, indent=2)) for name in COPY: shutil.copy2(src / name, dst / name) total = sum(f.stat().st_size for f in dst.glob("model-*.safetensors")) print(f"wrote {dst} in {time.time() - started:.0f}s: {total / 1e9:.2f} GB of backbone weights, " f"{len(quantized)} modules quantized ({len(raised)} of them at {args.high_bits} bits), " f"{len(skipped)} quantizable modules kept in {args.dtype}") if __name__ == "__main__": main()