File size: 8,714 Bytes
ae97c5d
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
"""Convert a Cloudflare Clairic checkpoint (PyTorch `Qwen3_5ForConditionalGeneration` + typed head) to MLX.

    python convert.py <src_checkpoint> <dst_dir> [--bits 4] [--group-size 64] [--mode affine]
                      [--keep in_proj_a,in_proj_b] [--high <regex> --high-bits 8] [--dtype bfloat16]

What it does:
- renames the backbone tensors to the `mlx_vlm.models.qwen3_5` layout with mlx-vlm's own `sanitize` (conv1d
  axis move, patch-embed transpose, +1 on the zero-centred RMSNorm weights) and drops the unused `mtp.*` draft
  head (the config sets `mtp_num_hidden_layers` to 0 and the runtime never calls it);
- stores the RMSNorm weights as exact fp32 `1 + w` (the runtime evaluates these norms in fp32);
- quantizes the language model's linear layers, token embedding and vocabulary head (`--bits 0` keeps
  everything in `--dtype`); the vision tower, norms, conv1d, `A_log` / `dt_bias` and any `--keep` module-path
  suffixes are never quantized;
- copies the typed head (`clair_head.safetensors`, fp32, unchanged), tokenizer, configs and licence.

The folder is not a stock mlx-lm / mlx-vlm chat model: answers come from the typed head and from
grammar-constrained decoding driven by the runtime shipped next to the weights.
"""

from __future__ import annotations

import argparse
import json
import re
import shutil
import time
from pathlib import Path

import mlx.core as mx
import mlx.nn as nn
from mlx.utils import tree_flatten
from mlx_vlm.models.qwen3_5 import Model, ModelConfig

SHARD_BYTES = 5 << 30
COPY = [
    "LICENSE", "chat_template.jinja", "clair_head.safetensors", "clair_head_config.json", "generation_config.json",
    "preprocessor_config.json", "tokenizer.json", "tokenizer_config.json",
]
NORM_SUFFIXES = (
    ".input_layernorm.weight", ".post_attention_layernorm.weight", "model.norm.weight", ".q_norm.weight", ".k_norm.weight",
)


def main() -> None:
    ap = argparse.ArgumentParser()
    ap.add_argument("src")
    ap.add_argument("dst")
    ap.add_argument("--bits", type=int, default=4, help="0 = no quantization")
    ap.add_argument("--group-size", type=int, default=64)
    ap.add_argument("--mode", default="affine")
    ap.add_argument("--keep", default="in_proj_a,in_proj_b", help="comma-separated module-path suffixes to leave unquantized")
    ap.add_argument("--high", default="", help="regular expression of module paths quantized with --high-bits instead")
    ap.add_argument("--high-bits", type=int, default=8)
    ap.add_argument("--dtype", default="bfloat16", choices=["bfloat16", "float32"])
    args = ap.parse_args()

    src, dst = Path(args.src), Path(args.dst)
    dst.mkdir(parents=True, exist_ok=True)
    dtype = getattr(mx, args.dtype)
    keep = tuple(s for s in args.keep.split(",") if s)
    config = json.loads((src / "config.json").read_text())
    started = time.time()

    # ---- load (lazily), rename and sanitize ----------------------------------------------------------
    raw: dict[str, mx.array] = {}
    for shard in sorted(src.glob("model*.safetensors")):
        raw.update(mx.load(str(shard)))
    dropped = [key for key in raw if key.startswith("mtp.")]
    if "lm_head.weight" not in raw or not any(key.startswith("model.visual.") for key in raw):
        raise ValueError("Expected a full Qwen3_5ForConditionalGeneration checkpoint (lm_head and vision tower)")
    # Copies: mlx-vlm's sanitize shifts the norm weights in place (`value += 1.0`).
    originals = {key: mx.array(value) for key, value in raw.items()
                 if key.startswith("model.language_model.") and (key.endswith(NORM_SUFFIXES) or "conv1d.weight" in key)}
    model = Model(ModelConfig.from_dict(config))
    weights = model.sanitize(raw)
    weights = model.vision_tower.sanitize(weights)

    # Zero-centred RMSNorm weights: the reference computes (1 + w) in fp32. mlx-vlm's sanitize adds the 1 in
    # bf16, which rounds weights above 2 to 1/64. Store the exact fp32 sum instead (a few hundred small vectors).
    norm_keys = set()
    for key, value in originals.items():
        new_key = "language_model.model." + key.removeprefix("model.language_model.")
        new = weights[new_key]
        if "conv1d.weight" in key:
            assert new.shape == (value.shape[0], value.shape[2], value.shape[1]), (key, new.shape)
            continue
        exact = value.astype(mx.float32) + 1.0
        # sanitize must have applied the shift itself (within bf16 rounding); otherwise its detection changed.
        assert mx.abs(new.astype(mx.float32) - exact).max().item() <= 0.04, f"norm weight {key} was not shifted by +1"
        weights[new_key] = exact
        norm_keys.add(new_key)
    print(f"sanitized {len(weights)} tensors ({len(dropped)} mtp tensors dropped); "
          f"{len(norm_keys)} RMSNorm weights stored as exact fp32 (1 + w)")

    def cast(key: str, value: mx.array) -> mx.array:
        if not mx.issubdtype(value.dtype, mx.floating) or key in norm_keys:
            return value
        return value.astype(dtype)

    weights = {key: cast(key, value) for key, value in weights.items()}
    model.load_weights(list(weights.items()), strict=True)

    # ---- quantize ------------------------------------------------------------------------------------
    quantized: list[str] = []
    skipped: list[str] = []
    raised: dict[str, dict] = {}
    high = re.compile(args.high) if args.high else None
    if args.bits:
        def predicate(path: str, module: nn.Module) -> bool | dict:
            if not hasattr(module, "to_quantized"):
                return False
            if path.startswith("vision_tower") or path.endswith(keep or ("\0",)):
                skipped.append(path)
                return False
            if module.weight.shape[-1] % args.group_size:
                skipped.append(path)
                return False
            quantized.append(path)
            settings = {"bits": args.bits, "group_size": args.group_size, "mode": args.mode}
            if high is not None and high.search(path):
                settings["bits"] = args.high_bits
                raised[path] = settings
            return settings

        nn.quantize(model, class_predicate=predicate)
        # The standard MLX layout: global settings, then per-module entries: `false` for language-model modules
        # left unquantized, and the settings of modules quantized with a different bit width.
        config["quantization"] = {"group_size": args.group_size, "bits": args.bits, "mode": args.mode,
                                  **{path: False for path in sorted(skipped) if path.startswith("language_model")},
                                  **{path: raised[path] for path in sorted(raised)}}
        config["quantization_config"] = config["quantization"]
    else:
        config.pop("quantization", None)
        config.pop("quantization_config", None)

    # ---- save shards (evaluating one shard at a time) -------------------------------------------------
    for old in dst.glob("model*.safetensors*"):
        old.unlink()
    params = dict(tree_flatten(model.parameters()))
    shards: list[dict[str, mx.array]] = [{}]
    size = 0
    for key in sorted(params):
        nbytes = params[key].nbytes
        if size and size + nbytes > SHARD_BYTES:
            shards.append({})
            size = 0
        shards[-1][key] = params[key]
        size += nbytes
    index = {"metadata": {"format": "mlx", "total_size": sum(v.nbytes for v in params.values()),
                          "total_parameters": sum(v.size for v in params.values())}, "weight_map": {}}
    for number, shard in enumerate(shards, 1):
        name = f"model-{number:05d}-of-{len(shards):05d}.safetensors"
        mx.eval(*shard.values())
        mx.save_safetensors(str(dst / name), shard, metadata={"format": "mlx"})
        for key in shard:
            index["weight_map"][key] = name
        print(f"  {name}: {len(shard)} tensors, {sum(v.nbytes for v in shard.values()) / 1e9:.2f} GB", flush=True)
        for key in shard:  # release
            params[key] = None
        shard.clear()
        mx.clear_cache()
    (dst / "model.safetensors.index.json").write_text(json.dumps(index, indent=2))
    (dst / "config.json").write_text(json.dumps(config, indent=2))
    for name in COPY:
        shutil.copy2(src / name, dst / name)

    total = sum(f.stat().st_size for f in dst.glob("model-*.safetensors"))
    print(f"wrote {dst} in {time.time() - started:.0f}s: {total / 1e9:.2f} GB of backbone weights, "
          f"{len(quantized)} modules quantized ({len(raised)} of them at {args.high_bits} bits), "
          f"{len(skipped)} quantizable modules kept in {args.dtype}")


if __name__ == "__main__":
    main()