diff --git a/mlx_vlm/convert.py b/mlx_vlm/convert.py index e7b032f5..0b884aea 100644 --- a/mlx_vlm/convert.py +++ b/mlx_vlm/convert.py @@ -318,6 +318,19 @@ def convert( model, config, processor = fetch_from_hub( model_path, lazy=True, trust_remote_code=trust_remote_code ) + # Native packed exports can declare only a generic quantization family in + # config.json, with the actual per-layer precision in a sidecar. Preserve + # that precision when saving the sanitized MLX weights. + metadata_path = model_path / "hf_quant_config.json" + if ( + "quantization" not in config + and hasattr(model, "quantization_from_metadata") + and metadata_path.exists() + ): + with metadata_path.open() as f: + native_quantization = model.quantization_from_metadata(json.load(f)) + if native_quantization is not None: + config["quantization"] = config["quantization_config"] = native_quantization model_quant_predicate = getattr(model, "quant_predicate", None) diff --git a/mlx_vlm/models/inkling2/config.py b/mlx_vlm/models/inkling2/config.py index 8e3b7b20..f6e4f1e4 100644 --- a/mlx_vlm/models/inkling2/config.py +++ b/mlx_vlm/models/inkling2/config.py @@ -40,6 +40,34 @@ class TextConfig(BaseModelConfig): mlp_gate_softcap: float = 4.0 mlp_up_softcap: float = 25.0 final_logit_softcapping: float = 30.0 + loop_config: Optional[dict] = None + + def __post_init__(self): + if self.loop_config is None: + return + cfg = self.loop_config + start, end = cfg["core_start"], cfg["core_end"] + block = self.n_layers_per_attn_res_block + if ( + not 0 < start < end <= self.num_hidden_layers + or start % block + or end % block + or cfg["num_passes"] < 1 + or cfg["gate_rank"] < 1 + or cfg.get("is_varloop", False) + ): + raise ValueError("Invalid or unsupported Inkling 2 loop configuration") + + @property + def layer_schedule(self): + if self.loop_config is None: + return list(range(self.num_hidden_layers)) + start, end = self.loop_config["core_start"], self.loop_config["core_end"] + return ( + list(range(start)) + + list(range(start, end)) * self.loop_config["num_passes"] + + list(range(end, self.num_hidden_layers)) + ) def layer_is_sliding(self, index): if self.layer_types is not None: diff --git a/mlx_vlm/models/inkling2/inkling2.py b/mlx_vlm/models/inkling2/inkling2.py index d2173a54..08c04909 100644 --- a/mlx_vlm/models/inkling2/inkling2.py +++ b/mlx_vlm/models/inkling2/inkling2.py @@ -114,6 +114,16 @@ class Model(nn.Module): if sub.startswith("mlp.experts."): experts.setdefault(layer, {})[sub[len("mlp.experts.") :]] = value continue + if sub.startswith("controllers."): + _, name, leaf = sub.split(".") + name = {"attn": "q", "k": "k", "v": "v", "mlp": "mlp"}[name] + leaf = { + "down_weight": "down_proj", + "forget_weight": "forget_gate_proj", + "gain_weight": "message_gate_proj", + }[leaf] + out[base + name + "_message_gate." + leaf + ".weight"] = value + continue if sub.startswith("attn."): sub = sub[len("attn.") :] name, _, leaf = sub.partition(".") @@ -156,6 +166,8 @@ class Model(nn.Module): out["language_model.model." + target] = value elif key.startswith("model.llm."): sub = key[len("model.llm.") :] + if sub.startswith("middle_cycle_cross_pass_"): + continue target = ( "language_model.lm_head.weight" if sub == "unembed.weight" @@ -181,6 +193,15 @@ class Model(nn.Module): continue else: out[key] = value + query_key = "model.llm.middle_cycle_cross_pass_queries" + kv_key = "model.llm.middle_cycle_cross_pass_kv_queries" + if query_key in weights or kv_key in weights: + hidden = self.config.text_config.hidden_size + queries = weights[query_key].reshape(-1, 2, hidden) + kv_queries = weights[kv_key].reshape(-1, 2, hidden) + out["language_model.model.message_input.query"] = mx.concatenate( + [queries[:, :1], kv_queries, queries[:, 1:]], axis=1 + ) for layer, tensors in experts.items(): base = f"language_model.model.layers.{layer}.mlp.switch_mlp." for name, projections in ( diff --git a/mlx_vlm/models/inkling2/language.py b/mlx_vlm/models/inkling2/language.py index 805dcc1f..023226db 100644 --- a/mlx_vlm/models/inkling2/language.py +++ b/mlx_vlm/models/inkling2/language.py @@ -537,6 +537,44 @@ class AttentionResidual(nn.Module): ) +class MessageGate(nn.Module): + """Mix a previous-pass message into the current attention-residual input.""" + + def __init__(self, config): + super().__init__() + rank = config.loop_config["gate_rank"] + self.eps = config.rms_norm_eps + self.down_proj = nn.Linear(2 * config.hidden_size, rank, bias=False) + self.forget_gate_proj = nn.Linear(rank, config.hidden_size, bias=False) + self.message_gate_proj = nn.Linear(rank, config.hidden_size, bias=False) + + def __call__(self, hidden, message): + hidden, message = hidden.astype(mx.float32), message.astype(mx.float32) + inputs = mx.concatenate( + [mx.fast.rms_norm(x, None, self.eps) for x in (hidden, message)], axis=-1 + ) + gate = self.down_proj(inputs.astype(self.down_proj.weight.dtype)).astype( + mx.float32 + ) + forget = mx.tanh(gate @ self.forget_gate_proj.weight.astype(mx.float32).T) + gain = mx.tanh(gate @ self.message_gate_proj.weight.astype(mx.float32).T) + return (1 - forget) * hidden + gain * message + + +class MessageAttentionResidual(nn.Module): + def __init__(self, config): + super().__init__() + loop = config.loop_config + self.query = mx.zeros( + (loop["core_end"] - loop["core_start"], 4, config.hidden_size), mx.float32 + ) + + def __call__(self, blocks, partial, layer_index): + return _attention_residual( + mx.stack([*blocks, partial], axis=-2), self.query[layer_index], True + ) + + class DecoderLayer(nn.Module): def __init__(self, config, layer_idx): super().__init__() @@ -563,8 +601,12 @@ class DecoderLayer(nn.Module): self.mlp_sconv = ShortConvolution( config.hidden_size, config.sconv_kernel_size, 3 ) + loop = config.loop_config + if loop and loop["core_start"] <= layer_idx < loop["core_end"]: + for name in ("q", "k", "v", "mlp"): + self[name + "_message_gate"] = MessageGate(config) - def __call__(self, x, blocks, cache=None): + def __call__(self, x, blocks, cache=None, messages=None): conv = cache[1] if cache is not None else None mask = _cache_padding_mask(conv, x.shape[1]) mixed = ( @@ -572,7 +614,7 @@ class DecoderLayer(nn.Module): if self.attention_input is not None else x[..., None, :] ) - if self.is_sliding: + if self.is_sliding and messages is None: attn = self.self_attn( self.input_layernorm(mixed[..., 0, :]).astype(x.dtype), cache=cache, @@ -580,17 +622,21 @@ class DecoderLayer(nn.Module): ) else: parts = [mixed[..., min(i, mixed.shape[-2] - 1), :] for i in range(3)] - parts = [ - norm(part).astype(x.dtype) - for norm, part in zip( - ( - self.q_input_layernorm, - self.k_input_layernorm, - self.v_input_layernorm, - ), - parts, + if messages is not None: + parts = [ + self[name + "_message_gate"](part, messages[..., i, :]) + for i, (name, part) in enumerate(zip(("q", "k", "v"), parts)) + ] + norms = ( + (self.input_layernorm,) * 3 + if self.is_sliding + else ( + self.q_input_layernorm, + self.k_input_layernorm, + self.v_input_layernorm, ) - ] + ) + parts = [norm(part).astype(x.dtype) for norm, part in zip(norms, parts)] attn = self.self_attn(*parts, cache=cache, conv_mask=mask) residual = self.attn_sconv(attn, conv, mask).astype(mx.float32) if self.is_block_boundary: @@ -598,6 +644,8 @@ class DecoderLayer(nn.Module): else: residual = residual + x.astype(mx.float32) hidden = self.mlp_input(blocks, residual)[..., 0, :] + if messages is not None: + hidden = self.mlp_message_gate(hidden, messages[..., 3, :]) hidden = self.mlp(self.post_attention_layernorm(hidden).astype(x.dtype)) out = (residual + self.mlp_sconv(hidden, conv, mask).astype(mx.float32)).astype( x.dtype @@ -610,11 +658,14 @@ class DecoderLayer(nn.Module): class TextModel(nn.Module): def __init__(self, config): super().__init__() + self.config = config self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size) self.embed_norm = nn.RMSNorm(config.hidden_size, eps=config.rms_norm_eps) self.layers = [DecoderLayer(config, i) for i in range(config.num_hidden_layers)] self.final_input = AttentionResidual(config) self.norm = nn.RMSNorm(config.hidden_size, eps=config.rms_norm_eps) + if config.loop_config: + self.message_input = MessageAttentionResidual(config) def embed(self, inputs): return self.embed_norm(self.embed_tokens(inputs)) @@ -632,7 +683,7 @@ class TextModel(nn.Module): if capture_layer_ids is not None and ( len(capture_layer_ids) != len(set(capture_layer_ids)) or any( - type(i) is not int or not 0 <= i <= len(self.layers) + type(i) is not int or not 0 <= i <= len(self.config.layer_schedule) for i in capture_layer_ids ) ): @@ -640,19 +691,35 @@ class TextModel(nn.Module): requested = set(capture_layer_ids or []) x = self.embed(inputs) if input_embeddings is None else input_embeddings blocks = [] - for i, (layer, c) in enumerate( - zip(self.layers, cache or [None] * len(self.layers)) - ): + schedule = self.config.layer_schedule + if cache is not None and len(cache) != len(schedule): + raise ValueError("Inkling 2 requires a separate cache for each layer visit") + caches = cache if cache is not None else [None] * len(schedule) + loop = self.config.loop_config + pre_loop = previous = None + for i, (layer_index, c) in enumerate(zip(schedule, caches)): + layer = self.layers[layer_index] + messages = None + if loop and layer_index == loop["core_start"]: + if pre_loop is None: + pre_loop = (x, list(blocks)) + else: + previous = (blocks[len(pre_loop[1]) + 1 :], x) + x, blocks = pre_loop[0], list(pre_loop[1]) + if previous is not None and layer_index < loop["core_end"]: + messages = self.message_input( + *previous, layer_index - loop["core_start"] + ) # DSpark taps the partial block entering the layer, not its output. if i in requested: captures[i] = x - x = layer(x, blocks, c) + x = layer(x, blocks, c, messages=messages) if speculative_verify and (i + 1) % 4 == 0: mx.async_eval(x) h = self.final_input(blocks, x)[..., 0, :] normalized = self.norm(h).astype(x.dtype) - if len(self.layers) in requested: - captures[len(self.layers)] = normalized + if len(schedule) in requested: + captures[len(schedule)] = normalized result = h if skip_final_norm else normalized if capture_layer_ids is not None: return result, [captures[i] for i in capture_layer_ids] @@ -830,7 +897,7 @@ class LanguageModel(InklingLanguageModel): ), ArraysCache(4), ) - for layer in self.model.layers + for layer in (self.model.layers[i] for i in self.config.layer_schedule) ] def speculative_verify_hidden(self, inputs, cache): diff --git a/mlx_vlm/models/inkling2/processing_inkling2.py b/mlx_vlm/models/inkling2/processing_inkling2.py index ffefc0a1..d640a644 100644 --- a/mlx_vlm/models/inkling2/processing_inkling2.py +++ b/mlx_vlm/models/inkling2/processing_inkling2.py @@ -206,4 +206,4 @@ class Inkling2Processor(ProcessorMixin): return ["input_ids", "attention_mask", "pixel_values", "image_grid_thw"] -install_auto_processor_patch(["inkling2"], Inkling2Processor) +install_auto_processor_patch(["inkling2", "inkling2_loop"], Inkling2Processor) diff --git a/mlx_vlm/prompt_utils.py b/mlx_vlm/prompt_utils.py index 79a6067b..387035a1 100644 --- a/mlx_vlm/prompt_utils.py +++ b/mlx_vlm/prompt_utils.py @@ -122,6 +122,7 @@ MODEL_CONFIG = { "inkling": MessageFormat.LIST_WITH_IMAGE_FIRST, "inkling_mm_model": MessageFormat.LIST_WITH_IMAGE_FIRST, "inkling2": MessageFormat.LIST_WITH_IMAGE_FIRST, + "inkling2_loop": MessageFormat.LIST_WITH_IMAGE_FIRST, # Prompt-only models "florence2": MessageFormat.PROMPT_ONLY, "plamo2vl": MessageFormat.PROMPT_ONLY, diff --git a/mlx_vlm/speculative/drafters/inkling2_dspark/config.py b/mlx_vlm/speculative/drafters/inkling2_dspark/config.py index 262597a0..15726d22 100644 --- a/mlx_vlm/speculative/drafters/inkling2_dspark/config.py +++ b/mlx_vlm/speculative/drafters/inkling2_dspark/config.py @@ -64,6 +64,6 @@ class ModelConfig(BaseModelConfig): len(ids) < 2 or ids != sorted(set(ids)) or ids[0] < 0 - or ids[-1] != text["num_hidden_layers"] + or ids[-1] != len(self.text_config.layer_schedule) ): raise ValueError("Invalid Inkling 2 DSpark target_layer_ids") diff --git a/mlx_vlm/speculative/drafters/inkling2_dspark/dspark.py b/mlx_vlm/speculative/drafters/inkling2_dspark/dspark.py index 197193be..b4ff9f21 100644 --- a/mlx_vlm/speculative/drafters/inkling2_dspark/dspark.py +++ b/mlx_vlm/speculative/drafters/inkling2_dspark/dspark.py @@ -211,7 +211,8 @@ class Model(nn.Module): args = getattr(target, "config", None) if ( args is None - or args.model_type != "inkling2_text" + or args.model_type not in ("inkling2_text", "inkling2_loop_text") + or args.loop_config != self.config.text_config.loop_config or any( getattr(args, name) != getattr(self.config.text_config, name) for name in ("hidden_size", "vocab_size", "num_hidden_layers") diff --git a/mlx_vlm/speculative/drafters/mtp_split.py b/mlx_vlm/speculative/drafters/mtp_split.py index 2c586aa2..a149c0ed 100644 --- a/mlx_vlm/speculative/drafters/mtp_split.py +++ b/mlx_vlm/speculative/drafters/mtp_split.py @@ -351,6 +351,8 @@ MTP_SPLITTERS: Dict[str, str] = { "inkling_mm_model": "mlx_vlm.speculative.drafters.inkling_mtp.split:InklingMTPSplitter", "inkling2": "mlx_vlm.speculative.drafters.inkling2_dspark.split:Inkling2DSparkSplitter", "inkling2_text": "mlx_vlm.speculative.drafters.inkling2_dspark.split:Inkling2DSparkSplitter", + "inkling2_loop": "mlx_vlm.speculative.drafters.inkling2_dspark.split:Inkling2DSparkSplitter", + "inkling2_loop_text": "mlx_vlm.speculative.drafters.inkling2_dspark.split:Inkling2DSparkSplitter", } diff --git a/mlx_vlm/tests/model_cases.json b/mlx_vlm/tests/model_cases.json index ce909d1a..3c21508c 100644 --- a/mlx_vlm/tests/model_cases.json +++ b/mlx_vlm/tests/model_cases.json @@ -1213,6 +1213,79 @@ "checks": ["forward_cache", "language", "vision", "input_embeddings"], "vision": {"input_shape": [4, 1176], "feature_layer": 0, "grid_thw": [[1, 2, 2]]} }, + { + "id": "inkling2_loop", + "language": { + "logits_dtype": "float32" + }, + "module": "inkling2", + "config": { + "image_token_id": 60, + "text_config": { + "hidden_size": 32, + "num_hidden_layers": 5, + "vocab_size": 64, + "unpadded_vocab_size": 64, + "num_attention_heads": 4, + "num_key_value_heads": 2, + "head_dim": 8, + "swa_num_attention_heads": 4, + "swa_num_key_value_heads": 2, + "swa_head_dim": 8, + "sliding_window_size": 4, + "local_layer_ids": [ + 0, + 2, + 4 + ], + "d_rel": 4, + "rel_extent": 4, + "sconv_kernel_size": 4, + "intermediate_size": 32, + "n_routed_experts": 4, + "num_experts_per_tok": 2, + "n_layers_per_attn_res_block": 2, + "loop_config": { + "core_start": 2, + "core_end": 4, + "num_passes": 3, + "gate_rank": 8, + "is_varloop": false + } + }, + "vision_config": { + "native_patch_size": 2, + "adapter_merge_size": 2, + "hidden_size": 16, + "num_hidden_layers": 2, + "num_attention_heads": 2, + "projection_hidden_dim": 32, + "posemb_grid_size": 2 + }, + "model_type": "inkling2_loop" + }, + "checks": [ + "forward_cache", + "language", + "vision" + ], + "vision": { + "input_shape": [ + 2, + 4, + 4, + 3 + ], + "feature_layer": null, + "grid_thw": [ + [ + 1, + 1, + 2 + ] + ] + } + }, { "id": "inkling2", "language": {"logits_dtype": "float32"}, diff --git a/mlx_vlm/tests/test_models.py b/mlx_vlm/tests/test_models.py index cc269be4..2adb290d 100644 --- a/mlx_vlm/tests/test_models.py +++ b/mlx_vlm/tests/test_models.py @@ -7696,11 +7696,12 @@ class TestGemma4PerLayerConfig: @pytest.fixture -def inkling2_model(): +def inkling2_model(request): from mlx_vlm.models.inkling2 import Model, ModelConfig mx.random.seed(194) - model = Model(ModelConfig.from_dict(model_case("inkling2")["config"])) + case = getattr(request, "param", "inkling2") + model = Model(ModelConfig.from_dict(model_case(case)["config"])) for path, value in tree_flatten(model.parameters()): if any(name in path for name in (".query", ".rel_proj", "gate_weight")): model.load_weights( @@ -7722,6 +7723,16 @@ def _inkling2_native_weights(model): key = f"model.llm.attn_res.attn_queries.{int(layer) - 1}" elif sub == "mlp_input.query": key = f"model.llm.attn_res.mlp_queries.{layer}" + elif "_message_gate." in sub: + gate, leaf, _ = sub.split(".") + name = gate.removesuffix("_message_gate") + name = "attn" if name == "q" else name + leaf = { + "down_proj": "down_weight", + "forget_gate_proj": "forget_weight", + "message_gate_proj": "gain_weight", + }[leaf] + key = f"model.llm.layers.{layer}.controllers.{name}.{leaf}" else: for src, dst in ( ("self_attn.q_proj", "attn.wq_du"), @@ -7752,6 +7763,14 @@ def _inkling2_native_weights(model): key = f"model.llm.layers.{layer}.{sub}" elif key.startswith("language_model.model."): sub = key[len("language_model.model.") :] + if sub == "message_input.query": + weights["model.llm.middle_cycle_cross_pass_queries"] = value[ + :, [0, 3], : + ].reshape(-1, value.shape[-1]) + weights["model.llm.middle_cycle_cross_pass_kv_queries"] = value[ + :, 1:3, : + ].reshape(-1, value.shape[-1]) + continue sub = sub.replace("embed_tokens.weight", "embed.weight") sub = sub.replace("final_input.query", "attn_res.output_query") key = "model.llm." + sub @@ -7786,6 +7805,7 @@ def _inkling2_native_weights(model): return weights +@pytest.mark.parametrize("inkling2_model", ["inkling2", "inkling2_loop"], indirect=True) def test_inkling2_native_checkpoint_and_mixed_quantization(inkling2_model, tmp_path): from mlx_vlm.convert import convert @@ -7803,7 +7823,9 @@ def test_inkling2_native_checkpoint_and_mixed_quantization(inkling2_model, tmp_p # Match the release's cast groups: queries and router bias stay FP32, # while dense weights (including router/conv weights) are stored in BF16. for key, value in raw.items(): - if not (".attn_res." in key or key.endswith(".mlp.gate.bias")): + if not ( + ".attn_res." in key or "cross_pass" in key or key.endswith(".mlp.gate.bias") + ): raw[key] = value.astype(mx.bfloat16) elif key.endswith(".mlp.gate.bias"): raw[key] = mx.arange(value.size, dtype=mx.float32) / 129 @@ -7860,6 +7882,11 @@ def test_inkling2_native_checkpoint_and_mixed_quantization(inkling2_model, tmp_p config = json.loads((tmp_path / "config.json").read_text()) config.pop("quantization") config.pop("quantization_config") + if model.config.text_config.loop_config: + config["quantization_config"] = { + "quant_method": "modelopt", + "quant_algo": "NVFP4", + } (tmp_path / "config.json").write_text(json.dumps(config)) mx.save_safetensors(str(tmp_path / "model.safetensors"), raw) loaded = load_model(tmp_path, strict=True) @@ -7909,7 +7936,6 @@ def test_inkling2_native_checkpoint_and_mixed_quantization(inkling2_model, tmp_p target = old[2:] if ".switch_mlp." in key else old assert mx.array_equal(value, target).item(), key converted = tmp_path / "converted" - config["quantization"] = model.quantization_from_metadata(metadata) expected_dtypes = {key: value.dtype for key, value in expected.items()} with patch("mlx_vlm.convert.fetch_from_hub", return_value=(loaded, config, None)): convert(str(tmp_path), str(converted), dtype="bfloat16") @@ -7952,6 +7978,7 @@ def test_inkling2_bf16_head_keeps_fp32_projection_scores(inkling2_model, softcap @pytest.mark.parametrize( "dtype,tolerance", [(mx.float32, 2e-5), (mx.float16, 1e-2), (mx.bfloat16, 8e-2)] ) +@pytest.mark.parametrize("inkling2_model", ["inkling2", "inkling2_loop"], indirect=True) def test_inkling2_cached_chunks_and_speculative_rollback( inkling2_model, dtype, tolerance ): @@ -7986,6 +8013,7 @@ def test_inkling2_cached_chunks_and_speculative_rollback( @pytest.mark.parametrize("dtype", [mx.float32, mx.float16, mx.bfloat16]) +@pytest.mark.parametrize("inkling2_model", ["inkling2", "inkling2_loop"], indirect=True) def test_inkling2_multiple_images_and_embedding_slots(inkling2_model, dtype): model = inkling2_model model.update(tree_map(lambda a: a.astype(dtype), model.parameters())) diff --git a/mlx_vlm/tests/test_speculative.py b/mlx_vlm/tests/test_speculative.py index f5950086..e4ab9688 100644 --- a/mlx_vlm/tests/test_speculative.py +++ b/mlx_vlm/tests/test_speculative.py @@ -60,13 +60,15 @@ DATA = json.loads(Path(__file__).with_name("model_cases.json").read_text())[ ] -def inkling2_dspark_case(dtype=mx.float32): +def inkling2_dspark_case(dtype=mx.float32, family="inkling2"): from mlx_vlm.models.inkling2 import Model, ModelConfig from mlx_vlm.speculative.drafters.inkling2_dspark import Model as Draft from mlx_vlm.speculative.drafters.inkling2_dspark import ModelConfig as DraftConfig - config = deepcopy(model_case("inkling2")["config"]) + config = deepcopy(model_case(family)["config"]) config["mtp_config"] = deepcopy(DATA["inkling2_dspark"]) + if family == "inkling2_loop": + config["mtp_config"]["target_layer_ids"] = [1, 3, 7, 9] target = Model(ModelConfig.from_dict(config)) draft = Draft( DraftConfig( @@ -461,11 +463,12 @@ def test_inkling2_dspark_markov_head_reference(dtype): @parametrize("temperature", [0, 0.7]) @parametrize("prefill_step_size", [None, 2]) @parametrize("dtype", [mx.float32, mx.bfloat16]) +@parametrize("family", ["inkling2", "inkling2_loop"]) def test_inkling2_dspark_standard_generation_and_reset( - temperature, prefill_step_size, dtype, monkeypatch + temperature, prefill_step_size, dtype, family, monkeypatch ): mx.random.seed(73) - target, draft = inkling2_dspark_case(dtype) + target, draft = inkling2_dspark_case(dtype, family=family) calls = [] forward = type(target.language_model).__call__ diff --git a/mlx_vlm/utils.py b/mlx_vlm/utils.py index 86e079e0..b43e7aa8 100644 --- a/mlx_vlm/utils.py +++ b/mlx_vlm/utils.py @@ -79,6 +79,7 @@ MODEL_REMAPPING = { "iquestcoder": "llama", "nemotron-nas": "nemotron_nas", "inkling_mm_model": "inkling", + "inkling2_loop": "inkling2", "lille-130m": "lille_130m", } @@ -547,7 +548,7 @@ python -m mlx_vlm.convert --hf-path --mlx-path if quantization_config is not None: config["quantization_config"] = quantization_config - if quantization_config is None: + if quantization_config is None or hasattr(model, "quantization_from_metadata"): hf_quant_path = model_path / "hf_quant_config.json" if hf_quant_path.exists(): with open(hf_quant_path) as f: @@ -566,6 +567,7 @@ python -m mlx_vlm.convert --hf-path --mlx-path config["quantization"] = config["quantization_config"] = ( native_quantization ) + quantization_config = None if quantization_config is not None: quant_method = quantization_config.get("quant_method") diff --git a/mlx_vlm/version.py b/mlx_vlm/version.py index 777f190d..2811cedf 100644 --- a/mlx_vlm/version.py +++ b/mlx_vlm/version.py @@ -1 +1 @@ -__version__ = "0.8.0" +__version__ = "0.8.0+inkling2mini"