Image-Text-to-Text
MLX
Safetensors
inkling2_loop
conversational
Mixture of Experts
nvfp4
mxfp8
4-bit precision
Instructions to use nativ-community/Inkling-2-Mini-nvfp4 with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- MLX
How to use nativ-community/Inkling-2-Mini-nvfp4 with MLX:
# Make sure mlx-vlm is installed # pip install --upgrade mlx-vlm from mlx_vlm import load, generate from mlx_vlm.prompt_utils import apply_chat_template from mlx_vlm.utils import load_config # Load the model model, processor = load("nativ-community/Inkling-2-Mini-nvfp4") config = load_config("nativ-community/Inkling-2-Mini-nvfp4") # Prepare input image = ["http://images.cocodataset.org/val2017/000000039769.jpg"] prompt = "Describe this image." # Apply chat template formatted_prompt = apply_chat_template( processor, config, prompt, num_images=1 ) # Generate output output = generate(model, processor, formatted_prompt, image) print(output) - Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- LM Studio
- Pi
How to use nativ-community/Inkling-2-Mini-nvfp4 with Pi:
Start the MLX server
# Install MLX LM: uv tool install mlx-lm # Start a local OpenAI-compatible server: mlx_lm.server --model "nativ-community/Inkling-2-Mini-nvfp4"
Configure the model in Pi
# Install Pi: npm install -g @earendil-works/pi-coding-agent # Add to ~/.pi/agent/models.json: { "providers": { "mlx-lm": { "baseUrl": "http://localhost:8080/v1", "api": "openai-completions", "apiKey": "none", "models": [ { "id": "nativ-community/Inkling-2-Mini-nvfp4" } ] } } }Run Pi
# Start Pi in your project directory: pi
- Hermes Agent
How to use nativ-community/Inkling-2-Mini-nvfp4 with Hermes Agent:
Start the MLX server
# Install MLX LM: uv tool install mlx-lm # Start a local OpenAI-compatible server: mlx_lm.server --model "nativ-community/Inkling-2-Mini-nvfp4"
Configure Hermes
# Install Hermes: curl -fsSL https://hermes-agent.nousresearch.com/install.sh | bash hermes setup # Point Hermes at the local server: hermes config set model.provider custom hermes config set model.base_url http://127.0.0.1:8080/v1 hermes config set model.default nativ-community/Inkling-2-Mini-nvfp4
Run Hermes
hermes
- Atomic Chat
- OpenClaw
How to use nativ-community/Inkling-2-Mini-nvfp4 with OpenClaw:
Start the MLX server
# Install MLX LM: uv tool install mlx-lm # Start a local OpenAI-compatible server: mlx_lm.server --model "nativ-community/Inkling-2-Mini-nvfp4"
Configure OpenClaw
# Install OpenClaw: npm install -g openclaw@latest # Register the local server and set it as the default model: openclaw onboard --non-interactive --mode local \ --auth-choice custom-api-key \ --custom-base-url http://127.0.0.1:8080/v1 \ --custom-model-id "nativ-community/Inkling-2-Mini-nvfp4" \ --custom-provider-id mlx-lm \ --custom-compatibility openai \ --custom-text-input \ --accept-risk \ --skip-health
Run OpenClaw
openclaw agent --local --agent main --message "Hello from Hugging Face"
Download runtime/inkling2-mini.patch from nativ-community/Inkling-2-Mini-nvfp4: direct link, hf CLI and curl.
- Browser
- Download file 27.3 kB
-
https://huggingface.co/nativ-community/Inkling-2-Mini-nvfp4/resolve/main/runtime/inkling2-mini.patch
- Command line
-
hf download hf://nativ-community/Inkling-2-Mini-nvfp4/runtime/inkling2-mini.patch
-
curl -L -o inkling2-mini.patch https://huggingface.co/nativ-community/Inkling-2-Mini-nvfp4/resolve/main/runtime/inkling2-mini.patch
27.3 kB
| diff --git a/mlx_vlm/convert.py b/mlx_vlm/convert.py | |
| index e7b032f5..0b884aea 100644 | |
| --- a/mlx_vlm/convert.py | |
| +++ b/mlx_vlm/convert.py | |
| def convert( | |
| model, config, processor = fetch_from_hub( | |
| model_path, lazy=True, trust_remote_code=trust_remote_code | |
| ) | |
| + # Native packed exports can declare only a generic quantization family in | |
| + # config.json, with the actual per-layer precision in a sidecar. Preserve | |
| + # that precision when saving the sanitized MLX weights. | |
| + metadata_path = model_path / "hf_quant_config.json" | |
| + if ( | |
| + "quantization" not in config | |
| + and hasattr(model, "quantization_from_metadata") | |
| + and metadata_path.exists() | |
| + ): | |
| + with metadata_path.open() as f: | |
| + native_quantization = model.quantization_from_metadata(json.load(f)) | |
| + if native_quantization is not None: | |
| + config["quantization"] = config["quantization_config"] = native_quantization | |
| model_quant_predicate = getattr(model, "quant_predicate", None) | |
| diff --git a/mlx_vlm/models/inkling2/config.py b/mlx_vlm/models/inkling2/config.py | |
| index 8e3b7b20..f6e4f1e4 100644 | |
| --- a/mlx_vlm/models/inkling2/config.py | |
| +++ b/mlx_vlm/models/inkling2/config.py | |
| class TextConfig(BaseModelConfig): | |
| mlp_gate_softcap: float = 4.0 | |
| mlp_up_softcap: float = 25.0 | |
| final_logit_softcapping: float = 30.0 | |
| + loop_config: Optional[dict] = None | |
| + | |
| + def __post_init__(self): | |
| + if self.loop_config is None: | |
| + return | |
| + cfg = self.loop_config | |
| + start, end = cfg["core_start"], cfg["core_end"] | |
| + block = self.n_layers_per_attn_res_block | |
| + if ( | |
| + not 0 < start < end <= self.num_hidden_layers | |
| + or start % block | |
| + or end % block | |
| + or cfg["num_passes"] < 1 | |
| + or cfg["gate_rank"] < 1 | |
| + or cfg.get("is_varloop", False) | |
| + ): | |
| + raise ValueError("Invalid or unsupported Inkling 2 loop configuration") | |
| + | |
| + @property | |
| + def layer_schedule(self): | |
| + if self.loop_config is None: | |
| + return list(range(self.num_hidden_layers)) | |
| + start, end = self.loop_config["core_start"], self.loop_config["core_end"] | |
| + return ( | |
| + list(range(start)) | |
| + + list(range(start, end)) * self.loop_config["num_passes"] | |
| + + list(range(end, self.num_hidden_layers)) | |
| + ) | |
| def layer_is_sliding(self, index): | |
| if self.layer_types is not None: | |
| diff --git a/mlx_vlm/models/inkling2/inkling2.py b/mlx_vlm/models/inkling2/inkling2.py | |
| index d2173a54..08c04909 100644 | |
| --- a/mlx_vlm/models/inkling2/inkling2.py | |
| +++ b/mlx_vlm/models/inkling2/inkling2.py | |
| class Model(nn.Module): | |
| if sub.startswith("mlp.experts."): | |
| experts.setdefault(layer, {})[sub[len("mlp.experts.") :]] = value | |
| continue | |
| + if sub.startswith("controllers."): | |
| + _, name, leaf = sub.split(".") | |
| + name = {"attn": "q", "k": "k", "v": "v", "mlp": "mlp"}[name] | |
| + leaf = { | |
| + "down_weight": "down_proj", | |
| + "forget_weight": "forget_gate_proj", | |
| + "gain_weight": "message_gate_proj", | |
| + }[leaf] | |
| + out[base + name + "_message_gate." + leaf + ".weight"] = value | |
| + continue | |
| if sub.startswith("attn."): | |
| sub = sub[len("attn.") :] | |
| name, _, leaf = sub.partition(".") | |
| class Model(nn.Module): | |
| out["language_model.model." + target] = value | |
| elif key.startswith("model.llm."): | |
| sub = key[len("model.llm.") :] | |
| + if sub.startswith("middle_cycle_cross_pass_"): | |
| + continue | |
| target = ( | |
| "language_model.lm_head.weight" | |
| if sub == "unembed.weight" | |
| class Model(nn.Module): | |
| continue | |
| else: | |
| out[key] = value | |
| + query_key = "model.llm.middle_cycle_cross_pass_queries" | |
| + kv_key = "model.llm.middle_cycle_cross_pass_kv_queries" | |
| + if query_key in weights or kv_key in weights: | |
| + hidden = self.config.text_config.hidden_size | |
| + queries = weights[query_key].reshape(-1, 2, hidden) | |
| + kv_queries = weights[kv_key].reshape(-1, 2, hidden) | |
| + out["language_model.model.message_input.query"] = mx.concatenate( | |
| + [queries[:, :1], kv_queries, queries[:, 1:]], axis=1 | |
| + ) | |
| for layer, tensors in experts.items(): | |
| base = f"language_model.model.layers.{layer}.mlp.switch_mlp." | |
| for name, projections in ( | |
| diff --git a/mlx_vlm/models/inkling2/language.py b/mlx_vlm/models/inkling2/language.py | |
| index 805dcc1f..023226db 100644 | |
| --- a/mlx_vlm/models/inkling2/language.py | |
| +++ b/mlx_vlm/models/inkling2/language.py | |
| class AttentionResidual(nn.Module): | |
| ) | |
| +class MessageGate(nn.Module): | |
| + """Mix a previous-pass message into the current attention-residual input.""" | |
| + | |
| + def __init__(self, config): | |
| + super().__init__() | |
| + rank = config.loop_config["gate_rank"] | |
| + self.eps = config.rms_norm_eps | |
| + self.down_proj = nn.Linear(2 * config.hidden_size, rank, bias=False) | |
| + self.forget_gate_proj = nn.Linear(rank, config.hidden_size, bias=False) | |
| + self.message_gate_proj = nn.Linear(rank, config.hidden_size, bias=False) | |
| + | |
| + def __call__(self, hidden, message): | |
| + hidden, message = hidden.astype(mx.float32), message.astype(mx.float32) | |
| + inputs = mx.concatenate( | |
| + [mx.fast.rms_norm(x, None, self.eps) for x in (hidden, message)], axis=-1 | |
| + ) | |
| + gate = self.down_proj(inputs.astype(self.down_proj.weight.dtype)).astype( | |
| + mx.float32 | |
| + ) | |
| + forget = mx.tanh(gate @ self.forget_gate_proj.weight.astype(mx.float32).T) | |
| + gain = mx.tanh(gate @ self.message_gate_proj.weight.astype(mx.float32).T) | |
| + return (1 - forget) * hidden + gain * message | |
| + | |
| + | |
| +class MessageAttentionResidual(nn.Module): | |
| + def __init__(self, config): | |
| + super().__init__() | |
| + loop = config.loop_config | |
| + self.query = mx.zeros( | |
| + (loop["core_end"] - loop["core_start"], 4, config.hidden_size), mx.float32 | |
| + ) | |
| + | |
| + def __call__(self, blocks, partial, layer_index): | |
| + return _attention_residual( | |
| + mx.stack([*blocks, partial], axis=-2), self.query[layer_index], True | |
| + ) | |
| + | |
| + | |
| class DecoderLayer(nn.Module): | |
| def __init__(self, config, layer_idx): | |
| super().__init__() | |
| class DecoderLayer(nn.Module): | |
| self.mlp_sconv = ShortConvolution( | |
| config.hidden_size, config.sconv_kernel_size, 3 | |
| ) | |
| + loop = config.loop_config | |
| + if loop and loop["core_start"] <= layer_idx < loop["core_end"]: | |
| + for name in ("q", "k", "v", "mlp"): | |
| + self[name + "_message_gate"] = MessageGate(config) | |
| - def __call__(self, x, blocks, cache=None): | |
| + def __call__(self, x, blocks, cache=None, messages=None): | |
| conv = cache[1] if cache is not None else None | |
| mask = _cache_padding_mask(conv, x.shape[1]) | |
| mixed = ( | |
| class DecoderLayer(nn.Module): | |
| if self.attention_input is not None | |
| else x[..., None, :] | |
| ) | |
| - if self.is_sliding: | |
| + if self.is_sliding and messages is None: | |
| attn = self.self_attn( | |
| self.input_layernorm(mixed[..., 0, :]).astype(x.dtype), | |
| cache=cache, | |
| class DecoderLayer(nn.Module): | |
| ) | |
| else: | |
| parts = [mixed[..., min(i, mixed.shape[-2] - 1), :] for i in range(3)] | |
| - parts = [ | |
| - norm(part).astype(x.dtype) | |
| - for norm, part in zip( | |
| - ( | |
| - self.q_input_layernorm, | |
| - self.k_input_layernorm, | |
| - self.v_input_layernorm, | |
| - ), | |
| - parts, | |
| + if messages is not None: | |
| + parts = [ | |
| + self[name + "_message_gate"](part, messages[..., i, :]) | |
| + for i, (name, part) in enumerate(zip(("q", "k", "v"), parts)) | |
| + ] | |
| + norms = ( | |
| + (self.input_layernorm,) * 3 | |
| + if self.is_sliding | |
| + else ( | |
| + self.q_input_layernorm, | |
| + self.k_input_layernorm, | |
| + self.v_input_layernorm, | |
| ) | |
| - ] | |
| + ) | |
| + parts = [norm(part).astype(x.dtype) for norm, part in zip(norms, parts)] | |
| attn = self.self_attn(*parts, cache=cache, conv_mask=mask) | |
| residual = self.attn_sconv(attn, conv, mask).astype(mx.float32) | |
| if self.is_block_boundary: | |
| class DecoderLayer(nn.Module): | |
| else: | |
| residual = residual + x.astype(mx.float32) | |
| hidden = self.mlp_input(blocks, residual)[..., 0, :] | |
| + if messages is not None: | |
| + hidden = self.mlp_message_gate(hidden, messages[..., 3, :]) | |
| hidden = self.mlp(self.post_attention_layernorm(hidden).astype(x.dtype)) | |
| out = (residual + self.mlp_sconv(hidden, conv, mask).astype(mx.float32)).astype( | |
| x.dtype | |
| class DecoderLayer(nn.Module): | |
| class TextModel(nn.Module): | |
| def __init__(self, config): | |
| super().__init__() | |
| + self.config = config | |
| self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size) | |
| self.embed_norm = nn.RMSNorm(config.hidden_size, eps=config.rms_norm_eps) | |
| self.layers = [DecoderLayer(config, i) for i in range(config.num_hidden_layers)] | |
| self.final_input = AttentionResidual(config) | |
| self.norm = nn.RMSNorm(config.hidden_size, eps=config.rms_norm_eps) | |
| + if config.loop_config: | |
| + self.message_input = MessageAttentionResidual(config) | |
| def embed(self, inputs): | |
| return self.embed_norm(self.embed_tokens(inputs)) | |
| class TextModel(nn.Module): | |
| if capture_layer_ids is not None and ( | |
| len(capture_layer_ids) != len(set(capture_layer_ids)) | |
| or any( | |
| - type(i) is not int or not 0 <= i <= len(self.layers) | |
| + type(i) is not int or not 0 <= i <= len(self.config.layer_schedule) | |
| for i in capture_layer_ids | |
| ) | |
| ): | |
| class TextModel(nn.Module): | |
| requested = set(capture_layer_ids or []) | |
| x = self.embed(inputs) if input_embeddings is None else input_embeddings | |
| blocks = [] | |
| - for i, (layer, c) in enumerate( | |
| - zip(self.layers, cache or [None] * len(self.layers)) | |
| - ): | |
| + schedule = self.config.layer_schedule | |
| + if cache is not None and len(cache) != len(schedule): | |
| + raise ValueError("Inkling 2 requires a separate cache for each layer visit") | |
| + caches = cache if cache is not None else [None] * len(schedule) | |
| + loop = self.config.loop_config | |
| + pre_loop = previous = None | |
| + for i, (layer_index, c) in enumerate(zip(schedule, caches)): | |
| + layer = self.layers[layer_index] | |
| + messages = None | |
| + if loop and layer_index == loop["core_start"]: | |
| + if pre_loop is None: | |
| + pre_loop = (x, list(blocks)) | |
| + else: | |
| + previous = (blocks[len(pre_loop[1]) + 1 :], x) | |
| + x, blocks = pre_loop[0], list(pre_loop[1]) | |
| + if previous is not None and layer_index < loop["core_end"]: | |
| + messages = self.message_input( | |
| + *previous, layer_index - loop["core_start"] | |
| + ) | |
| # DSpark taps the partial block entering the layer, not its output. | |
| if i in requested: | |
| captures[i] = x | |
| - x = layer(x, blocks, c) | |
| + x = layer(x, blocks, c, messages=messages) | |
| if speculative_verify and (i + 1) % 4 == 0: | |
| mx.async_eval(x) | |
| h = self.final_input(blocks, x)[..., 0, :] | |
| normalized = self.norm(h).astype(x.dtype) | |
| - if len(self.layers) in requested: | |
| - captures[len(self.layers)] = normalized | |
| + if len(schedule) in requested: | |
| + captures[len(schedule)] = normalized | |
| result = h if skip_final_norm else normalized | |
| if capture_layer_ids is not None: | |
| return result, [captures[i] for i in capture_layer_ids] | |
| class LanguageModel(InklingLanguageModel): | |
| ), | |
| ArraysCache(4), | |
| ) | |
| - for layer in self.model.layers | |
| + for layer in (self.model.layers[i] for i in self.config.layer_schedule) | |
| ] | |
| def speculative_verify_hidden(self, inputs, cache): | |
| diff --git a/mlx_vlm/models/inkling2/processing_inkling2.py b/mlx_vlm/models/inkling2/processing_inkling2.py | |
| index ffefc0a1..d640a644 100644 | |
| --- a/mlx_vlm/models/inkling2/processing_inkling2.py | |
| +++ b/mlx_vlm/models/inkling2/processing_inkling2.py | |
| class Inkling2Processor(ProcessorMixin): | |
| return ["input_ids", "attention_mask", "pixel_values", "image_grid_thw"] | |
| -install_auto_processor_patch(["inkling2"], Inkling2Processor) | |
| +install_auto_processor_patch(["inkling2", "inkling2_loop"], Inkling2Processor) | |
| diff --git a/mlx_vlm/prompt_utils.py b/mlx_vlm/prompt_utils.py | |
| index 79a6067b..387035a1 100644 | |
| --- a/mlx_vlm/prompt_utils.py | |
| +++ b/mlx_vlm/prompt_utils.py | |
| MODEL_CONFIG = { | |
| "inkling": MessageFormat.LIST_WITH_IMAGE_FIRST, | |
| "inkling_mm_model": MessageFormat.LIST_WITH_IMAGE_FIRST, | |
| "inkling2": MessageFormat.LIST_WITH_IMAGE_FIRST, | |
| + "inkling2_loop": MessageFormat.LIST_WITH_IMAGE_FIRST, | |
| # Prompt-only models | |
| "florence2": MessageFormat.PROMPT_ONLY, | |
| "plamo2vl": MessageFormat.PROMPT_ONLY, | |
| diff --git a/mlx_vlm/speculative/drafters/inkling2_dspark/config.py b/mlx_vlm/speculative/drafters/inkling2_dspark/config.py | |
| index 262597a0..15726d22 100644 | |
| --- a/mlx_vlm/speculative/drafters/inkling2_dspark/config.py | |
| +++ b/mlx_vlm/speculative/drafters/inkling2_dspark/config.py | |
| class ModelConfig(BaseModelConfig): | |
| len(ids) < 2 | |
| or ids != sorted(set(ids)) | |
| or ids[0] < 0 | |
| - or ids[-1] != text["num_hidden_layers"] | |
| + or ids[-1] != len(self.text_config.layer_schedule) | |
| ): | |
| raise ValueError("Invalid Inkling 2 DSpark target_layer_ids") | |
| diff --git a/mlx_vlm/speculative/drafters/inkling2_dspark/dspark.py b/mlx_vlm/speculative/drafters/inkling2_dspark/dspark.py | |
| index 197193be..b4ff9f21 100644 | |
| --- a/mlx_vlm/speculative/drafters/inkling2_dspark/dspark.py | |
| +++ b/mlx_vlm/speculative/drafters/inkling2_dspark/dspark.py | |
| class Model(nn.Module): | |
| args = getattr(target, "config", None) | |
| if ( | |
| args is None | |
| - or args.model_type != "inkling2_text" | |
| + or args.model_type not in ("inkling2_text", "inkling2_loop_text") | |
| + or args.loop_config != self.config.text_config.loop_config | |
| or any( | |
| getattr(args, name) != getattr(self.config.text_config, name) | |
| for name in ("hidden_size", "vocab_size", "num_hidden_layers") | |
| diff --git a/mlx_vlm/speculative/drafters/mtp_split.py b/mlx_vlm/speculative/drafters/mtp_split.py | |
| index 2c586aa2..a149c0ed 100644 | |
| --- a/mlx_vlm/speculative/drafters/mtp_split.py | |
| +++ b/mlx_vlm/speculative/drafters/mtp_split.py | |
| MTP_SPLITTERS: Dict[str, str] = { | |
| "inkling_mm_model": "mlx_vlm.speculative.drafters.inkling_mtp.split:InklingMTPSplitter", | |
| "inkling2": "mlx_vlm.speculative.drafters.inkling2_dspark.split:Inkling2DSparkSplitter", | |
| "inkling2_text": "mlx_vlm.speculative.drafters.inkling2_dspark.split:Inkling2DSparkSplitter", | |
| + "inkling2_loop": "mlx_vlm.speculative.drafters.inkling2_dspark.split:Inkling2DSparkSplitter", | |
| + "inkling2_loop_text": "mlx_vlm.speculative.drafters.inkling2_dspark.split:Inkling2DSparkSplitter", | |
| } | |
| diff --git a/mlx_vlm/tests/model_cases.json b/mlx_vlm/tests/model_cases.json | |
| index ce909d1a..3c21508c 100644 | |
| --- a/mlx_vlm/tests/model_cases.json | |
| +++ b/mlx_vlm/tests/model_cases.json | |
| "checks": ["forward_cache", "language", "vision", "input_embeddings"], | |
| "vision": {"input_shape": [4, 1176], "feature_layer": 0, "grid_thw": [[1, 2, 2]]} | |
| }, | |
| + { | |
| + "id": "inkling2_loop", | |
| + "language": { | |
| + "logits_dtype": "float32" | |
| + }, | |
| + "module": "inkling2", | |
| + "config": { | |
| + "image_token_id": 60, | |
| + "text_config": { | |
| + "hidden_size": 32, | |
| + "num_hidden_layers": 5, | |
| + "vocab_size": 64, | |
| + "unpadded_vocab_size": 64, | |
| + "num_attention_heads": 4, | |
| + "num_key_value_heads": 2, | |
| + "head_dim": 8, | |
| + "swa_num_attention_heads": 4, | |
| + "swa_num_key_value_heads": 2, | |
| + "swa_head_dim": 8, | |
| + "sliding_window_size": 4, | |
| + "local_layer_ids": [ | |
| + 0, | |
| + 2, | |
| + 4 | |
| + ], | |
| + "d_rel": 4, | |
| + "rel_extent": 4, | |
| + "sconv_kernel_size": 4, | |
| + "intermediate_size": 32, | |
| + "n_routed_experts": 4, | |
| + "num_experts_per_tok": 2, | |
| + "n_layers_per_attn_res_block": 2, | |
| + "loop_config": { | |
| + "core_start": 2, | |
| + "core_end": 4, | |
| + "num_passes": 3, | |
| + "gate_rank": 8, | |
| + "is_varloop": false | |
| + } | |
| + }, | |
| + "vision_config": { | |
| + "native_patch_size": 2, | |
| + "adapter_merge_size": 2, | |
| + "hidden_size": 16, | |
| + "num_hidden_layers": 2, | |
| + "num_attention_heads": 2, | |
| + "projection_hidden_dim": 32, | |
| + "posemb_grid_size": 2 | |
| + }, | |
| + "model_type": "inkling2_loop" | |
| + }, | |
| + "checks": [ | |
| + "forward_cache", | |
| + "language", | |
| + "vision" | |
| + ], | |
| + "vision": { | |
| + "input_shape": [ | |
| + 2, | |
| + 4, | |
| + 4, | |
| + 3 | |
| + ], | |
| + "feature_layer": null, | |
| + "grid_thw": [ | |
| + [ | |
| + 1, | |
| + 1, | |
| + 2 | |
| + ] | |
| + ] | |
| + } | |
| + }, | |
| { | |
| "id": "inkling2", | |
| "language": {"logits_dtype": "float32"}, | |
| diff --git a/mlx_vlm/tests/test_models.py b/mlx_vlm/tests/test_models.py | |
| index cc269be4..2adb290d 100644 | |
| --- a/mlx_vlm/tests/test_models.py | |
| +++ b/mlx_vlm/tests/test_models.py | |
| class TestGemma4PerLayerConfig: | |
| @pytest.fixture | |
| -def inkling2_model(): | |
| +def inkling2_model(request): | |
| from mlx_vlm.models.inkling2 import Model, ModelConfig | |
| mx.random.seed(194) | |
| - model = Model(ModelConfig.from_dict(model_case("inkling2")["config"])) | |
| + case = getattr(request, "param", "inkling2") | |
| + model = Model(ModelConfig.from_dict(model_case(case)["config"])) | |
| for path, value in tree_flatten(model.parameters()): | |
| if any(name in path for name in (".query", ".rel_proj", "gate_weight")): | |
| model.load_weights( | |
| def _inkling2_native_weights(model): | |
| key = f"model.llm.attn_res.attn_queries.{int(layer) - 1}" | |
| elif sub == "mlp_input.query": | |
| key = f"model.llm.attn_res.mlp_queries.{layer}" | |
| + elif "_message_gate." in sub: | |
| + gate, leaf, _ = sub.split(".") | |
| + name = gate.removesuffix("_message_gate") | |
| + name = "attn" if name == "q" else name | |
| + leaf = { | |
| + "down_proj": "down_weight", | |
| + "forget_gate_proj": "forget_weight", | |
| + "message_gate_proj": "gain_weight", | |
| + }[leaf] | |
| + key = f"model.llm.layers.{layer}.controllers.{name}.{leaf}" | |
| else: | |
| for src, dst in ( | |
| ("self_attn.q_proj", "attn.wq_du"), | |
| def _inkling2_native_weights(model): | |
| key = f"model.llm.layers.{layer}.{sub}" | |
| elif key.startswith("language_model.model."): | |
| sub = key[len("language_model.model.") :] | |
| + if sub == "message_input.query": | |
| + weights["model.llm.middle_cycle_cross_pass_queries"] = value[ | |
| + :, [0, 3], : | |
| + ].reshape(-1, value.shape[-1]) | |
| + weights["model.llm.middle_cycle_cross_pass_kv_queries"] = value[ | |
| + :, 1:3, : | |
| + ].reshape(-1, value.shape[-1]) | |
| + continue | |
| sub = sub.replace("embed_tokens.weight", "embed.weight") | |
| sub = sub.replace("final_input.query", "attn_res.output_query") | |
| key = "model.llm." + sub | |
| def _inkling2_native_weights(model): | |
| return weights | |
| +@pytest.mark.parametrize("inkling2_model", ["inkling2", "inkling2_loop"], indirect=True) | |
| def test_inkling2_native_checkpoint_and_mixed_quantization(inkling2_model, tmp_path): | |
| from mlx_vlm.convert import convert | |
| def test_inkling2_native_checkpoint_and_mixed_quantization(inkling2_model, tmp_p | |
| # Match the release's cast groups: queries and router bias stay FP32, | |
| # while dense weights (including router/conv weights) are stored in BF16. | |
| for key, value in raw.items(): | |
| - if not (".attn_res." in key or key.endswith(".mlp.gate.bias")): | |
| + if not ( | |
| + ".attn_res." in key or "cross_pass" in key or key.endswith(".mlp.gate.bias") | |
| + ): | |
| raw[key] = value.astype(mx.bfloat16) | |
| elif key.endswith(".mlp.gate.bias"): | |
| raw[key] = mx.arange(value.size, dtype=mx.float32) / 129 | |
| def test_inkling2_native_checkpoint_and_mixed_quantization(inkling2_model, tmp_p | |
| config = json.loads((tmp_path / "config.json").read_text()) | |
| config.pop("quantization") | |
| config.pop("quantization_config") | |
| + if model.config.text_config.loop_config: | |
| + config["quantization_config"] = { | |
| + "quant_method": "modelopt", | |
| + "quant_algo": "NVFP4", | |
| + } | |
| (tmp_path / "config.json").write_text(json.dumps(config)) | |
| mx.save_safetensors(str(tmp_path / "model.safetensors"), raw) | |
| loaded = load_model(tmp_path, strict=True) | |
| def test_inkling2_native_checkpoint_and_mixed_quantization(inkling2_model, tmp_p | |
| target = old[2:] if ".switch_mlp." in key else old | |
| assert mx.array_equal(value, target).item(), key | |
| converted = tmp_path / "converted" | |
| - config["quantization"] = model.quantization_from_metadata(metadata) | |
| expected_dtypes = {key: value.dtype for key, value in expected.items()} | |
| with patch("mlx_vlm.convert.fetch_from_hub", return_value=(loaded, config, None)): | |
| convert(str(tmp_path), str(converted), dtype="bfloat16") | |
| def test_inkling2_bf16_head_keeps_fp32_projection_scores(inkling2_model, softcap | |
| @pytest.mark.parametrize( | |
| "dtype,tolerance", [(mx.float32, 2e-5), (mx.float16, 1e-2), (mx.bfloat16, 8e-2)] | |
| ) | |
| +@pytest.mark.parametrize("inkling2_model", ["inkling2", "inkling2_loop"], indirect=True) | |
| def test_inkling2_cached_chunks_and_speculative_rollback( | |
| inkling2_model, dtype, tolerance | |
| ): | |
| def test_inkling2_cached_chunks_and_speculative_rollback( | |
| @pytest.mark.parametrize("dtype", [mx.float32, mx.float16, mx.bfloat16]) | |
| +@pytest.mark.parametrize("inkling2_model", ["inkling2", "inkling2_loop"], indirect=True) | |
| def test_inkling2_multiple_images_and_embedding_slots(inkling2_model, dtype): | |
| model = inkling2_model | |
| model.update(tree_map(lambda a: a.astype(dtype), model.parameters())) | |
| diff --git a/mlx_vlm/tests/test_speculative.py b/mlx_vlm/tests/test_speculative.py | |
| index f5950086..e4ab9688 100644 | |
| --- a/mlx_vlm/tests/test_speculative.py | |
| +++ b/mlx_vlm/tests/test_speculative.py | |
| DATA = json.loads(Path(__file__).with_name("model_cases.json").read_text())[ | |
| ] | |
| -def inkling2_dspark_case(dtype=mx.float32): | |
| +def inkling2_dspark_case(dtype=mx.float32, family="inkling2"): | |
| from mlx_vlm.models.inkling2 import Model, ModelConfig | |
| from mlx_vlm.speculative.drafters.inkling2_dspark import Model as Draft | |
| from mlx_vlm.speculative.drafters.inkling2_dspark import ModelConfig as DraftConfig | |
| - config = deepcopy(model_case("inkling2")["config"]) | |
| + config = deepcopy(model_case(family)["config"]) | |
| config["mtp_config"] = deepcopy(DATA["inkling2_dspark"]) | |
| + if family == "inkling2_loop": | |
| + config["mtp_config"]["target_layer_ids"] = [1, 3, 7, 9] | |
| target = Model(ModelConfig.from_dict(config)) | |
| draft = Draft( | |
| DraftConfig( | |
| def test_inkling2_dspark_markov_head_reference(dtype): | |
| @parametrize("temperature", [0, 0.7]) | |
| @parametrize("prefill_step_size", [None, 2]) | |
| @parametrize("dtype", [mx.float32, mx.bfloat16]) | |
| +@parametrize("family", ["inkling2", "inkling2_loop"]) | |
| def test_inkling2_dspark_standard_generation_and_reset( | |
| - temperature, prefill_step_size, dtype, monkeypatch | |
| + temperature, prefill_step_size, dtype, family, monkeypatch | |
| ): | |
| mx.random.seed(73) | |
| - target, draft = inkling2_dspark_case(dtype) | |
| + target, draft = inkling2_dspark_case(dtype, family=family) | |
| calls = [] | |
| forward = type(target.language_model).__call__ | |
| diff --git a/mlx_vlm/utils.py b/mlx_vlm/utils.py | |
| index 86e079e0..b43e7aa8 100644 | |
| --- a/mlx_vlm/utils.py | |
| +++ b/mlx_vlm/utils.py | |
| MODEL_REMAPPING = { | |
| "iquestcoder": "llama", | |
| "nemotron-nas": "nemotron_nas", | |
| "inkling_mm_model": "inkling", | |
| + "inkling2_loop": "inkling2", | |
| "lille-130m": "lille_130m", | |
| } | |
| python -m mlx_vlm.convert --hf-path <local_dir> --mlx-path <mlx_dir> | |
| if quantization_config is not None: | |
| config["quantization_config"] = quantization_config | |
| - if quantization_config is None: | |
| + if quantization_config is None or hasattr(model, "quantization_from_metadata"): | |
| hf_quant_path = model_path / "hf_quant_config.json" | |
| if hf_quant_path.exists(): | |
| with open(hf_quant_path) as f: | |
| python -m mlx_vlm.convert --hf-path <local_dir> --mlx-path <mlx_dir> | |
| config["quantization"] = config["quantization_config"] = ( | |
| native_quantization | |
| ) | |
| + quantization_config = None | |
| if quantization_config is not None: | |
| quant_method = quantization_config.get("quant_method") | |
| diff --git a/mlx_vlm/version.py b/mlx_vlm/version.py | |
| index 777f190d..2811cedf 100644 | |
| --- a/mlx_vlm/version.py | |
| +++ b/mlx_vlm/version.py | |
| @@ -1 +1 @@ | |
| -__version__ = "0.8.0" | |
| +__version__ = "0.8.0+inkling2mini" | |