Text Generation
MLX
Safetensors
modilify_mk2
diffusion
mixture-of-experts
custom-code
modilify-mk2
conversational
Instructions to use modilify/Modilify-Mk2-mlx with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- MLX
How to use modilify/Modilify-Mk2-mlx with MLX:
# Make sure mlx-lm is installed # pip install --upgrade mlx-lm # Generate text with mlx-lm from mlx_lm import load, generate model, tokenizer = load("modilify/Modilify-Mk2-mlx") prompt = "Write a story about Einstein" messages = [{"role": "user", "content": prompt}] prompt = tokenizer.apply_chat_template( messages, add_generation_prompt=True ) text = generate(model, tokenizer, prompt=prompt, verbose=True) - Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- LM Studio
- Pi
How to use modilify/Modilify-Mk2-mlx with Pi:
Start the MLX server
# Install MLX LM: uv tool install mlx-lm # Start a local OpenAI-compatible server: mlx_lm.server --model "modilify/Modilify-Mk2-mlx"
Configure the model in Pi
# Install Pi: npm install -g @earendil-works/pi-coding-agent # Add to ~/.pi/agent/models.json: { "providers": { "mlx-lm": { "baseUrl": "http://localhost:8080/v1", "api": "openai-completions", "apiKey": "none", "models": [ { "id": "modilify/Modilify-Mk2-mlx" } ] } } }Run Pi
# Start Pi in your project directory: pi
- MLX LM
How to use modilify/Modilify-Mk2-mlx with MLX LM:
Generate or start a chat session
# Install MLX LM uv tool install mlx-lm # Interactive chat REPL mlx_lm.chat --model "modilify/Modilify-Mk2-mlx"
Run an OpenAI-compatible server
# Install MLX LM uv tool install mlx-lm # Start the server mlx_lm.server --model "modilify/Modilify-Mk2-mlx" # Calling the OpenAI-compatible server with curl curl -X POST "http://localhost:8080/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "modilify/Modilify-Mk2-mlx", "messages": [ {"role": "user", "content": "Hello"} ] }' - Hermes Agent
How to use modilify/Modilify-Mk2-mlx with Hermes Agent:
Start the MLX server
# Install MLX LM: uv tool install mlx-lm # Start a local OpenAI-compatible server: mlx_lm.server --model "modilify/Modilify-Mk2-mlx"
Configure Hermes
# Install Hermes: curl -fsSL https://hermes-agent.nousresearch.com/install.sh | bash hermes setup # Point Hermes at the local server: hermes config set model.provider custom hermes config set model.base_url http://127.0.0.1:8080/v1 hermes config set model.default modilify/Modilify-Mk2-mlx
Run Hermes
hermes
- Atomic Chat
- OpenClaw
How to use modilify/Modilify-Mk2-mlx with OpenClaw:
Start the MLX server
# Install MLX LM: uv tool install mlx-lm # Start a local OpenAI-compatible server: mlx_lm.server --model "modilify/Modilify-Mk2-mlx"
Configure OpenClaw
# Install OpenClaw: npm install -g openclaw@latest # Register the local server and set it as the default model: openclaw onboard --non-interactive --mode local \ --auth-choice custom-api-key \ --custom-base-url http://127.0.0.1:8080/v1 \ --custom-model-id "modilify/Modilify-Mk2-mlx" \ --custom-provider-id mlx-lm \ --custom-compatibility openai \ --custom-text-input \ --accept-risk \ --skip-health
Run OpenClaw
openclaw agent --local --agent main --message "Hello from Hugging Face"
Download modilify_mk2_mlx/model.py from modilify/Modilify-Mk2-mlx: direct link, hf CLI and curl.
- Browser
- Download file 35.3 kB
-
https://huggingface.co/modilify/Modilify-Mk2-mlx/resolve/main/modilify_mk2_mlx/model.py
- Command line
-
hf download hf://modilify/Modilify-Mk2-mlx/modilify_mk2_mlx/model.py
-
curl -L -o model.py https://huggingface.co/modilify/Modilify-Mk2-mlx/resolve/main/modilify_mk2_mlx/model.py
35.3 kB
| """Schema25 MLX model, trajectory memory, and strict release loading.""" | |
| from __future__ import annotations | |
| from collections import Counter, defaultdict | |
| from dataclasses import dataclass, replace | |
| import hashlib | |
| import json | |
| import math | |
| from pathlib import Path | |
| from types import SimpleNamespace | |
| from typing import Any | |
| import weakref | |
| import mlx.core as mx | |
| from mlx import nn | |
| from mlx.utils import tree_flatten | |
| from mlx_lm.tuner.lora import LoRALinear, LoRASwitchLinear | |
| from mlx_vlm.models.diffusion_gemma.config import ModelConfig | |
| from mlx_vlm.models.diffusion_gemma.diffusion_gemma import Model | |
| from mlx_vlm.models.diffusion_gemma.language import DecoderLayer, Router | |
| def _l2(value: mx.array) -> mx.array: | |
| return value * mx.rsqrt(mx.maximum(mx.sum(mx.square(value), axis=-1, keepdims=True), 1e-12)) | |
| class _GateProjection(nn.Module): | |
| def __init__(self, source: int, target: int, rank: int) -> None: | |
| super().__init__() | |
| self.down = nn.Linear(source, rank, bias=False) | |
| self.up = nn.Linear(rank, target, bias=False) | |
| def __call__(self, source: mx.array) -> mx.array: | |
| return self.up(self.down(source)) | |
| class GDN2Memory(nn.Module): | |
| def __init__(self, input_dim: int, heads: int, key_dim: int, value_dim: int, | |
| *, observation_dim: int | None = None) -> None: | |
| super().__init__() | |
| if min(input_dim, heads, key_dim, value_dim) <= 0: | |
| raise ValueError("GDN2 dimensions must be positive.") | |
| self.heads, self.key_dim, self.value_dim = heads, key_dim, value_dim | |
| observation_dim = input_dim if observation_dim is None else observation_dim | |
| if observation_dim <= 0: | |
| raise ValueError("GDN2 observation width must be positive.") | |
| self.q_proj = nn.Linear(input_dim, heads * key_dim, bias=False) | |
| self.k_proj = nn.Linear(observation_dim, heads * key_dim, bias=False) | |
| self.v_proj = nn.Linear(observation_dim, heads * value_dim, bias=False) | |
| self.f_proj = _GateProjection(observation_dim, heads * key_dim, min(observation_dim, key_dim)) | |
| self.b_proj = nn.Linear(observation_dim, heads * key_dim, bias=False) | |
| self.w_proj = nn.Linear(observation_dim, heads * value_dim, bias=False) | |
| self.g_proj = _GateProjection(input_dim, heads * value_dim, min(input_dim, value_dim)) | |
| self.o_proj = nn.Linear(heads * value_dim, input_dim, bias=False) | |
| self.a_log = mx.zeros((heads,), mx.float32) | |
| self.dt_bias = mx.full((heads, key_dim), -6.906255, mx.float32) | |
| def empty(self, *leading: int) -> mx.array: | |
| return mx.zeros((*leading, self.heads, self.key_dim, self.value_dim), mx.float32) | |
| def _projections(self, source: mx.array): | |
| normalized = source.astype(mx.float32) | |
| source = (normalized * mx.rsqrt(mx.mean(mx.square(normalized), axis=-1, keepdims=True) + 1e-6)).astype(source.dtype) | |
| shape = source.shape[:-1] | |
| key_shape = (*shape, self.heads, self.key_dim) | |
| value_shape = (*shape, self.heads, self.value_dim) | |
| k = _l2(nn.silu(self.k_proj(source).astype(mx.float32)).reshape(key_shape)) | |
| v = nn.silu(self.v_proj(source).astype(mx.float32)).reshape(value_shape) | |
| head_rate = mx.exp(self.a_log.astype(mx.float32)).reshape( | |
| *((1,) * (source.ndim - 1)), self.heads, 1) | |
| decay = mx.exp(-head_rate * nn.softplus( | |
| self.f_proj(source).astype(mx.float32).reshape(key_shape) + self.dt_bias.astype(mx.float32))) | |
| erase = mx.sigmoid(self.b_proj(source).astype(mx.float32).reshape(key_shape)) | |
| write = mx.sigmoid(self.w_proj(source).astype(mx.float32).reshape(value_shape)) | |
| return k, v, decay, erase, write | |
| def _query(self, source: mx.array) -> mx.array: | |
| return _l2(nn.silu(self.q_proj(source).astype(mx.float32)).reshape( | |
| *source.shape[:-1], self.heads, self.key_dim)) | |
| def _output(self, value: mx.array, source: mx.array) -> mx.array: | |
| gate = nn.silu(self.g_proj(source).astype(mx.float32).reshape(value.shape)) | |
| value = value * mx.rsqrt(mx.mean(mx.square(value), axis=-1, keepdims=True) + 1e-6) * gate | |
| return self.o_proj(value.reshape(*source.shape[:-1], -1).astype(source.dtype)) | |
| def read(self, state: mx.array, source: mx.array) -> mx.array: | |
| if state.shape != (*source.shape[:-1], self.heads, self.key_dim, self.value_dim): | |
| raise ValueError("GDN2 state and query leading dimensions differ.") | |
| value = mx.sum(self._query(source)[..., None] * state.astype(mx.float32), axis=-2) | |
| return self._output(value, source) | |
| def read_shared(self, state: mx.array, source: mx.array) -> mx.array: | |
| if source.ndim != 3 or state.shape != (source.shape[0], self.heads, self.key_dim, self.value_dim): | |
| raise ValueError("Shared GDN2 read requires [batch, canvas, width] queries.") | |
| query = self._query(source).transpose(0, 2, 1, 3) | |
| value = (query @ state.astype(mx.float32)).transpose(0, 2, 1, 3) | |
| return self._output(value, source) | |
| def _transition(state, k, v, decay, erase, write, valid): | |
| decayed = state.astype(mx.float32) * decay[..., None] | |
| old = mx.sum((erase * k)[..., None] * decayed, axis=-2) | |
| candidate = decayed + k[..., None] * (write * v - old)[..., None, :] | |
| return candidate if valid is None else mx.where( | |
| valid[..., None, None, None], candidate, state.astype(mx.float32)) | |
| def transition(self, state: mx.array, source: mx.array, | |
| valid: mx.array | None = None) -> mx.array: | |
| if state.shape != (*source.shape[:-1], self.heads, self.key_dim, self.value_dim): | |
| raise ValueError("GDN2 state and observation leading dimensions differ.") | |
| k, v, decay, erase, write = self._projections(source) | |
| if valid is not None: | |
| if valid.shape != source.shape[:-1]: | |
| raise ValueError("GDN2 valid mask must match observation rows.") | |
| return self._transition(state, k, v, decay, erase, write, valid) | |
| def write_sequence(self, state: mx.array, source: mx.array, | |
| valid: mx.array) -> mx.array: | |
| if source.ndim != 3 or valid.shape != source.shape[:2]: | |
| raise ValueError("GDN2 sequence and mask must share [batch, length].") | |
| if state.shape != (source.shape[0], self.heads, self.key_dim, self.value_dim): | |
| raise ValueError("GDN2 sequence state shape differs.") | |
| projected = self._projections(source) | |
| for index in range(source.shape[1]): | |
| state = self._transition(state, *(part[:, index] for part in projected), valid[:, index]) | |
| return state | |
| class GDN2TrajectoryState: | |
| cells: mx.array | |
| row: mx.array | |
| persistent: mx.array | |
| seen: mx.array | |
| def clear_refill(self, lengths: mx.array, head: mx.array) -> "GDN2TrajectoryState": | |
| batch, canvas = self.seen.shape | |
| if lengths.shape != (batch,) or head.shape != (batch,): | |
| raise ValueError("Commit lengths and canvas heads must be per row.") | |
| physical = mx.arange(canvas)[None, :] | |
| overwritten = ((physical - head[:, None]) % canvas) < lengths[:, None] | |
| return GDN2TrajectoryState( | |
| mx.where(overwritten[..., None, None, None], 0.0, self.cells), | |
| self.row, self.persistent, self.seen & ~overwritten, | |
| ) | |
| class GDN2TrajectoryMemory(nn.Module): | |
| def __init__(self, width: int, *, probes: int = 4, | |
| working_heads: int = 16, working_key: int = 64, | |
| working_value: int = 64, persistent_heads: int = 16, | |
| persistent_key: int = 128, persistent_value: int = 128, | |
| persistent_observation_dim: int | None = None) -> None: | |
| super().__init__() | |
| if probes <= 0: | |
| raise ValueError("Probe count must be positive.") | |
| self.probes = probes | |
| self.cell = GDN2Memory(width, working_heads, working_key, working_value) | |
| self.row = GDN2Memory(width, working_heads, working_key, working_value) | |
| self.persistent = GDN2Memory(width, persistent_heads, persistent_key, persistent_value, | |
| observation_dim=persistent_observation_dim) | |
| self.probe_embed = nn.Embedding(probes, width) | |
| self.probe_embed.weight = mx.random.normal((probes, width)) * 0.02 | |
| def read(self, state: GDN2TrajectoryState, query: mx.array) -> mx.array: | |
| result = (self.cell.read(state.cells, query) | |
| + self.row.read_shared(state.row, query) | |
| + self.persistent.read_shared(state.persistent, query)) | |
| return mx.where(state.seen[..., None], result, 0.0) | |
| def observe(self, state: GDN2TrajectoryState, observation: mx.array, | |
| live: mx.array, head: mx.array) -> GDN2TrajectoryState: | |
| batch, canvas, width = observation.shape | |
| if live.shape != (batch, canvas) or head.shape != (batch,): | |
| raise ValueError("Observation mask and head have incorrect shapes.") | |
| cells = self.cell.transition(state.cells, observation, live) | |
| seen = state.seen | live | |
| logical_idx = (head[:, None] + mx.arange(canvas)[None, :]) % canvas | |
| logical = mx.take_along_axis(observation, | |
| mx.broadcast_to(logical_idx[..., None], observation.shape), axis=1) | |
| logical_live = mx.take_along_axis(live, logical_idx, axis=1) | |
| row_state = state.row | |
| for probe in range(self.probes): | |
| lo = canvas * probe // self.probes | |
| hi = canvas * (probe + 1) // self.probes | |
| selected = logical_live[:, lo:hi] | |
| count = mx.sum(selected.astype(mx.float32), axis=1, keepdims=True) | |
| pooled = mx.sum(logical[:, lo:hi].astype(mx.float32) | |
| * selected[..., None], axis=1) | |
| pooled = (pooled / mx.maximum(count, 1.0)).astype(observation.dtype) | |
| pooled = pooled + self.probe_embed.weight[probe].astype(observation.dtype) | |
| row_state = self.row.transition(row_state, pooled, count[:, 0] > 0) | |
| return GDN2TrajectoryState(cells, row_state, state.persistent, seen) | |
| def fp32_attention( | |
| query: mx.array, keys: mx.array, values: mx.array, | |
| *, attn_mask: mx.array | None = None, | |
| ) -> mx.array: | |
| """Small memory-attention reductions in FP32 with model-dtype output.""" | |
| dtype = query.dtype | |
| if query.ndim == 4 and keys.ndim == 4 and keys.shape[-2] >= 64: | |
| # The working bus attends over the full 256-token canvas. Metal's | |
| # fused attention avoids materializing its FP32 score/probability grid | |
| # while retaining FP32 accumulation. | |
| mask = attn_mask | |
| if mask is not None: | |
| mask = (mx.where(mask, 0.0, -1.0e9) | |
| if mask.dtype == mx.bool_ else mask.astype(mx.float32)) | |
| return mx.fast.scaled_dot_product_attention( | |
| query.astype(mx.float32), keys.astype(mx.float32), | |
| values.astype(mx.float32), | |
| scale=1.0 / math.sqrt(max(query.shape[-1], 1)), mask=mask, | |
| ).astype(dtype) | |
| scores = mx.matmul(query.astype(mx.float32), mx.swapaxes(keys.astype(mx.float32), -1, -2)) | |
| scores = scores / math.sqrt(max(query.shape[-1], 1)) | |
| if attn_mask is not None: | |
| if attn_mask.dtype == mx.bool_: | |
| scores = mx.where(attn_mask, scores, -1.0e9) | |
| else: | |
| scores = scores + attn_mask.astype(mx.float32) | |
| probabilities = mx.nan_to_num(mx.softmax(scores, axis=-1), nan=0.0) | |
| return mx.matmul(probabilities, values.astype(mx.float32)).astype(dtype) | |
| class RMSNorm(nn.Module): | |
| def __init__(self, dim: int, eps: float = 1.0e-6) -> None: | |
| super().__init__() | |
| self.weight = mx.ones((dim,)) | |
| self.eps = eps | |
| def __call__(self, hidden: mx.array) -> mx.array: | |
| value = hidden.astype(mx.float32) | |
| scale = mx.rsqrt(mx.mean(mx.square(value), axis=-1, keepdims=True) + self.eps) | |
| return (value * scale * self.weight.astype(mx.float32)).astype(hidden.dtype) | |
| class SwiGLU(nn.Module): | |
| def __init__(self, dim: int, hidden: int) -> None: | |
| super().__init__() | |
| self.gate = nn.Linear(dim, hidden, bias=False) | |
| self.up = nn.Linear(dim, hidden, bias=False) | |
| self.down = nn.Linear(hidden, dim, bias=False) | |
| def __call__(self, hidden: mx.array) -> mx.array: | |
| return self.down(nn.silu(self.gate(hidden)) * self.up(hidden)) | |
| class RankAttention(nn.Module): | |
| def __init__(self, dim: int, num_heads: int, kv_rank: int) -> None: | |
| super().__init__() | |
| if kv_rank % num_heads: | |
| raise ValueError("Attention K/V rank must divide heads.") | |
| self.num_heads, self.kv_rank = num_heads, kv_rank | |
| self.head_dim = kv_rank // num_heads | |
| self.q_proj = nn.Linear(dim, kv_rank, bias=False) | |
| self.k_proj = nn.Linear(dim, kv_rank, bias=False) | |
| self.v_proj = nn.Linear(dim, kv_rank, bias=False) | |
| self.o_proj = nn.Linear(kv_rank, dim, bias=False) | |
| self.q_norm = RMSNorm(dim) | |
| self.k_norm = RMSNorm(dim) | |
| def __call__(self, query: mx.array, keys: mx.array, values: mx.array, | |
| attn_mask: mx.array | None = None) -> mx.array: | |
| batch, queries, _ = query.shape | |
| key_len = keys.shape[1] | |
| heads, head_dim = self.num_heads, self.head_dim | |
| reshape_q = lambda x: x.reshape(batch, queries, heads, head_dim).transpose(0, 2, 1, 3) | |
| reshape_k = lambda x: x.reshape(batch, key_len, heads, head_dim).transpose(0, 2, 1, 3) | |
| q = reshape_q(self.q_proj(self.q_norm(query))) | |
| k = reshape_k(self.k_proj(self.k_norm(keys))) | |
| v = reshape_k(self.v_proj(values)) | |
| mask = attn_mask | |
| if mask is not None and mask.ndim == 2: | |
| mask = mask[None, None, :, :] | |
| elif mask is not None and mask.ndim == 3: | |
| mask = mask[:, None] | |
| context = fp32_attention(q, k, v, attn_mask=mask) | |
| return self.o_proj(context.transpose(0, 2, 1, 3).reshape(batch, queries, self.kv_rank)) | |
| class _WorkingBus(nn.Module): | |
| def __init__(self, width: int, heads: int, readers: int, rank: int, canvas: int) -> None: | |
| super().__init__() | |
| self.num_readers, self.heads, self.rank = readers, heads, rank | |
| self.head_dim = rank // heads | |
| self.memory_norm = RMSNorm(width) | |
| # Retain the published address norm tensor in the weight topology. | |
| self.address_norm = RMSNorm(width) | |
| self.k_proj = nn.Linear(width, rank, bias=False) | |
| self.v_proj = nn.Linear(width, rank, bias=False) | |
| self.q_norm = RMSNorm(width) | |
| self.q_proj = [nn.Linear(width, rank, bias=False) for _ in range(readers)] | |
| self.o_proj = [nn.Linear(rank, width, bias=False) for _ in range(readers)] | |
| self.alpha = mx.zeros((max(readers, 1), heads)) | |
| self.rel_bias = mx.zeros((heads, 2 * canvas - 1)) | |
| def prepare_kv(self, working: mx.array, seen: mx.array): | |
| if not self.num_readers: | |
| return None | |
| normed = self.memory_norm(working) | |
| batch, canvas, _ = working.shape | |
| shape = (batch, canvas, self.heads, self.head_dim) | |
| keys = self.k_proj(normed).reshape(shape).transpose(0, 2, 1, 3) | |
| values = self.v_proj(normed).reshape(shape).transpose(0, 2, 1, 3) | |
| positions = mx.arange(canvas) | |
| relative = positions[:, None] - positions[None, :] + canvas - 1 | |
| bias = mx.take(self.rel_bias, relative, axis=1)[None].astype(mx.float32) | |
| bias = bias + mx.where(seen[:, None, None, :], 0.0, -1e9).astype(mx.float32) | |
| return keys, values, bias, seen | |
| def read(self, hidden, reader, keys, values, bias, seen): | |
| batch, canvas, _ = hidden.shape | |
| query = self.q_proj[reader](self.q_norm(hidden)) | |
| query = query.reshape(batch, canvas, self.heads, self.head_dim).transpose(0, 2, 1, 3) | |
| context = fp32_attention(query, keys, values, attn_mask=bias) | |
| context = context * mx.tanh(self.alpha[reader]).astype(hidden.dtype)[None, :, None, None] | |
| context = context.transpose(0, 2, 1, 3).reshape(batch, canvas, self.rank) | |
| return mx.where(seen[..., None], hidden + self.o_proj[reader](context), hidden) | |
| class _CanvasBlock(nn.Module): | |
| def __init__(self, width: int, heads: int, rank: int, ffn: int, | |
| window: int, global_attention: bool) -> None: | |
| super().__init__() | |
| self.norm = RMSNorm(width) | |
| self.attn = RankAttention(width, heads, rank) | |
| self.ff_norm = RMSNorm(width) | |
| self.ff = SwiGLU(width, ffn) | |
| self.window = window | |
| self.global_attention = global_attention | |
| def __call__(self, hidden: mx.array, seen: mx.array, | |
| offsets: mx.array) -> mx.array: | |
| allowed = mx.broadcast_to(seen[:, None, :], (hidden.shape[0], hidden.shape[1], hidden.shape[1])) | |
| if not self.global_attention: | |
| allowed = allowed & (mx.abs(offsets[:, :, None] - offsets[:, None, :]) < self.window) | |
| mask = mx.where(allowed, 0.0, -1e9).astype(mx.float32) | |
| normed = self.norm(hidden) | |
| hidden = hidden + self.attn(normed, normed, normed, mask) | |
| hidden = hidden + self.ff(self.ff_norm(hidden)) | |
| return mx.where(seen[..., None], hidden, mx.zeros_like(hidden)) | |
| class _PersistentBus(nn.Module): | |
| def __init__(self, memory: nn.Module, readers: int) -> None: | |
| super().__init__() | |
| object.__setattr__(self, "_memory_ref", weakref.ref(memory)) | |
| self.num_readers = readers | |
| self.alpha = mx.zeros((max(readers, 1), 1)) | |
| def prepare_kv(self, memory: mx.array, | |
| seen: mx.array | None = None): | |
| if self.num_readers <= 0: | |
| return None | |
| return memory, seen | |
| def read(self, hidden: mx.array, reader_index: int, | |
| memory: mx.array, seen: mx.array | None) -> mx.array: | |
| delta = self._memory_ref().read_shared(memory, hidden) | |
| if seen is not None: | |
| delta = delta * seen[..., None].astype(delta.dtype) | |
| return hidden + mx.tanh(self.alpha[reader_index]).astype(hidden.dtype) * delta | |
| class LatentDeliberationTransformer(nn.Module): | |
| def __init__(self, *, hidden_size: int, | |
| latent_dim: int, ffn_dim: int, | |
| num_layers: int, num_heads: int, local_attention_window: int, | |
| tape_probes: int, | |
| history_kv_rank: int, num_working_readers: int, | |
| num_persistent_readers: int, | |
| working_last_block_global: bool, | |
| commit_sequence_dim: int | None = None, | |
| max_canvas_length: int = 256) -> None: | |
| super().__init__() | |
| if hidden_size != latent_dim: | |
| raise ValueError("Schema25 GDN2 requires hidden_size == latent_dim.") | |
| packet_dim = int(commit_sequence_dim or history_kv_rank) | |
| self.trajectory = GDN2TrajectoryMemory( | |
| hidden_size, probes=tape_probes, persistent_observation_dim=packet_dim, | |
| ) | |
| self.blocks = [ | |
| _CanvasBlock(hidden_size, num_heads, history_kv_rank, ffn_dim, | |
| local_attention_window, | |
| bool(working_last_block_global and i == num_layers - 1)) | |
| for i in range(num_layers) | |
| ] | |
| self.output_norm = RMSNorm(hidden_size) | |
| self.working_memory_bus = _WorkingBus( | |
| hidden_size, num_heads, num_working_readers, history_kv_rank, max_canvas_length, | |
| ) | |
| self.persistent_memory_bus = _PersistentBus( | |
| self.trajectory.persistent, num_persistent_readers, | |
| ) | |
| self.experience_in = nn.Linear(hidden_size * 3, packet_dim, bias=False) | |
| self.reason_embed = nn.Embedding(6, packet_dim) | |
| def __call__(self, token_embeddings: mx.array, | |
| memory: GDN2TrajectoryState, canvas_head: mx.array) -> mx.array: | |
| hidden = self.trajectory.read(memory, token_embeddings) | |
| offsets = (mx.arange(memory.seen.shape[1])[None, :] - canvas_head[:, None]) % memory.seen.shape[1] | |
| for block in self.blocks: | |
| hidden = block(hidden, memory.seen, offsets) | |
| return self.output_norm(hidden) * memory.seen[..., None].astype(hidden.dtype) | |
| def commit_write(self, *, memory: mx.array, working_state: mx.array, | |
| heavy_hidden: mx.array, | |
| committed_token_embeddings: mx.array, | |
| canvas_head: mx.array) -> mx.array: | |
| batch, canvas, width = working_state.shape | |
| count = committed_token_embeddings.shape[1] | |
| index = (mx.arange(count)[None, :] + canvas_head[:, None]) % canvas | |
| gather = mx.broadcast_to(index[..., None], (batch, count, width)) | |
| selected_working = mx.take_along_axis(working_state, gather, axis=1) | |
| selected_heavy = mx.take_along_axis(heavy_hidden, gather, axis=1) | |
| def normalize(value): | |
| fp32 = value.astype(mx.float32) | |
| return (fp32 * mx.rsqrt(mx.mean(mx.square(fp32), axis=-1, keepdims=True) + 1.0)).astype(value.dtype) | |
| roles = tuple(normalize(value) for value in ( | |
| selected_heavy, selected_working, committed_token_embeddings, | |
| )) | |
| packet = self.experience_in(mx.concatenate(roles, axis=-1)) | |
| # Schema25 maps both continuing commit reasons to embedding row 5. | |
| packet = packet + self.reason_embed.weight[5] | |
| valid = mx.ones((batch, count), mx.bool_) | |
| return self.trajectory.persistent.write_sequence(memory, packet, valid) | |
| def create_mlx_latent(config: Any) -> LatentDeliberationTransformer: | |
| readers = sum(kind == "full_attention" for kind in config.text_config.layer_types) | |
| return LatentDeliberationTransformer( | |
| hidden_size=config.text_config.hidden_size, | |
| latent_dim=config.latent_dim, | |
| ffn_dim=config.latent_ffn_dim, | |
| num_layers=config.latent_num_layers, | |
| num_heads=config.latent_num_heads, | |
| local_attention_window=config.latent_local_attention_window, | |
| tape_probes=config.latent_tape_probes, | |
| history_kv_rank=config.latent_history_kv_rank, | |
| num_working_readers=readers if config.working_memory_bus else 0, | |
| num_persistent_readers=readers if config.persistent_memory_bus else 0, | |
| working_last_block_global=config.latent_working_last_block_global, | |
| commit_sequence_dim=config.commit_sequence_dim, | |
| max_canvas_length=config.canvas_length, | |
| ) | |
| class MLXCanvasOutput: | |
| heavy_hidden: mx.array | |
| working_state: mx.array | |
| class MLXModilifyMk2(nn.Module): | |
| """Shared DiffusionGemma trunk plus native commit-only trajectory module.""" | |
| def __init__(self, backbone: Any, config: Any) -> None: | |
| super().__init__() | |
| if int(config.state_schema_version) != 25: | |
| raise RuntimeError("Native MLX ModilifyMk2 requires schema25.") | |
| self.model = backbone.model | |
| self.latent_deliberation = create_mlx_latent(config) | |
| self.config = config | |
| def _merge_context(self, token_embeddings: mx.array, | |
| context: mx.array) -> mx.array: | |
| """Frozen self-conditioning bridge with the schema25 residual cap.""" | |
| mapper = self.model.decoder.self_conditioning | |
| normed = mapper.pre_norm(context.astype(token_embeddings.dtype)) | |
| mapped = mapper.down_proj( | |
| nn.gelu_approx(mapper.gate_proj(normed)) * mapper.up_proj(normed) | |
| ) | |
| mapped_fp32 = mapped.astype(mx.float32) | |
| energy = mx.mean(mx.square(mapped_fp32), axis=-1, keepdims=True) | |
| token_rms = mx.sqrt(mx.mean(mx.square(token_embeddings.astype(mx.float32)), | |
| axis=-1, keepdims=True)) | |
| cap = 0.5 * token_rms | |
| scale = cap / mx.sqrt(energy + mx.square(cap) + 1.0e-12) | |
| combined = token_embeddings + (mapped_fp32 * scale).astype(mapped.dtype) | |
| return mapper.post_norm(combined) | |
| def forward_canvas(self, canvas: mx.array, *, prefix_cache: list[Any], | |
| state: GDN2TrajectoryState, | |
| canvas_head: mx.array, first_step: bool = False, | |
| has_commits: bool = True) -> MLXCanvasOutput: | |
| """Denoise physical ring state through logical-order trunk positions.""" | |
| batch, length = canvas.shape | |
| decoder = self.model.decoder | |
| token_embeddings = decoder.embed_tokens(canvas) * decoder.embed_scale | |
| working = (mx.zeros_like(token_embeddings) if first_step | |
| else self.latent_deliberation(token_embeddings, state, canvas_head)) | |
| physical = (canvas_head[:, None] + mx.arange(length)[None, :]) % length | |
| hidden_width = token_embeddings.shape[-1] | |
| gather = mx.broadcast_to(physical[:, :, None], (batch, length, hidden_width)) | |
| logical_tokens = mx.take_along_axis(token_embeddings, gather, axis=1) | |
| logical_working = mx.take_along_axis(working, gather, axis=1) | |
| hidden = self._merge_context(logical_tokens, logical_working) | |
| prefix = int(prefix_cache[0].offset) | |
| mask = mx.ones((batch, prefix + length), mx.bool_) | |
| masks = decoder._make_decoder_masks(hidden, prefix_cache, mask) | |
| offset = prefix_cache[0].offset | |
| working_bus = self.latent_deliberation.working_memory_bus | |
| persistent_bus = self.latent_deliberation.persistent_memory_bus | |
| logical_seen = mx.take_along_axis(state.seen, physical, axis=1) | |
| working_kv = None if first_step else working_bus.prepare_kv(logical_working, logical_seen) | |
| persistent_kv = persistent_bus.prepare_kv(state.persistent, logical_seen) if has_commits else None | |
| reader = 0 | |
| for layer, layer_cache in zip(decoder.layers, prefix_cache): | |
| hidden = layer(hidden, masks[layer.layer_type], layer_cache, | |
| decoder=True, offset=offset) | |
| if layer.layer_type == "full_attention": | |
| if working_kv is not None and reader < working_bus.num_readers: | |
| hidden = working_bus.read(hidden, reader, *working_kv) | |
| if persistent_kv is not None and reader < persistent_bus.num_readers: | |
| hidden = persistent_bus.read(hidden, reader, *persistent_kv) | |
| reader += 1 | |
| hidden = decoder.norm(hidden) | |
| inverse = (mx.arange(length)[None, :] - canvas_head[:, None]) % length | |
| heavy = mx.take_along_axis(hidden, mx.broadcast_to(inverse[:, :, None], hidden.shape), axis=1) | |
| return MLXCanvasOutput(heavy, working) | |
| def _install_sliding_encoder_masks(encoder: Any) -> None: | |
| """Align mlx-vlm's full-length masks with trimmed sliding KV chunks.""" | |
| if getattr(encoder, "_modilify_sliding_masks", False): | |
| return | |
| original = encoder._make_encoder_masks | |
| window = int(encoder.text_config.sliding_window) | |
| def make_masks(h, cache, attention_mask=None, mm_token_type_ids=None): | |
| masks = original(h, cache, attention_mask, mm_token_type_ids) | |
| if isinstance(masks, dict): | |
| return masks | |
| query_length = int(h.shape[1]) | |
| maximum = window - 1 + query_length | |
| return [ | |
| mask[..., -maximum:] if ( | |
| layer.layer_type == "sliding_attention" | |
| and isinstance(mask, mx.array) and mask.shape[-1] > maximum | |
| ) else mask | |
| for layer, mask in zip(encoder.decoder.layers, masks) | |
| ] | |
| encoder._make_encoder_masks = make_masks | |
| encoder._modilify_sliding_masks = True | |
| def _read_json(path: Path) -> dict: | |
| return json.loads(path.read_text(encoding="utf-8")) | |
| class _ReleaseRouter(Router): | |
| """Route with the published model's FP32 probabilities and weights.""" | |
| def __call__(self, x: mx.array) -> tuple[mx.array, mx.array]: | |
| normalized = mx.fast.rms_norm(x.astype(mx.float32), None, self.eps).astype(x.dtype) | |
| scores = self.proj(normalized * self.scale * self._root_size) | |
| probabilities = mx.softmax(scores.astype(mx.float32), axis=-1, precise=True) | |
| k = self.config.top_k_experts | |
| indices = mx.argpartition(-probabilities, kth=k - 1, axis=-1)[..., :k] | |
| weights = mx.take_along_axis(probabilities, indices, axis=-1) | |
| weights = weights / mx.maximum(weights.sum(axis=-1, keepdims=True), 1.17549435e-38) | |
| return indices, weights * self.per_expert_scale[indices] | |
| class _ReleaseLayer(DecoderLayer): | |
| """Keep expert weighting in FP32 and return the sum in activation dtype.""" | |
| def __call__(self, x, mask=None, cache=None, *, decoder=False, offset=None, layer_scalar=None): | |
| attention = self.self_attn(self.input_layernorm(x), mask, cache, decoder=decoder, offset=offset) | |
| residual = x + self.post_attention_layernorm(attention) | |
| dense = self.post_feedforward_layernorm_1(self.mlp(self.pre_feedforward_layernorm(residual))) | |
| flat = residual.reshape(-1, residual.shape[-1]) | |
| indices, weights = self.router(flat) | |
| routed = self.experts(self.pre_feedforward_layernorm_2(flat), indices) | |
| routed = (routed * weights[..., None]).sum(axis=-2).astype(x.dtype).reshape(x.shape) | |
| routed = self.post_feedforward_layernorm_2(routed) | |
| output = residual + self.post_feedforward_layernorm(dense + routed) | |
| return output * (self.layer_scalar if layer_scalar is None else layer_scalar) | |
| def _create_model(config: dict) -> MLXModilifyMk2: | |
| text_config = dict(config["text_config"], model_type="diffusion_gemma_text") | |
| backbone = Model(ModelConfig.from_dict({ | |
| "model_type": "diffusion_gemma", "text_config": text_config, | |
| "vision_config": None, | |
| })) | |
| lora = config["lora_config"] | |
| targets = set(lora["target_modules"]) | |
| for layer in backbone.model.decoder.layers: | |
| layer.__class__ = _ReleaseLayer | |
| layer.router = _ReleaseRouter(layer.router.config) | |
| for parent in (layer.self_attn, layer.mlp): | |
| for name, module in list(parent.named_modules()): | |
| if name in targets: | |
| setattr(parent, name, LoRALinear.from_base( | |
| module, r=lora["r"], scale=lora["alpha"] / lora["r"], | |
| )) | |
| for name in ("gate_up_proj", "down_proj"): | |
| module = getattr(layer.experts, name) | |
| setattr(layer.experts, name, LoRASwitchLinear.from_base( | |
| module, r=lora["expert_r"], scale=lora["expert_alpha"] / lora["expert_r"], | |
| )) | |
| settings = SimpleNamespace(**dict(config, text_config=SimpleNamespace(**text_config))) | |
| model = MLXModilifyMk2(backbone, settings) | |
| _install_sliding_encoder_masks(model.model.encoder) | |
| model.eval() | |
| model.freeze() | |
| return model | |
| def _published_name(native: str) -> tuple[str, bool]: | |
| """Return the release key and whether it requires dense LoRA transposition.""" | |
| if not native.startswith("model.decoder.layers."): | |
| return native, False | |
| if ".experts." in native: | |
| for projection, role in (("gate_up_proj", "gate_up"), ("down_proj", "down")): | |
| prefix = f".experts.{projection}." | |
| if prefix in native: | |
| if native.endswith(".linear.weight"): | |
| return native.replace(prefix + "linear.weight", f".experts.{projection}"), False | |
| return native.replace(prefix + "lora_", f".experts.lora_{role}_"), False | |
| return native.replace(".linear.weight", ".base.weight"), ".lora_" in native | |
| def _expected_dtype(name: str) -> mx.Dtype: | |
| if name.startswith("latent_deliberation.") and ( | |
| name.endswith((".dt_bias", ".a_log")) | |
| or (name.endswith(".weight") and "norm" in name.rsplit(".", 2)[-2]) | |
| ): | |
| return mx.float32 | |
| return mx.bfloat16 | |
| def _verify_shard(path: Path, info: dict) -> None: | |
| if path.stat().st_size != info["bytes"]: | |
| raise ValueError(f"Incorrect shard size: {path.name}") | |
| with path.open("rb") as source: | |
| if hashlib.file_digest(source, "sha256").hexdigest() != info["sha256"]: | |
| raise ValueError(f"Incorrect shard SHA-256: {path.name}") | |
| def load_model(root: Path) -> MLXModilifyMk2: | |
| config = _read_json(root / "config.json") | |
| manifest = _read_json(root / "export_manifest.json") | |
| protocol = {"state_schema_version": 25, "memory_architecture": "compact_gdn2_v2", | |
| "memory_scheme": "dual_timescale_gdn2_trajectory_memory"} | |
| if (manifest.get("format") != "modilify_mk2_mlx_inference_v1" | |
| or any(config.get(k) != v or manifest.get(k) != v for k, v in protocol.items()) | |
| or config.get("checkpoint_step") != 4400 or manifest.get("global_step") != 4400 | |
| or config.get("canvas_length") != 256 or config.get("dtype") != "bfloat16" | |
| or manifest.get("precision_policy") != "gdn2_small_fp32_v1"): | |
| raise ValueError("Expected the step 4400 schema25 compact_gdn2_v2 MLX release.") | |
| if not mx.metal.is_available(): | |
| raise RuntimeError("Modilify Mk2 MLX requires an Apple Silicon Metal device.") | |
| model = _create_model(config) | |
| weight_map = _read_json(root / "model.safetensors.index.json")["weight_map"] | |
| tensors = {} | |
| for native, value in tree_flatten(model.parameters()): | |
| name, transpose = _published_name(native) | |
| if name in tensors: | |
| raise ValueError(f"Duplicate release weight mapping: {name}") | |
| tensors[name] = (native, tuple(value.shape), _expected_dtype(name), transpose) | |
| if set(tensors) != set(weight_map) or len(tensors) != manifest.get("total_tensor_count") or len(tensors) != 1323: | |
| raise ValueError( | |
| f"Weight index mismatch. Missing: {sorted(set(tensors) - set(weight_map))[:8]}. " | |
| f"Unexpected: {sorted(set(weight_map) - set(tensors))[:8]}." | |
| ) | |
| by_shard = defaultdict(set) | |
| for name, shard in weight_map.items(): | |
| by_shard[shard].add(name) | |
| shards = {entry["file"]: entry for entry in manifest["shards"]} | |
| expected_files = {f"model-{i:05d}-of-00016.safetensors" for i in range(1, 17)} | |
| if set(shards) != expected_files or set(by_shard) != expected_files or len(manifest["shards"]) != 16: | |
| raise ValueError("Release manifest and index must describe all 16 shards.") | |
| counts = Counter() | |
| for shard in sorted(by_shard): | |
| path = root / shard | |
| _verify_shard(path, shards[shard]) | |
| raw = mx.load(str(path)) | |
| if set(raw) != by_shard[shard]: | |
| raise ValueError(f"Weight index and shard disagree: {shard}") | |
| updates = [] | |
| for name, value in raw.items(): | |
| native, shape, dtype, transpose = tensors[name] | |
| if transpose: | |
| value = value.T | |
| if value.shape != shape or value.dtype != dtype: | |
| raise ValueError(f"Invalid shape or dtype for {name}: {value.shape}, {value.dtype}") | |
| updates.append((native, value)) | |
| counts["float32" if dtype == mx.float32 else "bfloat16"] += 1 | |
| model.load_weights(updates, strict=False) | |
| mx.eval(*(value for _, value in updates)) | |
| del raw, updates | |
| if counts != {"bfloat16": 1297, "float32": 26} or counts != manifest.get("tensor_dtypes"): | |
| raise ValueError(f"Unexpected release precision counts: {dict(counts)}") | |
| return model | |