Text Generation
MLX
Safetensors
modilify_mk2
diffusion
mixture-of-experts
custom-code
modilify-mk2
conversational
Instructions to use modilify/Modilify-Mk2-mlx with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- MLX
How to use modilify/Modilify-Mk2-mlx with MLX:
# Make sure mlx-lm is installed # pip install --upgrade mlx-lm # Generate text with mlx-lm from mlx_lm import load, generate model, tokenizer = load("modilify/Modilify-Mk2-mlx") prompt = "Write a story about Einstein" messages = [{"role": "user", "content": prompt}] prompt = tokenizer.apply_chat_template( messages, add_generation_prompt=True ) text = generate(model, tokenizer, prompt=prompt, verbose=True) - Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- LM Studio
- Pi
How to use modilify/Modilify-Mk2-mlx with Pi:
Start the MLX server
# Install MLX LM: uv tool install mlx-lm # Start a local OpenAI-compatible server: mlx_lm.server --model "modilify/Modilify-Mk2-mlx"
Configure the model in Pi
# Install Pi: npm install -g @earendil-works/pi-coding-agent # Add to ~/.pi/agent/models.json: { "providers": { "mlx-lm": { "baseUrl": "http://localhost:8080/v1", "api": "openai-completions", "apiKey": "none", "models": [ { "id": "modilify/Modilify-Mk2-mlx" } ] } } }Run Pi
# Start Pi in your project directory: pi
- MLX LM
How to use modilify/Modilify-Mk2-mlx with MLX LM:
Generate or start a chat session
# Install MLX LM uv tool install mlx-lm # Interactive chat REPL mlx_lm.chat --model "modilify/Modilify-Mk2-mlx"
Run an OpenAI-compatible server
# Install MLX LM uv tool install mlx-lm # Start the server mlx_lm.server --model "modilify/Modilify-Mk2-mlx" # Calling the OpenAI-compatible server with curl curl -X POST "http://localhost:8080/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "modilify/Modilify-Mk2-mlx", "messages": [ {"role": "user", "content": "Hello"} ] }' - Hermes Agent
How to use modilify/Modilify-Mk2-mlx with Hermes Agent:
Start the MLX server
# Install MLX LM: uv tool install mlx-lm # Start a local OpenAI-compatible server: mlx_lm.server --model "modilify/Modilify-Mk2-mlx"
Configure Hermes
# Install Hermes: curl -fsSL https://hermes-agent.nousresearch.com/install.sh | bash hermes setup # Point Hermes at the local server: hermes config set model.provider custom hermes config set model.base_url http://127.0.0.1:8080/v1 hermes config set model.default modilify/Modilify-Mk2-mlx
Run Hermes
hermes
- Atomic Chat
- OpenClaw
How to use modilify/Modilify-Mk2-mlx with OpenClaw:
Start the MLX server
# Install MLX LM: uv tool install mlx-lm # Start a local OpenAI-compatible server: mlx_lm.server --model "modilify/Modilify-Mk2-mlx"
Configure OpenClaw
# Install OpenClaw: npm install -g openclaw@latest # Register the local server and set it as the default model: openclaw onboard --non-interactive --mode local \ --auth-choice custom-api-key \ --custom-base-url http://127.0.0.1:8080/v1 \ --custom-model-id "modilify/Modilify-Mk2-mlx" \ --custom-provider-id mlx-lm \ --custom-compatibility openai \ --custom-text-input \ --accept-risk \ --skip-health
Run OpenClaw
openclaw agent --local --agent main --message "Hello from Hugging Face"
Download modilify_mk2_mlx/runtime.py from modilify/Modilify-Mk2-mlx: direct link, hf CLI and curl.
- Browser
- Download file 18 kB
-
https://huggingface.co/modilify/Modilify-Mk2-mlx/resolve/main/modilify_mk2_mlx/runtime.py
- Command line
-
hf download hf://modilify/Modilify-Mk2-mlx/modilify_mk2_mlx/runtime.py
-
curl -L -o runtime.py https://huggingface.co/modilify/Modilify-Mk2-mlx/resolve/main/modilify_mk2_mlx/runtime.py
18 kB
| """Text generation and commit policy for the schema25 MLX release.""" | |
| from __future__ import annotations | |
| from collections.abc import Callable, Sequence | |
| from dataclasses import dataclass, replace | |
| import math | |
| from pathlib import Path | |
| from types import SimpleNamespace | |
| import mlx.core as mx | |
| from tokenizers.decoders import DecodeStream | |
| from transformers import GemmaTokenizer | |
| from .model import GDN2TrajectoryState, MLXModilifyMk2, _read_json, load_model | |
| JUMP_FAILURE_BUDGET = 2.0 | |
| FUSED_EPS = 1.0e-6 | |
| def commit_target_confidence_bias( | |
| target_confidence: float | None, | |
| failure_budget: float = 0.2, | |
| budget_safety_ratio: float = 0.85, | |
| ) -> float: | |
| if target_confidence is None or target_confidence <= 0.0 or target_confidence >= 1.0: | |
| return 0.0 | |
| target_failure = min(budget_safety_ratio * failure_budget, 1.0 - target_confidence) | |
| target_failure = max(target_failure, 1.0e-4) | |
| target_conf = 1.0 - target_failure | |
| logit_c = math.log(target_conf / (1.0 - target_conf)) | |
| logit_p = math.log(target_confidence / (1.0 - target_confidence)) | |
| return max(logit_c - logit_p, 0.0) | |
| def fused_commit_confidence( | |
| proposal_confidence: mx.array, | |
| token_entropy: mx.array, | |
| *, | |
| eps: float = FUSED_EPS, | |
| entropy_weight: float = 1.0, | |
| confidence_power: float = 2.0, | |
| top_k: int | None = None, | |
| min_p: float | None = None, | |
| target_confidence: float | None = None, | |
| failure_budget: float = 0.2, | |
| ) -> mx.array: | |
| """Match the schema25 excess-entropy sigmoid and checkpoint power.""" | |
| p = mx.clip(mx.nan_to_num(proposal_confidence.astype(mx.float32), nan=0.5), eps, 1.0 - eps) | |
| entropy = mx.maximum(mx.nan_to_num(token_entropy.astype(mx.float32), nan=0.0), 0.0) | |
| binary_entropy = -p * mx.log(p) - (1.0 - p) * mx.log1p(-p) | |
| excess = mx.maximum(entropy - binary_entropy, 0.0) | |
| k_eff = None | |
| if top_k is not None and top_k > 0: | |
| k_eff = mx.full(p.shape, float(top_k), dtype=mx.float32) | |
| if min_p is not None and min_p > 0: | |
| thresh = mx.maximum(float(min_p) * p, 1.0e-6) | |
| k_min_p = mx.maximum((1.0 - p) / thresh, 1.0) | |
| k_eff = k_min_p if k_eff is None else mx.minimum(k_eff, k_min_p) | |
| if k_eff is not None: | |
| max_excess = (1.0 - p) * mx.log(k_eff) | |
| excess = mx.minimum(excess, max_excess) | |
| bias = commit_target_confidence_bias(target_confidence, failure_budget=failure_budget) | |
| fused_logit = mx.log(p) - mx.log1p(-p) + bias - entropy_weight * excess | |
| return mx.clip(mx.sigmoid(fused_logit) ** confidence_power, eps, 1.0 - eps) | |
| def fused_commit_failure_rate( | |
| proposal_confidence: mx.array, token_entropy: mx.array, **kwargs: object | |
| ) -> mx.array: | |
| return 1.0 - fused_commit_confidence(proposal_confidence, token_entropy, **kwargs) | |
| def _terminal_mask(tokens: mx.array, stop_token_id: int | Sequence[int]) -> mx.array: | |
| ids = (stop_token_id,) if isinstance(stop_token_id, int) else tuple(dict.fromkeys(stop_token_id)) | |
| if not ids: | |
| raise ValueError("At least one stop token ID is required.") | |
| matches = tokens == int(ids[0]) | |
| for value in ids[1:]: | |
| matches = matches | (tokens == int(value)) | |
| return matches | |
| def _prefix_length(risk: mx.array, valid: mx.array, budget: float) -> mx.array: | |
| # Nonnegative risk and contiguous validity make the allowed prefix monotonic. | |
| allowed = (mx.cumsum(mx.clip(risk, 0.0, 1.0) * valid, axis=-1) < budget) & valid | |
| return mx.sum(allowed, axis=-1).astype(mx.int32) | |
| def _commit( | |
| sampled, risk, previous_risk, greedy, greedy_risk, *, | |
| remaining, stops, budget, ponder, stagnation, generation, | |
| ): | |
| positions = mx.arange(sampled.shape[-1])[None, :] | |
| valid = positions < remaining | |
| def bounded(ids, rates, limit): | |
| length = _prefix_length(rates, valid, limit) | |
| terminal = _terminal_mask(ids, stops) & (positions < length[:, None]) | |
| first = mx.min(mx.where(terminal, positions + 1, sampled.shape[-1]), axis=-1) | |
| return mx.minimum(length, first) | |
| normal = bounded(sampled, risk, budget) | |
| previous = _prefix_length(previous_risk, valid, budget) | |
| frontier = mx.minimum(mx.maximum(previous, normal) + 1, mx.sum(valid, axis=-1)) | |
| progress = mx.sum((previous_risk - risk) * (positions < frontier[:, None]), axis=-1) | |
| progress = progress / mx.maximum(frontier, 1) | |
| next_ponder = mx.where(normal > 0, 0, ponder + 1) | |
| next_stagnation = mx.where(normal > 0, 0, stagnation + 1) | |
| jump = (normal == 0) & ( | |
| ((next_stagnation >= generation.jump_on_no_progress_after) | |
| & (progress <= generation.min_trajectory_progress)) | |
| | (next_ponder >= generation.max_ponder_steps) | |
| ) | |
| lengths = mx.where(jump, bounded(greedy, greedy_risk, JUMP_FAILURE_BUDGET), normal) | |
| return (lengths, mx.where(jump[:, None], greedy, sampled), | |
| mx.where(lengths > 0, 0, next_ponder), mx.where(lengths > 0, 0, next_stagnation)) | |
| def seed_key(seed: int, salt: int) -> mx.array: | |
| return mx.random.key((seed + salt) % (2**32)) | |
| def proposal_statistics( | |
| hidden: mx.array, weight: mx.array, *, seed: int, step: int, | |
| softcap: float, temperature: float, top_k: int, min_p: float, | |
| repetition_penalty: float, repetition_seen: set[int], chunk_size: int = 32768, | |
| ) -> tuple[mx.array, ...]: | |
| canvas = hidden.shape[1] | |
| flat = hidden[0] | |
| log_z = mx.full((canvas,), -mx.inf, mx.float32) | |
| greedy_score = log_z | |
| greedy = mx.zeros((canvas,), mx.int32) | |
| moment_sum = mx.zeros_like(log_z) | |
| moment_weighted = mx.zeros_like(log_z) | |
| candidates = [] | |
| candidate_ids = [] | |
| if repetition_penalty != 1: | |
| seen = mx.zeros((weight.shape[0],), mx.bool_) | |
| if repetition_seen: | |
| seen[mx.array(sorted(repetition_seen), mx.int32)] = True | |
| for start in range(0, weight.shape[0], chunk_size): | |
| stop = min(start + chunk_size, weight.shape[0]) | |
| score = mx.tanh((flat @ weight[start:stop].T).astype(mx.float32) / softcap) * softcap | |
| if repetition_penalty != 1: | |
| penalized = mx.where(score < 0, score * repetition_penalty, score / repetition_penalty) | |
| score = mx.where(seen[start:stop], penalized, score) | |
| score = score / temperature | |
| log_z = mx.logaddexp(log_z, mx.logsumexp(score, axis=-1)) | |
| local_max = mx.max(score, axis=-1) | |
| greedy = mx.where(local_max > greedy_score, mx.argmax(score, axis=-1) + start, greedy) | |
| next_max = mx.maximum(greedy_score, local_max) | |
| previous_scale = mx.exp(greedy_score - next_max) | |
| local_weights = mx.exp(score - next_max[:, None]) | |
| moment_sum = moment_sum * previous_scale + mx.sum(local_weights, axis=-1) | |
| moment_weighted = moment_weighted * previous_scale + mx.sum(local_weights * score, axis=-1) | |
| greedy_score = next_max | |
| k = min(top_k, stop - start) | |
| indices = mx.argpartition(-score, kth=k - 1, axis=-1)[:, :k] | |
| candidates.append(mx.take_along_axis(score, indices, axis=-1)) | |
| candidate_ids.append(indices + start) | |
| mx.async_eval(log_z, greedy, greedy_score, moment_sum, moment_weighted, | |
| candidates[-1], candidate_ids[-1]) | |
| all_scores = mx.concatenate(candidates, axis=-1) | |
| all_ids = mx.concatenate(candidate_ids, axis=-1) | |
| k = min(top_k, all_scores.shape[-1]) | |
| indices = mx.argpartition(-all_scores, kth=k - 1, axis=-1)[:, :k] | |
| scores = mx.take_along_axis(all_scores, indices, axis=-1) | |
| ids = mx.take_along_axis(all_ids, indices, axis=-1) | |
| eligible = mx.where(scores >= greedy_score[:, None] + math.log(min_p), scores, -mx.inf) | |
| uniform = mx.random.uniform(shape=(canvas, k), key=seed_key(seed, 0xC09A + step * 1000003)) | |
| uniform = mx.clip(uniform, 1.17549435e-38, 1.0 - 1.19209290e-7) | |
| chosen = mx.argmax(eligible - mx.log(-mx.log(uniform)), axis=-1, keepdims=True) | |
| token_ids = mx.take_along_axis(ids, chosen, axis=-1)[:, 0] | |
| chosen_score = mx.take_along_axis(scores, chosen, axis=-1)[:, 0] | |
| confidence = mx.clip(mx.exp(chosen_score - log_z), 0.0, 1.0) | |
| greedy_confidence = mx.clip(mx.exp(greedy_score - log_z), 0.0, 1.0) | |
| entropy = log_z - moment_weighted / moment_sum | |
| return tuple(value[None] for value in (token_ids, confidence, entropy, greedy, greedy_confidence)) | |
| class GenerationResult: | |
| text: str | |
| token_ids: tuple[int, ...] | |
| denoise_steps: int | |
| stop_reason: str | |
| def _positive_int(value: int, name: str) -> None: | |
| if type(value) is not int or value < 1: | |
| raise ValueError(f"{name} must be a positive integer.") | |
| class Runtime: | |
| def __init__(self, model: MLXModilifyMk2, tokenizer, generation: dict): | |
| self.model = model | |
| self.tokenizer = tokenizer | |
| self.generation = SimpleNamespace(**generation) | |
| def generate( | |
| self, messages: Sequence[dict[str, str]], *, enable_thinking: bool = True, | |
| seed: int = 42, max_new_tokens: int = 256, max_denoising_steps: int = 256, | |
| on_text: Callable[[str], None] | None = None, | |
| ) -> GenerationResult: | |
| """Generate one text response and optionally emit committed text increments.""" | |
| _positive_int(max_new_tokens, "max_new_tokens") | |
| _positive_int(max_denoising_steps, "max_denoising_steps") | |
| if type(seed) is not int: | |
| raise ValueError("seed must be an integer.") | |
| if not messages or any( | |
| not isinstance(message, dict) | |
| or message.get("role") not in ("system", "user", "assistant") | |
| or not isinstance(message.get("content"), str) | |
| for message in messages | |
| ): | |
| raise ValueError("Expected a non-empty sequence of system, user or assistant text messages.") | |
| prompt_ids = self.tokenizer.apply_chat_template( | |
| list(messages), tokenize=True, add_generation_prompt=True, | |
| enable_thinking=enable_thinking, return_dict=True, | |
| )["input_ids"] | |
| max_positions = self.model.config.text_config.max_position_embeddings | |
| if not prompt_ids or len(prompt_ids) + max_new_tokens > max_positions: | |
| raise ValueError("Prompt and response budget must fit within the model position limit.") | |
| with mx.stream(mx.gpu): | |
| return self._generate(prompt_ids, seed, max_new_tokens, max_denoising_steps, on_text) | |
| def _generate( | |
| self, prompt_ids: list[int], seed: int, max_new_tokens: int, | |
| max_denoising_steps: int, on_text: Callable[[str], None] | None, | |
| ) -> GenerationResult: | |
| config = self.model.config | |
| decoder = self.model.model.decoder | |
| encoder = self.model.model.encoder | |
| cache = encoder.make_cache() | |
| for start in range(0, len(prompt_ids), 256): | |
| _, cache = encoder(mx.array([prompt_ids[start:start + 256]], mx.int32), cache=cache) | |
| mx.eval(*(value for layer in cache for value in (layer.keys, layer.values) | |
| if isinstance(value, mx.array))) | |
| generation = self.generation | |
| vocab = config.text_config.vocab_size | |
| canvas = config.canvas_length | |
| positions = mx.arange(canvas)[None, :] | |
| unknown_entropy = math.log(vocab) | |
| memory = GDN2TrajectoryState( | |
| mx.zeros((1, canvas, 16, 64, 64), mx.float32), | |
| mx.zeros((1, 16, 64, 64), mx.float32), | |
| mx.zeros((1, 16, 128, 128), mx.float32), | |
| mx.zeros((1, canvas), mx.bool_), | |
| ) | |
| confidence = mx.zeros((1, canvas), mx.float32) | |
| entropy = mx.full((1, canvas), unknown_entropy, mx.float32) | |
| clocks = mx.zeros((1,), mx.int32) | |
| ponder = stagnation = head = clocks | |
| def noise(step: int) -> mx.array: | |
| return mx.random.randint(0, vocab, (1, canvas), key=seed_key(seed, 0x51A7 + step * 1000003)) | |
| tokens = mx.where(positions < max_new_tokens, noise(0), generation.pad_token_id) | |
| turn_end = generation.turn_end_token_id | |
| eos = generation.eos_token_id | |
| stops = tuple(dict.fromkeys((turn_end, *(eos if isinstance(eos, list) else [eos])))) | |
| excluded = set(generation.repetition_penalty_exclude_token_ids) | |
| excluded.update((generation.pad_token_id, generation.bos_token_id, *stops)) | |
| seen = set(prompt_ids) - excluded | |
| generated = [] | |
| stream = DecodeStream(skip_special_tokens=True) if on_text is not None else None | |
| emitted = "" | |
| watchdog = max_new_tokens * generation.max_ponder_steps | |
| def failure(confidence: mx.array, entropy: mx.array) -> mx.array: | |
| return fused_commit_failure_rate( | |
| confidence, entropy, entropy_weight=config.commit_entropy_weight, | |
| confidence_power=config.commit_confidence_power, top_k=config.commit_top_k, | |
| min_p=config.commit_min_p, target_confidence=config.commit_target_confidence, | |
| failure_budget=config.commit_failure_budget, | |
| ) | |
| limit = min(max_denoising_steps, watchdog) | |
| for step in range(1, limit + 1): | |
| output = self.model.forward_canvas( | |
| tokens, prefix_cache=cache, state=memory, canvas_head=head, | |
| first_step=step == 1, has_commits=bool(generated), | |
| ) | |
| sampled, next_confidence, next_entropy, greedy, greedy_confidence = proposal_statistics( | |
| output.heavy_hidden, decoder.embed_tokens.weight, seed=seed, step=step - 1, | |
| softcap=config.text_config.final_logit_softcapping, temperature=generation.t_min, | |
| top_k=config.commit_top_k, min_p=config.commit_min_p, | |
| repetition_penalty=generation.repetition_penalty, repetition_seen=seen, | |
| ) | |
| remaining = max_new_tokens - len(generated) | |
| logical_order = (head[:, None] + positions) % canvas | |
| physical_offsets = (positions - head[:, None]) % canvas | |
| def logical(values: mx.array) -> mx.array: | |
| return mx.take_along_axis(values, logical_order, axis=1) | |
| lengths, selected, ponder, stagnation = _commit( | |
| logical(sampled), logical(failure(next_confidence, next_entropy)), | |
| logical(failure(confidence, entropy)), | |
| logical(greedy), logical(failure(greedy_confidence, next_entropy)), | |
| ponder=ponder, stagnation=stagnation, remaining=remaining, | |
| budget=config.commit_failure_budget, stops=stops, generation=generation, | |
| ) | |
| mx.eval(lengths, selected) | |
| count = int(lengths.item()) | |
| block = selected[0, :count].tolist() | |
| generated.extend(block) | |
| seen.update(set(block) - excluded) | |
| if stream is not None and block: | |
| delta = stream.step(self.tokenizer.backend_tokenizer, block) | |
| if delta: | |
| on_text(delta) | |
| emitted += delta | |
| if turn_end in block: | |
| reason = "turn_end" | |
| break | |
| if any(token in stops for token in block): | |
| reason = "eos" | |
| break | |
| if len(generated) >= max_new_tokens: | |
| reason = "max_new_tokens" | |
| break | |
| if step == limit: | |
| continue | |
| memory = self.model.latent_deliberation.trajectory.observe( | |
| memory, output.heavy_hidden + output.working_state, physical_offsets < remaining, head, | |
| ) | |
| tokens, confidence, entropy = sampled, next_confidence, next_entropy | |
| if count: | |
| committed_ids = selected[:, :count] | |
| persistent = self.model.latent_deliberation.commit_write( | |
| memory=memory.persistent, working_state=output.working_state, | |
| heavy_hidden=output.heavy_hidden, | |
| committed_token_embeddings=decoder.embed_tokens(committed_ids) * decoder.embed_scale, | |
| canvas_head=head, | |
| ) | |
| memory = replace(memory.clear_refill(lengths, head), persistent=persistent) | |
| _, cache = encoder(committed_ids, cache=cache) | |
| head = (head + count) % canvas | |
| refill = physical_offsets < count | |
| tail = mx.take_along_axis(noise(step), physical_offsets, axis=1) | |
| pad = ((positions - head[:, None]) % canvas) >= remaining - count | |
| tokens = mx.where(refill, mx.where(pad, generation.pad_token_id, tail), tokens) | |
| confidence = mx.where(refill, 0.0, confidence) | |
| entropy = mx.where(refill, unknown_entropy, entropy) | |
| mx.async_eval(tokens, confidence, entropy, memory.cells, memory.row, memory.persistent, memory.seen, | |
| *(value for layer in cache for value in (layer.keys, layer.values) | |
| if isinstance(value, mx.array))) | |
| else: | |
| reason = "max_denoising_steps" if max_denoising_steps <= watchdog else "episode_watchdog" | |
| text = self.tokenizer.backend_tokenizer.decode(generated, skip_special_tokens=True) | |
| if on_text is not None and emitted != text: | |
| if not text.startswith(emitted): | |
| raise RuntimeError("Tokenizer streaming output diverged from complete decode.") | |
| on_text(text[len(emitted):]) | |
| return GenerationResult(text, tuple(generated), step, reason) | |
| def load(model_dir: str | Path | None = None) -> Runtime: | |
| """Open a local release directory with model computation on Metal.""" | |
| root = Path(model_dir).expanduser().resolve() if model_dir is not None else Path(__file__).resolve().parents[1] | |
| generation = _read_json(root / "generation_config.json") | |
| # Keep the released Gemma tokenizer regex as stored in tokenizer.json. | |
| tokenizer = GemmaTokenizer.from_pretrained( | |
| root, local_files_only=True, fix_mistral_regex=False, | |
| ) | |
| with mx.stream(mx.gpu): | |
| model = load_model(root) | |
| return Runtime(model, tokenizer, generation) | |