"""Clairic inference on Apple silicon with MLX. Not part of the Cloudflare release: this module replaces its PyTorch engine (`native.py`, `math.py`, `readout.py`, the typed head in `model.py` and the CUDA kernels). It keeps the same two entry points the request pipeline calls, with the same inputs and outputs: - `read_typed(item, media)`: one forward pass of the backbone over the typed-field prompt; the final hidden states feed the typed head (dates, times, durations, computed numbers, estimates, forecasts); - `async_generate(...)`: greedy decoding constrained by an xgrammar grammar (JSON schema, regex or EBNF), one prompt per field or list item. Token log-probabilities are taken after the grammar mask, as upstream does. The backbone is mlx-vlm's `qwen3_5` model (causal full attention every fourth layer, Gated DeltaNet linear attention elsewhere) run through an explicit layer loop with a key/value cache for the attention layers and a recurrent state plus convolution window for the linear layers. """ from __future__ import annotations import asyncio import concurrent.futures import hashlib import json import threading from collections import defaultdict from collections.abc import Callable from dataclasses import dataclass, field from pathlib import Path from typing import Any import mlx.core as mx import mlx.nn as nn import numpy as np from mlx_vlm.models.qwen3_5 import Model, ModelConfig from mlx_vlm.models.qwen3_5.gated_delta import gated_delta_update from . import model as cm ATTENTION_ELEMENTS = 1 << 27 # attention scores per call (batch x heads x queries x keys); bounds temporaries SEGMENT_TOKENS = 1024 # tokens (rows x length) run through the layers at once; longer inputs go in segments MEMORY_FRACTION = 0.75 # default ceiling on MLX memory, as a fraction of the machine's RAM ROPE_MODES = ("mrope", "temporal") class PreciseRMSNorm(nn.Module): """RMSNorm computed in fp32 and cast back, as the reference does. The weight is stored as (1 + w) in fp32.""" def __init__(self, dims: int, eps: float) -> None: super().__init__() self.weight = mx.ones((dims,)) self.eps = eps def __call__(self, x: mx.array) -> mx.array: return mx.fast.rms_norm(x.astype(mx.float32), self.weight.astype(mx.float32), self.eps).astype(x.dtype) @dataclass class Cache: """Per-layer state after a prefix: attention keys/values, or the linear layers' memory and conv window.""" keys: list[mx.array | None] values: list[mx.array | None] memory: list[mx.array | None] window: list[mx.array | None] valid: mx.array | None = None # [B, T] False on padding; None when no row is padded length: int = 0 position: mx.array | None = None # [B] rotary position of the next token @classmethod def empty(cls, layers: int) -> Cache: return cls([None] * layers, [None] * layers, [None] * layers, [None] * layers) def arrays(self) -> list[mx.array]: return [a for group in (self.keys, self.values, self.memory, self.window) for a in group if a is not None] # ---------------------------------------------------------------------------------------------- typed head def _normalize(x: mx.array) -> mx.array: return x / mx.maximum(mx.sqrt((x * x).sum(axis=-1, keepdims=True)), 1e-12) def _span_scores(start: mx.array, end: mx.array, length_bias: mx.array) -> mx.array: """scores[q, s, l] = start[q, s] + end[q, s + l] + bias[l]; spans running past the input are -inf.""" width, tokens = length_bias.shape[0], end.shape[1] padded = mx.concatenate([end, mx.full((end.shape[0], width - 1), -mx.inf, dtype=end.dtype)], axis=1) index = mx.arange(tokens)[:, None] + mx.arange(width)[None, :] return start[:, :, None] + padded[:, index] + length_bias def _two_digits(slots: mx.array, values: range) -> mx.array: tens = mx.array([value // 10 for value in values]) ones = mx.array([value % 10 for value in values]) return slots[:, 0, :][:, tens] + slots[:, 1, :][:, ones] def _years(slots: mx.array) -> mx.array: index = [[int(f"{year:04d}"[position]) for year in range(cm.YEAR_LOW, cm.YEAR_HIGH + 1)] for position in range(cm.YEAR_DIGITS)] inside = sum(slots[:, position, :][:, mx.array(index[position])] for position in range(cm.YEAR_DIGITS)) total = mx.logsumexp(slots, axis=-1).sum(axis=-1) gap = mx.minimum(mx.logsumexp(inside, axis=-1) - total, -1e-06) outside = total + mx.log(-mx.expm1(gap)) return mx.concatenate([inside, outside[:, None]], axis=-1) class TypedHead: """The Clairic typed head (`clair_head.safetensors`) with the lexical readout, evaluated in fp32. A port of upstream `ClairicHead.forward_lexical` and `answer_slot_states`: field queries are routed through two pre-norm transformer decoder layers that cross-attend the prompt, then read out as span pointers, digit / month / sign slots, estimate bins and list-item states. """ REQUIRED = {"readout": "lexical", "estimate_lexical": True, "span_trim": True, "answer_slots": True, "answer_slot_routing": True, "number_groups": True, "number_pointer": True} def __init__(self, path: Path, round_bf16: bool) -> None: self.config = json.loads((path / "clair_head_config.json").read_text()) for name, value in self.REQUIRED.items(): if self.config.get(name) != value: raise ValueError(f"unsupported typed head: {name} must be {value!r}") if self.config.get("loop_feedback"): raise ValueError("unsupported typed head: loop_feedback") weights = mx.load(str(path / "clair_head.safetensors")) if round_bf16: # Upstream serving builds the head in bf16 before widening it to fp32 (the length bias excepted). weights = {name: value if name == "length_bias" else value.astype(mx.bfloat16) for name, value in weights.items()} self.w = {name: value.astype(mx.float32) for name, value in weights.items()} mx.eval(self.w) self.bins = self.w["bins.weight"].shape[0] self.pointer_width = int(self.config["pointer_width"]) self.heads = int(self.config["route_heads"]) self.layers = int(self.config["route_layers"]) self.slot_rank = int(self.config["slot_rank"]) def eval(self) -> TypedHead: return self # ---- building blocks def linear(self, name: str, x: mx.array) -> mx.array: y = x @ self.w[f"{name}.weight"].T bias = self.w.get(f"{name}.bias") return y if bias is None else y + bias def norm(self, name: str, x: mx.array) -> mx.array: return mx.fast.layer_norm(x, self.w[f"{name}.weight"], self.w[f"{name}.bias"], 1e-5) def _project(self, prefix: str, part: int, x: mx.array) -> mx.array: weight, bias = self.w[f"{prefix}.in_proj_weight"], self.w[f"{prefix}.in_proj_bias"] width = weight.shape[1] y = x @ weight[part * width:(part + 1) * width].T + bias[part * width:(part + 1) * width] return y.reshape(*x.shape[:-1], self.heads, -1).transpose(0, 2, 1, 3) def _attention(self, prefix: str, queries: mx.array, keys: mx.array, values: mx.array, mask: mx.array | None) -> mx.array: batch = queries.shape[0] if keys.shape[0] != batch: keys = mx.broadcast_to(keys, (batch, *keys.shape[1:])) values = mx.broadcast_to(values, (batch, *values.shape[1:])) out = mx.fast.scaled_dot_product_attention(queries, keys, values, scale=queries.shape[-1] ** -0.5, mask=mask) out = out.transpose(0, 2, 1, 3).reshape(batch, queries.shape[2], -1) return self.linear(f"{prefix}.out_proj", out) def _memory(self, hidden: mx.array, stop: int) -> list[tuple[mx.array, mx.array]]: """The prompt as cross-attention memory: each route layer's keys and values, computed once per read.""" memory = self.linear("memory_in", self.norm("token_norm", hidden[:stop]))[None] projected = [] for layer in range(self.layers): prefix = f"route_layers.{layer}.multihead_attn" projected.append((self._project(prefix, 1, memory), self._project(prefix, 2, memory))) mx.eval(projected) return projected def _route_layers(self, queries: mx.array, memory: list[tuple[mx.array, mx.array]], padding: mx.array | None = None) -> mx.array: """`nn.TransformerDecoderLayer(norm_first=True, activation="gelu")` stack; `padding` marks ignored keys.""" mask = None if padding is None else mx.logical_not(padding)[:, None, None, :] for layer, (keys, values) in enumerate(memory): prefix = f"route_layers.{layer}" x = self.norm(f"{prefix}.norm1", queries) queries = queries + self._attention( f"{prefix}.self_attn", self._project(f"{prefix}.self_attn", 0, x), self._project(f"{prefix}.self_attn", 1, x), self._project(f"{prefix}.self_attn", 2, x), mask) x = self.norm(f"{prefix}.norm2", queries) queries = queries + self._attention( f"{prefix}.multihead_attn", self._project(f"{prefix}.multihead_attn", 0, x), keys, values, None) x = self.norm(f"{prefix}.norm3", queries) queries = queries + self.linear(f"{prefix}.linear2", nn.gelu(self.linear(f"{prefix}.linear1", x))) return queries # ---- readouts def typed_logits(self, raw: mx.array, routed: mx.array) -> tuple[mx.array, mx.array, mx.array]: count, slots = raw.shape[0], cm.DIGIT_SLOTS scale = mx.exp(mx.minimum(self.w["lexical_scale"], float(np.log(100.0)))) unit = _normalize(raw) down = (raw @ self.w["slot_down.weight"].T).reshape(count, slots + 2, self.slot_rank) digit_prior = (unit @ _normalize(self.w["digit_rows"]).T)[:, None, :] month_prior = unit @ _normalize(self.w["month_rows"]).T sign_prior = unit @ _normalize(self.w["sign_rows"]).T residual = mx.sigmoid(self.w["residual_gate"]) * self.linear("slot_residual", routed) digit_residual = residual[:, : slots * 10].reshape(count, slots, 10) month_residual, sign_residual = residual[:, slots * 10: slots * 10 + 12], residual[:, slots * 10 + 12:] return ( scale[0] * digit_prior + mx.einsum("nsr,srd->nsd", down[:, :slots], self.w["slot_up"]) + digit_residual, scale[1] * month_prior + down[:, slots] @ self.w["month_up"] + month_residual, scale[2] * sign_prior + down[:, slots + 1] @ self.w["sign_up"] + sign_residual, ) def typed_outputs(self, raw: mx.array, states: mx.array, routed: mx.array) -> dict[str, mx.array]: digits, months, signs = self.typed_logits(raw, routed) count = raw.shape[0] year_slots = digits[:, cm.YEAR_SLOT: cm.YEAR_SLOT + cm.YEAR_DIGITS] return { "number": mx.concatenate([signs, self.linear("exponent", states), digits[:, : cm.NUMBER_DIGITS].reshape(count, -1)], axis=-1), "date": mx.concatenate([year_slots.reshape(count, -1), _years(year_slots), months, _two_digits(digits[:, cm.DAY_SLOT: cm.DAY_SLOT + 2], range(1, 32))], axis=-1), "time": mx.concatenate([_two_digits(digits[:, cm.HOUR_SLOT: cm.HOUR_SLOT + 2], range(24)), _two_digits(digits[:, cm.MINUTE_SLOT: cm.MINUTE_SLOT + 2], range(60)), _two_digits(digits[:, cm.SECOND_SLOT: cm.SECOND_SLOT + 2], range(60))], axis=-1), } def group_scores(self, states: mx.array) -> mx.array: return self.linear("number_group_up", nn.gelu(self.linear("number_group_down", states))).reshape( -1, cm.NUMBER_GROUPS, cm.GROUP_CLASSES) def temper_number(self, logits: mx.array, spread: mx.array) -> mx.array: classes, digits = cm.EXPONENT_CLASSES, cm.NUMBER_DIGITS sign, exponent = logits[:, :2], logits[:, 2: 2 + classes] slots = logits[:, 2 + classes:].reshape(-1, digits, 10) limit = cm.ESTIMATE_SPREAD_LIMIT temperature = mx.exp(limit * mx.tanh(spread / limit)) slots = mx.concatenate([slots[:, : cm.LEXICAL_DIGITS] / temperature[:, 1:, None], slots[:, cm.LEXICAL_DIGITS:]], axis=1) return mx.concatenate([sign, exponent / temperature[:, :1], slots.reshape(slots.shape[0], -1)], axis=-1) def forward(self, hidden: mx.array, start: int, stop: int, views: dict[str, np.ndarray], memory: list[tuple[mx.array, mx.array]]) -> dict[str, mx.array]: """Upstream `forward_lexical`: `hidden` is [tokens, width] in fp32, `views` the index arrays.""" if any(name.endswith("_prefix_length") for name in views): raise ValueError("prefixed number readouts are not supported") rows = {name: np.asarray(value) for name, value in views.items()} queries = rows["query_positions"] item_kinds = [kind for kind in cm.LIST_KINDS if rows[f"{kind}_rows"].shape[0]] positions = np.concatenate([queries] + [rows[f"{kind}_items"].reshape(-1) for kind in item_kinds]) kinds = np.zeros(positions.shape[0], dtype=np.int64) for index, group in enumerate(cm.ROUTE_KINDS): kinds[rows[f"{group}_rows"]] = index offset = queries.shape[0] for kind in item_kinds: size = rows[f"{kind}_items"].size kinds[offset: offset + size] = len(cm.ROUTE_KINDS) + cm.LIST_KINDS.index(kind) offset += size def take(values: mx.array, index: np.ndarray) -> mx.array: return values[mx.array(index.astype(np.int32))] raw = take(hidden, positions) routed = self._route_layers( (self.linear("route_in", self.norm("norm", raw)) + take(self.w["kind_embedding.weight"], kinds))[None], memory) routed = self.norm("route_norm", routed[0]) states = self.norm("norm", raw) + mx.sigmoid(self.w["route_gate"]) * self.linear("route_out", routed) count = queries.shape[0] output: dict[str, mx.array] = {"null": self.linear("null", states[:count])[:, 0]} field_states, field_raw, field_routed = states[:count], raw[:count], routed[:count] text, tags, numbers = rows["text_rows"], rows["tags_rows"], rows["number_rows"] scale = 1.0 / float(np.sqrt(self.pointer_width)) tokens = None if text.shape[0] or tags.shape[0] or numbers.shape[0]: tokens = self.norm("token_norm", hidden[start:stop]) if text.shape[0] or tags.shape[0]: output["trim"] = self.linear("trim", tokens).reshape(-1, 2, cm.TRIM_CLASSES) def pointer(query: str, selected: mx.array) -> mx.array: query_start, query_end = mx.split(self.linear(query, selected), 2, axis=-1) key_start, key_end = mx.split(self.linear("span_key", tokens), 2, axis=-1) return _span_scores(query_start @ key_start.T * scale, query_end @ key_end.T * scale, self.w["length_bias"]) if numbers.shape[0]: number_states = take(field_states, numbers) output["number_copy"] = pointer("copy_query", number_states) output["number_copy_gate"] = self.linear("copy_gate", number_states)[:, 0] if text.shape[0]: output["text"] = pointer("span_query", take(field_states, text)) if tags.shape[0]: query_begin, query_inside = mx.split(self.linear("tag_query", take(field_states, tags)), 2, axis=-1) keys = self.linear("tag_key", tokens) begin = query_begin @ keys.T * scale + self.w["tag_bias"][0] inside = query_inside @ keys.T * scale + self.w["tag_bias"][1] output["tags"] = mx.stack([mx.zeros_like(begin), begin, inside], axis=-1) typed_rows = [rows[f"{group}_rows"] for group in ("number", "date", "time")] if any(selected.shape[0] for selected in typed_rows): selected = np.concatenate(typed_rows) typed = self.typed_outputs(take(field_raw, selected), take(field_states, selected), take(field_routed, selected)) offset = 0 for group, selected in zip(("number", "date", "time"), typed_rows): if selected.shape[0]: output[group] = typed[group][offset: offset + selected.shape[0]] offset += selected.shape[0] if "number" in output: output["number_groups"] = self.group_scores(take(field_states, numbers)) estimates = rows["estimate_rows"] if estimates.shape[0]: estimate_states = take(field_states, estimates) output["estimate"] = self.linear("bins", estimate_states) output["estimate_offset"] = self.linear("offset", estimate_states) output["estimate_overflow"] = self.linear("overflow", estimate_states) number_logits = self.typed_outputs(take(field_raw, estimates), estimate_states, take(field_routed, estimates))["number"] output["estimate_number_raw"] = number_logits output["estimate_number"] = self.temper_number(number_logits, self.linear("estimate_spread", estimate_states)) output["estimate_gate"] = self.linear("estimate_gate", estimate_states)[:, 0] offset = count for kind in item_kinds: shape = rows[f"{kind}_items"].shape size = shape[0] * shape[1] item_states, item_raw, item_routed = ( states[offset: offset + size], raw[offset: offset + size], routed[offset: offset + size]) offset += size output[f"{kind}_end"] = self.linear("item_end", item_states)[:, 0].reshape(shape) group = "number" if kind == "number_list" else "date" output[kind] = self.typed_outputs(item_raw, item_states, item_routed)[group].reshape(*shape, -1) if kind == "number_list": output["number_list_groups"] = self.group_scores(item_states).reshape( *shape, cm.NUMBER_GROUPS, cm.GROUP_CLASSES) return output def answer_slot_states(self, hidden: mx.array, field_queries: np.ndarray, kinds: np.ndarray, positions: np.ndarray, lengths: np.ndarray, stop: int, memory: list[tuple[mx.array, mx.array]], chunk: int = 256) -> mx.array: """Upstream `answer_slot_states`: each answer-slot position corrected by evidence routed from the prompt.""" width = positions.shape[1] if (width < 1 or positions.shape[0] != field_queries.shape[0] or (lengths < 1).any() or (lengths > width).any() or not 0 < stop <= hidden.shape[0]): raise ValueError("invalid answer-slot routing layout") gate = mx.sigmoid(self.w["route_gate"]) output = [] field_batch = max(1, min(chunk // (width + 1), 262144 // stop)) for first in range(0, positions.shape[0], field_batch): last = first + field_batch slot_positions = mx.array(positions[first:last].astype(np.int32)) query_positions = mx.concatenate([mx.array(field_queries[first:last, None].astype(np.int32)), slot_positions], axis=1) row_lengths = mx.array(lengths[first:last].astype(np.int32)) row_kinds = mx.array(kinds[first:last].astype(np.int32)) padding = mx.arange(width + 1)[None, :] > row_lengths[:, None] queries = self.linear("route_in", self.norm("norm", hidden[query_positions])) queries = self._route_layers(queries + self.w["kind_embedding.weight"][row_kinds][:, None, :], memory, padding) evidence = self.linear("route_out", self.norm("route_norm", queries[:, 1:])) scale = mx.tanh(self.w["slot_route_scale"][row_kinds])[:, None, None] corrected = hidden[slot_positions] + scale * gate * evidence index = mx.minimum(mx.arange(width)[None, :], row_lengths[:, None] - 1) output.append(mx.take_along_axis(corrected, index[:, :, None], axis=1)) mx.eval(output[-1]) return mx.concatenate(output, axis=0) # ---------------------------------------------------------------------------------------------- engine @dataclass class _Rows: """One right-padded batch: token ids, which positions are real, and the rows' image inputs in order.""" ids: np.ndarray valid: np.ndarray media: list[dict[str, Any]] = field(default_factory=list) @property def lengths(self) -> np.ndarray: return self.valid.sum(axis=1) def _common_prefix(sequences: list[list[int]]) -> int: first, last = min(sequences), max(sequences) return next((i for i, (a, b) in enumerate(zip(first, last)) if a != b), min(len(first), len(last))) class MLXEngine: """Share one MLX backbone between grammar-constrained extraction and typed readouts.""" def __init__(self, path: Path, tokenizer: Any, *, batch_size: int = 8, batch_token_budget: int = 8192, rope: str = "temporal", device: str = "gpu", memory_limit_gb: float | None = None) -> None: import xgrammar as xgr if rope not in ROPE_MODES: raise ValueError(f"rope must be one of {ROPE_MODES}") self.path, self.tokenizer, self.rope = Path(path), tokenizer, rope self.batch_size, self.batch_token_budget = max(1, int(batch_size)), max(1, int(batch_token_budget)) self.text_batch_size = self.batch_size self.device = device # "gpu", or "cpu" for bit-accurate float32 checks (slow) self.capture: Callable[[str, Any], None] | None = None # test hook: receives intermediate arrays # A request that would need more memory than this fails with an error instead of exhausting the machine. self.memory_cap = int((memory_limit_gb * 1e9) if memory_limit_gb else MEMORY_FRACTION * mx.device_info()["memory_size"]) self._worker = concurrent.futures.ThreadPoolExecutor(max_workers=1, thread_name_prefix="clairic-model") self._features: dict[str, mx.array] = {} self._prefixes: dict[tuple, Cache] = {} self.prefix_cache = True # read a request's images once and continue every prompt from that state # Every MLX call happens on the worker thread, including loading the weights. self._worker.submit(self._load).result() self.eos = tokenizer.convert_tokens_to_ids("<|im_end|>") self.pad = tokenizer.pad_token_id if tokenizer.pad_token_id is not None else self.eos self.vocab = self.lm.lm_head.weight.shape[0] info = xgr.TokenizerInfo.from_huggingface(tokenizer, vocab_size=self.vocab, stop_token_ids=[self.eos]) self.compiler = xgr.GrammarCompiler(info, max_threads=2) self._xgr = xgr # ---- loading def _load(self) -> None: path = self.path if self.device == "cpu": mx.set_default_device(mx.cpu) config = json.loads((path / "config.json").read_text()) self.model = Model(ModelConfig.from_dict(config)) self.lm = self.model.language_model text = self.lm.model hidden, eps = config["text_config"]["hidden_size"], config["text_config"]["rms_norm_eps"] text.norm = PreciseRMSNorm(hidden, eps) for layer in text.layers: layer.input_layernorm = PreciseRMSNorm(hidden, eps) layer.post_attention_layernorm = PreciseRMSNorm(hidden, eps) if not layer.is_linear: layer.self_attn.q_norm = PreciseRMSNorm(layer.self_attn.head_dim, eps) layer.self_attn.k_norm = PreciseRMSNorm(layer.self_attn.head_dim, eps) weights: dict[str, mx.array] = {} for shard in sorted(path.glob("model*.safetensors")): weights.update(mx.load(str(shard))) if quantization := config.get("quantization"): def quantized(path: str, module: nn.Module) -> bool | dict: if not hasattr(module, "to_quantized") or f"{path}.scales" not in weights: return False settings = quantization.get(path) # modules with their own bit width return settings if isinstance(settings, dict) else True nn.quantize(self.model, group_size=quantization["group_size"], bits=quantization["bits"], mode=quantization.get("mode", "affine"), class_predicate=quantized) self.model.load_weights(list(weights.items()), strict=True) self.model.eval() mx.eval(self.model.parameters()) self.image_token = int(config["image_token_id"]) self.vision_end = int(config["vision_end_token_id"]) self.dtype = text.layers[0].linear_attn.conv1d.weight.dtype # never quantized: bf16 (release) or fp32 self.backbone = self.model self.head = TypedHead(path, round_bf16=self.dtype != mx.float32) # ---- request scope (image features are kept for one request) def begin_request(self) -> None: self._features, self._prefixes = {}, {} async def end_request(self) -> None: await asyncio.get_running_loop().run_in_executor(self._worker, self._clear) def _clear(self) -> None: self._features, self._prefixes = {}, {} mx.clear_cache() def shutdown(self) -> None: self._worker.shutdown(wait=True, cancel_futures=True) self._features, self._prefixes = {}, {} def _check_memory(self) -> None: if mx.get_active_memory() > self.memory_cap: mx.clear_cache() if mx.get_active_memory() > self.memory_cap: raise MemoryError( f"this request needs more than the {self.memory_cap / 1e9:.0f} GB memory limit; " "shorten the input, lower batch_token_budget, or raise memory_limit_gb") # ---- backbone def _image_features(self, mapping: dict[str, Any]) -> mx.array: key = mapping.get("clair_pixels_key") if key is None: pixels = np.asarray(mapping["pixel_values"]) key = hashlib.sha1(pixels.tobytes() + np.asarray(mapping["image_grid_thw"]).tobytes()).hexdigest() if key not in self._features: tower = self.model.vision_tower pixels = mx.array(mapping["pixel_values"]).astype(tower.patch_embed.proj.weight.dtype) features, _ = tower(pixels, mx.array(np.asarray(mapping["image_grid_thw"]))) mx.eval(features) self._features[key] = features return self._features[key] def _rows(self, sequences: list[list[int]], media: list[Any]) -> _Rows: width = max(map(len, sequences)) ids = np.full((len(sequences), width), self.pad, dtype=np.int32) valid = np.zeros((len(sequences), width), dtype=bool) for row, sequence in enumerate(sequences): ids[row, : len(sequence)] = sequence valid[row, : len(sequence)] = True return _Rows(ids, valid, [mapping for payload in media if payload for mapping in payload]) def _embed(self, rows: _Rows) -> tuple[mx.array, mx.array]: """Input embeddings (image features merged in) and rotary positions [3, B, L] of a right-padded batch.""" tokens, mask = mx.array(rows.ids), mx.array(rows.valid.astype(np.int32)) if rows.media: features = mx.concatenate([self._image_features(mapping) for mapping in rows.media], axis=0) grids = mx.array(np.concatenate([np.asarray(mapping["image_grid_thw"]) for mapping in rows.media])) embedded = self.model.get_input_embeddings( tokens, features, image_grid_thw=grids, mask=mask, cached_image_features=features) else: embedded = self.model.get_input_embeddings(tokens, None, mask=mask) positions = embedded.position_ids if positions.ndim == 2: # text only: the same position on the three rotary axes positions = mx.broadcast_to(positions[None], (3, *positions.shape)) return embedded.inputs_embeds, positions def _rotary(self, positions: mx.array) -> mx.array: # "temporal" (default) is what the upstream runtime does in bfloat16, its serving configuration: the # rotary table is indexed by the first (temporal) position axis for every frequency, so the tokens of an # image share one rotary position. "mrope" is the stock Transformers 3-axis rotary, which upstream runs # in float32. Text tokens are identical in both. if self.rope == "temporal": return mx.broadcast_to(positions[:1], positions.shape) return positions def _linear(self, attention: nn.Module, x: mx.array, valid: mx.array | None, window: mx.array | None, memory: mx.array | None, lengths: mx.array | None = None) -> tuple[mx.array, mx.array, mx.array]: """A Gated DeltaNet layer continuing from a convolution window and recurrent memory (padding skipped).""" batch, length, _ = x.shape mixed = attention.in_proj_qkv(x) z = attention.in_proj_z(x).reshape(batch, length, -1, attention.head_v_dim) b, a = attention.in_proj_b(x), attention.in_proj_a(x) if valid is not None: mixed = mx.where(valid[..., None], mixed, 0) if window is None: window = mx.zeros((batch, attention.conv_kernel_size - 1, attention.conv_dim), dtype=mixed.dtype) conv_input = mx.concatenate([window.astype(mixed.dtype), mixed], axis=1) conv = nn.silu(attention.conv1d(conv_input)) q, k, v = ( t.reshape(batch, length, heads, width) for t, heads, width in zip( mx.split(conv, [attention.key_dim, 2 * attention.key_dim], -1), [attention.num_k_heads, attention.num_k_heads, attention.num_v_heads], [attention.head_k_dim, attention.head_k_dim, attention.head_v_dim], strict=True) ) q, k = attention._normalize_qk(q, k) out, memory = gated_delta_update(q, k, v, a, b, attention.A_log, attention.dt_bias, state=memory, mask=valid, use_kernel=True) out = attention.out_proj(attention.norm(out, z).reshape(batch, length, -1)) taps = attention.conv_kernel_size - 1 if lengths is None: return out, memory, conv_input[:, -taps:] # Right-padded rows: keep the last real inputs of each row, not the padding after them. index = lengths[:, None] + mx.arange(taps)[None, :] index = mx.broadcast_to(index[:, :, None], (batch, taps, conv_input.shape[-1])) return out, memory, mx.take_along_axis(conv_input, index, axis=1) def _attention(self, attention: nn.Module, x: mx.array, positions: mx.array, keys: mx.array | None, values: mx.array | None, valid: mx.array | None, offset: int) -> tuple[mx.array, mx.array, mx.array]: """Causal attention of new tokens over the cached and new keys; padded keys are masked.""" batch, length, _ = x.shape queries, new_keys, new_values, gate, _ = attention._prepare_projected_qkv( attention.q_proj(x), attention.k_proj(x), attention.v_proj(x), None, positions, None, None) if keys is not None: new_keys = mx.concatenate([keys, new_keys], axis=2) new_values = mx.concatenate([values, new_values], axis=2) total = new_keys.shape[2] # The score matrix is queries x keys per head. Long inputs are attended in slices of query rows so its # size stays bounded (a slice's output depends only on its own rows); unsliced, a 40k-token input needs # tens of GB per layer. rows = max(1, ATTENTION_ELEMENTS // (batch * attention.num_attention_heads * total)) outputs = [] for start in range(0, length, rows): stop = min(length, start + rows) mask = None if length > 1: key_index = mx.arange(total)[None, :] row_index = mx.arange(offset + start, offset + stop)[:, None] mask = (key_index <= row_index)[None, None] if valid is not None: # a padded query still sees itself, so no row is fully masked mask = mask & (valid[:, None, None, :] | (key_index == row_index)[None, None]) elif valid is not None: mask = valid[:, None, None, :] outputs.append(mx.fast.scaled_dot_product_attention( queries[:, :, start:stop], new_keys, new_values, scale=attention.scale, mask=mask)) if rows < length: mx.eval(outputs[-1]) self._check_memory() output = outputs[0] if len(outputs) == 1 else mx.concatenate(outputs, axis=2) output = output.transpose(0, 2, 1, 3).reshape(batch, length, -1) return attention.o_proj(output * mx.sigmoid(gate)), new_keys, new_values def _forward(self, h: mx.array, positions: mx.array, lengths: np.ndarray | None, cache: Cache | None, keep: bool = True) -> tuple[mx.array, Cache | None]: """Run the layers over new tokens, optionally continuing from a cache. `lengths` gives each row's number of real tokens when the batch is right-padded (None: no padding). Returns the hidden states before the final norm and, with `keep`, the cache after each row's last real token (padded positions stay in the attention cache, marked invalid). """ layers = self.lm.model.layers cache = cache or Cache.empty(len(layers)) batch, length, _ = h.shape valid = row_lengths = None if lengths is not None and (lengths < length).any(): row_lengths = mx.array(lengths.astype(np.int32)) valid = mx.arange(length)[None, :] < row_lengths[:, None] all_valid = None if cache.valid is not None or valid is not None: before = cache.valid if cache.valid is not None else mx.ones((batch, cache.length), dtype=mx.bool_) new = valid if valid is not None else mx.ones((batch, length), dtype=mx.bool_) all_valid = mx.concatenate([before, new], axis=1) rotary = self._rotary(positions) updated = Cache.empty(len(layers)) if keep else None stepwise = batch * length > 4096 # keep the lazy graph small on long inputs for index, layer in enumerate(layers): x = layer.input_layernorm(h) if layer.is_linear: r, memory, window = self._linear(layer.linear_attn, x, valid, cache.window[index], cache.memory[index], row_lengths) if keep: updated.memory[index], updated.window[index] = memory, window else: r, keys, values = self._attention(layer.self_attn, x, rotary, cache.keys[index], cache.values[index], all_valid, cache.length) if keep: updated.keys[index], updated.values[index] = keys, values h = h + r h = h + layer.mlp(layer.post_attention_layernorm(h)) if stepwise: mx.eval(h, *(updated.arrays() if keep else [])) self._check_memory() if keep: updated.valid, updated.length = all_valid, cache.length + length return h, updated def _run(self, h: mx.array, positions: mx.array, lengths: np.ndarray | None, cache: Cache | None, keep: bool = True) -> tuple[mx.array, Cache | None]: """`_forward` over a whole input. Long inputs go through the layers in segments, each continuing from the cache of the previous one, so working memory depends on the segment size rather than the input length (attention is causal and the linear layers are recurrent, so the result is the same). With `keep`, the returned cache has `position` set to each row's next rotary position.""" batch, length, _ = h.shape segment = max(256, SEGMENT_TOKENS // batch) if length <= segment: out, cache = self._forward(h, positions, lengths, cache, keep) else: outputs = [] for start in range(0, length, segment): stop = min(length, start + segment) inside = None if lengths is None else np.clip(lengths - start, 0, stop - start) part, cache = self._forward(h[:, start:stop], positions[:, :, start:stop], inside, cache) mx.eval(part, *cache.arrays()) self._check_memory() outputs.append(part) out = mx.concatenate(outputs, axis=1) cache = cache if keep else None if keep: last = mx.array((lengths if lengths is not None else np.full(batch, length)).astype(np.int32)) - 1 index = mx.broadcast_to(last[None, :, None], (positions.shape[0], batch, 1)) cache.position = mx.take_along_axis(positions, index, axis=2)[:, :, 0].max(axis=0) + 1 return out, cache def _hidden(self, sequences: list[list[int]], media: list[Any]) -> mx.array: """Final-norm hidden states of an unpadded single sequence.""" rows = self._rows(sequences, media) h, positions = self._embed(rows) h, _ = self._run(h, positions, None, None, keep=False) return self.lm.model.norm(h) # ---- typed readout (upstream `read_record`) def _read_record(self, states: mx.array, item: Any) -> dict[str, np.ndarray]: start, length, kinds, queries, items, slot_fields, slots = cm.parse_prediction_layout( cm.prediction_layout(item), len(item.input_ids), with_slots=True) views = cm.prediction_views(kinds, queries, items, None) hidden = states.astype(mx.float32) stop = start + length memory = self.head._memory(hidden, stop) output = self.head.forward(hidden, start, stop, views, memory) mx.eval(output) values: dict[str, np.ndarray] = {name: np.array(value) for name, value in output.items()} if slots: width = max(map(len, slots)) positions = np.array([row + [row[-1]] * (width - len(row)) for row in slots], dtype=np.int64) field_queries = np.asarray(views["query_positions"])[slot_fields] slot_kinds = np.array([cm.ROUTE_KINDS.index("tags" if kinds[row] == "text_list" else kinds[row]) for row in slot_fields], dtype=np.int64) lengths = np.array([len(row) for row in slots], dtype=np.int64) slot_states = self.head.answer_slot_states(hidden, field_queries, slot_kinds, positions, lengths, stop, memory) slot_states = slot_states.reshape(-1, slot_states.shape[-1]).astype(states.dtype) ids, logprobs = [], [] for first in range(0, slot_states.shape[0], 256): logits = self.lm.lm_head(slot_states[first: first + 256]).astype(mx.float32) best = mx.argmax(logits, axis=-1) ids.append(best) logprobs.append(mx.take_along_axis(logits, best[:, None], axis=-1)[:, 0] - mx.logsumexp(logits, axis=-1)) mx.eval(ids[-1], logprobs[-1]) values["slot_ids"] = np.array(mx.concatenate(ids)).reshape(positions.shape) values["slot_log_probs"] = np.array(mx.concatenate(logprobs)).reshape(positions.shape) return values def _read_typed(self, item: Any, media: Any, _cancelled: threading.Event | None = None) -> dict[str, dict[str, Any]]: if _cancelled is not None and _cancelled.is_set(): raise RuntimeError("Inference cancelled") states = self._hidden([list(item.input_ids)], [media])[0] mx.eval(states) if _cancelled is not None and _cancelled.is_set(): raise RuntimeError("Inference cancelled") values = self._read_record(states, item) compact = cm.compact_output(values) if self.capture is not None: self.capture("hidden", np.array(states.astype(mx.float32))) self.capture("compact", np.asarray(compact, dtype=np.float32)) if "slot_ids" in values: self.capture("slots", values["slot_ids"]) if media: self.capture("pixels", np.asarray(media[0]["pixel_values"]).astype(np.float16)) self.capture("grid", np.asarray(media[0]["image_grid_thw"])) values = cm.expand_output(compact, item, bins=self.head.bins, estimate_lexical=True, span_trim=True, number_groups=True, number_pointer=True) return cm.decode_record(values, item, self.tokenizer) async def read_typed(self, item: Any, media: Any = None) -> dict[str, dict[str, Any]]: return await self._submit(lambda cancelled: self._read_typed(item, media, _cancelled=cancelled)) # ---- constrained generation def _grammar(self, parameters: dict[str, Any]) -> Any: if "json_schema" in parameters: value = parameters["json_schema"] return self.compiler.compile_json_schema(json.loads(value) if isinstance(value, str) else value, any_whitespace=True) if "regex" in parameters: return self.compiler.compile_regex(parameters["regex"]) if "ebnf" in parameters: return self.compiler.compile_grammar(parameters["ebnf"]) raise ValueError("Extraction continuations require a JSON schema, regex, or grammar") def _logits(self, h: mx.array) -> np.ndarray: """Vocabulary logits (fp32) from hidden states [B, 1, width] before the final norm.""" return np.array(self.lm.lm_head(self.lm.model.norm(h))[:, 0].astype(mx.float32)) @staticmethod def _media_key(payload: Any) -> tuple | None: return tuple(mapping.get("clair_pixels_key") or id(mapping["pixel_values"]) for mapping in payload) if payload else None def _image_prefix(self, sequences: list[list[int]], media: list[Any]) -> tuple[Cache | None, int]: """The state after the images shared by every prompt of a batch, computed once per request. Image prompts start with the same image tokens and differ only afterwards. The shared part, up to the last `<|vision_end|>` common to all rows, is run once and each row continues from its cache; attention is causal and the linear layers are recurrent, so this equals running every full prompt. """ if not self.prefix_cache or not media[0] or any(self._media_key(m) != self._media_key(media[0]) for m in media): return None, 0 common = sequences[0][: _common_prefix(sequences)] if self.vision_end not in common: return None, 0 cut = len(common) - common[::-1].index(self.vision_end) if any(self.image_token in sequence[cut:] for sequence in sequences): return None, 0 key = (self._media_key(media[0]), hashlib.sha1(np.asarray(common[:cut], dtype=np.int32).tobytes()).hexdigest()) if key not in self._prefixes: rows = self._rows([common[:cut]], [media[0]]) h, positions = self._embed(rows) _, cache = self._run(h, positions, None, None) mx.eval(cache.position, *cache.arrays()) self._prefixes[key] = cache return self._prefixes[key], cut def _prefill(self, sequences: list[list[int]], media: list[Any]) -> tuple[mx.array, Cache]: """Run the prompts of one batch. Returns each row's last hidden state [B, 1, width] and the cache.""" size = len(sequences) prefix, cut = self._image_prefix(sequences, media) if prefix is None: rows = self._rows(sequences, media) h, positions = self._embed(rows) cache = None else: rows = self._rows([sequence[cut:] for sequence in sequences], [None] * size) h = self.lm.model.embed_tokens(mx.array(rows.ids)) positions = prefix.position[0] + mx.arange(rows.ids.shape[1]) positions = mx.broadcast_to(positions[None, None, :], (3, size, rows.ids.shape[1])) def spread(array: mx.array | None) -> mx.array | None: return None if array is None else mx.broadcast_to(array, (size, *array.shape[1:])) cache = Cache([spread(a) for a in prefix.keys], [spread(a) for a in prefix.values], [spread(a) for a in prefix.memory], [spread(a) for a in prefix.window], None, prefix.length) lengths = rows.lengths h, cache = self._run(h, positions, lengths, cache) last = mx.array((lengths - 1).astype(np.int32)) index = mx.broadcast_to(last[:, None, None], (size, 1, h.shape[-1])) return mx.take_along_axis(h, index, axis=1), cache def _decode(self, sequences: list[list[int]], media: list[Any], grammars: list[Any], maximum: int, stop_ids: tuple[int, ...], score_ids: list[int], cancelled: threading.Event | None, ) -> list[tuple[list[int], list[float], list[float]]]: """Greedy decoding of one batch under per-row grammars. Returns (tokens, log-probabilities, first-step log-probabilities of `score_ids`) per row; tokens end with the stop token when one was produced.""" size = len(sequences) h, cache = self._prefill(sequences, media) mx.eval(h, cache.position, *cache.arrays()) scores = self._logits(h) matchers = [self._xgr.GrammarMatcher(grammar) for grammar in grammars] bitmask = np.zeros(self._xgr.get_bitmask_shape(size, self.vocab), dtype=np.int32) tokens: list[list[int]] = [[] for _ in range(size)] logprobs: list[list[float]] = [[] for _ in range(size)] first: list[list[float]] = [[] for _ in range(size)] done = [False] * size for step in range(maximum): for row, matcher in enumerate(matchers): if not matcher.is_terminated(): matcher.fill_next_token_bitmask(bitmask, row) allowed = np.unpackbits(bitmask.view(np.uint8), axis=-1, bitorder="little")[:, : self.vocab].astype(bool) masked = np.where(allowed, scores, -np.inf).astype(np.float32) peak = masked.max(axis=-1, keepdims=True) with np.errstate(divide="ignore"): log_probabilities = masked - peak - np.log(np.exp(masked - peak).sum(axis=-1, keepdims=True, dtype=np.float32)) choices = masked.argmax(axis=-1) for row in range(size): if step == 0: first[row] = [float(log_probabilities[row, token]) for token in score_ids] if done[row]: continue token = int(choices[row]) tokens[row].append(token) logprobs[row].append(float(log_probabilities[row, token])) if token in stop_ids: done[row] = True if not matchers[row].is_terminated() and not matchers[row].accept_token(token): raise RuntimeError("Generated token rejected by its grammar") if all(done) or step + 1 == maximum: break if cancelled is not None and cancelled.is_set(): raise RuntimeError("Inference cancelled") feed = mx.array([[self.pad if done[row] else tokens[row][-1]] for row in range(size)]) position = cache.position h, cache = self._forward(self.lm.model.embed_tokens(feed), mx.broadcast_to(position[None, :, None], (3, size, 1)), None, cache) cache.position = position + 1 mx.eval(h, cache.position, *cache.arrays()) scores = self._logits(h) return list(zip(tokens, logprobs, first, strict=True)) def _generate(self, *, input_ids: list[int] | list[list[int]], image_data: Any = None, sampling_params: dict[str, Any] | list[dict[str, Any]], rid: Any = None, token_ids_logprob: list[int] | None = None, _cancelled: threading.Event | None = None, **kwargs: Any) -> Any: if _cancelled is not None and _cancelled.is_set(): raise RuntimeError("Inference cancelled") if not input_ids: return [] single = isinstance(input_ids[0], int) sequences = [input_ids] if single else input_ids size = len(sequences) parameters = [sampling_params] * size if isinstance(sampling_params, dict) else sampling_params media = [image_data] if single else (image_data or [None] * size) ids = [rid] if single else (rid or [None] * size) if not (len(parameters) == len(media) == len(ids) == size): raise ValueError("Extraction batch arguments have inconsistent lengths") results: list[Any] = [None] * size groups: dict[tuple[int, tuple[int, ...]], list[int]] = defaultdict(list) for index, params in enumerate(parameters): if params.get("temperature", 0) != 0: raise ValueError("Clairic extraction uses deterministic decoding") groups[(int(params.get("max_new_tokens", 256)), tuple(params.get("stop_token_ids", [self.eos])))].append(index) for (maximum, stop_ids), indices in groups.items(): # Rows with the same images and similar length share a batch; a row's result does not depend on # its batch. Rows that share images are charged for the image prefix once. indices = sorted(indices, key=lambda index: (str(self._media_key(media[index])), len(sequences[index]))) chunks: list[list[int]] = [] pending: list[int] = [] for index in indices: if pending and (len(pending) >= self.batch_size or self._media_key(media[index]) != self._media_key(media[pending[0]]) or self._cost([sequences[i] for i in (*pending, index)], media[index]) > self.batch_token_budget): chunks.append(pending) pending = [] pending.append(index) if pending: chunks.append(pending) for chunk in chunks: if _cancelled is not None and _cancelled.is_set(): raise RuntimeError("Inference cancelled") decoded = self._decode([sequences[i] for i in chunk], [media[i] for i in chunk], [self._grammar(parameters[i]) for i in chunk], maximum, stop_ids, list(token_ids_logprob or []), _cancelled) for index, (output, logprobs, first) in zip(chunk, decoded, strict=True): stopped = bool(output) and output[-1] in stop_ids results[index] = { "output_ids": output, "text": self.tokenizer.decode(output[:-1] if stopped else output, skip_special_tokens=False, clean_up_tokenization_spaces=False), "meta_info": { "id": ids[index], "completion_tokens": len(output), "finish_reason": {"type": "stop", "matched": output[-1]} if stopped else {"type": "length"}, "output_token_logprobs": [(value, token, None) for value, token in zip(logprobs, output, strict=True)], "output_token_ids_logprobs": [[(value, token, None) for value, token in zip(first, token_ids_logprob or [], strict=True)]], }, } return results[0] if single else results def _cost(self, sequences: list[list[int]], payload: Any) -> int: """Tokens a batch runs through the model: the padded width times its rows (shared image tokens once).""" shared = 0 if payload and self.prefix_cache and len(sequences) > 1: common = sequences[0][: _common_prefix(sequences)] shared = len(common) - common[::-1].index(self.vision_end) if self.vision_end in common else 0 return (max(map(len, sequences)) - shared) * len(sequences) async def async_generate(self, **kwargs: Any) -> Any: return await self._submit(lambda cancelled: self._generate(_cancelled=cancelled, **kwargs)) async def _submit(self, operation: Callable[[threading.Event], Any]) -> Any: cancelled = threading.Event() future = asyncio.get_running_loop().run_in_executor(self._worker, lambda: operation(cancelled)) try: return await asyncio.shield(future) except asyncio.CancelledError: cancelled.set() # Admission stays occupied until the worker has actually stopped. while not future.done(): try: await asyncio.shield(future) except asyncio.CancelledError: continue except Exception: break if not future.cancelled(): future.exception() raise