# Copyright 2025 The Qwen Team and The HuggingFace Inc. team. All rights reserved. # Copyright 2026 Cloudflare, Inc. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 # Unless required by applicable law or agreed to in writing, software distributed # under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR # CONDITIONS OF ANY KIND, either express or implied. See the License for the # specific language governing permissions and limitations under the License. """Numerical operations for the Transformers Clairic backbone.""" from __future__ import annotations from collections.abc import Callable from types import MethodType from typing import Any import torch from torch import nn from torch.nn import functional as F from torch.utils._python_dispatch import TorchDispatchMode from transformers.cache_utils import Cache from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS from transformers.models.qwen3_5.modeling_qwen3_5 import ( Qwen3_5Attention, Qwen3_5DecoderLayer, Qwen3_5ForConditionalGeneration, Qwen3_5GatedDeltaNet, Qwen3_5MLP, Qwen3_5RMSNorm, Qwen3_5TextRotaryEmbedding, apply_mask_to_padding_states, eager_attention_forward, torch_causal_conv1d_update, ) from .attention_metadata import DecodeMetadata from .recurrent_metadata import scope as recurrent_metadata_scope def _matmul(a: torch.Tensor, b: torch.Tensor, bias: torch.Tensor | None = None) -> torch.Tensor: if a.ndim != 2 or b.ndim != 2 or a.shape[1] != b.shape[0] or a.dtype != b.dtype: raise ValueError("Expected aligned matrices with equal dtype") if (a.dtype == torch.bfloat16 and b.dtype == torch.bfloat16 and a.is_contiguous() and b.transpose(0, 1).is_contiguous() and b.shape[1] >= 16): from deep_gemm import bf16_gemm_nn output = torch.empty((a.shape[0], b.shape[1]), device=a.device, dtype=a.dtype) bf16_gemm_nn(a, b, output) if bias is not None: output += bias return output from .kernels import _matmul_persistent_triton return _matmul_persistent_triton(a, b, bias) def _first_tensor(values: Any) -> torch.Tensor | None: if isinstance(values, torch.Tensor): return values if isinstance(values, (list, tuple)): return next((found for value in values if (found := _first_tensor(value)) is not None), None) return None class ReadoutMath(TorchDispatchMode): """Apply CUDA readout operations within the current worker call.""" def __torch_dispatch__(self: ReadoutMath, function: Any, types: Any, args: tuple[Any, ...] = (), kwargs: dict[str, Any] | None = None) -> Any: kwargs = {} if kwargs is None else kwargs first = _first_tensor(args) eligible = first is not None and first.is_cuda and first.dtype in (torch.bfloat16, torch.float32) if eligible and function in (torch.ops.aten.linear.default, torch.ops.aten.matmul.default, torch.ops.aten.einsum.default, torch.ops.aten.scaled_dot_product_attention.default, torch.ops.aten._scaled_dot_product_attention_math.default): with self: result = function.decompose(*args, **kwargs) if result is NotImplemented: raise RuntimeError(f"Missing matrix decomposition: {function}") return result if eligible and function is torch.ops.aten._native_multi_head_attention.default: with self: return function._op_dk(torch._C.DispatchKey.CUDA, *args, **kwargs) if eligible and function is torch.ops.aten.mm.default: return _matmul(args[0], args[1]) if eligible and function is torch.ops.aten.addmm.default: if kwargs.get("alpha", 1) != 1 or kwargs.get("beta", 1) != 1: raise ValueError("Expected unit addmm alpha and beta") return _matmul(args[1], args[2], args[0]) if eligible and function is torch.ops.aten.mm.dtype: return _matmul(args[0].contiguous(), args[1].contiguous()).to(kwargs["out_dtype"]) if eligible and function is torch.ops.aten.bmm.default: from .kernels import bmm_batch_invariant return bmm_batch_invariant(args[0], args[1]) if eligible and function is torch.ops.aten.mean.dim: from .kernels import mean_batch_invariant return mean_batch_invariant(args[0], args[1], args[2] if len(args) > 2 else False, dtype=kwargs.get("dtype")) return function(*args, **kwargs) def _linear_forward(original: Callable[..., Any], lm_head: bool, transpose_caches: list[dict[str, Any]] | None = None) -> Callable[..., torch.Tensor]: cache: dict[str, Any] = {} if lm_head and transpose_caches is not None: transpose_caches.append(cache) def forward(self: Any, input: torch.Tensor) -> torch.Tensor: if (not input.is_cuda or input.dtype not in (torch.bfloat16, torch.float32) or input.dtype != self.weight.dtype): cache.clear() return original(input) shape = input.shape[:-1] flat = input.reshape(-1, input.shape[-1]) if lm_head: weight = self.weight version = None if not torch.is_grad_enabled() and not torch.compiler.is_compiling(): try: version = weight._version except RuntimeError: pass if version is None: cache.clear() transposed = weight.T.contiguous() else: stream = torch.cuda.current_stream(weight.device).cuda_stream if weight.is_cuda else None key = (id(weight), version, weight.data_ptr(), weight.storage_offset(), weight.device, weight.dtype, tuple(weight.shape), tuple(weight.stride()), stream) if cache.get("weight") is not weight or cache.get("key") != key: cache.clear() cache["transposed"] = weight.T.contiguous() cache["key"] = key cache["weight"] = weight transposed = cache["transposed"] output = _matmul(flat.contiguous(), transposed).float() else: output = _matmul(flat, self.weight.T, self.bias) return output.reshape(*shape, self.weight.shape[0]) return forward def _norm(self: Qwen3_5RMSNorm, x: torch.Tensor) -> torch.Tensor: if x.is_cuda and x.dtype in (torch.bfloat16, torch.float32): from .normalization import rms_norm return rms_norm(x, self.weight, self.eps) value = x.float() value = value * torch.rsqrt(value.square().mean(-1, keepdim=True) + self.eps) return (value * (1.0 + self.weight.float())).to(self.weight.dtype) def _residual_sum(first: torch.Tensor, second: torch.Tensor, dtype: torch.dtype) -> torch.Tensor: if (dtype == torch.bfloat16 and first.is_cuda and second.is_cuda and first.device == second.device and first.shape == second.shape and first.dtype in (torch.bfloat16, torch.float32) and second.dtype == torch.bfloat16 and first.is_contiguous() and second.is_contiguous() and first.numel() <= 2**31 - 1 and not torch.is_grad_enabled() and not torch.compiler.is_compiling()): from .kernels import residual_sum return residual_sum(first, second) return first.to(dtype).float() + second.float() def _decoder(self: Qwen3_5DecoderLayer, hidden_states: torch.Tensor, position_embeddings: tuple[torch.Tensor, torch.Tensor], attention_mask: torch.Tensor | None = None, position_ids: torch.Tensor | None = None, past_key_values: Cache | None = None, **kwargs: Any) -> torch.Tensor: dtype = self.input_layernorm.weight.dtype residual = hidden_states.to(dtype) states = self.input_layernorm(hidden_states) if self.layer_type == "linear_attention": states = self.linear_attn(hidden_states=states, cache_params=past_key_values, attention_mask=attention_mask, **kwargs) else: states, _ = self.self_attn(hidden_states=states, attention_mask=attention_mask, position_ids=position_ids, past_key_values=past_key_values, position_embeddings=position_embeddings, **kwargs) merged = _residual_sum(residual, states, dtype) states = self.post_attention_layernorm(merged) return _residual_sum(merged, self.mlp(states), dtype) def _mlp(self: Qwen3_5MLP, x: torch.Tensor) -> torch.Tensor: gate, up = self.gate_proj(x), self.up_proj(x) if (gate.is_cuda and gate.dtype == up.dtype == torch.bfloat16 and gate.is_contiguous() and up.is_contiguous()): from .kernels import silu_mul return self.down_proj(silu_mul(gate, up)) return self.down_proj((F.silu(gate.float()) * up.float()).to(gate.dtype)) def _rotate(q: torch.Tensor, k: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: dim = cos.shape[-1] half = dim // 2 c, s = cos[:, None, :, :half], sin[:, None, :, :half] def apply(value: torch.Tensor) -> torch.Tensor: first, second = value[..., :half].float(), value[..., half:dim].float() rotated = torch.cat((first * c - second * s, second * c + first * s), -1) return torch.cat((rotated.to(value.dtype), value[..., dim:]), -1) return apply(q), apply(k) def _dense(self: Qwen3_5Attention, hidden_states: torch.Tensor, position_embeddings: tuple[torch.Tensor, torch.Tensor], attention_mask: torch.Tensor | None, past_key_values: Cache | None = None, *, metadata: DecodeMetadata | None = None, **kwargs: Any) -> tuple[torch.Tensor, torch.Tensor | None]: input_shape = hidden_states.shape[:-1] hidden_shape = (*input_shape, -1, self.head_dim) q_gate, raw_key = self.q_proj(hidden_states), self.k_proj(hidden_states) if hidden_states.is_cuda: from .kernels import dense_qk query, key, gate = dense_qk(q_gate, raw_key, self.q_norm.weight, self.k_norm.weight, *position_embeddings, self.q_norm.eps) else: query, gate = torch.chunk(q_gate.view(*input_shape, -1, self.head_dim * 2), 2, dim=-1) gate = gate.reshape(*input_shape, -1) query = self.q_norm(query.view(hidden_shape)).transpose(1, 2) key = self.k_norm(raw_key.view(hidden_shape)).transpose(1, 2) query, key = _rotate(query, key, *position_embeddings) value = self.v_proj(hidden_states).view(hidden_shape).transpose(1, 2) if past_key_values is not None: key, value = past_key_values.update(key, value, self.layer_idx) interface = ALL_ATTENTION_FUNCTIONS.get_interface( self.config._attn_implementation, eager_attention_forward) if metadata is not None: output, weights = metadata.attend(interface, self, query, key, value, attention_mask, dropout=0.0 if not self.training else self.attention_dropout, scaling=self.scaling, **kwargs) else: output, weights = interface(self, query, key, value, attention_mask, dropout=0.0 if not self.training else self.attention_dropout, scaling=self.scaling, **kwargs) output = output.reshape(*input_shape, -1).contiguous() if output.is_cuda and output.dtype == torch.bfloat16: from .kernels import fused_sigmoid_mul output = fused_sigmoid_mul(output, gate, inplace=True) else: output = (output.float() * torch.sigmoid(gate.float())).to(output.dtype) return self.o_proj(output), weights def _gdn(self: Qwen3_5GatedDeltaNet, hidden_states: torch.Tensor, cache_params: Cache | None = None, attention_mask: torch.Tensor | None = None, **kwargs: Any) -> torch.Tensor: hidden_states = apply_mask_to_padding_states(hidden_states, attention_mask) batch, length, _ = hidden_states.shape cached = cache_params is not None and cache_params.has_previous_state(self.layer_idx) if cached: conv_state = cache_params.layers[self.layer_idx].conv_states recurrent_state = cache_params.layers[self.layer_idx].recurrent_states mixed = self.in_proj_qkv(hidden_states).transpose(1, 2) z = self.in_proj_z(hidden_states).reshape(batch, length, -1, self.head_v_dim) b, a = self.in_proj_b(hidden_states), self.in_proj_a(hidden_states) if cached and length == 1: mixed = self.causal_conv1d_update(mixed, conv_state, self.conv1d.weight.squeeze(1), self.conv1d.bias, self.activation) else: if cached: mixed = torch.cat((conv_state, mixed), -1) if cache_params is not None: state = F.pad(mixed, (self.conv_kernel_size - mixed.shape[-1], 0)) cache_params.update_conv_state(state, self.layer_idx) if self.causal_conv1d_fn is not None: mixed = self.causal_conv1d_fn(x=mixed, weight=self.conv1d.weight.squeeze(1), bias=self.conv1d.bias, activation=self.activation, seq_idx=kwargs.get("seq_idx")) else: mixed = F.silu(self.conv1d(mixed)[:, :, :mixed.shape[-1]]) if cached: mixed = mixed[:, :, -length:] from .gdn.adapter import decode, prefill, store_state if cached and length == 1: output, state = decode(mixed.transpose(1, 2), a, b, self.A_log, self.dt_bias, recurrent_state) else: output, state = prefill(mixed.transpose(1, 2), a, b, self.A_log, self.dt_bias, self.num_k_heads, self.num_v_heads, self.head_k_dim, self.head_v_dim, recurrent_state if cached else None, attention_mask) if cache_params is not None: store_state(cache_params, self.layer_idx, state) output = self.norm(output.reshape(-1, self.head_v_dim), z.reshape(-1, self.head_v_dim)) return self.out_proj(output.reshape(batch, length, -1)) def _gated_norm(self: Any, x: torch.Tensor, z: torch.Tensor) -> torch.Tensor: epsilon = getattr(self, "eps", getattr(self, "variance_epsilon", None)) if (x.ndim != 2 or z.shape != x.shape or self.weight.shape != (128,) or x.shape[-1] != 128 or x.shape[0] < 1 or epsilon != 1e-6 or x.dtype != z.dtype or self.weight.dtype != x.dtype or any(v.device != x.device or v.stride(-1) != 1 for v in (x, z, self.weight))): raise ValueError("Unsupported gated normalization inputs") from .kernels import gated_norm return gated_norm(x, z, self.weight, epsilon) def _nonoverlapping(value: torch.Tensor) -> None: span = 1 for stride, size in sorted((s, n) for s, n in zip(value.stride(), value.shape, strict=True) if n > 1): if stride < span: raise ValueError("Overlapping convolution tensor") span += (size - 1) * stride def _validate_conv(x: torch.Tensor, weight: torch.Tensor, bias: torch.Tensor | None, activation: str | None) -> None: if (x.ndim != 3 or weight.ndim != 2 or tuple(weight.shape) != (x.shape[1], 4) or min(x.shape) < 1 or weight.dtype != x.dtype or weight.device != x.device or not weight.is_contiguous()): raise ValueError("Expected nonempty input[B,D,T] and matching contiguous weight[D,4]") if bias is not None and (tuple(bias.shape) != (x.shape[1],) or bias.dtype != x.dtype or bias.device != x.device or not bias.is_contiguous()): raise ValueError("Expected a matching contiguous bias[D]") if activation not in (None, "silu", "swish"): raise ValueError("Unsupported convolution activation") _nonoverlapping(x) def _prefill(x: torch.Tensor, weight: torch.Tensor, bias: torch.Tensor | None = None, activation: str | None = None, seq_idx: torch.Tensor | None = None ) -> torch.Tensor: if seq_idx is not None: raise ValueError("Packed convolution sequence indices are unsupported") if not x.is_cuda or x.dtype != torch.bfloat16: output = F.conv1d(x, weight.unsqueeze(1), bias, padding=weight.shape[-1] - 1, groups=x.shape[1])[:, :, :x.shape[2]] return F.silu(output) if activation is not None else output _validate_conv(x, weight, bias, activation) from .kernels import prefill return prefill(x, weight, bias, activation) def _update(x: torch.Tensor, state: torch.Tensor, weight: torch.Tensor, bias: torch.Tensor | None = None, activation: str | None = None) -> torch.Tensor: if not x.is_cuda or x.dtype != torch.bfloat16: return torch_causal_conv1d_update(x, state, weight, bias, activation) _validate_conv(x, weight, bias, activation) if (x.shape[2] != 1 or tuple(state.shape) != (x.shape[0], x.shape[1], 4) or state.dtype != x.dtype or state.device != x.device): raise ValueError("Expected singleton input and matching live cache[B,D,4]") _nonoverlapping(state) pointer = state.untyped_storage().data_ptr() if any(v.untyped_storage().data_ptr() == pointer for v in (x, weight, bias) if v is not None): raise ValueError("Convolution cache must not alias input, weight or bias") from .kernels import update return update(x, state, weight, bias, activation) def install(backbone: Qwen3_5ForConditionalGeneration, head: nn.Module | None = None ) -> Callable[[], None]: """Configure this backbone and return an idempotent cleanup function.""" def native_cleanup() -> None: return None language = backbone.model.language_model if language.embed_tokens.weight.dtype != torch.bfloat16: return native_cleanup rotary = language.rotary_emb rope = rotary.config.rope_parameters if (rope.get("rope_type") != "default" or rope.get("partial_rotary_factor") != 0.25 or rope.get("rope_theta") != 10000000.0 or rotary.config.head_dim != 256 or rope.get("mrope_interleaved") is not True or list(rope.get("mrope_section", ())) != [11, 11, 10] or language.config.hidden_act != "silu"): raise ValueError("Unsupported Clairic rotary configuration") forwards: list[tuple[nn.Module, bool, Any, Any]] = [] convolutions: list[tuple[Any, Any, Any]] = [] caches: dict[torch.device, torch.Tensor] = {} transpose_caches: list[dict[str, Any]] = [] attention_metadata = DecodeMetadata() closed = False def restore() -> None: nonlocal closed if closed: return closed = True attention_metadata.close() for module, original_prefill, original_update in reversed(convolutions): module.causal_conv1d_fn, module.causal_conv1d_update = original_prefill, original_update for module, had_override, original, installed in reversed(forwards): if module.forward == installed: if had_override: module.forward = original else: del module.forward caches.clear() for cache in transpose_caches: cache.clear() transpose_caches.clear() def bind(module: nn.Module, function: Callable[..., Any], argument: str = "hidden_states", parameter: torch.Tensor | None = None, *, cuda_only: bool = False) -> None: original = module.forward def forward(self: nn.Module, *args: Any, **kwargs: Any) -> Any: value = args[0] if args else kwargs[argument] dtype = parameter.dtype if parameter is not None else value.dtype active = dtype == torch.bfloat16 and (not cuda_only or value.is_cuda) return function(self, *args, **kwargs) if active else original(*args, **kwargs) installed = MethodType(forward, module) forwards.append((module, "forward" in module.__dict__, original, installed)) module.forward = installed def text_rotary(self: Qwen3_5TextRotaryEmbedding, x: torch.Tensor, position_ids: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: if x.device not in caches: dim = int(self.config.head_dim * rope["partial_rotary_factor"]) with torch.no_grad(), torch.autocast(device_type=x.device.type, enabled=False): inv = 1.0 / (rope["rope_theta"] ** ( torch.arange(0, dim, 2, dtype=torch.float32, device=x.device) / dim)) positions = torch.arange(self.config.max_position_embeddings, dtype=torch.float32, device=x.device) frequencies = torch.einsum("i,j -> ij", positions, inv) caches[x.device] = torch.cat((frequencies.cos(), frequencies.sin()), -1) positions = position_ids[0] if position_ids.ndim == 3 else position_ids cos, sin = caches[x.device][positions].chunk(2, -1) return torch.cat((cos, cos), -1), torch.cat((sin, sin), -1) try: for module in language.modules(): if isinstance(module, Qwen3_5RMSNorm): bind(module, _norm, "x", module.weight) elif isinstance(module, Qwen3_5DecoderLayer): bind(module, _decoder, parameter=module.input_layernorm.weight) elif isinstance(module, Qwen3_5MLP): bind(module, _mlp, "x", module.gate_proj.weight) elif isinstance(module, Qwen3_5Attention): def dense(self: Qwen3_5Attention, *args: Any, **kwargs: Any) -> Any: return _dense(self, *args, metadata=attention_metadata, **kwargs) bind(module, dense, parameter=module.q_proj.weight) elif isinstance(module, Qwen3_5GatedDeltaNet): bind(module, _gdn, parameter=module.in_proj_qkv.weight, cuda_only=True) bind(module.norm, _gated_norm, "x", module.norm.weight, cuda_only=True) convolutions.append((module, module.causal_conv1d_fn, module.causal_conv1d_update)) module.causal_conv1d_fn, module.causal_conv1d_update = _prefill, _update bind(rotary, text_rotary, "x") original_language_forward = language.forward def language_forward(self: nn.Module, *args: Any, **kwargs: Any) -> Any: with attention_metadata.scope(args, kwargs), recurrent_metadata_scope(self): return original_language_forward(*args, **kwargs) installed_language = MethodType(language_forward, language) forwards.append((language, "forward" in language.__dict__, original_language_forward, installed_language)) language.forward = installed_language lm_head = backbone.get_output_embeddings() roots = (backbone,) if head is None else (backbone, head) seen: set[int] = set() for root in roots: for module in root.modules(): if id(module) in seen or (type(module) is not nn.Linear and module is not lm_head): continue seen.add(id(module)) if module.weight.dtype not in (torch.bfloat16, torch.float32): continue original = module.forward installed = MethodType(_linear_forward(original, module is lm_head, transpose_caches), module) forwards.append((module, "forward" in module.__dict__, original, installed)) module.forward = installed except BaseException: restore() raise return restore