Image-Text-to-Text
Transformers
Safetensors
qwen3_5
clairic
cloudflare
systemone
multimodal
structured-output
information-extraction
custom-code
conversational
Instructions to use Cloudflare/clairic with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use Cloudflare/clairic with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("image-text-to-text", model="Cloudflare/clairic") messages = [ { "role": "user", "content": [ {"type": "image", "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/p-blog/candy.JPG"}, {"type": "text", "text": "What animal is on the candy?"} ] }, ] pipe(text=messages)# pip install -U transformers accelerate # Load model directly from transformers import AutoProcessor, AutoModelForMultimodalLM processor = AutoProcessor.from_pretrained("Cloudflare/clairic") model = AutoModelForMultimodalLM.from_pretrained("Cloudflare/clairic", device_map="auto") messages = [ { "role": "user", "content": [ {"type": "image", "url": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/p-blog/candy.JPG"}, {"type": "text", "text": "What animal is on the candy?"} ] }, ] inputs = processor.apply_chat_template( messages, add_generation_prompt=True, tokenize=True, return_dict=True, return_tensors="pt", ).to(model.device) outputs = model.generate(**inputs, max_new_tokens=256) print(processor.decode(outputs[0][inputs["input_ids"].shape[-1]:])) - Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- vLLM
How to use Cloudflare/clairic with vLLM:
Install from pip and serve model
# Install vLLM from pip: pip install vllm # Start the vLLM server: vllm serve "Cloudflare/clairic" # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:8000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "Cloudflare/clairic", "messages": [ { "role": "user", "content": [ { "type": "text", "text": "Describe this image in one sentence." }, { "type": "image_url", "image_url": { "url": "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg" } } ] } ] }'Use Docker
docker model run hf.co/Cloudflare/clairic
- SGLang
How to use Cloudflare/clairic with SGLang:
Install from pip and serve model
# Install SGLang from pip: pip install sglang # Start the SGLang server: python3 -m sglang.launch_server \ --model-path "Cloudflare/clairic" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "Cloudflare/clairic", "messages": [ { "role": "user", "content": [ { "type": "text", "text": "Describe this image in one sentence." }, { "type": "image_url", "image_url": { "url": "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg" } } ] } ] }'Use Docker images
docker run --gpus all \ --shm-size 32g \ -p 30000:30000 \ -v ~/.cache/huggingface:/root/.cache/huggingface \ --env "HF_TOKEN=<secret>" \ --ipc=host \ lmsysorg/sglang:latest \ python3 -m sglang.launch_server \ --model-path "Cloudflare/clairic" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/chat/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "Cloudflare/clairic", "messages": [ { "role": "user", "content": [ { "type": "text", "text": "Describe this image in one sentence." }, { "type": "image_url", "image_url": { "url": "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg" } } ] } ] }' - Docker Model Runner
How to use Cloudflare/clairic with Docker Model Runner:
docker model run hf.co/Cloudflare/clairic
Download clairic_runtime/math.py from Cloudflare/clairic: direct link, hf CLI and curl.
- Browser
- Download file 24 kB
-
https://huggingface.co/Cloudflare/clairic/resolve/main/clairic_runtime/math.py
- Command line
-
hf download hf://Cloudflare/clairic/clairic_runtime/math.py
-
curl -L -o math.py https://huggingface.co/Cloudflare/clairic/resolve/main/clairic_runtime/math.py
24 kB
| # Copyright 2025 The Qwen Team and The HuggingFace Inc. team. All rights reserved. | |
| # Copyright 2026 Cloudflare, Inc. | |
| # Licensed under the Apache License, Version 2.0 (the "License"); | |
| # you may not use this file except in compliance with the License. | |
| # You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 | |
| # Unless required by applicable law or agreed to in writing, software distributed | |
| # under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR | |
| # CONDITIONS OF ANY KIND, either express or implied. See the License for the | |
| # specific language governing permissions and limitations under the License. | |
| """Numerical operations for the Transformers Clairic backbone.""" | |
| from __future__ import annotations | |
| from collections.abc import Callable | |
| from types import MethodType | |
| from typing import Any | |
| import torch | |
| from torch import nn | |
| from torch.nn import functional as F | |
| from torch.utils._python_dispatch import TorchDispatchMode | |
| from transformers.cache_utils import Cache | |
| from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS | |
| from transformers.models.qwen3_5.modeling_qwen3_5 import ( | |
| Qwen3_5Attention, Qwen3_5DecoderLayer, Qwen3_5ForConditionalGeneration, | |
| Qwen3_5GatedDeltaNet, Qwen3_5MLP, Qwen3_5RMSNorm, | |
| Qwen3_5TextRotaryEmbedding, apply_mask_to_padding_states, | |
| eager_attention_forward, torch_causal_conv1d_update, | |
| ) | |
| from .attention_metadata import DecodeMetadata | |
| from .recurrent_metadata import scope as recurrent_metadata_scope | |
| def _matmul(a: torch.Tensor, b: torch.Tensor, bias: torch.Tensor | None = None) -> torch.Tensor: | |
| if a.ndim != 2 or b.ndim != 2 or a.shape[1] != b.shape[0] or a.dtype != b.dtype: | |
| raise ValueError("Expected aligned matrices with equal dtype") | |
| if (a.dtype == torch.bfloat16 and b.dtype == torch.bfloat16 | |
| and a.is_contiguous() and b.transpose(0, 1).is_contiguous() | |
| and b.shape[1] >= 16): | |
| from deep_gemm import bf16_gemm_nn | |
| output = torch.empty((a.shape[0], b.shape[1]), device=a.device, dtype=a.dtype) | |
| bf16_gemm_nn(a, b, output) | |
| if bias is not None: | |
| output += bias | |
| return output | |
| from .kernels import _matmul_persistent_triton | |
| return _matmul_persistent_triton(a, b, bias) | |
| def _first_tensor(values: Any) -> torch.Tensor | None: | |
| if isinstance(values, torch.Tensor): | |
| return values | |
| if isinstance(values, (list, tuple)): | |
| return next((found for value in values if (found := _first_tensor(value)) is not None), None) | |
| return None | |
| class ReadoutMath(TorchDispatchMode): | |
| """Apply CUDA readout operations within the current worker call.""" | |
| def __torch_dispatch__(self: ReadoutMath, function: Any, types: Any, args: tuple[Any, ...] = (), | |
| kwargs: dict[str, Any] | None = None) -> Any: | |
| kwargs = {} if kwargs is None else kwargs | |
| first = _first_tensor(args) | |
| eligible = first is not None and first.is_cuda and first.dtype in (torch.bfloat16, torch.float32) | |
| if eligible and function in (torch.ops.aten.linear.default, torch.ops.aten.matmul.default, | |
| torch.ops.aten.einsum.default, | |
| torch.ops.aten.scaled_dot_product_attention.default, | |
| torch.ops.aten._scaled_dot_product_attention_math.default): | |
| with self: | |
| result = function.decompose(*args, **kwargs) | |
| if result is NotImplemented: | |
| raise RuntimeError(f"Missing matrix decomposition: {function}") | |
| return result | |
| if eligible and function is torch.ops.aten._native_multi_head_attention.default: | |
| with self: | |
| return function._op_dk(torch._C.DispatchKey.CUDA, *args, **kwargs) | |
| if eligible and function is torch.ops.aten.mm.default: | |
| return _matmul(args[0], args[1]) | |
| if eligible and function is torch.ops.aten.addmm.default: | |
| if kwargs.get("alpha", 1) != 1 or kwargs.get("beta", 1) != 1: | |
| raise ValueError("Expected unit addmm alpha and beta") | |
| return _matmul(args[1], args[2], args[0]) | |
| if eligible and function is torch.ops.aten.mm.dtype: | |
| return _matmul(args[0].contiguous(), args[1].contiguous()).to(kwargs["out_dtype"]) | |
| if eligible and function is torch.ops.aten.bmm.default: | |
| from .kernels import bmm_batch_invariant | |
| return bmm_batch_invariant(args[0], args[1]) | |
| if eligible and function is torch.ops.aten.mean.dim: | |
| from .kernels import mean_batch_invariant | |
| return mean_batch_invariant(args[0], args[1], args[2] if len(args) > 2 else False, | |
| dtype=kwargs.get("dtype")) | |
| return function(*args, **kwargs) | |
| def _linear_forward(original: Callable[..., Any], lm_head: bool, | |
| transpose_caches: list[dict[str, Any]] | None = None) -> Callable[..., torch.Tensor]: | |
| cache: dict[str, Any] = {} | |
| if lm_head and transpose_caches is not None: | |
| transpose_caches.append(cache) | |
| def forward(self: Any, input: torch.Tensor) -> torch.Tensor: | |
| if (not input.is_cuda or input.dtype not in (torch.bfloat16, torch.float32) | |
| or input.dtype != self.weight.dtype): | |
| cache.clear() | |
| return original(input) | |
| shape = input.shape[:-1] | |
| flat = input.reshape(-1, input.shape[-1]) | |
| if lm_head: | |
| weight = self.weight | |
| version = None | |
| if not torch.is_grad_enabled() and not torch.compiler.is_compiling(): | |
| try: | |
| version = weight._version | |
| except RuntimeError: | |
| pass | |
| if version is None: | |
| cache.clear() | |
| transposed = weight.T.contiguous() | |
| else: | |
| stream = torch.cuda.current_stream(weight.device).cuda_stream if weight.is_cuda else None | |
| key = (id(weight), version, weight.data_ptr(), weight.storage_offset(), | |
| weight.device, weight.dtype, tuple(weight.shape), tuple(weight.stride()), stream) | |
| if cache.get("weight") is not weight or cache.get("key") != key: | |
| cache.clear() | |
| cache["transposed"] = weight.T.contiguous() | |
| cache["key"] = key | |
| cache["weight"] = weight | |
| transposed = cache["transposed"] | |
| output = _matmul(flat.contiguous(), transposed).float() | |
| else: | |
| output = _matmul(flat, self.weight.T, self.bias) | |
| return output.reshape(*shape, self.weight.shape[0]) | |
| return forward | |
| def _norm(self: Qwen3_5RMSNorm, x: torch.Tensor) -> torch.Tensor: | |
| if x.is_cuda and x.dtype in (torch.bfloat16, torch.float32): | |
| from .normalization import rms_norm | |
| return rms_norm(x, self.weight, self.eps) | |
| value = x.float() | |
| value = value * torch.rsqrt(value.square().mean(-1, keepdim=True) + self.eps) | |
| return (value * (1.0 + self.weight.float())).to(self.weight.dtype) | |
| def _residual_sum(first: torch.Tensor, second: torch.Tensor, | |
| dtype: torch.dtype) -> torch.Tensor: | |
| if (dtype == torch.bfloat16 and first.is_cuda and second.is_cuda | |
| and first.device == second.device and first.shape == second.shape | |
| and first.dtype in (torch.bfloat16, torch.float32) | |
| and second.dtype == torch.bfloat16 | |
| and first.is_contiguous() and second.is_contiguous() | |
| and first.numel() <= 2**31 - 1 | |
| and not torch.is_grad_enabled() and not torch.compiler.is_compiling()): | |
| from .kernels import residual_sum | |
| return residual_sum(first, second) | |
| return first.to(dtype).float() + second.float() | |
| def _decoder(self: Qwen3_5DecoderLayer, hidden_states: torch.Tensor, | |
| position_embeddings: tuple[torch.Tensor, torch.Tensor], | |
| attention_mask: torch.Tensor | None = None, | |
| position_ids: torch.Tensor | None = None, past_key_values: Cache | None = None, | |
| **kwargs: Any) -> torch.Tensor: | |
| dtype = self.input_layernorm.weight.dtype | |
| residual = hidden_states.to(dtype) | |
| states = self.input_layernorm(hidden_states) | |
| if self.layer_type == "linear_attention": | |
| states = self.linear_attn(hidden_states=states, cache_params=past_key_values, | |
| attention_mask=attention_mask, **kwargs) | |
| else: | |
| states, _ = self.self_attn(hidden_states=states, attention_mask=attention_mask, | |
| position_ids=position_ids, past_key_values=past_key_values, | |
| position_embeddings=position_embeddings, **kwargs) | |
| merged = _residual_sum(residual, states, dtype) | |
| states = self.post_attention_layernorm(merged) | |
| return _residual_sum(merged, self.mlp(states), dtype) | |
| def _mlp(self: Qwen3_5MLP, x: torch.Tensor) -> torch.Tensor: | |
| gate, up = self.gate_proj(x), self.up_proj(x) | |
| if (gate.is_cuda and gate.dtype == up.dtype == torch.bfloat16 | |
| and gate.is_contiguous() and up.is_contiguous()): | |
| from .kernels import silu_mul | |
| return self.down_proj(silu_mul(gate, up)) | |
| return self.down_proj((F.silu(gate.float()) * up.float()).to(gate.dtype)) | |
| def _rotate(q: torch.Tensor, k: torch.Tensor, cos: torch.Tensor, | |
| sin: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: | |
| dim = cos.shape[-1] | |
| half = dim // 2 | |
| c, s = cos[:, None, :, :half], sin[:, None, :, :half] | |
| def apply(value: torch.Tensor) -> torch.Tensor: | |
| first, second = value[..., :half].float(), value[..., half:dim].float() | |
| rotated = torch.cat((first * c - second * s, second * c + first * s), -1) | |
| return torch.cat((rotated.to(value.dtype), value[..., dim:]), -1) | |
| return apply(q), apply(k) | |
| def _dense(self: Qwen3_5Attention, hidden_states: torch.Tensor, | |
| position_embeddings: tuple[torch.Tensor, torch.Tensor], | |
| attention_mask: torch.Tensor | None, past_key_values: Cache | None = None, | |
| *, metadata: DecodeMetadata | None = None, | |
| **kwargs: Any) -> tuple[torch.Tensor, torch.Tensor | None]: | |
| input_shape = hidden_states.shape[:-1] | |
| hidden_shape = (*input_shape, -1, self.head_dim) | |
| q_gate, raw_key = self.q_proj(hidden_states), self.k_proj(hidden_states) | |
| if hidden_states.is_cuda: | |
| from .kernels import dense_qk | |
| query, key, gate = dense_qk(q_gate, raw_key, self.q_norm.weight, | |
| self.k_norm.weight, *position_embeddings, self.q_norm.eps) | |
| else: | |
| query, gate = torch.chunk(q_gate.view(*input_shape, -1, self.head_dim * 2), 2, dim=-1) | |
| gate = gate.reshape(*input_shape, -1) | |
| query = self.q_norm(query.view(hidden_shape)).transpose(1, 2) | |
| key = self.k_norm(raw_key.view(hidden_shape)).transpose(1, 2) | |
| query, key = _rotate(query, key, *position_embeddings) | |
| value = self.v_proj(hidden_states).view(hidden_shape).transpose(1, 2) | |
| if past_key_values is not None: | |
| key, value = past_key_values.update(key, value, self.layer_idx) | |
| interface = ALL_ATTENTION_FUNCTIONS.get_interface( | |
| self.config._attn_implementation, eager_attention_forward) | |
| if metadata is not None: | |
| output, weights = metadata.attend(interface, self, query, key, value, attention_mask, | |
| dropout=0.0 if not self.training else self.attention_dropout, | |
| scaling=self.scaling, **kwargs) | |
| else: | |
| output, weights = interface(self, query, key, value, attention_mask, | |
| dropout=0.0 if not self.training else self.attention_dropout, | |
| scaling=self.scaling, **kwargs) | |
| output = output.reshape(*input_shape, -1).contiguous() | |
| if output.is_cuda and output.dtype == torch.bfloat16: | |
| from .kernels import fused_sigmoid_mul | |
| output = fused_sigmoid_mul(output, gate, inplace=True) | |
| else: | |
| output = (output.float() * torch.sigmoid(gate.float())).to(output.dtype) | |
| return self.o_proj(output), weights | |
| def _gdn(self: Qwen3_5GatedDeltaNet, hidden_states: torch.Tensor, | |
| cache_params: Cache | None = None, attention_mask: torch.Tensor | None = None, | |
| **kwargs: Any) -> torch.Tensor: | |
| hidden_states = apply_mask_to_padding_states(hidden_states, attention_mask) | |
| batch, length, _ = hidden_states.shape | |
| cached = cache_params is not None and cache_params.has_previous_state(self.layer_idx) | |
| if cached: | |
| conv_state = cache_params.layers[self.layer_idx].conv_states | |
| recurrent_state = cache_params.layers[self.layer_idx].recurrent_states | |
| mixed = self.in_proj_qkv(hidden_states).transpose(1, 2) | |
| z = self.in_proj_z(hidden_states).reshape(batch, length, -1, self.head_v_dim) | |
| b, a = self.in_proj_b(hidden_states), self.in_proj_a(hidden_states) | |
| if cached and length == 1: | |
| mixed = self.causal_conv1d_update(mixed, conv_state, self.conv1d.weight.squeeze(1), | |
| self.conv1d.bias, self.activation) | |
| else: | |
| if cached: | |
| mixed = torch.cat((conv_state, mixed), -1) | |
| if cache_params is not None: | |
| state = F.pad(mixed, (self.conv_kernel_size - mixed.shape[-1], 0)) | |
| cache_params.update_conv_state(state, self.layer_idx) | |
| if self.causal_conv1d_fn is not None: | |
| mixed = self.causal_conv1d_fn(x=mixed, weight=self.conv1d.weight.squeeze(1), | |
| bias=self.conv1d.bias, activation=self.activation, seq_idx=kwargs.get("seq_idx")) | |
| else: | |
| mixed = F.silu(self.conv1d(mixed)[:, :, :mixed.shape[-1]]) | |
| if cached: | |
| mixed = mixed[:, :, -length:] | |
| from .gdn.adapter import decode, prefill, store_state | |
| if cached and length == 1: | |
| output, state = decode(mixed.transpose(1, 2), a, b, self.A_log, self.dt_bias, | |
| recurrent_state) | |
| else: | |
| output, state = prefill(mixed.transpose(1, 2), a, b, self.A_log, self.dt_bias, | |
| self.num_k_heads, self.num_v_heads, self.head_k_dim, self.head_v_dim, | |
| recurrent_state if cached else None, attention_mask) | |
| if cache_params is not None: | |
| store_state(cache_params, self.layer_idx, state) | |
| output = self.norm(output.reshape(-1, self.head_v_dim), z.reshape(-1, self.head_v_dim)) | |
| return self.out_proj(output.reshape(batch, length, -1)) | |
| def _gated_norm(self: Any, x: torch.Tensor, z: torch.Tensor) -> torch.Tensor: | |
| epsilon = getattr(self, "eps", getattr(self, "variance_epsilon", None)) | |
| if (x.ndim != 2 or z.shape != x.shape or self.weight.shape != (128,) | |
| or x.shape[-1] != 128 or x.shape[0] < 1 or epsilon != 1e-6 | |
| or x.dtype != z.dtype or self.weight.dtype != x.dtype | |
| or any(v.device != x.device or v.stride(-1) != 1 for v in (x, z, self.weight))): | |
| raise ValueError("Unsupported gated normalization inputs") | |
| from .kernels import gated_norm | |
| return gated_norm(x, z, self.weight, epsilon) | |
| def _nonoverlapping(value: torch.Tensor) -> None: | |
| span = 1 | |
| for stride, size in sorted((s, n) for s, n in zip(value.stride(), value.shape, strict=True) if n > 1): | |
| if stride < span: | |
| raise ValueError("Overlapping convolution tensor") | |
| span += (size - 1) * stride | |
| def _validate_conv(x: torch.Tensor, weight: torch.Tensor, bias: torch.Tensor | None, | |
| activation: str | None) -> None: | |
| if (x.ndim != 3 or weight.ndim != 2 or tuple(weight.shape) != (x.shape[1], 4) | |
| or min(x.shape) < 1 or weight.dtype != x.dtype or weight.device != x.device | |
| or not weight.is_contiguous()): | |
| raise ValueError("Expected nonempty input[B,D,T] and matching contiguous weight[D,4]") | |
| if bias is not None and (tuple(bias.shape) != (x.shape[1],) or bias.dtype != x.dtype | |
| or bias.device != x.device or not bias.is_contiguous()): | |
| raise ValueError("Expected a matching contiguous bias[D]") | |
| if activation not in (None, "silu", "swish"): | |
| raise ValueError("Unsupported convolution activation") | |
| _nonoverlapping(x) | |
| def _prefill(x: torch.Tensor, weight: torch.Tensor, bias: torch.Tensor | None = None, | |
| activation: str | None = None, seq_idx: torch.Tensor | None = None | |
| ) -> torch.Tensor: | |
| if seq_idx is not None: | |
| raise ValueError("Packed convolution sequence indices are unsupported") | |
| if not x.is_cuda or x.dtype != torch.bfloat16: | |
| output = F.conv1d(x, weight.unsqueeze(1), bias, padding=weight.shape[-1] - 1, | |
| groups=x.shape[1])[:, :, :x.shape[2]] | |
| return F.silu(output) if activation is not None else output | |
| _validate_conv(x, weight, bias, activation) | |
| from .kernels import prefill | |
| return prefill(x, weight, bias, activation) | |
| def _update(x: torch.Tensor, state: torch.Tensor, weight: torch.Tensor, | |
| bias: torch.Tensor | None = None, activation: str | None = None) -> torch.Tensor: | |
| if not x.is_cuda or x.dtype != torch.bfloat16: | |
| return torch_causal_conv1d_update(x, state, weight, bias, activation) | |
| _validate_conv(x, weight, bias, activation) | |
| if (x.shape[2] != 1 or tuple(state.shape) != (x.shape[0], x.shape[1], 4) | |
| or state.dtype != x.dtype or state.device != x.device): | |
| raise ValueError("Expected singleton input and matching live cache[B,D,4]") | |
| _nonoverlapping(state) | |
| pointer = state.untyped_storage().data_ptr() | |
| if any(v.untyped_storage().data_ptr() == pointer for v in (x, weight, bias) if v is not None): | |
| raise ValueError("Convolution cache must not alias input, weight or bias") | |
| from .kernels import update | |
| return update(x, state, weight, bias, activation) | |
| def install(backbone: Qwen3_5ForConditionalGeneration, head: nn.Module | None = None | |
| ) -> Callable[[], None]: | |
| """Configure this backbone and return an idempotent cleanup function.""" | |
| def native_cleanup() -> None: | |
| return None | |
| language = backbone.model.language_model | |
| if language.embed_tokens.weight.dtype != torch.bfloat16: | |
| return native_cleanup | |
| rotary = language.rotary_emb | |
| rope = rotary.config.rope_parameters | |
| if (rope.get("rope_type") != "default" or rope.get("partial_rotary_factor") != 0.25 | |
| or rope.get("rope_theta") != 10000000.0 or rotary.config.head_dim != 256 | |
| or rope.get("mrope_interleaved") is not True | |
| or list(rope.get("mrope_section", ())) != [11, 11, 10] | |
| or language.config.hidden_act != "silu"): | |
| raise ValueError("Unsupported Clairic rotary configuration") | |
| forwards: list[tuple[nn.Module, bool, Any, Any]] = [] | |
| convolutions: list[tuple[Any, Any, Any]] = [] | |
| caches: dict[torch.device, torch.Tensor] = {} | |
| transpose_caches: list[dict[str, Any]] = [] | |
| attention_metadata = DecodeMetadata() | |
| closed = False | |
| def restore() -> None: | |
| nonlocal closed | |
| if closed: | |
| return | |
| closed = True | |
| attention_metadata.close() | |
| for module, original_prefill, original_update in reversed(convolutions): | |
| module.causal_conv1d_fn, module.causal_conv1d_update = original_prefill, original_update | |
| for module, had_override, original, installed in reversed(forwards): | |
| if module.forward == installed: | |
| if had_override: | |
| module.forward = original | |
| else: | |
| del module.forward | |
| caches.clear() | |
| for cache in transpose_caches: | |
| cache.clear() | |
| transpose_caches.clear() | |
| def bind(module: nn.Module, function: Callable[..., Any], argument: str = "hidden_states", | |
| parameter: torch.Tensor | None = None, *, cuda_only: bool = False) -> None: | |
| original = module.forward | |
| def forward(self: nn.Module, *args: Any, **kwargs: Any) -> Any: | |
| value = args[0] if args else kwargs[argument] | |
| dtype = parameter.dtype if parameter is not None else value.dtype | |
| active = dtype == torch.bfloat16 and (not cuda_only or value.is_cuda) | |
| return function(self, *args, **kwargs) if active else original(*args, **kwargs) | |
| installed = MethodType(forward, module) | |
| forwards.append((module, "forward" in module.__dict__, original, installed)) | |
| module.forward = installed | |
| def text_rotary(self: Qwen3_5TextRotaryEmbedding, x: torch.Tensor, | |
| position_ids: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: | |
| if x.device not in caches: | |
| dim = int(self.config.head_dim * rope["partial_rotary_factor"]) | |
| with torch.no_grad(), torch.autocast(device_type=x.device.type, enabled=False): | |
| inv = 1.0 / (rope["rope_theta"] ** ( | |
| torch.arange(0, dim, 2, dtype=torch.float32, device=x.device) / dim)) | |
| positions = torch.arange(self.config.max_position_embeddings, | |
| dtype=torch.float32, device=x.device) | |
| frequencies = torch.einsum("i,j -> ij", positions, inv) | |
| caches[x.device] = torch.cat((frequencies.cos(), frequencies.sin()), -1) | |
| positions = position_ids[0] if position_ids.ndim == 3 else position_ids | |
| cos, sin = caches[x.device][positions].chunk(2, -1) | |
| return torch.cat((cos, cos), -1), torch.cat((sin, sin), -1) | |
| try: | |
| for module in language.modules(): | |
| if isinstance(module, Qwen3_5RMSNorm): | |
| bind(module, _norm, "x", module.weight) | |
| elif isinstance(module, Qwen3_5DecoderLayer): | |
| bind(module, _decoder, parameter=module.input_layernorm.weight) | |
| elif isinstance(module, Qwen3_5MLP): | |
| bind(module, _mlp, "x", module.gate_proj.weight) | |
| elif isinstance(module, Qwen3_5Attention): | |
| def dense(self: Qwen3_5Attention, *args: Any, **kwargs: Any) -> Any: | |
| return _dense(self, *args, metadata=attention_metadata, **kwargs) | |
| bind(module, dense, parameter=module.q_proj.weight) | |
| elif isinstance(module, Qwen3_5GatedDeltaNet): | |
| bind(module, _gdn, parameter=module.in_proj_qkv.weight, cuda_only=True) | |
| bind(module.norm, _gated_norm, "x", module.norm.weight, cuda_only=True) | |
| convolutions.append((module, module.causal_conv1d_fn, module.causal_conv1d_update)) | |
| module.causal_conv1d_fn, module.causal_conv1d_update = _prefill, _update | |
| bind(rotary, text_rotary, "x") | |
| original_language_forward = language.forward | |
| def language_forward(self: nn.Module, *args: Any, **kwargs: Any) -> Any: | |
| with attention_metadata.scope(args, kwargs), recurrent_metadata_scope(self): | |
| return original_language_forward(*args, **kwargs) | |
| installed_language = MethodType(language_forward, language) | |
| forwards.append((language, "forward" in language.__dict__, original_language_forward, installed_language)) | |
| language.forward = installed_language | |
| lm_head = backbone.get_output_embeddings() | |
| roots = (backbone,) if head is None else (backbone, head) | |
| seen: set[int] = set() | |
| for root in roots: | |
| for module in root.modules(): | |
| if id(module) in seen or (type(module) is not nn.Linear and module is not lm_head): | |
| continue | |
| seen.add(id(module)) | |
| if module.weight.dtype not in (torch.bfloat16, torch.float32): | |
| continue | |
| original = module.forward | |
| installed = MethodType(_linear_forward(original, module is lm_head, transpose_caches), module) | |
| forwards.append((module, "forward" in module.__dict__, original, installed)) | |
| module.forward = installed | |
| except BaseException: | |
| restore() | |
| raise | |
| return restore | |