"""Engine-owned native linear-cache storage; dense caches remain fresh native.""" from __future__ import annotations from typing import Any import torch class NativeStatePool: """One maxB allocation per linear layer, attached to fresh DynamicCache objects. The caller serializes requests on its model worker and must not reuse a cache after its request ends. The initialized flags prevent native prefill from allocating duplicate state; has_previous_state=False retains original prefill. """ def __init__(self, backbone: Any, max_batch: int) -> None: if max_batch < 1: raise ValueError("Positive maximum batch required") self.max_batch, self.closed = max_batch, False self.language = backbone.model.language_model self.states: dict[int, tuple[torch.Tensor, torch.Tensor]] = {} for index, layer in enumerate(self.language.layers): if layer.layer_type != "linear_attention": continue gdn = layer.linear_attn weight = gdn.in_proj_qkv.weight conv = torch.zeros(max_batch, gdn.conv_dim, gdn.conv_kernel_size, dtype=weight.dtype, device=weight.device) source = torch.zeros(max_batch, gdn.num_v_heads, gdn.head_v_dim, gdn.head_k_dim, dtype=torch.float32, device=weight.device) self.states[index] = (conv, source.transpose(-1, -2)) if not self.states: raise ValueError("Expected native linear-attention layers") def attach(self, cache: Any, batch: int, *, previous: bool = False) -> Any: if self.closed or not 1 <= batch <= self.max_batch: raise RuntimeError("Closed pool or batch outside owned state") if len(cache.layers) != len(self.language.layers): raise ValueError("Cache layer count differs from language model") for index, (conv, recurrent) in self.states.items(): layer = cache.layers[index] # Attach to empty native linear layers only; replacing live storage # would silently discard state or retain an unaccounted allocation. if layer.is_conv_states_initialized or layer.is_recurrent_states_initialized: raise ValueError("Fresh uninitialized native cache required") layer.conv_states = conv[:batch] layer.recurrent_states = recurrent[:batch] layer.is_conv_states_initialized = True layer.is_recurrent_states_initialized = True layer.has_previous_state = previous layer.dtype, layer.device = conv.dtype, conv.device layer.max_batch_size, layer.conv_kernel_size = batch, conv.shape[-1] return cache def new_cache(self, batch: int, *, previous: bool = False) -> Any: if self.closed: raise RuntimeError("Closed pool") from transformers.cache_utils import DynamicCache return self.attach(DynamicCache(config=self.language.config), batch, previous=previous) def owns(self, cache: Any, batch: int) -> bool: if self.closed or not 1 <= batch <= self.max_batch or cache is None: return False if len(cache.layers) != len(self.language.layers): return False for index, (conv, recurrent) in self.states.items(): layer = cache.layers[index] for actual, owned in ((layer.conv_states, conv), (layer.recurrent_states, recurrent)): if (actual is None or actual.data_ptr() != owned.data_ptr() or actual.untyped_storage().data_ptr() != owned.untyped_storage().data_ptr() or actual.dtype != owned.dtype or actual.device != owned.device or actual.storage_offset() != owned.storage_offset() or tuple(actual.shape) != (batch, *owned.shape[1:]) or tuple(actual.stride()) != tuple(owned.stride())): return False if not (layer.is_conv_states_initialized and layer.is_recurrent_states_initialized): return False return True @property def storage_bytes(self) -> int: return sum(value.untyped_storage().nbytes() for states in self.states.values() for value in states) def zero(self) -> None: if self.closed: raise RuntimeError("Closed pool") for conv, recurrent in self.states.values(): conv.zero_() recurrent.zero_() def close(self) -> None: self.states.clear() self.language = None self.closed = True