Blame for cache_utils.py

"""Shared Safin-1 cache utils for dense and MoE remote code."""

from __future__ import annotations


from types import MethodType
import torch
from torch import nn
from transformers.cache_utils import DynamicCache
from transformers.models.qwen3_5_moe import modeling_qwen3_5_moe as qwen35_moe_modeling


def _new_qwen35_cache(config):
    cache_cls = getattr(qwen35_moe_modeling, "Qwen3_5MoeDynamicCache", DynamicCache)
    try:
        cache = cache_cls(config=config)
    except TypeError:
        cache = cache_cls()
    return _patch_hsd_cache_get_seq_length(cache)


def _patch_hsd_cache_get_seq_length(cache):
    if cache is None:
        return cache
    _patch_hsd_cache_batch_ops(cache)
    if getattr(cache, "_historical_hsd_get_seq_length_patched", False):
        return cache
    original_get_seq_length = getattr(cache, "get_seq_length", None)
    if not callable(original_get_seq_length):
        return cache

    def get_seq_length(self, *args, **kwargs):
        try:
            return original_get_seq_length(*args, **kwargs)
        except ValueError as exc:
            message = str(exc)
            if "get_seq_length" not in message or "LinearAttention" not in message:
                raise
            value = getattr(self, "hsd_seen_expanded_tokens", None)
            if value is not None:
                return _length_value_to_int(value)
            value = getattr(self, "hsd_seen_text_tokens", None)
            if value is not None:
                return _length_value_to_int(value)
            return 0

    cache.get_seq_length = MethodType(get_seq_length, cache)
    cache._historical_hsd_get_seq_length_patched = True
    return cache


def _cache_has_previous_state(past_key_values) -> bool:
    if past_key_values is None:
        return False
    if _length_value_to_int(getattr(past_key_values, "hsd_seen_expanded_tokens", 0)) > 0:
        return True
    if _length_value_to_int(getattr(past_key_values, "hsd_seen_text_tokens", 0)) > 0:
        return True
    if _cache_text_length(past_key_values) > 0:
        return True
    has_previous_state = getattr(past_key_values, "has_previous_state", None)
    if callable(has_previous_state):
        try:
            return bool(has_previous_state())
        except TypeError:
            return bool(has_previous_state(0))
    if isinstance(has_previous_state, bool):
        return has_previous_state
    try:
        return int(past_key_values.get_seq_length()) > 0
    except Exception:
        return False


def _cache_text_length(past_key_values) -> int:
    if past_key_values is None:
        return 0
    value = getattr(past_key_values, "hsd_seen_text_tokens", None)
    if value is not None:
        return _length_value_to_int(value)
    layers = getattr(past_key_values, "layers", None)
    if layers is not None:
        for layer in layers:
            value = getattr(layer, "hsd_seen_text_tokens", None)
            if value is not None:
                return _length_value_to_int(value)
    layers = getattr(past_key_values, "_hsd_layer_states", None)
    if layers is not None:
        for layer in layers:
            value = getattr(layer, "hsd_seen_text_tokens", None)
            if value is not None:
                return _length_value_to_int(value)
    try:
        return int(past_key_values.get_seq_length())
    except Exception:
        return 0


def _cache_expanded_length(past_key_values) -> int:
    if past_key_values is None:
        return 0
    value = getattr(past_key_values, "hsd_seen_expanded_tokens", None)
    if value is not None:
        return _length_value_to_int(value)
    try:
        value = int(past_key_values.get_seq_length())
        if value > 0:
            return value
    except Exception:
        pass
    text_len = _cache_text_length(past_key_values)
    return text_len


def _length_value_to_int(value) -> int:
    if value is None:
        return 0
    if torch.is_tensor(value):
        if value.numel() == 0:
            return 0
        return int(value.detach().max().item())
    return int(value or 0)


def _cache_lengths(
    past_key_values,
    *,
    base_attr: str,
    per_row_attr: str,
    batch_size: int,
    device: torch.device,
) -> torch.Tensor:
    if past_key_values is None:
        return torch.zeros(batch_size, dtype=torch.long, device=device)
    for source in (past_key_values,):
        value = getattr(source, per_row_attr, None)
        if value is not None:
            tensor = torch.as_tensor(value, dtype=torch.long, device=device).reshape(-1)
            if tensor.numel() == batch_size:
                return tensor
            if tensor.numel() == 1:
                return tensor.expand(batch_size)
    for attr_name in ("layers", "_hsd_layer_states"):
        layers = getattr(past_key_values, attr_name, None)
        if layers is None:
            continue
        for layer in layers:
            value = getattr(layer, per_row_attr, None)
            if value is not None:
                tensor = torch.as_tensor(value, dtype=torch.long, device=device).reshape(-1)
                if tensor.numel() == batch_size:
                    return tensor
                if tensor.numel() == 1:
                    return tensor.expand(batch_size)
            value = getattr(layer, base_attr, None)
            if value is not None:
                scalar = _length_value_to_int(value)
                return torch.full((batch_size,), scalar, dtype=torch.long, device=device)
    value = getattr(past_key_values, base_attr, None)
    if value is not None:
        scalar = _length_value_to_int(value)
        return torch.full((batch_size,), scalar, dtype=torch.long, device=device)
    scalar = _cache_text_length(past_key_values) if base_attr == "hsd_seen_text_tokens" else _cache_expanded_length(past_key_values)
    return torch.full((batch_size,), scalar, dtype=torch.long, device=device)


def _cache_text_lengths(past_key_values, batch_size: int, device: torch.device) -> torch.Tensor:
    return _cache_lengths(
        past_key_values,
        base_attr="hsd_seen_text_tokens",
        per_row_attr="hsd_seen_text_tokens_per_row",
        batch_size=batch_size,
        device=device,
    )


def _cache_expanded_lengths(past_key_values, batch_size: int, device: torch.device) -> torch.Tensor:
    return _cache_lengths(
        past_key_values,
        base_attr="hsd_seen_expanded_tokens",
        per_row_attr="hsd_seen_expanded_tokens_per_row",
        batch_size=batch_size,
        device=device,
    )


def _set_root_cache_lengths(
    past_key_values,
    *,
    text_lengths: torch.Tensor,
    expanded_lengths: torch.Tensor,
) -> None:
    if past_key_values is None:
        return
    text_cpu = text_lengths.detach().to(dtype=torch.long)
    expanded_cpu = expanded_lengths.detach().to(dtype=torch.long)
    past_key_values.hsd_seen_text_tokens_per_row = text_cpu
    past_key_values.hsd_seen_expanded_tokens_per_row = expanded_cpu
    past_key_values.hsd_seen_text_tokens = _length_value_to_int(text_cpu)
    past_key_values.hsd_seen_expanded_tokens = _length_value_to_int(expanded_cpu)


class _RowFullAttentionCache:
    def __init__(self, layer_idx: int, keys: torch.Tensor | None = None, values: torch.Tensor | None = None):
        self.layer_idx = int(layer_idx)
        self.keys = keys
        self.values = values

    def get_seq_length(self, layer_idx: int = 0) -> int:
        # This adapter represents one compacted row of one full-attention layer.
        return int(self.keys.shape[-2]) if torch.is_tensor(self.keys) else 0

    def get_mask_sizes(self, query_length: int | torch.Tensor, layer_idx: int = 0) -> tuple[int, int]:
        length = int(query_length.numel()) if torch.is_tensor(query_length) else int(query_length)
        return self.get_seq_length() + length, 0

    def update(self, key_states: torch.Tensor, value_states: torch.Tensor, layer_idx: int, *args, **kwargs):
        if int(layer_idx) != self.layer_idx:
            raise ValueError(f"row full-attention cache received layer {layer_idx}, expected {self.layer_idx}.")
        if self.keys is None or self.values is None or self.keys.numel() == 0:
            self.keys = key_states
            self.values = value_states
        else:
            self.keys = torch.cat([self.keys.to(key_states.device), key_states], dim=-2)
            self.values = torch.cat([self.values.to(value_states.device), value_states], dim=-2)
        return self.keys, self.values


def _full_attention_layer_idx(layer: nn.Module) -> int:
    attn = getattr(layer, "self_attn", None)
    return int(getattr(attn, "layer_idx", getattr(layer, "_historical_hsd_layer_idx", getattr(layer, "layer_idx", 0))))


def _slice_row_full_attention_cache(past_key_values, layer_idx: int, batch_idx: int, batch_size: int) -> _RowFullAttentionCache | None:
    if past_key_values is None:
        return None
    layers = getattr(past_key_values, "layers", None)
    if layers is None or len(layers) <= layer_idx:
        return _RowFullAttentionCache(layer_idx)
    layer_cache = layers[layer_idx]
    keys = getattr(layer_cache, "keys", None)
    values = getattr(layer_cache, "values", None)
    if not torch.is_tensor(keys) or not torch.is_tensor(values) or keys.numel() == 0:
        return _RowFullAttentionCache(layer_idx)
    lengths = getattr(layer_cache, "hsd_text_cache_lengths_per_row", None)
    if torch.is_tensor(lengths) and lengths.numel() > batch_idx:
        row_len = int(lengths.reshape(-1)[batch_idx].item())
    else:
        row_len = int(keys.shape[-2])
    return _RowFullAttentionCache(
        layer_idx,
        keys[batch_idx : batch_idx + 1, :, :row_len].contiguous(),
        values[batch_idx : batch_idx + 1, :, :row_len].contiguous(),
    )


def _merge_row_full_attention_caches(past_key_values, layer_idx: int, row_caches: list[_RowFullAttentionCache | None]) -> None:
    if past_key_values is None:
        return
    layers = getattr(past_key_values, "layers", None)
    if layers is None or len(layers) <= layer_idx:
        return
    keyed = [cache for cache in row_caches if cache is not None and torch.is_tensor(cache.keys)]
    if not keyed:
        return
    sample_keys = keyed[0].keys
    sample_values = keyed[0].values
    max_len = max(int(cache.keys.shape[-2]) if cache is not None and torch.is_tensor(cache.keys) else 0 for cache in row_caches)
    batch_size = len(row_caches)
    merged_keys = sample_keys.new_zeros(batch_size, sample_keys.shape[1], max_len, sample_keys.shape[-1])
    merged_values = sample_values.new_zeros(batch_size, sample_values.shape[1], max_len, sample_values.shape[-1])
    lengths = torch.zeros(batch_size, dtype=torch.long, device=merged_keys.device)
    for batch_idx, cache in enumerate(row_caches):
        if cache is None or not torch.is_tensor(cache.keys) or not torch.is_tensor(cache.values):
            continue
        row_len = int(cache.keys.shape[-2])
        merged_keys[batch_idx : batch_idx + 1, :, :row_len] = cache.keys.to(merged_keys.dtype)
        merged_values[batch_idx : batch_idx + 1, :, :row_len] = cache.values.to(merged_values.dtype)
        lengths[batch_idx] = row_len
    layer_cache = layers[layer_idx]
    layer_cache.keys = merged_keys
    layer_cache.values = merged_values
    layer_cache.is_initialized = True
    layer_cache.dtype = merged_keys.dtype
    layer_cache.device = merged_keys.device
    layer_cache.hsd_text_cache_lengths_per_row = lengths


def _reorder_batch_tensor(value, beam_idx: torch.LongTensor, *, min_ndim=1):
    """Select rows of a declared batch tensor, including expansion and pruning."""
    if torch.is_tensor(value) and value.ndim >= min_ndim:
        return value.index_select(0, beam_idx.to(value.device))
    if isinstance(value, list):
        return [_reorder_batch_tensor(item, beam_idx, min_ndim=min_ndim) for item in value]
    if isinstance(value, tuple):
        return tuple(_reorder_batch_tensor(item, beam_idx, min_ndim=min_ndim) for item in value)
    return value


_PER_ROW_LENGTHS = ("hsd_seen_text_tokens_per_row", "hsd_seen_expanded_tokens_per_row",
                    "hsd_text_cache_lengths_per_row")
_ANCHOR_BATCH_FIELDS = ("hsd_anchor_states", "hsd_anchor_keys", "hsd_anchor_boundaries",
                       "hsd_anchor_valid_mask", "hsd_anchor_persistent_memory_mask")


def _hsd_cache_objects(cache):
    seen = set()
    for obj in [cache, *(getattr(cache, "_hsd_layer_states", None) or []), *(getattr(cache, "layers", None) or [])]:
        if obj is not None and id(obj) not in seen:
            seen.add(id(obj))
            yield obj


def _hsd_cache_batch_size(cache):
    for obj in _hsd_cache_objects(cache):
        for name in (*_PER_ROW_LENGTHS, "hsd_anchor_states", "conv_states", "recurrent_states", "keys"):
            value = getattr(obj, name, None)
            if torch.is_tensor(value) and value.ndim > 0:
                return int(value.shape[0])
    for name in ("_hsd_conv_states", "_hsd_recurrent_states"):
        for value in getattr(cache, name, []) or []:
            if torch.is_tensor(value) and value.ndim > 0:
                return int(value.shape[0])
    return None


def _validate_cache_indices(cache, indices):
    if not torch.is_tensor(indices) or indices.ndim != 1 or indices.dtype not in (torch.int32, torch.int64):
        raise ValueError("cache row indices must be a one-dimensional integer tensor")
    batch_size = _hsd_cache_batch_size(cache)
    if indices.numel() and (int(indices.min()) < 0 or
                           (batch_size is not None and int(indices.max()) >= batch_size)):
        raise IndexError(f"cache row indices are out of range for batch size {batch_size}")


def _reorder_hsd_sidecar_cache(past_key_values, beam_idx: torch.LongTensor):
    """Reorder MARCH sidecars only; upstream owns native KV/conv/recurrent layers."""
    if past_key_values is None:
        return past_key_values
    _validate_cache_indices(past_key_values, beam_idx)
    for obj in _hsd_cache_objects(past_key_values):
        for name in _PER_ROW_LENGTHS:
            if hasattr(obj, name):
                setattr(obj, name, _reorder_batch_tensor(getattr(obj, name), beam_idx))
        for name in _ANCHOR_BATCH_FIELDS:
            if hasattr(obj, name):
                # One-dimensional boundaries/masks are shared across the batch.
                # Their length can coincide with B; that does not make them batch tensors.
                setattr(obj, name, _reorder_batch_tensor(getattr(obj, name), beam_idx, min_ndim=2))
        for base in ("hsd_seen_text_tokens", "hsd_seen_expanded_tokens"):
            lengths = getattr(obj, base + "_per_row", None)
            if torch.is_tensor(lengths):
                setattr(obj, base, _length_value_to_int(lengths))
    for name in ("_hsd_conv_states", "_hsd_recurrent_states"):
        states = getattr(past_key_values, name, None)
        if states is not None:
            setattr(past_key_values, name, _reorder_batch_tensor(states, beam_idx))
    return past_key_values


def _patch_hsd_cache_batch_ops(cache):
    """Give all HF cache row operations one native reorder and one sidecar reorder.

    LinearAttentionLayer lacks the upstream repeat/select methods. Implement
    those through its native reorder_cache, which also handles size changes.
    """
    if getattr(cache, "_march_batch_ops_patched", False):
        return cache
    original = getattr(cache, "reorder_cache", None)
    if not callable(original):
        return cache

    def reorder_cache(self, indices):
        _validate_cache_indices(self, indices)
        # Select sidecars before native layers change B, so validation sees old B.
        _reorder_hsd_sidecar_cache(self, indices)
        return original(indices)

    def batch_select_indices(self, indices):
        return self.reorder_cache(indices)

    def batch_repeat_interleave(self, repeats):
        if not isinstance(repeats, int) or isinstance(repeats, bool) or repeats < 1:
            raise ValueError("cache repeats must be a positive integer")
        batch_size = _hsd_cache_batch_size(self)
        if batch_size is None:
            return None
        indices = torch.arange(batch_size, dtype=torch.long).repeat_interleave(repeats)
        return self.reorder_cache(indices)

    cache.reorder_cache = MethodType(reorder_cache, cache)
    cache.batch_select_indices = MethodType(batch_select_indices, cache)
    cache.batch_repeat_interleave = MethodType(batch_repeat_interleave, cache)
    cache._march_batch_ops_patched = True
    return cache