File size: 4,653 Bytes
24191a8
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
"""Engine-owned native linear-cache storage; dense caches remain fresh native."""
from __future__ import annotations

from typing import Any

import torch


class NativeStatePool:
    """One maxB allocation per linear layer, attached to fresh DynamicCache objects.

    The caller serializes requests on its model worker and must not reuse a cache
    after its request ends. The initialized flags prevent native prefill from
    allocating duplicate state; has_previous_state=False retains original prefill.
    """

    def __init__(self, backbone: Any, max_batch: int) -> None:
        if max_batch < 1:
            raise ValueError("Positive maximum batch required")
        self.max_batch, self.closed = max_batch, False
        self.language = backbone.model.language_model
        self.states: dict[int, tuple[torch.Tensor, torch.Tensor]] = {}
        for index, layer in enumerate(self.language.layers):
            if layer.layer_type != "linear_attention":
                continue
            gdn = layer.linear_attn
            weight = gdn.in_proj_qkv.weight
            conv = torch.zeros(max_batch, gdn.conv_dim, gdn.conv_kernel_size,
                               dtype=weight.dtype, device=weight.device)
            source = torch.zeros(max_batch, gdn.num_v_heads, gdn.head_v_dim, gdn.head_k_dim,
                                 dtype=torch.float32, device=weight.device)
            self.states[index] = (conv, source.transpose(-1, -2))
        if not self.states:
            raise ValueError("Expected native linear-attention layers")

    def attach(self, cache: Any, batch: int, *, previous: bool = False) -> Any:
        if self.closed or not 1 <= batch <= self.max_batch:
            raise RuntimeError("Closed pool or batch outside owned state")
        if len(cache.layers) != len(self.language.layers):
            raise ValueError("Cache layer count differs from language model")
        for index, (conv, recurrent) in self.states.items():
            layer = cache.layers[index]
            # Attach to empty native linear layers only; replacing live storage
            # would silently discard state or retain an unaccounted allocation.
            if layer.is_conv_states_initialized or layer.is_recurrent_states_initialized:
                raise ValueError("Fresh uninitialized native cache required")
            layer.conv_states = conv[:batch]
            layer.recurrent_states = recurrent[:batch]
            layer.is_conv_states_initialized = True
            layer.is_recurrent_states_initialized = True
            layer.has_previous_state = previous
            layer.dtype, layer.device = conv.dtype, conv.device
            layer.max_batch_size, layer.conv_kernel_size = batch, conv.shape[-1]
        return cache

    def new_cache(self, batch: int, *, previous: bool = False) -> Any:
        if self.closed:
            raise RuntimeError("Closed pool")
        from transformers.cache_utils import DynamicCache

        return self.attach(DynamicCache(config=self.language.config), batch, previous=previous)

    def owns(self, cache: Any, batch: int) -> bool:
        if self.closed or not 1 <= batch <= self.max_batch or cache is None:
            return False
        if len(cache.layers) != len(self.language.layers):
            return False
        for index, (conv, recurrent) in self.states.items():
            layer = cache.layers[index]
            for actual, owned in ((layer.conv_states, conv), (layer.recurrent_states, recurrent)):
                if (actual is None or actual.data_ptr() != owned.data_ptr()
                        or actual.untyped_storage().data_ptr() != owned.untyped_storage().data_ptr()
                        or actual.dtype != owned.dtype or actual.device != owned.device
                        or actual.storage_offset() != owned.storage_offset()
                        or tuple(actual.shape) != (batch, *owned.shape[1:])
                        or tuple(actual.stride()) != tuple(owned.stride())):
                    return False
            if not (layer.is_conv_states_initialized and layer.is_recurrent_states_initialized):
                return False
        return True

    @property
    def storage_bytes(self) -> int:
        return sum(value.untyped_storage().nbytes() for states in self.states.values() for value in states)

    def zero(self) -> None:
        if self.closed:
            raise RuntimeError("Closed pool")
        for conv, recurrent in self.states.values():
            conv.zero_()
            recurrent.zero_()

    def close(self) -> None:
        self.states.clear()
        self.language = None
        self.closed = True