Spaces:
Running on Zero
Running on Zero
File size: 5,241 Bytes
878a616 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 | from __future__ import annotations
import logging
from collections import OrderedDict
from collections.abc import Callable
from dataclasses import dataclass
from typing import Generic, TypeVar
import torch
from .kv_cache import StaticShiftKVCache
logger = logging.getLogger(__name__)
T = TypeVar("T")
@dataclass
class ConvStateBlock:
cache_buf: torch.Tensor
left_cache_len: int
@dataclass
class TransConvStateBlock:
cache_buf: torch.Tensor
left_cache_len: int
@dataclass
class RequestStateSlot:
req_id: str | None
next_step: int
tail_flushed: bool
conv1d: dict[str, ConvStateBlock]
tconv1d: dict[str, TransConvStateBlock]
kv: StaticShiftKVCache
def reset_request_state(slot: RequestStateSlot) -> None:
slot.req_id = None
slot.next_step = 0
slot.tail_flushed = False
for block in slot.conv1d.values():
block.cache_buf.zero_()
for block in slot.tconv1d.values():
block.cache_buf.zero_()
for layer in slot.kv.layers:
layer.k.zero_()
layer.v.zero_()
class RequestStatePool(Generic[T]):
def __init__(
self,
create_slot: Callable[[], T],
reset_slot: Callable[[T], None],
max_active_reqs: int | None = None,
on_evict: Callable[[str], None] | None = None,
):
if max_active_reqs is not None and max_active_reqs <= 0:
raise ValueError(
f"max_active_reqs must be > 0 or None, got {max_active_reqs}"
)
self._create_slot = create_slot
self._reset_slot = reset_slot
self.max_active_reqs = max_active_reqs
self._on_evict = on_evict
self._req_to_slot: OrderedDict[str, int] = OrderedDict()
self._slot_to_req: list[str | None] = []
self._slots: list[T] = []
self._pinned: set[str] = set()
self._dynamic: dict[str, T] = {}
if self.max_active_reqs is not None:
self._slots = [self._create_slot() for _ in range(self.max_active_reqs)]
self._slot_to_req = [None for _ in range(self.max_active_reqs)]
def __len__(self) -> int:
return (
len(self._req_to_slot)
if self.max_active_reqs is not None
else len(self._dynamic)
)
def active_req_ids(self) -> list[str]:
if self.max_active_reqs is None:
return list(self._dynamic.keys())
return list(self._req_to_slot.keys())
def num_free_slots(self) -> int | None:
if self.max_active_reqs is None:
return None
return sum(1 for req_id in self._slot_to_req if req_id is None)
def pin(self, req_id: str) -> None:
self._pinned.add(req_id)
def unpin(self, req_id: str) -> None:
self._pinned.discard(req_id)
def _find_free_slot(self) -> int | None:
for idx, req_id in enumerate(self._slot_to_req):
if req_id is None:
return idx
return None
def _evict_lru(self) -> int:
victim_req_id: str | None = None
for req_id in self._req_to_slot.keys():
if req_id not in self._pinned:
victim_req_id = req_id
break
if victim_req_id is None:
logger.error(
"RequestStatePool found no evictable request. max_active_reqs=%s active_req_ids=%s pinned=%s",
self.max_active_reqs,
list(self._req_to_slot.keys()),
sorted(self._pinned),
)
raise RuntimeError("No evictable request state slot available.")
slot_idx = self._req_to_slot.pop(victim_req_id)
self._slot_to_req[slot_idx] = None
self._reset_slot(self._slots[slot_idx])
if self._on_evict is not None:
self._on_evict(victim_req_id)
logger.info(
"Evicted request state due to LRU capacity limit: req_id=%s slot_idx=%d",
victim_req_id,
slot_idx,
)
return slot_idx
def get(self, req_id: str) -> T:
if self.max_active_reqs is None:
value = self._dynamic.get(req_id)
if value is None:
value = self._create_slot()
self._dynamic[req_id] = value
return value
slot_idx = self._req_to_slot.get(req_id)
if slot_idx is None:
slot_idx = self._find_free_slot()
if slot_idx is None:
slot_idx = self._evict_lru()
self._slot_to_req[slot_idx] = req_id
self._req_to_slot[req_id] = slot_idx
self._reset_slot(self._slots[slot_idx])
self._req_to_slot.move_to_end(req_id)
return self._slots[slot_idx]
def pop(self, req_id: str) -> None:
self._pinned.discard(req_id)
if self.max_active_reqs is None:
self._dynamic.pop(req_id, None)
return
slot_idx = self._req_to_slot.pop(req_id, None)
if slot_idx is None:
return
self._reset_slot(self._slots[slot_idx])
self._slot_to_req[slot_idx] = None
def get_slot_index(self, req_id: str) -> int | None:
if self.max_active_reqs is None:
return None
return self._req_to_slot.get(req_id)
|