clairic-flash / clairic_runtime /decoder_graph.py
areneau's picture
Update Clairic Transformers runtime
24191a8 verified
Raw History Blame Contribute Delete
14.4 kB
"""Generic startup GDN decoder graphs with owned native state prefixes."""
from __future__ import annotations
from contextlib import contextmanager
import os
import threading
import time
from types import MethodType
from typing import Any, Iterator
import weakref
import torch
from .decoder_capture import DecoderGraph, TorchBackend, callable_key
from .gdn_pool import NativeStatePool
class _SelectedCache:
"""Expose one real native layer to the proved single-layer capture adapter."""
def __init__(self, cache: Any, index: int) -> None:
self.cache, self.index = cache, index
self.layers = [cache.layers[index]]
def has_previous_state(self, index: int) -> bool:
assert index == 0
return self.cache.has_previous_state(self.index)
def update_recurrent_state(self, state: torch.Tensor, index: int) -> torch.Tensor:
assert index == 0
return self.cache.update_recurrent_state(state, self.index)
class DecoderGraphs:
"""One engine, CUDA stream and worker; native prefill and dense attention stay eager."""
def __init__(self, backbone: Any, max_batch: int, buckets: tuple[int, ...]) -> None:
self.backbone = backbone
self.thread = threading.get_ident()
self.device = backbone.device
self.stream = torch.cuda.current_stream(self.device)
self.max_batch, self.buckets = max_batch, buckets
self.pool = NativeStatePool(backbone, max_batch)
self.graphs: dict[tuple[int, int], DecoderGraph] = {}
self.wrappers: list[tuple[Any, Any, Any]] = []
self.active: tuple[Any, int] | None = None
self.closed = False
self.preparing = True
self.stats = {"replay": 0, "eager": 0, "retired": 0}
self.cleanup: list[dict[str, Any]] = []
self.preparation: list[dict[str, Any]] = []
self.observe_requests = False
self.last_request: dict[str, Any] | None = None
self.active_methods: dict[int, Any] = {}
def _owner(self) -> bool:
return (not self.closed and threading.get_ident() == self.thread
and torch.cuda.current_stream(self.device).cuda_stream == self.stream.cuda_stream)
@staticmethod
def _runtime() -> tuple[Any, ...]:
import deep_gemm as dg
return (dg.get_num_sms(), dg.get_tc_util(), dg.get_pdl(),
tuple(sorted((key, value) for key, value in os.environ.items() if key.startswith("DG_"))))
def prepare(self) -> None:
if not self._owner():
raise RuntimeError("Graph preparation must use the engine CUDA worker")
backend = TorchBackend(torch, self._runtime)
try:
for index in self.pool.states:
layer = self.pool.language.layers[index]
eager = layer.forward
weak_manager = weakref.ref(self)
def forward(this: Any, *args: Any, _index: int = index,
_eager: Any = eager, _manager: Any = weak_manager, **kwargs: Any) -> Any:
manager = _manager()
if manager is None:
return _eager(*args, **kwargs)
return manager._forward(_index, _eager, args, kwargs)
installed = MethodType(forward, layer)
self.wrappers.append((layer, eager, installed))
layer.forward = installed
dtype = torch.bfloat16 if index == 0 else torch.float32
for bucket in self.buckets:
began = time.perf_counter_ns()
dummy = self.pool.new_cache(bucket, previous=True)
selected = _SelectedCache(dummy, index)
def operation(value: Any, _eager: Any = eager, _cache: Any = dummy) -> Any:
return _eager(value, position_embeddings=(), past_key_values=_cache, attention_mask=None)
zero = torch.zeros(bucket, 1, layer.input_layernorm.weight.numel(),
dtype=dtype, device=self.device)
self.graphs[index, bucket] = DecoderGraph(backend, layer, selected, zero,
operation, exclusive_process=True)
entry = self.graphs[index, bucket].entry
self.preparation.append({"layer": index, "bucket": bucket, "dtype": str(dtype),
"capture_ms": (time.perf_counter_ns() - began) / 1e6,
"generic_zero_input": True,
"capture_mutation_restored": entry["capture_mutation_restored"],
"pool": list(entry["pool"]),
"pool_reserved_bytes": entry["pool_at_capture"]["reserved_bytes"],
"static_input_storage_bytes": entry["static_input_storage_bytes"]})
self.pool.zero()
self.stream.synchronize()
self.preparing = False
except BaseException:
self.close()
raise
@contextmanager
def request(self, batch: int) -> Iterator[Any]:
if (not self._owner() or self.preparing or self.active is not None
or not 1 <= batch <= self.max_batch or torch.is_grad_enabled()
or torch.compiler.is_compiling() or torch.cuda.is_current_stream_capturing()):
yield None
return
cache = self.pool.new_cache(batch)
if not self.pool.owns(cache, batch):
raise RuntimeError("Fresh cache must use owned native state prefixes")
self.active = (cache, batch)
self.active_methods = {index: self._cache_methods(cache, index) for index in self.pool.states}
completed = False
try:
yield cache
completed = True
finally:
try:
if self.observe_requests:
self.last_request = {"batch": batch, "completed": completed,
"cache_type": type(cache).__name__,
"native_prefixes_owned": self.pool.owns(cache, batch),
"layers": [{"layer": index, "conv_shape": list(cache.layers[index].conv_states.shape),
"conv_stride": list(cache.layers[index].conv_states.stride()),
"conv_dtype": str(cache.layers[index].conv_states.dtype),
"recurrent_shape": list(cache.layers[index].recurrent_states.shape),
"recurrent_stride": list(cache.layers[index].recurrent_states.stride()),
"recurrent_dtype": str(cache.layers[index].recurrent_states.dtype),
"has_previous_state": bool(cache.layers[index].has_previous_state)}
for index in self.pool.states]}
finally:
self.active = None
self.active_methods.clear()
@staticmethod
def _cache_methods(cache: Any, index: int) -> tuple[Any, ...]:
layer = cache.layers[index]
return (callable_key(cache.update_recurrent_state), callable_key(cache.has_previous_state),
callable_key(layer.update_recurrent_state), callable_key(layer.update_conv_state),
layer.dtype, layer.device, layer.max_batch_size, layer.conv_kernel_size)
def diagnostics(self) -> dict[str, Any]:
"""Primitive-only snapshot for separate untimed diagnostics and shutdown."""
return {"closed": self.closed, "preparing": self.preparing,
"active": self.active is not None, "owner_thread": self.thread,
"stream": self.stream.cuda_stream, "max_batch": self.max_batch,
"buckets": list(self.buckets), "graphs": len(self.graphs),
"arena_storage_bytes": self.pool.storage_bytes,
"stats": dict(self.stats), "preparation": list(self.preparation),
"observe_requests": self.observe_requests,
"last_request": self.last_request, "cleanup": list(self.cleanup)}
def _retire(self, index: int) -> None:
for key in tuple(self.graphs):
if key[0] == index:
graph = self.graphs[key]
details = graph.close()
del self.graphs[key]
self.cleanup.append({"layer": key[0], "bucket": key[1], **details})
if not details["passed"]:
raise RuntimeError("Decoder graph retirement failed strict cleanup")
self.stats["retired"] += 1
def _owns_layer(self, cache: Any, batch: int, index: int) -> bool:
layer = cache.layers[index]
for actual, owned in zip((layer.conv_states, layer.recurrent_states), self.pool.states[index], strict=True):
if (actual is None or actual.data_ptr() != owned.data_ptr()
or actual.untyped_storage().data_ptr() != owned.untyped_storage().data_ptr()
or actual.dtype != owned.dtype or actual.device != owned.device
or actual.storage_offset() != owned.storage_offset()
or tuple(actual.shape) != (batch, *owned.shape[1:])
or tuple(actual.stride()) != tuple(owned.stride())):
return False
return layer.is_conv_states_initialized and layer.is_recurrent_states_initialized
def _forward(self, index: int, eager: Any, args: Any, kwargs: Any) -> Any:
value = args[0] if args else kwargs.get("hidden_states")
cache = kwargs.get("past_key_values", args[4] if len(args) > 4 else None)
mask = kwargs.get("attention_mask", args[2] if len(args) > 2 else None)
batch = value.shape[0] if isinstance(value, torch.Tensor) and value.ndim == 3 else 0
expected_dtype = torch.bfloat16 if index == 0 else torch.float32
admitted = (not self.preparing and self._owner() and self.active is not None
and cache is self.active[0] and batch == self.active[1]
and value.shape[1] == 1 and value.is_contiguous() and value.dtype == expected_dtype
and value.device == self.device and mask is None and not torch.is_grad_enabled()
and not torch.compiler.is_compiling() and not torch.cuda.is_current_stream_capturing()
and set(kwargs) <= {"hidden_states", "position_embeddings", "attention_mask", "position_ids",
"past_key_values", "use_cache", "cache_position"}
and self._cache_methods(cache, index) == self.active_methods.get(index)
and self._owns_layer(cache, batch, index) and cache.has_previous_state(index))
if not admitted:
self.stats["eager"] += 1
return eager(*args, **kwargs)
bucket = next((bound for bound in self.buckets if batch <= bound), None)
graph = self.graphs.get((index, bucket))
if graph is None:
self.stats["eager"] += 1
return eager(*args, **kwargs)
entry = graph.entry
try:
graph.check(entry["static_input"])
except RuntimeError:
self._retire(index)
self.stats["eager"] += 1
return eager(*args, **kwargs)
entry["static_input"][:batch].copy_(value)
if batch < bucket:
entry["static_input"][batch:].zero_()
entry["graph"].replay()
self.stats["replay"] += 1
return entry["static_output"][:batch].clone()
def close(self) -> None:
if self.closed:
return
if not self._owner():
raise RuntimeError("Graph cleanup must use the engine CUDA worker")
self.active = None
self.active_methods.clear()
error = None
try:
for index in tuple(self.pool.states):
try:
self._retire(index)
except BaseException as exc:
if error is None:
error = exc
finally:
for layer, original, installed in reversed(self.wrappers):
if layer.forward is installed:
layer.forward = original
self.wrappers.clear()
self.pool.close()
self.backbone = None
self.closed = True
if error is not None:
raise error
def _admissible(backbone: Any, max_batch: int) -> bool:
"""Reject unsupported contexts before allocating arenas or installing wrappers."""
config = backbone.config.get_text_config()
limit = 64 if config.hidden_size == 4096 else 32 if config.hidden_size == 5120 else 0
if (backbone.device.type != "cuda" or backbone.dtype != torch.bfloat16
or not 1 <= max_batch <= limit):
return False
registry = torch.nn.modules.module
if registry._global_forward_hooks or registry._global_forward_pre_hooks:
return False
for module in backbone.modules():
if module._forward_hooks or module._forward_pre_hooks:
return False
for layer in backbone.model.language_model.layers:
if layer.layer_type != "linear_attention":
continue
for value in layer.parameters():
try:
version = value._version
except RuntimeError:
return False
if version is None:
return False
return not torch.compiler.is_compiling() and not torch.cuda.is_current_stream_capturing()
def prepare(backbone: Any, max_batch: int) -> DecoderGraphs | None:
"""Run on the existing worker after math installation; unsupported engines stay eager."""
if not _admissible(backbone, max_batch):
return None
largest = next(bound for bound in (2, 8, 32, 64) if max_batch <= bound)
buckets = tuple(bound for bound in (2, 8, 32, 64) if bound <= largest)
with torch.no_grad():
manager = DecoderGraphs(backbone, max_batch=largest, buckets=buckets)
manager.prepare()
return manager