"""Generic startup GDN decoder graphs with owned native state prefixes.""" from __future__ import annotations from contextlib import contextmanager import os import threading import time from types import MethodType from typing import Any, Iterator import weakref import torch from .decoder_capture import DecoderGraph, TorchBackend, callable_key from .gdn_pool import NativeStatePool class _SelectedCache: """Expose one real native layer to the proved single-layer capture adapter.""" def __init__(self, cache: Any, index: int) -> None: self.cache, self.index = cache, index self.layers = [cache.layers[index]] def has_previous_state(self, index: int) -> bool: assert index == 0 return self.cache.has_previous_state(self.index) def update_recurrent_state(self, state: torch.Tensor, index: int) -> torch.Tensor: assert index == 0 return self.cache.update_recurrent_state(state, self.index) class DecoderGraphs: """One engine, CUDA stream and worker; native prefill and dense attention stay eager.""" def __init__(self, backbone: Any, max_batch: int, buckets: tuple[int, ...]) -> None: self.backbone = backbone self.thread = threading.get_ident() self.device = backbone.device self.stream = torch.cuda.current_stream(self.device) self.max_batch, self.buckets = max_batch, buckets self.pool = NativeStatePool(backbone, max_batch) self.graphs: dict[tuple[int, int], DecoderGraph] = {} self.wrappers: list[tuple[Any, Any, Any]] = [] self.active: tuple[Any, int] | None = None self.closed = False self.preparing = True self.stats = {"replay": 0, "eager": 0, "retired": 0} self.cleanup: list[dict[str, Any]] = [] self.preparation: list[dict[str, Any]] = [] self.observe_requests = False self.last_request: dict[str, Any] | None = None self.active_methods: dict[int, Any] = {} def _owner(self) -> bool: return (not self.closed and threading.get_ident() == self.thread and torch.cuda.current_stream(self.device).cuda_stream == self.stream.cuda_stream) @staticmethod def _runtime() -> tuple[Any, ...]: import deep_gemm as dg return (dg.get_num_sms(), dg.get_tc_util(), dg.get_pdl(), tuple(sorted((key, value) for key, value in os.environ.items() if key.startswith("DG_")))) def prepare(self) -> None: if not self._owner(): raise RuntimeError("Graph preparation must use the engine CUDA worker") backend = TorchBackend(torch, self._runtime) try: for index in self.pool.states: layer = self.pool.language.layers[index] eager = layer.forward weak_manager = weakref.ref(self) def forward(this: Any, *args: Any, _index: int = index, _eager: Any = eager, _manager: Any = weak_manager, **kwargs: Any) -> Any: manager = _manager() if manager is None: return _eager(*args, **kwargs) return manager._forward(_index, _eager, args, kwargs) installed = MethodType(forward, layer) self.wrappers.append((layer, eager, installed)) layer.forward = installed dtype = torch.bfloat16 if index == 0 else torch.float32 for bucket in self.buckets: began = time.perf_counter_ns() dummy = self.pool.new_cache(bucket, previous=True) selected = _SelectedCache(dummy, index) def operation(value: Any, _eager: Any = eager, _cache: Any = dummy) -> Any: return _eager(value, position_embeddings=(), past_key_values=_cache, attention_mask=None) zero = torch.zeros(bucket, 1, layer.input_layernorm.weight.numel(), dtype=dtype, device=self.device) self.graphs[index, bucket] = DecoderGraph(backend, layer, selected, zero, operation, exclusive_process=True) entry = self.graphs[index, bucket].entry self.preparation.append({"layer": index, "bucket": bucket, "dtype": str(dtype), "capture_ms": (time.perf_counter_ns() - began) / 1e6, "generic_zero_input": True, "capture_mutation_restored": entry["capture_mutation_restored"], "pool": list(entry["pool"]), "pool_reserved_bytes": entry["pool_at_capture"]["reserved_bytes"], "static_input_storage_bytes": entry["static_input_storage_bytes"]}) self.pool.zero() self.stream.synchronize() self.preparing = False except BaseException: self.close() raise @contextmanager def request(self, batch: int) -> Iterator[Any]: if (not self._owner() or self.preparing or self.active is not None or not 1 <= batch <= self.max_batch or torch.is_grad_enabled() or torch.compiler.is_compiling() or torch.cuda.is_current_stream_capturing()): yield None return cache = self.pool.new_cache(batch) if not self.pool.owns(cache, batch): raise RuntimeError("Fresh cache must use owned native state prefixes") self.active = (cache, batch) self.active_methods = {index: self._cache_methods(cache, index) for index in self.pool.states} completed = False try: yield cache completed = True finally: try: if self.observe_requests: self.last_request = {"batch": batch, "completed": completed, "cache_type": type(cache).__name__, "native_prefixes_owned": self.pool.owns(cache, batch), "layers": [{"layer": index, "conv_shape": list(cache.layers[index].conv_states.shape), "conv_stride": list(cache.layers[index].conv_states.stride()), "conv_dtype": str(cache.layers[index].conv_states.dtype), "recurrent_shape": list(cache.layers[index].recurrent_states.shape), "recurrent_stride": list(cache.layers[index].recurrent_states.stride()), "recurrent_dtype": str(cache.layers[index].recurrent_states.dtype), "has_previous_state": bool(cache.layers[index].has_previous_state)} for index in self.pool.states]} finally: self.active = None self.active_methods.clear() @staticmethod def _cache_methods(cache: Any, index: int) -> tuple[Any, ...]: layer = cache.layers[index] return (callable_key(cache.update_recurrent_state), callable_key(cache.has_previous_state), callable_key(layer.update_recurrent_state), callable_key(layer.update_conv_state), layer.dtype, layer.device, layer.max_batch_size, layer.conv_kernel_size) def diagnostics(self) -> dict[str, Any]: """Primitive-only snapshot for separate untimed diagnostics and shutdown.""" return {"closed": self.closed, "preparing": self.preparing, "active": self.active is not None, "owner_thread": self.thread, "stream": self.stream.cuda_stream, "max_batch": self.max_batch, "buckets": list(self.buckets), "graphs": len(self.graphs), "arena_storage_bytes": self.pool.storage_bytes, "stats": dict(self.stats), "preparation": list(self.preparation), "observe_requests": self.observe_requests, "last_request": self.last_request, "cleanup": list(self.cleanup)} def _retire(self, index: int) -> None: for key in tuple(self.graphs): if key[0] == index: graph = self.graphs[key] details = graph.close() del self.graphs[key] self.cleanup.append({"layer": key[0], "bucket": key[1], **details}) if not details["passed"]: raise RuntimeError("Decoder graph retirement failed strict cleanup") self.stats["retired"] += 1 def _owns_layer(self, cache: Any, batch: int, index: int) -> bool: layer = cache.layers[index] for actual, owned in zip((layer.conv_states, layer.recurrent_states), self.pool.states[index], strict=True): if (actual is None or actual.data_ptr() != owned.data_ptr() or actual.untyped_storage().data_ptr() != owned.untyped_storage().data_ptr() or actual.dtype != owned.dtype or actual.device != owned.device or actual.storage_offset() != owned.storage_offset() or tuple(actual.shape) != (batch, *owned.shape[1:]) or tuple(actual.stride()) != tuple(owned.stride())): return False return layer.is_conv_states_initialized and layer.is_recurrent_states_initialized def _forward(self, index: int, eager: Any, args: Any, kwargs: Any) -> Any: value = args[0] if args else kwargs.get("hidden_states") cache = kwargs.get("past_key_values", args[4] if len(args) > 4 else None) mask = kwargs.get("attention_mask", args[2] if len(args) > 2 else None) batch = value.shape[0] if isinstance(value, torch.Tensor) and value.ndim == 3 else 0 expected_dtype = torch.bfloat16 if index == 0 else torch.float32 admitted = (not self.preparing and self._owner() and self.active is not None and cache is self.active[0] and batch == self.active[1] and value.shape[1] == 1 and value.is_contiguous() and value.dtype == expected_dtype and value.device == self.device and mask is None and not torch.is_grad_enabled() and not torch.compiler.is_compiling() and not torch.cuda.is_current_stream_capturing() and set(kwargs) <= {"hidden_states", "position_embeddings", "attention_mask", "position_ids", "past_key_values", "use_cache", "cache_position"} and self._cache_methods(cache, index) == self.active_methods.get(index) and self._owns_layer(cache, batch, index) and cache.has_previous_state(index)) if not admitted: self.stats["eager"] += 1 return eager(*args, **kwargs) bucket = next((bound for bound in self.buckets if batch <= bound), None) graph = self.graphs.get((index, bucket)) if graph is None: self.stats["eager"] += 1 return eager(*args, **kwargs) entry = graph.entry try: graph.check(entry["static_input"]) except RuntimeError: self._retire(index) self.stats["eager"] += 1 return eager(*args, **kwargs) entry["static_input"][:batch].copy_(value) if batch < bucket: entry["static_input"][batch:].zero_() entry["graph"].replay() self.stats["replay"] += 1 return entry["static_output"][:batch].clone() def close(self) -> None: if self.closed: return if not self._owner(): raise RuntimeError("Graph cleanup must use the engine CUDA worker") self.active = None self.active_methods.clear() error = None try: for index in tuple(self.pool.states): try: self._retire(index) except BaseException as exc: if error is None: error = exc finally: for layer, original, installed in reversed(self.wrappers): if layer.forward is installed: layer.forward = original self.wrappers.clear() self.pool.close() self.backbone = None self.closed = True if error is not None: raise error def _admissible(backbone: Any, max_batch: int) -> bool: """Reject unsupported contexts before allocating arenas or installing wrappers.""" config = backbone.config.get_text_config() limit = 64 if config.hidden_size == 4096 else 32 if config.hidden_size == 5120 else 0 if (backbone.device.type != "cuda" or backbone.dtype != torch.bfloat16 or not 1 <= max_batch <= limit): return False registry = torch.nn.modules.module if registry._global_forward_hooks or registry._global_forward_pre_hooks: return False for module in backbone.modules(): if module._forward_hooks or module._forward_pre_hooks: return False for layer in backbone.model.language_model.layers: if layer.layer_type != "linear_attention": continue for value in layer.parameters(): try: version = value._version except RuntimeError: return False if version is None: return False return not torch.compiler.is_compiling() and not torch.cuda.is_current_stream_capturing() def prepare(backbone: Any, max_batch: int) -> DecoderGraphs | None: """Run on the existing worker after math installation; unsupported engines stay eager.""" if not _admissible(backbone, max_batch): return None largest = next(bound for bound in (2, 8, 32, 64) if max_batch <= bound) buckets = tuple(bound for bound in (2, 8, 32, 64) if bound <= largest) with torch.no_grad(): manager = DecoderGraphs(backbone, max_batch=largest, buckets=buckets) manager.prepare() return manager