agent.voice / models /cudagraph /backbone_graph.py
multimodalart's picture
multimodalart HF Staff
Upload folder using huggingface_hub
878a616 verified
Raw History Blame Contribute Delete
15.8 kB
"""
CUDA graph capture for the Breeze backbone's single-token decode step,
using transformers StaticCache.
The backbone (BreezeBackboneModel) predicts the first codebook token at each step.
For CFG, we run batch=2 (cond + uncond) and apply the guidance formula inside
the graph so it's captured as a single replay.
Strategy:
- Use transformers StaticCache for KV cache management
- Use the model's forward method (handles mask, RoPE, attention internally)
- Capture single-token decode + lm_head + CFG as a CUDA graph
- Update cache_position buffer between replays
- Batch size changes trigger automatic recapture via ensure_batch_size()
"""
import torch
from transformers import StaticCache
from transformers.masking_utils import create_causal_mask
class BackboneGraph:
"""
Captures the Breeze backbone's single-token decode step as a CUDA graph,
using the model's own forward with transformers StaticCache.
Supports batch=2 for CFG (cond=batch[0], uncond=batch[1]).
Batch size changes trigger automatic recapture:
graph.ensure_batch_size(4)
guidance_scale is runtime-mutable without recapture:
graph.guidance_scale.fill_(5.0)
"""
def __init__(
self,
backbone_model,
lm_head,
embed_tokens,
config,
device="cuda:0",
dtype=torch.bfloat16,
max_seq_len=512,
guidance_scale=3.0,
debug: bool = False,
batch_size: int = 1,
):
self.device = device
self.dtype = dtype
self.debug = debug
self.no_graph = (
False # runtime flag: True = skip graph replay (for layer-diff hooks)
)
self.max_seq_len = max_seq_len
self.hidden_size = config.hidden_size
self.num_layers = config.num_hidden_layers
self.vocab_size = getattr(lm_head, "out_features", config.vocab_size)
self.num_codebooks = config.num_codebooks
self.config = config
self.batch_size = batch_size
self.half = self._real_batch_size(batch_size)
# Keep references to model components
self.model = backbone_model # BreezeBackboneModel
self.lm_head = lm_head # nn.Linear(hidden_size, vocab_size)
self.embed_tokens = embed_tokens # BreezeBackboneModelEmbeddings
# Cast lm_head to fp32 for numerical stability (hooks still fire through module forward)
self.lm_head = self.lm_head.float()
# Transformers StaticCache β€” batch=2 for CFG
self.static_cache = StaticCache(
config=config, max_cache_len=max_seq_len, batch_size=self.batch_size
)
# Cache position buffer β€” [1] for KV cache slot (shared across batch)
self.cache_position = torch.zeros(1, dtype=torch.long, device=device)
# Per-batch position_ids for RoPE β€” [batch_size, 1]
self.position_ids = torch.zeros(
self.batch_size, 1, dtype=torch.long, device=device
)
# Base position for each batch (set by set_generation_state from attention_mask)
self._base_position = torch.zeros(
self.batch_size, dtype=torch.long, device=device
)
# Guidance scale buffer β€” can be updated at runtime via fill_()
self.guidance_scale = torch.tensor(
[guidance_scale], dtype=torch.float32, device=device
)
self._alloc_buffers()
self.graph = None
self.captured = False
self.attn_mask = None
# Per-batch left-padding lengths for vectorized mask construction
self._pad_lens = torch.zeros(self.batch_size, dtype=torch.long, device=device)
# Pre-allocated index range [0, 1, ..., max_seq_len-1] for broadcasting
self._kv_indices = torch.arange(max_seq_len, dtype=torch.long, device=device)
@staticmethod
def _real_batch_size(batch_size: int) -> int:
if batch_size <= 1:
return batch_size
return batch_size // 2
# ------------------------------------------------------------------
# Buffer allocation (called from __init__ and _rebuild_for_batch)
# ------------------------------------------------------------------
def _alloc_buffers(self):
"""Allocate / re-allocate I/O buffers for current batch_size."""
# Input: codebook IDs [batch, 1, num_codebooks]
self.input_ids_buf = torch.zeros(
self.batch_size, 1, self.num_codebooks, dtype=torch.long, device=self.device
)
# Hidden states output [batch, 1, hidden_size]
self.hidden_buf = torch.zeros(
self.batch_size, 1, self.hidden_size, dtype=self.dtype, device=self.device
)
# Raw logits [batch, vocab_size]
self.logits_buf = torch.zeros(
self.batch_size, self.vocab_size, dtype=torch.float32, device=self.device
)
# CFG-applied logits [half, vocab_size]
self.cfg_logits_buf = torch.zeros(
self.half, self.vocab_size, dtype=torch.float32, device=self.device
)
# ------------------------------------------------------------------
# Batch size management β€” auto-recapture on change
# ------------------------------------------------------------------
@torch.inference_mode()
def ensure_batch_size(self, batch_size: int):
"""No-op if batch_size matches current captured graph; else rebuild and recapture."""
if batch_size == self.batch_size:
return
self._rebuild_for_batch(batch_size)
@torch.inference_mode()
def _rebuild_for_batch(self, batch_size: int):
self.batch_size = batch_size
self.half = self._real_batch_size(batch_size)
self.static_cache = StaticCache(
config=self.config, max_cache_len=self.max_seq_len, batch_size=batch_size
)
# Re-allocate buffers for new batch_size
self.cache_position = torch.zeros(1, dtype=torch.long, device=self.device)
self.position_ids = torch.zeros(
batch_size, 1, dtype=torch.long, device=self.device
)
self._base_position = torch.zeros(
batch_size, dtype=torch.long, device=self.device
)
self._alloc_buffers()
self.captured = False
self.graph = None
self.attn_mask = None
self._pad_lens = torch.zeros(batch_size, dtype=torch.long, device=self.device)
self._kv_indices = torch.arange(
self.max_seq_len, dtype=torch.long, device=self.device
)
if self.no_graph:
self.prepare_eager()
else:
self.capture()
@torch.inference_mode()
def prepare_eager(self):
"""Initialize fixed eager buffers without compile or CUDA Graph capture."""
self.no_graph = True
self._init_cache_layers()
self._build_initial_attention_mask()
self.captured = False
return self
def _init_cache_layers(self):
"""Force lazy initialization of StaticCache layers before graph capture."""
config = self.model.config
num_kv_heads = getattr(
config, "num_key_value_heads", config.num_attention_heads
)
head_dim = getattr(
config, "head_dim", config.hidden_size // config.num_attention_heads
)
dummy_k = torch.zeros(
self.batch_size,
num_kv_heads,
1,
head_dim,
dtype=self.dtype,
device=self.device,
)
for layer in self.static_cache.layers:
if not layer.is_initialized:
layer.lazy_initialization(dummy_k)
def _build_initial_attention_mask(self):
"""Build a default causal mask (no padding) for graph capture/warmup.
Uses create_causal_mask once at position 0 to determine the correct
shape and dtype, then allocates self.attn_mask as a static buffer.
"""
dummy = torch.zeros(
self.batch_size, 1, self.hidden_size, dtype=self.dtype, device=self.device
)
pos = torch.tensor([0], device=self.device)
mask = create_causal_mask(
config=self.model.config,
input_embeds=dummy,
attention_mask=None,
cache_position=pos,
past_key_values=self.static_cache,
)
# mask shape: [batch, 1, 1, max_seq_len], dtype may be float or bool
if mask is None or mask.dtype == torch.bool:
# Some attn implementations return bool or None; allocate float mask
self.attn_mask = torch.zeros(
self.batch_size,
1,
1,
self.max_seq_len,
dtype=self.dtype,
device=self.device,
)
elif self.attn_mask is None:
self.attn_mask = mask.clone()
else:
self.attn_mask.copy_(mask)
print(
f"[BackboneGraph] Initial attention mask shape: {self.attn_mask.shape}, dtype: {self.attn_mask.dtype}"
)
self._mask_min_val = torch.finfo(self.attn_mask.dtype).min
def _set_attention_mask(self, position: int):
"""Construct attention mask in-place from pad_lens and cache_position.
For each batch i, the valid KV range is [pad_lens[i], position].
Everything outside is masked with -inf. Fully vectorized, no loops.
attn_mask shape: [batch, 1, 1, max_seq_len]
"""
# kv_indices: [max_seq_len], pad_lens: [batch, 1], position: scalar
kv = self._kv_indices # [S]
lo = self._pad_lens.unsqueeze(1) # [B, 1]
valid = (kv >= lo) & (kv <= position) # [B, S]
self.attn_mask.fill_(self._mask_min_val)
self.attn_mask[:, 0, 0, :].masked_fill_(valid, 0.0)
def _decode_step(self):
"""Single-token decode: embed β†’ backbone forward β†’ lm_head β†’ CFG."""
# 1. Embed codebook tokens (sum of num_codebooks embeddings)
inputs_embeds = self.embed_tokens(self.input_ids_buf) # [batch, 1, H]
# 2. Backbone forward with per-batch position_ids and shared cache_position
# position_ids: [batch_size, 1] β€” per-batch logical position for RoPE
# cache_position: [1] β€” shared KV cache slot for all batches
out = self.model(
inputs_embeds=inputs_embeds,
attention_mask=self.attn_mask,
past_key_values=self.static_cache,
position_ids=self.position_ids,
cache_position=self.cache_position,
use_cache=True,
)
self.hidden_buf.copy_(out.last_hidden_state)
# 3. lm_head β†’ logits (lm_head is fp32, input cast to fp32)
logits = self.lm_head(self.hidden_buf[:, -1, :].float()) # [batch, vocab] fp32
self.logits_buf.copy_(logits)
# 4. CFG when paired cond/uncond rows are present; otherwise pass through.
if self.batch_size >= 2:
cond_logits = self.logits_buf[: self.half]
uncond_logits = self.logits_buf[self.half :]
cfg_result = uncond_logits + self.guidance_scale * (
cond_logits - uncond_logits
)
else:
cfg_result = self.logits_buf[: self.half]
self.cfg_logits_buf.copy_(cfg_result)
@torch.inference_mode()
def capture(self, prefill_len=100, num_warmup=3):
"""Capture CUDA graph for single-token decode."""
print(f"Warming up backbone graph ({num_warmup} runs)...")
self._init_cache_layers()
self._build_initial_attention_mask()
# Initialize positions for warmup
self.cache_position[0] = prefill_len
self.position_ids.fill_(prefill_len)
self._pad_lens.zero_()
self._set_attention_mask(prefill_len)
for _ in range(num_warmup):
self._decode_step()
torch.cuda.synchronize(device=self.device)
print("Capturing CUDA graph for backbone decode...")
self.graph = torch.cuda.CUDAGraph()
s = torch.cuda.Stream(device=self.device)
s.wait_stream(torch.cuda.current_stream())
with torch.cuda.stream(s):
self._decode_step()
torch.cuda.synchronize(device=self.device)
with torch.cuda.graph(self.graph):
self._decode_step()
torch.cuda.current_stream().wait_stream(s)
torch.cuda.synchronize(device=self.device)
self.captured = True
print("Backbone CUDA graph captured!")
@torch.inference_mode()
def reset(self):
"""Reset cache for new sequence."""
self.static_cache.reset()
@torch.inference_mode()
def finish_direct_prefill(self, seq_len: int) -> int:
self._prefill_len = int(seq_len)
return self._prefill_len
@torch.inference_mode()
def prefill_kv(self, past_key_values):
"""
Copy HF DynamicCache from prefill into our StaticCache.
past_key_values: DynamicCache with num_layers layers of [batch, kv_heads, seq_len, head_dim]
For CFG: the DynamicCache should already have batch=2.
"""
self.static_cache.reset()
seq_len = 0
for li in range(self.num_layers):
k, v = past_key_values[li]
seq_len = k.shape[2]
if seq_len > self.max_seq_len:
raise RuntimeError(
f"Input too long: prefill has {seq_len} tokens but max_seq_len={self.max_seq_len}."
)
cache_pos = torch.arange(seq_len, device=self.device)
self.static_cache.update(k, v, li, {"cache_position": cache_pos})
self._prefill_len = seq_len # remember for decode cache_position
return seq_len
@torch.inference_mode()
def set_generation_state(self, attention_mask=None):
"""Initialize per-batch position_ids and pad_lens from attention_mask.
For left-padded inputs, pad_lens[i] = number of leading zeros in
attention_mask[i]. The mask is then constructed on-the-fly in
_set_attention_mask() using pad_lens β€” no 512-iter table rebuild.
"""
if attention_mask is not None:
per_batch_pos = attention_mask.sum(dim=1).long() # [batch_size]
self._base_position.copy_(per_batch_pos)
self.position_ids.copy_(per_batch_pos.unsqueeze(-1))
self.cache_position[0] = per_batch_pos.max().item()
# Compute per-batch left-padding length:
# pad_lens[i] = seq_len - num_valid_tokens[i]
seq_len = attention_mask.shape[1]
self._pad_lens.copy_(seq_len - per_batch_pos)
else:
self._pad_lens.zero_()
@torch.inference_mode()
def run(self, input_ids, step_idx):
"""
Run one decode step.
input_ids: [batch, 1, num_codebooks] long tensor
step_idx: decode step index (0-based), added to per-batch base position
Returns: (hidden_states [batch, 1, H], cfg_logits [half, vocab])
"""
self.input_ids_buf.copy_(input_ids)
# Update per-batch position_ids: base_position + step_idx
self.position_ids.copy_((self._base_position + step_idx).unsqueeze(-1))
# KV cache slot: prefill_len + step_idx (append after prefill KV, never overwrite)
self.cache_position[0] = self._prefill_len + step_idx
self._set_attention_mask(self.cache_position[0].item())
if self.no_graph:
self._decode_step()
else:
self.graph.replay()
if self.debug:
# Fake forward: trigger lm_head hooks using buffer data from replay.
# Result is discarded β€” only hook side-effects matter.
self.lm_head(self.hidden_buf[:, -1, :].float())
return self.hidden_buf, self.cfg_logits_buf