Spaces:
Running on Zero
Running on Zero
Download models/cudagraph/backbone_graph.py from Quantumbraid/agent.voice: direct link, hf CLI and curl.
- Browser
- Download file 15.8 kB
-
https://huggingface.co/spaces/Quantumbraid/agent.voice/resolve/main/models/cudagraph/backbone_graph.py
- Command line
-
hf download hf://spaces/Quantumbraid/agent.voice/models/cudagraph/backbone_graph.py
-
curl -L -o backbone_graph.py https://huggingface.co/spaces/Quantumbraid/agent.voice/resolve/main/models/cudagraph/backbone_graph.py
15.8 kB
| """ | |
| CUDA graph capture for the Breeze backbone's single-token decode step, | |
| using transformers StaticCache. | |
| The backbone (BreezeBackboneModel) predicts the first codebook token at each step. | |
| For CFG, we run batch=2 (cond + uncond) and apply the guidance formula inside | |
| the graph so it's captured as a single replay. | |
| Strategy: | |
| - Use transformers StaticCache for KV cache management | |
| - Use the model's forward method (handles mask, RoPE, attention internally) | |
| - Capture single-token decode + lm_head + CFG as a CUDA graph | |
| - Update cache_position buffer between replays | |
| - Batch size changes trigger automatic recapture via ensure_batch_size() | |
| """ | |
| import torch | |
| from transformers import StaticCache | |
| from transformers.masking_utils import create_causal_mask | |
| class BackboneGraph: | |
| """ | |
| Captures the Breeze backbone's single-token decode step as a CUDA graph, | |
| using the model's own forward with transformers StaticCache. | |
| Supports batch=2 for CFG (cond=batch[0], uncond=batch[1]). | |
| Batch size changes trigger automatic recapture: | |
| graph.ensure_batch_size(4) | |
| guidance_scale is runtime-mutable without recapture: | |
| graph.guidance_scale.fill_(5.0) | |
| """ | |
| def __init__( | |
| self, | |
| backbone_model, | |
| lm_head, | |
| embed_tokens, | |
| config, | |
| device="cuda:0", | |
| dtype=torch.bfloat16, | |
| max_seq_len=512, | |
| guidance_scale=3.0, | |
| debug: bool = False, | |
| batch_size: int = 1, | |
| ): | |
| self.device = device | |
| self.dtype = dtype | |
| self.debug = debug | |
| self.no_graph = ( | |
| False # runtime flag: True = skip graph replay (for layer-diff hooks) | |
| ) | |
| self.max_seq_len = max_seq_len | |
| self.hidden_size = config.hidden_size | |
| self.num_layers = config.num_hidden_layers | |
| self.vocab_size = getattr(lm_head, "out_features", config.vocab_size) | |
| self.num_codebooks = config.num_codebooks | |
| self.config = config | |
| self.batch_size = batch_size | |
| self.half = self._real_batch_size(batch_size) | |
| # Keep references to model components | |
| self.model = backbone_model # BreezeBackboneModel | |
| self.lm_head = lm_head # nn.Linear(hidden_size, vocab_size) | |
| self.embed_tokens = embed_tokens # BreezeBackboneModelEmbeddings | |
| # Cast lm_head to fp32 for numerical stability (hooks still fire through module forward) | |
| self.lm_head = self.lm_head.float() | |
| # Transformers StaticCache β batch=2 for CFG | |
| self.static_cache = StaticCache( | |
| config=config, max_cache_len=max_seq_len, batch_size=self.batch_size | |
| ) | |
| # Cache position buffer β [1] for KV cache slot (shared across batch) | |
| self.cache_position = torch.zeros(1, dtype=torch.long, device=device) | |
| # Per-batch position_ids for RoPE β [batch_size, 1] | |
| self.position_ids = torch.zeros( | |
| self.batch_size, 1, dtype=torch.long, device=device | |
| ) | |
| # Base position for each batch (set by set_generation_state from attention_mask) | |
| self._base_position = torch.zeros( | |
| self.batch_size, dtype=torch.long, device=device | |
| ) | |
| # Guidance scale buffer β can be updated at runtime via fill_() | |
| self.guidance_scale = torch.tensor( | |
| [guidance_scale], dtype=torch.float32, device=device | |
| ) | |
| self._alloc_buffers() | |
| self.graph = None | |
| self.captured = False | |
| self.attn_mask = None | |
| # Per-batch left-padding lengths for vectorized mask construction | |
| self._pad_lens = torch.zeros(self.batch_size, dtype=torch.long, device=device) | |
| # Pre-allocated index range [0, 1, ..., max_seq_len-1] for broadcasting | |
| self._kv_indices = torch.arange(max_seq_len, dtype=torch.long, device=device) | |
| def _real_batch_size(batch_size: int) -> int: | |
| if batch_size <= 1: | |
| return batch_size | |
| return batch_size // 2 | |
| # ------------------------------------------------------------------ | |
| # Buffer allocation (called from __init__ and _rebuild_for_batch) | |
| # ------------------------------------------------------------------ | |
| def _alloc_buffers(self): | |
| """Allocate / re-allocate I/O buffers for current batch_size.""" | |
| # Input: codebook IDs [batch, 1, num_codebooks] | |
| self.input_ids_buf = torch.zeros( | |
| self.batch_size, 1, self.num_codebooks, dtype=torch.long, device=self.device | |
| ) | |
| # Hidden states output [batch, 1, hidden_size] | |
| self.hidden_buf = torch.zeros( | |
| self.batch_size, 1, self.hidden_size, dtype=self.dtype, device=self.device | |
| ) | |
| # Raw logits [batch, vocab_size] | |
| self.logits_buf = torch.zeros( | |
| self.batch_size, self.vocab_size, dtype=torch.float32, device=self.device | |
| ) | |
| # CFG-applied logits [half, vocab_size] | |
| self.cfg_logits_buf = torch.zeros( | |
| self.half, self.vocab_size, dtype=torch.float32, device=self.device | |
| ) | |
| # ------------------------------------------------------------------ | |
| # Batch size management β auto-recapture on change | |
| # ------------------------------------------------------------------ | |
| def ensure_batch_size(self, batch_size: int): | |
| """No-op if batch_size matches current captured graph; else rebuild and recapture.""" | |
| if batch_size == self.batch_size: | |
| return | |
| self._rebuild_for_batch(batch_size) | |
| def _rebuild_for_batch(self, batch_size: int): | |
| self.batch_size = batch_size | |
| self.half = self._real_batch_size(batch_size) | |
| self.static_cache = StaticCache( | |
| config=self.config, max_cache_len=self.max_seq_len, batch_size=batch_size | |
| ) | |
| # Re-allocate buffers for new batch_size | |
| self.cache_position = torch.zeros(1, dtype=torch.long, device=self.device) | |
| self.position_ids = torch.zeros( | |
| batch_size, 1, dtype=torch.long, device=self.device | |
| ) | |
| self._base_position = torch.zeros( | |
| batch_size, dtype=torch.long, device=self.device | |
| ) | |
| self._alloc_buffers() | |
| self.captured = False | |
| self.graph = None | |
| self.attn_mask = None | |
| self._pad_lens = torch.zeros(batch_size, dtype=torch.long, device=self.device) | |
| self._kv_indices = torch.arange( | |
| self.max_seq_len, dtype=torch.long, device=self.device | |
| ) | |
| if self.no_graph: | |
| self.prepare_eager() | |
| else: | |
| self.capture() | |
| def prepare_eager(self): | |
| """Initialize fixed eager buffers without compile or CUDA Graph capture.""" | |
| self.no_graph = True | |
| self._init_cache_layers() | |
| self._build_initial_attention_mask() | |
| self.captured = False | |
| return self | |
| def _init_cache_layers(self): | |
| """Force lazy initialization of StaticCache layers before graph capture.""" | |
| config = self.model.config | |
| num_kv_heads = getattr( | |
| config, "num_key_value_heads", config.num_attention_heads | |
| ) | |
| head_dim = getattr( | |
| config, "head_dim", config.hidden_size // config.num_attention_heads | |
| ) | |
| dummy_k = torch.zeros( | |
| self.batch_size, | |
| num_kv_heads, | |
| 1, | |
| head_dim, | |
| dtype=self.dtype, | |
| device=self.device, | |
| ) | |
| for layer in self.static_cache.layers: | |
| if not layer.is_initialized: | |
| layer.lazy_initialization(dummy_k) | |
| def _build_initial_attention_mask(self): | |
| """Build a default causal mask (no padding) for graph capture/warmup. | |
| Uses create_causal_mask once at position 0 to determine the correct | |
| shape and dtype, then allocates self.attn_mask as a static buffer. | |
| """ | |
| dummy = torch.zeros( | |
| self.batch_size, 1, self.hidden_size, dtype=self.dtype, device=self.device | |
| ) | |
| pos = torch.tensor([0], device=self.device) | |
| mask = create_causal_mask( | |
| config=self.model.config, | |
| input_embeds=dummy, | |
| attention_mask=None, | |
| cache_position=pos, | |
| past_key_values=self.static_cache, | |
| ) | |
| # mask shape: [batch, 1, 1, max_seq_len], dtype may be float or bool | |
| if mask is None or mask.dtype == torch.bool: | |
| # Some attn implementations return bool or None; allocate float mask | |
| self.attn_mask = torch.zeros( | |
| self.batch_size, | |
| 1, | |
| 1, | |
| self.max_seq_len, | |
| dtype=self.dtype, | |
| device=self.device, | |
| ) | |
| elif self.attn_mask is None: | |
| self.attn_mask = mask.clone() | |
| else: | |
| self.attn_mask.copy_(mask) | |
| print( | |
| f"[BackboneGraph] Initial attention mask shape: {self.attn_mask.shape}, dtype: {self.attn_mask.dtype}" | |
| ) | |
| self._mask_min_val = torch.finfo(self.attn_mask.dtype).min | |
| def _set_attention_mask(self, position: int): | |
| """Construct attention mask in-place from pad_lens and cache_position. | |
| For each batch i, the valid KV range is [pad_lens[i], position]. | |
| Everything outside is masked with -inf. Fully vectorized, no loops. | |
| attn_mask shape: [batch, 1, 1, max_seq_len] | |
| """ | |
| # kv_indices: [max_seq_len], pad_lens: [batch, 1], position: scalar | |
| kv = self._kv_indices # [S] | |
| lo = self._pad_lens.unsqueeze(1) # [B, 1] | |
| valid = (kv >= lo) & (kv <= position) # [B, S] | |
| self.attn_mask.fill_(self._mask_min_val) | |
| self.attn_mask[:, 0, 0, :].masked_fill_(valid, 0.0) | |
| def _decode_step(self): | |
| """Single-token decode: embed β backbone forward β lm_head β CFG.""" | |
| # 1. Embed codebook tokens (sum of num_codebooks embeddings) | |
| inputs_embeds = self.embed_tokens(self.input_ids_buf) # [batch, 1, H] | |
| # 2. Backbone forward with per-batch position_ids and shared cache_position | |
| # position_ids: [batch_size, 1] β per-batch logical position for RoPE | |
| # cache_position: [1] β shared KV cache slot for all batches | |
| out = self.model( | |
| inputs_embeds=inputs_embeds, | |
| attention_mask=self.attn_mask, | |
| past_key_values=self.static_cache, | |
| position_ids=self.position_ids, | |
| cache_position=self.cache_position, | |
| use_cache=True, | |
| ) | |
| self.hidden_buf.copy_(out.last_hidden_state) | |
| # 3. lm_head β logits (lm_head is fp32, input cast to fp32) | |
| logits = self.lm_head(self.hidden_buf[:, -1, :].float()) # [batch, vocab] fp32 | |
| self.logits_buf.copy_(logits) | |
| # 4. CFG when paired cond/uncond rows are present; otherwise pass through. | |
| if self.batch_size >= 2: | |
| cond_logits = self.logits_buf[: self.half] | |
| uncond_logits = self.logits_buf[self.half :] | |
| cfg_result = uncond_logits + self.guidance_scale * ( | |
| cond_logits - uncond_logits | |
| ) | |
| else: | |
| cfg_result = self.logits_buf[: self.half] | |
| self.cfg_logits_buf.copy_(cfg_result) | |
| def capture(self, prefill_len=100, num_warmup=3): | |
| """Capture CUDA graph for single-token decode.""" | |
| print(f"Warming up backbone graph ({num_warmup} runs)...") | |
| self._init_cache_layers() | |
| self._build_initial_attention_mask() | |
| # Initialize positions for warmup | |
| self.cache_position[0] = prefill_len | |
| self.position_ids.fill_(prefill_len) | |
| self._pad_lens.zero_() | |
| self._set_attention_mask(prefill_len) | |
| for _ in range(num_warmup): | |
| self._decode_step() | |
| torch.cuda.synchronize(device=self.device) | |
| print("Capturing CUDA graph for backbone decode...") | |
| self.graph = torch.cuda.CUDAGraph() | |
| s = torch.cuda.Stream(device=self.device) | |
| s.wait_stream(torch.cuda.current_stream()) | |
| with torch.cuda.stream(s): | |
| self._decode_step() | |
| torch.cuda.synchronize(device=self.device) | |
| with torch.cuda.graph(self.graph): | |
| self._decode_step() | |
| torch.cuda.current_stream().wait_stream(s) | |
| torch.cuda.synchronize(device=self.device) | |
| self.captured = True | |
| print("Backbone CUDA graph captured!") | |
| def reset(self): | |
| """Reset cache for new sequence.""" | |
| self.static_cache.reset() | |
| def finish_direct_prefill(self, seq_len: int) -> int: | |
| self._prefill_len = int(seq_len) | |
| return self._prefill_len | |
| def prefill_kv(self, past_key_values): | |
| """ | |
| Copy HF DynamicCache from prefill into our StaticCache. | |
| past_key_values: DynamicCache with num_layers layers of [batch, kv_heads, seq_len, head_dim] | |
| For CFG: the DynamicCache should already have batch=2. | |
| """ | |
| self.static_cache.reset() | |
| seq_len = 0 | |
| for li in range(self.num_layers): | |
| k, v = past_key_values[li] | |
| seq_len = k.shape[2] | |
| if seq_len > self.max_seq_len: | |
| raise RuntimeError( | |
| f"Input too long: prefill has {seq_len} tokens but max_seq_len={self.max_seq_len}." | |
| ) | |
| cache_pos = torch.arange(seq_len, device=self.device) | |
| self.static_cache.update(k, v, li, {"cache_position": cache_pos}) | |
| self._prefill_len = seq_len # remember for decode cache_position | |
| return seq_len | |
| def set_generation_state(self, attention_mask=None): | |
| """Initialize per-batch position_ids and pad_lens from attention_mask. | |
| For left-padded inputs, pad_lens[i] = number of leading zeros in | |
| attention_mask[i]. The mask is then constructed on-the-fly in | |
| _set_attention_mask() using pad_lens β no 512-iter table rebuild. | |
| """ | |
| if attention_mask is not None: | |
| per_batch_pos = attention_mask.sum(dim=1).long() # [batch_size] | |
| self._base_position.copy_(per_batch_pos) | |
| self.position_ids.copy_(per_batch_pos.unsqueeze(-1)) | |
| self.cache_position[0] = per_batch_pos.max().item() | |
| # Compute per-batch left-padding length: | |
| # pad_lens[i] = seq_len - num_valid_tokens[i] | |
| seq_len = attention_mask.shape[1] | |
| self._pad_lens.copy_(seq_len - per_batch_pos) | |
| else: | |
| self._pad_lens.zero_() | |
| def run(self, input_ids, step_idx): | |
| """ | |
| Run one decode step. | |
| input_ids: [batch, 1, num_codebooks] long tensor | |
| step_idx: decode step index (0-based), added to per-batch base position | |
| Returns: (hidden_states [batch, 1, H], cfg_logits [half, vocab]) | |
| """ | |
| self.input_ids_buf.copy_(input_ids) | |
| # Update per-batch position_ids: base_position + step_idx | |
| self.position_ids.copy_((self._base_position + step_idx).unsqueeze(-1)) | |
| # KV cache slot: prefill_len + step_idx (append after prefill KV, never overwrite) | |
| self.cache_position[0] = self._prefill_len + step_idx | |
| self._set_attention_mask(self.cache_position[0].item()) | |
| if self.no_graph: | |
| self._decode_step() | |
| else: | |
| self.graph.replay() | |
| if self.debug: | |
| # Fake forward: trigger lm_head hooks using buffer data from replay. | |
| # Result is discarded β only hook side-effects matter. | |
| self.lm_head(self.hidden_buf[:, -1, :].float()) | |
| return self.hidden_buf, self.cfg_logits_buf | |