"""Grouped-query attention with q/k RMSNorm and SDPA.""" from __future__ import annotations import torch import torch.nn.functional as F from torch import nn from .config import MyModelConfig from .norm import RMSNorm from .rope import apply_rotary_pos_emb def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor: if n_rep == 1: return hidden_states batch, seq_len, num_kv_heads, head_dim = hidden_states.shape hidden_states = hidden_states[:, :, :, None, :].expand( batch, seq_len, num_kv_heads, n_rep, head_dim ) return hidden_states.reshape(batch, seq_len, num_kv_heads * n_rep, head_dim) def causal_mask( q_len: int, kv_len: int, device: torch.device, ) -> torch.Tensor: """Boolean mask for a query block aligned to the end of the KV cache.""" rows = torch.arange(q_len, device=device).unsqueeze(1) + (kv_len - q_len) cols = torch.arange(kv_len, device=device).unsqueeze(0) return cols <= rows class Attention(nn.Module): def __init__(self, config: MyModelConfig): super().__init__() self.num_heads = config.num_attention_heads self.num_kv_heads = config.num_key_value_heads self.head_dim = config.head_dim self.num_kv_groups = self.num_heads // self.num_kv_heads self.dropout = config.dropout self.q_proj = nn.Linear( config.hidden_size, self.num_heads * self.head_dim, bias=config.attention_bias, ) self.k_proj = nn.Linear( config.hidden_size, self.num_kv_heads * self.head_dim, bias=config.attention_bias, ) self.v_proj = nn.Linear( config.hidden_size, self.num_kv_heads * self.head_dim, bias=config.attention_bias, ) self.o_proj = nn.Linear( self.num_heads * self.head_dim, config.hidden_size, bias=config.attention_bias, ) self.q_norm = RMSNorm(self.head_dim, eps=config.rms_norm_eps) if config.qk_norm else nn.Identity() self.k_norm = RMSNorm(self.head_dim, eps=config.rms_norm_eps) if config.qk_norm else nn.Identity() def forward( self, hidden_states: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor, past_key_value: tuple[torch.Tensor, torch.Tensor] | None = None, use_cache: bool = False, attention_mask: torch.Tensor | None = None, ) -> tuple[torch.Tensor, tuple[torch.Tensor, torch.Tensor] | None]: batch, seq_len, _ = hidden_states.shape query = self.q_proj(hidden_states).view(batch, seq_len, self.num_heads, self.head_dim) key = self.k_proj(hidden_states).view(batch, seq_len, self.num_kv_heads, self.head_dim) value = self.v_proj(hidden_states).view(batch, seq_len, self.num_kv_heads, self.head_dim) query = self.q_norm(query) key = self.k_norm(key) query, key = apply_rotary_pos_emb(query, key, cos, sin) if past_key_value is not None: key = torch.cat((past_key_value[0], key), dim=1) value = torch.cat((past_key_value[1], value), dim=1) present = (key, value) if use_cache else None kv_len = key.shape[1] key = repeat_kv(key, self.num_kv_groups).transpose(1, 2) value = repeat_kv(value, self.num_kv_groups).transpose(1, 2) query = query.transpose(1, 2) dropout_p = self.dropout if self.training else 0.0 if attention_mask is None and past_key_value is None: output = F.scaled_dot_product_attention( query, key, value, dropout_p=dropout_p, is_causal=True, ) else: mask = causal_mask(seq_len, kv_len, hidden_states.device) if attention_mask is not None: if attention_mask.shape != (batch, kv_len): raise ValueError( f"attention_mask must have shape {(batch, kv_len)}, " f"got {tuple(attention_mask.shape)}" ) mask = mask.unsqueeze(0).unsqueeze(0) & attention_mask[:, None, None, :].bool() output = F.scaled_dot_product_attention( query, key, value, attn_mask=mask, dropout_p=dropout_p, ) output = output.transpose(1, 2).reshape(batch, seq_len, -1) return self.o_proj(output), present