MyModel-sft / attention.py
HappyBulingling's picture
Upload MyModel checkpoint
72f5707 verified
Raw History Blame Contribute Delete
4.56 kB
"""Grouped-query attention with q/k RMSNorm and SDPA."""
from __future__ import annotations
import torch
import torch.nn.functional as F
from torch import nn
from .config import MyModelConfig
from .norm import RMSNorm
from .rope import apply_rotary_pos_emb
def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor:
if n_rep == 1:
return hidden_states
batch, seq_len, num_kv_heads, head_dim = hidden_states.shape
hidden_states = hidden_states[:, :, :, None, :].expand(
batch, seq_len, num_kv_heads, n_rep, head_dim
)
return hidden_states.reshape(batch, seq_len, num_kv_heads * n_rep, head_dim)
def causal_mask(
q_len: int,
kv_len: int,
device: torch.device,
) -> torch.Tensor:
"""Boolean mask for a query block aligned to the end of the KV cache."""
rows = torch.arange(q_len, device=device).unsqueeze(1) + (kv_len - q_len)
cols = torch.arange(kv_len, device=device).unsqueeze(0)
return cols <= rows
class Attention(nn.Module):
def __init__(self, config: MyModelConfig):
super().__init__()
self.num_heads = config.num_attention_heads
self.num_kv_heads = config.num_key_value_heads
self.head_dim = config.head_dim
self.num_kv_groups = self.num_heads // self.num_kv_heads
self.dropout = config.dropout
self.q_proj = nn.Linear(
config.hidden_size,
self.num_heads * self.head_dim,
bias=config.attention_bias,
)
self.k_proj = nn.Linear(
config.hidden_size,
self.num_kv_heads * self.head_dim,
bias=config.attention_bias,
)
self.v_proj = nn.Linear(
config.hidden_size,
self.num_kv_heads * self.head_dim,
bias=config.attention_bias,
)
self.o_proj = nn.Linear(
self.num_heads * self.head_dim,
config.hidden_size,
bias=config.attention_bias,
)
self.q_norm = RMSNorm(self.head_dim, eps=config.rms_norm_eps) if config.qk_norm else nn.Identity()
self.k_norm = RMSNorm(self.head_dim, eps=config.rms_norm_eps) if config.qk_norm else nn.Identity()
def forward(
self,
hidden_states: torch.Tensor,
cos: torch.Tensor,
sin: torch.Tensor,
past_key_value: tuple[torch.Tensor, torch.Tensor] | None = None,
use_cache: bool = False,
attention_mask: torch.Tensor | None = None,
) -> tuple[torch.Tensor, tuple[torch.Tensor, torch.Tensor] | None]:
batch, seq_len, _ = hidden_states.shape
query = self.q_proj(hidden_states).view(batch, seq_len, self.num_heads, self.head_dim)
key = self.k_proj(hidden_states).view(batch, seq_len, self.num_kv_heads, self.head_dim)
value = self.v_proj(hidden_states).view(batch, seq_len, self.num_kv_heads, self.head_dim)
query = self.q_norm(query)
key = self.k_norm(key)
query, key = apply_rotary_pos_emb(query, key, cos, sin)
if past_key_value is not None:
key = torch.cat((past_key_value[0], key), dim=1)
value = torch.cat((past_key_value[1], value), dim=1)
present = (key, value) if use_cache else None
kv_len = key.shape[1]
key = repeat_kv(key, self.num_kv_groups).transpose(1, 2)
value = repeat_kv(value, self.num_kv_groups).transpose(1, 2)
query = query.transpose(1, 2)
dropout_p = self.dropout if self.training else 0.0
if attention_mask is None and past_key_value is None:
output = F.scaled_dot_product_attention(
query,
key,
value,
dropout_p=dropout_p,
is_causal=True,
)
else:
mask = causal_mask(seq_len, kv_len, hidden_states.device)
if attention_mask is not None:
if attention_mask.shape != (batch, kv_len):
raise ValueError(
f"attention_mask must have shape {(batch, kv_len)}, "
f"got {tuple(attention_mask.shape)}"
)
mask = mask.unsqueeze(0).unsqueeze(0) & attention_mask[:, None, None, :].bool()
output = F.scaled_dot_product_attention(
query,
key,
value,
attn_mask=mask,
dropout_p=dropout_p,
)
output = output.transpose(1, 2).reshape(batch, seq_len, -1)
return self.o_proj(output), present