File size: 4,564 Bytes
72f5707
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
"""Grouped-query attention with q/k RMSNorm and SDPA."""

from __future__ import annotations

import torch
import torch.nn.functional as F
from torch import nn

from .config import MyModelConfig
from .norm import RMSNorm
from .rope import apply_rotary_pos_emb


def repeat_kv(hidden_states: torch.Tensor, n_rep: int) -> torch.Tensor:
    if n_rep == 1:
        return hidden_states
    batch, seq_len, num_kv_heads, head_dim = hidden_states.shape
    hidden_states = hidden_states[:, :, :, None, :].expand(
        batch, seq_len, num_kv_heads, n_rep, head_dim
    )
    return hidden_states.reshape(batch, seq_len, num_kv_heads * n_rep, head_dim)


def causal_mask(
    q_len: int,
    kv_len: int,
    device: torch.device,
) -> torch.Tensor:
    """Boolean mask for a query block aligned to the end of the KV cache."""
    rows = torch.arange(q_len, device=device).unsqueeze(1) + (kv_len - q_len)
    cols = torch.arange(kv_len, device=device).unsqueeze(0)
    return cols <= rows


class Attention(nn.Module):
    def __init__(self, config: MyModelConfig):
        super().__init__()
        self.num_heads = config.num_attention_heads
        self.num_kv_heads = config.num_key_value_heads
        self.head_dim = config.head_dim
        self.num_kv_groups = self.num_heads // self.num_kv_heads
        self.dropout = config.dropout

        self.q_proj = nn.Linear(
            config.hidden_size,
            self.num_heads * self.head_dim,
            bias=config.attention_bias,
        )
        self.k_proj = nn.Linear(
            config.hidden_size,
            self.num_kv_heads * self.head_dim,
            bias=config.attention_bias,
        )
        self.v_proj = nn.Linear(
            config.hidden_size,
            self.num_kv_heads * self.head_dim,
            bias=config.attention_bias,
        )
        self.o_proj = nn.Linear(
            self.num_heads * self.head_dim,
            config.hidden_size,
            bias=config.attention_bias,
        )
        self.q_norm = RMSNorm(self.head_dim, eps=config.rms_norm_eps) if config.qk_norm else nn.Identity()
        self.k_norm = RMSNorm(self.head_dim, eps=config.rms_norm_eps) if config.qk_norm else nn.Identity()

    def forward(
        self,
        hidden_states: torch.Tensor,
        cos: torch.Tensor,
        sin: torch.Tensor,
        past_key_value: tuple[torch.Tensor, torch.Tensor] | None = None,
        use_cache: bool = False,
        attention_mask: torch.Tensor | None = None,
    ) -> tuple[torch.Tensor, tuple[torch.Tensor, torch.Tensor] | None]:
        batch, seq_len, _ = hidden_states.shape

        query = self.q_proj(hidden_states).view(batch, seq_len, self.num_heads, self.head_dim)
        key = self.k_proj(hidden_states).view(batch, seq_len, self.num_kv_heads, self.head_dim)
        value = self.v_proj(hidden_states).view(batch, seq_len, self.num_kv_heads, self.head_dim)

        query = self.q_norm(query)
        key = self.k_norm(key)
        query, key = apply_rotary_pos_emb(query, key, cos, sin)

        if past_key_value is not None:
            key = torch.cat((past_key_value[0], key), dim=1)
            value = torch.cat((past_key_value[1], value), dim=1)

        present = (key, value) if use_cache else None
        kv_len = key.shape[1]

        key = repeat_kv(key, self.num_kv_groups).transpose(1, 2)
        value = repeat_kv(value, self.num_kv_groups).transpose(1, 2)
        query = query.transpose(1, 2)

        dropout_p = self.dropout if self.training else 0.0
        if attention_mask is None and past_key_value is None:
            output = F.scaled_dot_product_attention(
                query,
                key,
                value,
                dropout_p=dropout_p,
                is_causal=True,
            )
        else:
            mask = causal_mask(seq_len, kv_len, hidden_states.device)
            if attention_mask is not None:
                if attention_mask.shape != (batch, kv_len):
                    raise ValueError(
                        f"attention_mask must have shape {(batch, kv_len)}, "
                        f"got {tuple(attention_mask.shape)}"
                    )
                mask = mask.unsqueeze(0).unsqueeze(0) & attention_mask[:, None, None, :].bool()
            output = F.scaled_dot_product_attention(
                query,
                key,
                value,
                attn_mask=mask,
                dropout_p=dropout_p,
            )

        output = output.transpose(1, 2).reshape(batch, seq_len, -1)
        return self.o_proj(output), present