File size: 1,911 Bytes
ad63418
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
import torch
import torch.nn as nn
import torch.nn.functional as F
import math
from . import positional

def compute_attention(Q, K, V, d_k, mask=None):
    K_T = torch.transpose(K, -2, -1)
    QK_T = torch.matmul(Q, K_T)
    scores = QK_T / math.sqrt(d_k)
    if mask is not None:
        scores = scores.masked_fill(mask == 0, float('-inf'))
    softmax_scores = F.softmax(scores, dim=-1)
    final_scores = torch.matmul(softmax_scores, V)
    return final_scores

class MultiHeadAttention(nn.Module):
    def __init__(self, d, n_heads, use_rope = True, max_len = 10000):
        super().__init__()

        assert d % n_heads == 0

        self.d = d
        self.n_heads = n_heads
        self.d_k = d // n_heads

        self.W_Q = nn.Linear(d, d)
        self.W_K = nn.Linear(d, d)
        self.W_V = nn.Linear(d, d)
        self.W_O = nn.Linear(d, d)

        self.rope = positional.RoPE(self.d_k, max_len) if use_rope else None

    def forward(self, query, key, value, mask=None):
        batch = query.shape[0]
        seq_len_q = query.shape[1]
        seq_len_k = key.shape[1]


        Q = self.W_Q(query)
        K = self.W_K(key)
        V = self.W_V(value)


        Q = Q.view(batch, seq_len_q, self.n_heads, self.d_k)
        Q = torch.transpose(Q, 1, 2)

        K = K.view(batch, seq_len_k, self.n_heads, self.d_k)
        K = torch.transpose(K, 1, 2)

        V = V.view(batch, seq_len_k, self.n_heads, self.d_k)
        V = torch.transpose(V, 1, 2)

        if self.rope is not None:
            Q = self.rope(Q)
            K = self.rope(K)

        attention_outputs = compute_attention(Q, K, V, self.d_k, mask)


        attention_outputs = torch.transpose(attention_outputs, 1, 2)
        attention_outputs = attention_outputs.contiguous()
        attention_outputs = attention_outputs.view(batch, seq_len_q, self.d)


        output = self.W_O(attention_outputs)
        return output