import torch import torch.nn as nn import torch.nn.functional as F import math from . import positional def compute_attention(Q, K, V, d_k, mask=None): K_T = torch.transpose(K, -2, -1) QK_T = torch.matmul(Q, K_T) scores = QK_T / math.sqrt(d_k) if mask is not None: scores = scores.masked_fill(mask == 0, float('-inf')) softmax_scores = F.softmax(scores, dim=-1) final_scores = torch.matmul(softmax_scores, V) return final_scores class MultiHeadAttention(nn.Module): def __init__(self, d, n_heads, use_rope = True, max_len = 10000): super().__init__() assert d % n_heads == 0 self.d = d self.n_heads = n_heads self.d_k = d // n_heads self.W_Q = nn.Linear(d, d) self.W_K = nn.Linear(d, d) self.W_V = nn.Linear(d, d) self.W_O = nn.Linear(d, d) self.rope = positional.RoPE(self.d_k, max_len) if use_rope else None def forward(self, query, key, value, mask=None): batch = query.shape[0] seq_len_q = query.shape[1] seq_len_k = key.shape[1] Q = self.W_Q(query) K = self.W_K(key) V = self.W_V(value) Q = Q.view(batch, seq_len_q, self.n_heads, self.d_k) Q = torch.transpose(Q, 1, 2) K = K.view(batch, seq_len_k, self.n_heads, self.d_k) K = torch.transpose(K, 1, 2) V = V.view(batch, seq_len_k, self.n_heads, self.d_k) V = torch.transpose(V, 1, 2) if self.rope is not None: Q = self.rope(Q) K = self.rope(K) attention_outputs = compute_attention(Q, K, V, self.d_k, mask) attention_outputs = torch.transpose(attention_outputs, 1, 2) attention_outputs = attention_outputs.contiguous() attention_outputs = attention_outputs.view(batch, seq_len_q, self.d) output = self.W_O(attention_outputs) return output