Download src/part2/attention.py from Chechaaa09/anlp-a2-models: direct link, hf CLI and curl.
- Browser
- Download file 1.91 kB
-
https://huggingface.co/Chechaaa09/anlp-a2-models/resolve/main/src/part2/attention.py
- Command line
-
hf download hf://Chechaaa09/anlp-a2-models/src/part2/attention.py
-
curl -L -o attention.py https://huggingface.co/Chechaaa09/anlp-a2-models/resolve/main/src/part2/attention.py
1.91 kB
| import torch | |
| import torch.nn as nn | |
| import torch.nn.functional as F | |
| import math | |
| from . import positional | |
| def compute_attention(Q, K, V, d_k, mask=None): | |
| K_T = torch.transpose(K, -2, -1) | |
| QK_T = torch.matmul(Q, K_T) | |
| scores = QK_T / math.sqrt(d_k) | |
| if mask is not None: | |
| scores = scores.masked_fill(mask == 0, float('-inf')) | |
| softmax_scores = F.softmax(scores, dim=-1) | |
| final_scores = torch.matmul(softmax_scores, V) | |
| return final_scores | |
| class MultiHeadAttention(nn.Module): | |
| def __init__(self, d, n_heads, use_rope = True, max_len = 10000): | |
| super().__init__() | |
| assert d % n_heads == 0 | |
| self.d = d | |
| self.n_heads = n_heads | |
| self.d_k = d // n_heads | |
| self.W_Q = nn.Linear(d, d) | |
| self.W_K = nn.Linear(d, d) | |
| self.W_V = nn.Linear(d, d) | |
| self.W_O = nn.Linear(d, d) | |
| self.rope = positional.RoPE(self.d_k, max_len) if use_rope else None | |
| def forward(self, query, key, value, mask=None): | |
| batch = query.shape[0] | |
| seq_len_q = query.shape[1] | |
| seq_len_k = key.shape[1] | |
| Q = self.W_Q(query) | |
| K = self.W_K(key) | |
| V = self.W_V(value) | |
| Q = Q.view(batch, seq_len_q, self.n_heads, self.d_k) | |
| Q = torch.transpose(Q, 1, 2) | |
| K = K.view(batch, seq_len_k, self.n_heads, self.d_k) | |
| K = torch.transpose(K, 1, 2) | |
| V = V.view(batch, seq_len_k, self.n_heads, self.d_k) | |
| V = torch.transpose(V, 1, 2) | |
| if self.rope is not None: | |
| Q = self.rope(Q) | |
| K = self.rope(K) | |
| attention_outputs = compute_attention(Q, K, V, self.d_k, mask) | |
| attention_outputs = torch.transpose(attention_outputs, 1, 2) | |
| attention_outputs = attention_outputs.contiguous() | |
| attention_outputs = attention_outputs.view(batch, seq_len_q, self.d) | |
| output = self.W_O(attention_outputs) | |
| return output |