Download src/part1/moe.py from Chechaaa09/anlp-a2-models: direct link, hf CLI and curl.
- Browser
- Download file 4.2 kB
-
https://huggingface.co/Chechaaa09/anlp-a2-models/resolve/main/src/part1/moe.py
- Command line
-
hf download hf://Chechaaa09/anlp-a2-models/src/part1/moe.py
-
curl -L -o moe.py https://huggingface.co/Chechaaa09/anlp-a2-models/resolve/main/src/part1/moe.py
4.2 kB
| import torch | |
| import torch.nn as nn | |
| import torch.nn.functional as F | |
| class PositionWiseFFN(nn.Module): | |
| def __init__(self, d_model, d_ff, dropout=0.1): | |
| super().__init__() | |
| self.linear1 = nn.Linear(d_model, d_ff) | |
| self.linear2 = nn.Linear(d_ff, d_model) | |
| self.dropout = nn.Dropout(dropout) | |
| def forward(self, x): | |
| x = self.linear1(x) | |
| x = F.relu(x) | |
| x = self.dropout(x) | |
| x = self.linear2(x) | |
| return x | |
| class TopKRouter(nn.Module): | |
| def __init__(self, d_model, num_experts, top_k): | |
| super().__init__() | |
| self.num_experts = num_experts | |
| self.top_k = top_k | |
| self.gate = nn.Linear(d_model, num_experts, bias = False) | |
| def forward(self, x): | |
| logits = self.gate(x) | |
| values, indices = torch.topk(logits, self.top_k, dim = -1) | |
| routing_weights = F.softmax(values, dim = -1) | |
| return routing_weights, indices | |
| class MoELayer(nn.Module): | |
| def __init__(self, d_model, d_expert, num_experts = 4, top_k = 1, use_shared_expert = False, dropout = 0.1): | |
| super().__init__() | |
| self.d_model = d_model | |
| self.d_expert = d_expert | |
| self.num_experts = num_experts | |
| self.top_k = top_k | |
| self.use_shared_expert = use_shared_expert | |
| self.register_buffer("expert_counts", torch.zeros(num_experts, dtype = torch.long)) | |
| self.track_usage = False | |
| if self.use_shared_expert == True: | |
| self.shared_expert = PositionWiseFFN(self.d_model, self.d_expert, dropout) | |
| self.experts = nn.ModuleList([PositionWiseFFN(self.d_model, self.d_expert, dropout) for _ in range(num_experts - 1)]) | |
| self.router = TopKRouter(self.d_model, self.num_experts - 1, top_k = 1) | |
| else: | |
| self.shared_expert = None | |
| self.experts = nn.ModuleList([PositionWiseFFN(self.d_model, self.d_expert, dropout) for _ in range(num_experts)]) | |
| self.router = TopKRouter(self.d_model, self.num_experts, self.top_k) | |
| def reset_usage_counts(self): | |
| self.expert_counts.zero_() | |
| def forward(self, x): | |
| batch_size, seq_len, d_model = x.shape | |
| x_flat = x.view(-1, d_model) | |
| routing_weights, selected_experts = self.router(x_flat) | |
| if self.track_usage: | |
| with torch.no_grad(): | |
| counts = torch.bincount(selected_experts.flatten(), minlength = self.num_experts) | |
| self.expert_counts += counts | |
| out = torch.zeros_like(x_flat) | |
| for i, expert in enumerate(self.experts): | |
| token_idx, top_k_idx = torch.where(selected_experts == i) | |
| if token_idx.numel() == 0: | |
| continue | |
| tokens_for_expert = x_flat[token_idx] | |
| expert_out = expert(tokens_for_expert) | |
| weights = routing_weights[token_idx, top_k_idx].unsqueeze(-1) | |
| weighted_out = expert_out * weights | |
| out[token_idx] += weighted_out | |
| if self.use_shared_expert: | |
| shared_out = self.shared_expert(x_flat) | |
| out = out + shared_out | |
| if self.track_usage: | |
| with torch.no_grad(): | |
| self.expert_counts[-1] += x_flat.size(0) | |
| out = out.view(batch_size, seq_len, d_model) | |
| return out | |
| def build_ffn_variant(variant, d_model = 384, d_ff = 1536, dropout = 0.1): | |
| if variant == 1: | |
| return PositionWiseFFN(d_model = d_model, d_ff = d_ff, dropout = dropout) | |
| elif variant == 2: | |
| return MoELayer(d_model = d_model, d_expert = d_ff // 4, num_experts = 4, top_k = 1, use_shared_expert = False, dropout = dropout) | |
| elif variant == 3: | |
| return MoELayer(d_model = d_model, d_expert = d_ff // 4, num_experts = 4, top_k = 2, use_shared_expert = False, dropout = dropout) | |
| elif variant == 4: | |
| return MoELayer(d_model = d_model, d_expert = d_ff // 4, num_experts = 4, top_k = 1, use_shared_expert = True, dropout = dropout) | |
| elif variant == 5: | |
| return MoELayer(d_model = d_model, d_expert = d_ff // 2, num_experts = 4, top_k = 2, use_shared_expert = False, dropout = dropout) | |
| else: | |
| raise ValueError(f"unknown variant {variant}. Expected 1, 2, 3, 4, or 5") | |