import torch import torch.nn as nn import torch.nn.functional as F class PositionWiseFFN(nn.Module): def __init__(self, d_model, d_ff, dropout=0.1): super().__init__() self.linear1 = nn.Linear(d_model, d_ff) self.linear2 = nn.Linear(d_ff, d_model) self.dropout = nn.Dropout(dropout) def forward(self, x): x = self.linear1(x) x = F.relu(x) x = self.dropout(x) x = self.linear2(x) return x class TopKRouter(nn.Module): def __init__(self, d_model, num_experts, top_k): super().__init__() self.num_experts = num_experts self.top_k = top_k self.gate = nn.Linear(d_model, num_experts, bias = False) def forward(self, x): logits = self.gate(x) values, indices = torch.topk(logits, self.top_k, dim = -1) routing_weights = F.softmax(values, dim = -1) return routing_weights, indices class MoELayer(nn.Module): def __init__(self, d_model, d_expert, num_experts = 4, top_k = 1, use_shared_expert = False, dropout = 0.1): super().__init__() self.d_model = d_model self.d_expert = d_expert self.num_experts = num_experts self.top_k = top_k self.use_shared_expert = use_shared_expert self.register_buffer("expert_counts", torch.zeros(num_experts, dtype = torch.long)) self.track_usage = False if self.use_shared_expert == True: self.shared_expert = PositionWiseFFN(self.d_model, self.d_expert, dropout) self.experts = nn.ModuleList([PositionWiseFFN(self.d_model, self.d_expert, dropout) for _ in range(num_experts - 1)]) self.router = TopKRouter(self.d_model, self.num_experts - 1, top_k = 1) else: self.shared_expert = None self.experts = nn.ModuleList([PositionWiseFFN(self.d_model, self.d_expert, dropout) for _ in range(num_experts)]) self.router = TopKRouter(self.d_model, self.num_experts, self.top_k) def reset_usage_counts(self): self.expert_counts.zero_() def forward(self, x): batch_size, seq_len, d_model = x.shape x_flat = x.view(-1, d_model) routing_weights, selected_experts = self.router(x_flat) if self.track_usage: with torch.no_grad(): counts = torch.bincount(selected_experts.flatten(), minlength = self.num_experts) self.expert_counts += counts out = torch.zeros_like(x_flat) for i, expert in enumerate(self.experts): token_idx, top_k_idx = torch.where(selected_experts == i) if token_idx.numel() == 0: continue tokens_for_expert = x_flat[token_idx] expert_out = expert(tokens_for_expert) weights = routing_weights[token_idx, top_k_idx].unsqueeze(-1) weighted_out = expert_out * weights out[token_idx] += weighted_out if self.use_shared_expert: shared_out = self.shared_expert(x_flat) out = out + shared_out if self.track_usage: with torch.no_grad(): self.expert_counts[-1] += x_flat.size(0) out = out.view(batch_size, seq_len, d_model) return out def build_ffn_variant(variant, d_model = 384, d_ff = 1536, dropout = 0.1): if variant == 1: return PositionWiseFFN(d_model = d_model, d_ff = d_ff, dropout = dropout) elif variant == 2: return MoELayer(d_model = d_model, d_expert = d_ff // 4, num_experts = 4, top_k = 1, use_shared_expert = False, dropout = dropout) elif variant == 3: return MoELayer(d_model = d_model, d_expert = d_ff // 4, num_experts = 4, top_k = 2, use_shared_expert = False, dropout = dropout) elif variant == 4: return MoELayer(d_model = d_model, d_expert = d_ff // 4, num_experts = 4, top_k = 1, use_shared_expert = True, dropout = dropout) elif variant == 5: return MoELayer(d_model = d_model, d_expert = d_ff // 2, num_experts = 4, top_k = 2, use_shared_expert = False, dropout = dropout) else: raise ValueError(f"unknown variant {variant}. Expected 1, 2, 3, 4, or 5")