Chechaaa09's picture
Upload Task 1 checkpoints, tokenizer, configurations and source
ad63418 verified
Raw History Blame Contribute Delete
4.2 kB
import torch
import torch.nn as nn
import torch.nn.functional as F
class PositionWiseFFN(nn.Module):
def __init__(self, d_model, d_ff, dropout=0.1):
super().__init__()
self.linear1 = nn.Linear(d_model, d_ff)
self.linear2 = nn.Linear(d_ff, d_model)
self.dropout = nn.Dropout(dropout)
def forward(self, x):
x = self.linear1(x)
x = F.relu(x)
x = self.dropout(x)
x = self.linear2(x)
return x
class TopKRouter(nn.Module):
def __init__(self, d_model, num_experts, top_k):
super().__init__()
self.num_experts = num_experts
self.top_k = top_k
self.gate = nn.Linear(d_model, num_experts, bias = False)
def forward(self, x):
logits = self.gate(x)
values, indices = torch.topk(logits, self.top_k, dim = -1)
routing_weights = F.softmax(values, dim = -1)
return routing_weights, indices
class MoELayer(nn.Module):
def __init__(self, d_model, d_expert, num_experts = 4, top_k = 1, use_shared_expert = False, dropout = 0.1):
super().__init__()
self.d_model = d_model
self.d_expert = d_expert
self.num_experts = num_experts
self.top_k = top_k
self.use_shared_expert = use_shared_expert
self.register_buffer("expert_counts", torch.zeros(num_experts, dtype = torch.long))
self.track_usage = False
if self.use_shared_expert == True:
self.shared_expert = PositionWiseFFN(self.d_model, self.d_expert, dropout)
self.experts = nn.ModuleList([PositionWiseFFN(self.d_model, self.d_expert, dropout) for _ in range(num_experts - 1)])
self.router = TopKRouter(self.d_model, self.num_experts - 1, top_k = 1)
else:
self.shared_expert = None
self.experts = nn.ModuleList([PositionWiseFFN(self.d_model, self.d_expert, dropout) for _ in range(num_experts)])
self.router = TopKRouter(self.d_model, self.num_experts, self.top_k)
def reset_usage_counts(self):
self.expert_counts.zero_()
def forward(self, x):
batch_size, seq_len, d_model = x.shape
x_flat = x.view(-1, d_model)
routing_weights, selected_experts = self.router(x_flat)
if self.track_usage:
with torch.no_grad():
counts = torch.bincount(selected_experts.flatten(), minlength = self.num_experts)
self.expert_counts += counts
out = torch.zeros_like(x_flat)
for i, expert in enumerate(self.experts):
token_idx, top_k_idx = torch.where(selected_experts == i)
if token_idx.numel() == 0:
continue
tokens_for_expert = x_flat[token_idx]
expert_out = expert(tokens_for_expert)
weights = routing_weights[token_idx, top_k_idx].unsqueeze(-1)
weighted_out = expert_out * weights
out[token_idx] += weighted_out
if self.use_shared_expert:
shared_out = self.shared_expert(x_flat)
out = out + shared_out
if self.track_usage:
with torch.no_grad():
self.expert_counts[-1] += x_flat.size(0)
out = out.view(batch_size, seq_len, d_model)
return out
def build_ffn_variant(variant, d_model = 384, d_ff = 1536, dropout = 0.1):
if variant == 1:
return PositionWiseFFN(d_model = d_model, d_ff = d_ff, dropout = dropout)
elif variant == 2:
return MoELayer(d_model = d_model, d_expert = d_ff // 4, num_experts = 4, top_k = 1, use_shared_expert = False, dropout = dropout)
elif variant == 3:
return MoELayer(d_model = d_model, d_expert = d_ff // 4, num_experts = 4, top_k = 2, use_shared_expert = False, dropout = dropout)
elif variant == 4:
return MoELayer(d_model = d_model, d_expert = d_ff // 4, num_experts = 4, top_k = 1, use_shared_expert = True, dropout = dropout)
elif variant == 5:
return MoELayer(d_model = d_model, d_expert = d_ff // 2, num_experts = 4, top_k = 2, use_shared_expert = False, dropout = dropout)
else:
raise ValueError(f"unknown variant {variant}. Expected 1, 2, 3, 4, or 5")