import torch import torch.nn as nn import math from src.part2.attention import MultiHeadAttention from src.part2.norm import LayerNorm from .moe import build_ffn_variant, MoELayer class DecoderBlock(nn.Module): def __init__(self, d_model, n_heads, d_ff, variant = 1, dropout = 0.1, use_rope = True): super().__init__() self.norm1 = LayerNorm(d_model) self.norm2 = LayerNorm(d_model) self.attention = MultiHeadAttention(d_model, n_heads, use_rope) self.dropout = nn.Dropout(dropout) self.ffn = build_ffn_variant(variant, d_model, d_ff, dropout) def forward(self, x, mask): normed = self.norm1(x) attention_output = self.attention(normed, normed, normed, mask) x = x + self.dropout(attention_output) normed = self.norm2(x) ffn_out = self.ffn(normed) result = x + self.dropout(ffn_out) return result class DecoderOnlyTransformer(nn.Module): def __init__(self, vocab_size, d_model = 384, n_heads = 6, d_ff = 1536, n_layers = 6, variant = 1, max_seq_len = 256, dropout = 0.1, use_rope = True): super().__init__() self.d_model = d_model self.vocab_size = vocab_size self.variant = variant self.token_embeddings = nn.Embedding(vocab_size, d_model) self.dropout = nn.Dropout(dropout) self.layers = nn.ModuleList([DecoderBlock(d_model, n_heads, d_ff, variant, dropout, use_rope) for _ in range(n_layers)]) self.final_norm = LayerNorm(d_model) self.lm_head = nn.Linear(d_model, vocab_size, bias = False) self.lm_head.weight = self.token_embeddings.weight def set_moe_tracking(self, track = True): for layer in self.layers: if isinstance(layer.ffn, MoELayer): layer.ffn.track_usage = track def reset_moe_usage(self): for layer in self.layers: if isinstance(layer.ffn, MoELayer): layer.ffn.reset_usage_counts() def get_moe_usage(self): usages = [layer.ffn.expert_counts for layer in self.layers if isinstance(layer.ffn, MoELayer)] if not usages: return None return torch.stack(usages) def forward(self, input_ids): batch_size, seq_len = input_ids.shape lower_t = torch.tril(torch.ones((seq_len, seq_len), device=input_ids.device)) x = self.token_embeddings(input_ids) * math.sqrt(self.d_model) x = self.dropout(x) for layer in self.layers: x = layer(x, lower_t) x = self.final_norm(x) logits = self.lm_head(x) return logits