anlp-a2-models / src /part1 /model.py
Chechaaa09's picture
Upload Task 1 checkpoints, tokenizer, configurations and source
ad63418 verified
Raw History Blame Contribute Delete
2.63 kB
import torch
import torch.nn as nn
import math
from src.part2.attention import MultiHeadAttention
from src.part2.norm import LayerNorm
from .moe import build_ffn_variant, MoELayer
class DecoderBlock(nn.Module):
def __init__(self, d_model, n_heads, d_ff, variant = 1, dropout = 0.1, use_rope = True):
super().__init__()
self.norm1 = LayerNorm(d_model)
self.norm2 = LayerNorm(d_model)
self.attention = MultiHeadAttention(d_model, n_heads, use_rope)
self.dropout = nn.Dropout(dropout)
self.ffn = build_ffn_variant(variant, d_model, d_ff, dropout)
def forward(self, x, mask):
normed = self.norm1(x)
attention_output = self.attention(normed, normed, normed, mask)
x = x + self.dropout(attention_output)
normed = self.norm2(x)
ffn_out = self.ffn(normed)
result = x + self.dropout(ffn_out)
return result
class DecoderOnlyTransformer(nn.Module):
def __init__(self, vocab_size, d_model = 384, n_heads = 6, d_ff = 1536, n_layers = 6, variant = 1, max_seq_len = 256, dropout = 0.1, use_rope = True):
super().__init__()
self.d_model = d_model
self.vocab_size = vocab_size
self.variant = variant
self.token_embeddings = nn.Embedding(vocab_size, d_model)
self.dropout = nn.Dropout(dropout)
self.layers = nn.ModuleList([DecoderBlock(d_model, n_heads, d_ff, variant, dropout, use_rope) for _ in range(n_layers)])
self.final_norm = LayerNorm(d_model)
self.lm_head = nn.Linear(d_model, vocab_size, bias = False)
self.lm_head.weight = self.token_embeddings.weight
def set_moe_tracking(self, track = True):
for layer in self.layers:
if isinstance(layer.ffn, MoELayer):
layer.ffn.track_usage = track
def reset_moe_usage(self):
for layer in self.layers:
if isinstance(layer.ffn, MoELayer):
layer.ffn.reset_usage_counts()
def get_moe_usage(self):
usages = [layer.ffn.expert_counts for layer in self.layers if isinstance(layer.ffn, MoELayer)]
if not usages:
return None
return torch.stack(usages)
def forward(self, input_ids):
batch_size, seq_len = input_ids.shape
lower_t = torch.tril(torch.ones((seq_len, seq_len), device=input_ids.device))
x = self.token_embeddings(input_ids) * math.sqrt(self.d_model)
x = self.dropout(x)
for layer in self.layers:
x = layer(x, lower_t)
x = self.final_norm(x)
logits = self.lm_head(x)
return logits