import torch import torch.nn as nn import torch.nn.functional as F import math from .attention import MultiHeadAttention from .ffn import PositionWiseFFN from .norm import LayerNorm class DecoderBlock(nn.Module): def __init__(self, d_model, n_heads, d_ff, dropout = 0.1, use_rope = True): super().__init__() self.norm1 = LayerNorm(d_model) self.norm2 = LayerNorm(d_model) self.attention = MultiHeadAttention(d_model, n_heads, use_rope) self.ffn = PositionWiseFFN(d_model, d_ff) self.dropout = nn.Dropout(dropout) def forward(self, x, mask): normed = self.norm1(x) attention_output = self.attention(normed, normed, normed, mask) x = x + self.dropout(attention_output) normed = self.norm2(x) ffn_out = self.ffn(normed) result = x + self.dropout(ffn_out) return result class DecoderOnlyTransformer(nn.Module): def __init__(self, vocab_size, d_model, n_heads, d_ff, n_layers, max_seq_len = 1024, dropout = 0.1, use_rope = True): super().__init__() self.d_model = d_model self.vocab_size = vocab_size self.max_seq_len = max_seq_len self.token_embeddings = nn.Embedding(vocab_size, d_model) self.dropout = nn.Dropout(dropout) self.layers = nn.ModuleList([DecoderBlock(d_model, n_heads, d_ff, dropout, use_rope) for _ in range(n_layers)]) self.final_norm = LayerNorm(d_model) self.lm_head = nn.Linear(d_model, vocab_size, bias = False) self.lm_head.weight = self.token_embeddings.weight def forward(self, input_ids): batch_size, seq_len = input_ids.shape lower_t = torch.tril(torch.ones((seq_len, seq_len), device=input_ids.device)) x = self.token_embeddings(input_ids) * math.sqrt(self.d_model) x = self.dropout(x) for layer in self.layers: x = layer(x, lower_t) x = self.final_norm(x) logits = self.lm_head(x) return logits