"""Transformer block: pre-norm GQA + SwiGLU with two residual paths.""" from __future__ import annotations import torch from torch import nn from transformers.activations import ACT2FN from .attention import Attention from .config import MyModelConfig from .norm import RMSNorm class FeedForward(nn.Module): def __init__(self, config: MyModelConfig): super().__init__() self.gate_proj = nn.Linear( config.hidden_size, config.intermediate_size, bias=config.mlp_bias, ) self.up_proj = nn.Linear( config.hidden_size, config.intermediate_size, bias=config.mlp_bias, ) self.down_proj = nn.Linear( config.intermediate_size, config.hidden_size, bias=config.mlp_bias, ) self.act_fn = ACT2FN[config.hidden_act] def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: return self.down_proj(self.act_fn(self.gate_proj(hidden_states)) * self.up_proj(hidden_states)) class Block(nn.Module): def __init__(self, config: MyModelConfig): super().__init__() self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) self.self_attn = Attention(config) self.post_attention_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) self.mlp = FeedForward(config) def forward( self, hidden_states: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor, past_key_value: tuple[torch.Tensor, torch.Tensor] | None = None, use_cache: bool = False, attention_mask: torch.Tensor | None = None, ) -> tuple[torch.Tensor, tuple[torch.Tensor, torch.Tensor] | None]: attention_output, present = self.self_attn( self.input_layernorm(hidden_states), cos, sin, past_key_value, use_cache, attention_mask, ) hidden_states = hidden_states + attention_output hidden_states = hidden_states + self.mlp(self.post_attention_layernorm(hidden_states)) return hidden_states, present