MyModel-pretrain / block.py
HappyBulingling's picture
Upload MyModel checkpoint
9b53d42 verified
Raw History Blame Contribute Delete
2.18 kB
"""Transformer block: pre-norm GQA + SwiGLU with two residual paths."""
from __future__ import annotations
import torch
from torch import nn
from transformers.activations import ACT2FN
from .attention import Attention
from .config import MyModelConfig
from .norm import RMSNorm
class FeedForward(nn.Module):
def __init__(self, config: MyModelConfig):
super().__init__()
self.gate_proj = nn.Linear(
config.hidden_size,
config.intermediate_size,
bias=config.mlp_bias,
)
self.up_proj = nn.Linear(
config.hidden_size,
config.intermediate_size,
bias=config.mlp_bias,
)
self.down_proj = nn.Linear(
config.intermediate_size,
config.hidden_size,
bias=config.mlp_bias,
)
self.act_fn = ACT2FN[config.hidden_act]
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
return self.down_proj(self.act_fn(self.gate_proj(hidden_states)) * self.up_proj(hidden_states))
class Block(nn.Module):
def __init__(self, config: MyModelConfig):
super().__init__()
self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.self_attn = Attention(config)
self.post_attention_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
self.mlp = FeedForward(config)
def forward(
self,
hidden_states: torch.Tensor,
cos: torch.Tensor,
sin: torch.Tensor,
past_key_value: tuple[torch.Tensor, torch.Tensor] | None = None,
use_cache: bool = False,
attention_mask: torch.Tensor | None = None,
) -> tuple[torch.Tensor, tuple[torch.Tensor, torch.Tensor] | None]:
attention_output, present = self.self_attn(
self.input_layernorm(hidden_states),
cos,
sin,
past_key_value,
use_cache,
attention_mask,
)
hidden_states = hidden_states + attention_output
hidden_states = hidden_states + self.mlp(self.post_attention_layernorm(hidden_states))
return hidden_states, present