File size: 564 Bytes
fb67af8 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 |
"""Model components for WikiMini 95M."""
from .rmsnorm import RMSNorm, RMSNormOptimized
from .rope import RotaryPositionEmbeddings, RotaryPositionEmbeddingsComplex
from .swiglu import SwiGLU, SwiGLUParallel, GeGLU
from .attention import MultiHeadAttention
from .transformer_block import TransformerBlock, WikiMiniModel
__all__ = [
"RMSNorm",
"RMSNormOptimized",
"RotaryPositionEmbeddings",
"RotaryPositionEmbeddingsComplex",
"SwiGLU",
"SwiGLUParallel",
"GeGLU",
"MultiHeadAttention",
"TransformerBlock",
"WikiMiniModel",
] |