File size: 564 Bytes
fb67af8
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
"""Model components for WikiMini 95M."""

from .rmsnorm import RMSNorm, RMSNormOptimized
from .rope import RotaryPositionEmbeddings, RotaryPositionEmbeddingsComplex
from .swiglu import SwiGLU, SwiGLUParallel, GeGLU
from .attention import MultiHeadAttention
from .transformer_block import TransformerBlock, WikiMiniModel

__all__ = [
    "RMSNorm",
    "RMSNormOptimized",
    "RotaryPositionEmbeddings",
    "RotaryPositionEmbeddingsComplex",
    "SwiGLU",
    "SwiGLUParallel",
    "GeGLU",
    "MultiHeadAttention",
    "TransformerBlock",
    "WikiMiniModel",
]