File size: 2,695 Bytes
dc64c03 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 | """Standalone Transformers configuration for Limen0.2B."""
from __future__ import annotations
from transformers import PretrainedConfig
DEFAULT_VOCAB_SIZE = 16_384
DEFAULT_HIDDEN_SIZE = 768
DEFAULT_NUM_HIDDEN_LAYERS = 35
DEFAULT_NUM_ATTENTION_HEADS = 6
DEFAULT_NUM_KEY_VALUE_HEADS = 2
DEFAULT_HEAD_DIM = DEFAULT_HIDDEN_SIZE // DEFAULT_NUM_ATTENTION_HEADS
DEFAULT_INTERMEDIATE_SIZE = DEFAULT_HIDDEN_SIZE * 5 // 2
DEFAULT_BLOCK_SIZE = 1024
DEFAULT_ROPE_THETA = 100_000.0
class GPTConfig(PretrainedConfig):
"""Configuration for the Limen0.2B decoder-only language model."""
model_type = "gpt"
def __init__(
self,
vocab_size: int = DEFAULT_VOCAB_SIZE,
hidden_size: int = DEFAULT_HIDDEN_SIZE,
num_hidden_layers: int = DEFAULT_NUM_HIDDEN_LAYERS,
num_attention_heads: int = DEFAULT_NUM_ATTENTION_HEADS,
num_key_value_heads: int | None = DEFAULT_NUM_KEY_VALUE_HEADS,
intermediate_size: int | None = DEFAULT_INTERMEDIATE_SIZE,
head_dim: int | None = None,
block_size: int = DEFAULT_BLOCK_SIZE,
rope_theta: float = DEFAULT_ROPE_THETA,
rms_norm_eps: float = 1e-6,
xsa_projection: bool = True,
tie_word_embeddings: bool = True,
labels_are_shifted: bool = False,
**kwargs,
):
if num_key_value_heads is None:
num_key_value_heads = num_attention_heads
if head_dim is None:
if hidden_size % num_attention_heads != 0:
raise ValueError("hidden_size must be divisible by num_attention_heads")
head_dim = hidden_size // num_attention_heads
if intermediate_size is None:
intermediate_size = hidden_size * 4
if num_attention_heads % num_key_value_heads != 0:
raise ValueError("num_attention_heads must be divisible by num_key_value_heads")
if head_dim % 2 != 0:
raise ValueError("head_dim must be even for RoPE")
super().__init__(tie_word_embeddings=tie_word_embeddings, **kwargs)
self.vocab_size = int(vocab_size)
self.hidden_size = int(hidden_size)
self.num_hidden_layers = int(num_hidden_layers)
self.num_attention_heads = int(num_attention_heads)
self.num_key_value_heads = int(num_key_value_heads)
self.intermediate_size = int(intermediate_size)
self.head_dim = int(head_dim)
self.block_size = int(block_size)
self.max_position_embeddings = int(block_size)
self.rope_theta = float(rope_theta)
self.rms_norm_eps = float(rms_norm_eps)
self.xsa_projection = bool(xsa_projection)
self.labels_are_shifted = bool(labels_are_shifted)
|