"""Model configuration and the two presets used by this project.""" from __future__ import annotations from transformers import PretrainedConfig PRESETS = { "tiny": { "hidden_size": 256, "num_hidden_layers": 4, "num_attention_heads": 8, "num_key_value_heads": 1, "head_dim": 32, "intermediate_size": 896, "vocab_size": 32768, }, "base": { "hidden_size": 1024, "num_hidden_layers": 20, "num_attention_heads": 16, "num_key_value_heads": 4, "head_dim": 64, "intermediate_size": 2816, "vocab_size": 32768, }, } class MyModelConfig(PretrainedConfig): """Configuration for the 259M dense text backbone. Defaults: 20 layers, d=1024, GQA 16:4, SwiGLU 2816, 32K vocabulary, tied embeddings. """ model_type = "mymodel" def __init__( self, hidden_size: int = 1024, num_hidden_layers: int = 20, num_attention_heads: int = 16, num_key_value_heads: int = 4, head_dim: int = 64, intermediate_size: int = 2816, vocab_size: int = 32768, hidden_act: str = "silu", rms_norm_eps: float = 1e-6, qk_norm: bool = True, rope_theta: float = 1_000_000.0, max_position_embeddings: int = 4096, rope_scaling: dict | None = None, tie_word_embeddings: bool = True, attention_bias: bool = False, mlp_bias: bool = False, dropout: float = 0.0, ffn_type: str = "dense", parametrization: str = "standard", initializer_range: float = 0.02, bos_token_id: int = 2, eos_token_id: int = 0, im_end_token_id: int = 5, pad_token_id: int = 1, unk_token_id: int = 3, **kwargs, ): super().__init__( bos_token_id=bos_token_id, eos_token_id=eos_token_id, pad_token_id=pad_token_id, unk_token_id=unk_token_id, tie_word_embeddings=tie_word_embeddings, **kwargs, ) self.im_end_token_id = im_end_token_id if hidden_size % num_attention_heads != 0: raise ValueError("hidden_size must be divisible by num_attention_heads") if num_attention_heads % num_key_value_heads != 0: raise ValueError("num_attention_heads must be divisible by num_key_value_heads") if head_dim * num_attention_heads != hidden_size: raise ValueError("head_dim * num_attention_heads must equal hidden_size") self.hidden_size = hidden_size self.num_hidden_layers = num_hidden_layers self.num_attention_heads = num_attention_heads self.num_key_value_heads = num_key_value_heads self.head_dim = head_dim self.intermediate_size = intermediate_size self.vocab_size = vocab_size self.hidden_act = hidden_act self.rms_norm_eps = rms_norm_eps self.qk_norm = qk_norm self.rope_theta = rope_theta self.max_position_embeddings = max_position_embeddings self.rope_scaling = rope_scaling self.attention_bias = attention_bias self.mlp_bias = mlp_bias self.dropout = dropout self.ffn_type = ffn_type self.parametrization = parametrization self.initializer_range = initializer_range if self.ffn_type != "dense": raise ValueError(f"unsupported ffn_type: {self.ffn_type}") if self.parametrization != "standard": raise ValueError(f"unsupported parametrization: {self.parametrization}") @classmethod def from_preset(cls, name: str, **overrides) -> "MyModelConfig": if name not in PRESETS: raise ValueError(f"unknown preset {name!r}; choose from {sorted(PRESETS)}") values = dict(PRESETS[name]) values.update(overrides) return cls(**values) def estimate_parameters(self) -> int: """Formula-based parameter count, excluding q/k norm parameters.""" d = self.hidden_size attn = 2 * d * (self.num_attention_heads * self.head_dim) attn += 2 * d * (self.num_key_value_heads * self.head_dim) ffn = 3 * d * self.intermediate_size norms = self.num_hidden_layers * 2 * d + d if self.qk_norm: norms += self.num_hidden_layers * 2 * self.head_dim embedding = self.vocab_size * d return self.num_hidden_layers * (attn + ffn) + norms + embedding