File size: 4,499 Bytes
72f5707
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
"""Model configuration and the two presets used by this project."""

from __future__ import annotations

from transformers import PretrainedConfig


PRESETS = {
    "tiny": {
        "hidden_size": 256,
        "num_hidden_layers": 4,
        "num_attention_heads": 8,
        "num_key_value_heads": 1,
        "head_dim": 32,
        "intermediate_size": 896,
        "vocab_size": 32768,
    },
    "base": {
        "hidden_size": 1024,
        "num_hidden_layers": 20,
        "num_attention_heads": 16,
        "num_key_value_heads": 4,
        "head_dim": 64,
        "intermediate_size": 2816,
        "vocab_size": 32768,
    },
}


class MyModelConfig(PretrainedConfig):
    """Configuration for the 259M dense text backbone.

    Defaults: 20 layers, d=1024, GQA 16:4, SwiGLU 2816, 32K vocabulary,
    tied embeddings.
    """

    model_type = "mymodel"

    def __init__(
        self,
        hidden_size: int = 1024,
        num_hidden_layers: int = 20,
        num_attention_heads: int = 16,
        num_key_value_heads: int = 4,
        head_dim: int = 64,
        intermediate_size: int = 2816,
        vocab_size: int = 32768,
        hidden_act: str = "silu",
        rms_norm_eps: float = 1e-6,
        qk_norm: bool = True,
        rope_theta: float = 1_000_000.0,
        max_position_embeddings: int = 4096,
        rope_scaling: dict | None = None,
        tie_word_embeddings: bool = True,
        attention_bias: bool = False,
        mlp_bias: bool = False,
        dropout: float = 0.0,
        ffn_type: str = "dense",
        parametrization: str = "standard",
        initializer_range: float = 0.02,
        bos_token_id: int = 2,
        eos_token_id: int = 0,
        im_end_token_id: int = 5,
        pad_token_id: int = 1,
        unk_token_id: int = 3,
        **kwargs,
    ):
        super().__init__(
            bos_token_id=bos_token_id,
            eos_token_id=eos_token_id,
            pad_token_id=pad_token_id,
            unk_token_id=unk_token_id,
            tie_word_embeddings=tie_word_embeddings,
            **kwargs,
        )
        self.im_end_token_id = im_end_token_id
        if hidden_size % num_attention_heads != 0:
            raise ValueError("hidden_size must be divisible by num_attention_heads")
        if num_attention_heads % num_key_value_heads != 0:
            raise ValueError("num_attention_heads must be divisible by num_key_value_heads")
        if head_dim * num_attention_heads != hidden_size:
            raise ValueError("head_dim * num_attention_heads must equal hidden_size")

        self.hidden_size = hidden_size
        self.num_hidden_layers = num_hidden_layers
        self.num_attention_heads = num_attention_heads
        self.num_key_value_heads = num_key_value_heads
        self.head_dim = head_dim
        self.intermediate_size = intermediate_size
        self.vocab_size = vocab_size
        self.hidden_act = hidden_act
        self.rms_norm_eps = rms_norm_eps
        self.qk_norm = qk_norm
        self.rope_theta = rope_theta
        self.max_position_embeddings = max_position_embeddings
        self.rope_scaling = rope_scaling
        self.attention_bias = attention_bias
        self.mlp_bias = mlp_bias
        self.dropout = dropout
        self.ffn_type = ffn_type
        self.parametrization = parametrization
        self.initializer_range = initializer_range

        if self.ffn_type != "dense":
            raise ValueError(f"unsupported ffn_type: {self.ffn_type}")
        if self.parametrization != "standard":
            raise ValueError(f"unsupported parametrization: {self.parametrization}")

    @classmethod
    def from_preset(cls, name: str, **overrides) -> "MyModelConfig":
        if name not in PRESETS:
            raise ValueError(f"unknown preset {name!r}; choose from {sorted(PRESETS)}")
        values = dict(PRESETS[name])
        values.update(overrides)
        return cls(**values)

    def estimate_parameters(self) -> int:
        """Formula-based parameter count, excluding q/k norm parameters."""
        d = self.hidden_size
        attn = 2 * d * (self.num_attention_heads * self.head_dim)
        attn += 2 * d * (self.num_key_value_heads * self.head_dim)
        ffn = 3 * d * self.intermediate_size
        norms = self.num_hidden_layers * 2 * d + d
        if self.qk_norm:
            norms += self.num_hidden_layers * 2 * self.head_dim
        embedding = self.vocab_size * d
        return self.num_hidden_layers * (attn + ffn) + norms + embedding