File size: 20,394 Bytes
b9ee6b4
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
"""Decoder-only Transformer from SPEC section 4: pre-RMSNorm, RoPE, SwiGLU, tied embeddings, no biases.

Options from the 600-update comparisons (see WORKING_RECORD.md), all on in config.json: qk_norm, attention_gate,
local_window with global_every, and the hashed n-gram embedding (ngram, ngram_orders, ngram_rows, ngram_dim).
With every option off, the model is the agreed 33.3M-parameter design."""
import math

import torch
import torch.nn as nn
import torch.nn.functional as F

OPTIONS = ("qk_norm", "attention_gate", "local_window", "global_every", "ngram", "ngram_orders", "ngram_rows",
           "ngram_dim", "ngram_heads", "ngram_layers", "key_offset", "document_mask", "global_kv_heads",
           "learn_key_offset")


class RMSNorm(nn.Module):
    def __init__(self, width, eps):
        super().__init__()
        self.eps = eps
        self.weight = nn.Parameter(torch.ones(width))

    def forward(self, x):
        # Statistics in float32 even under autocast.
        h = x.float()
        h = h * torch.rsqrt(h.pow(2).mean(-1, keepdim=True) + self.eps)
        return (h * self.weight).to(x.dtype)


def rotary_tables(length, dim, base):
    inverse = 1.0 / (base ** (torch.arange(0, dim, 2, dtype=torch.float32) / dim))
    angles = torch.outer(torch.arange(length, dtype=torch.float32), inverse)
    return angles.cos(), angles.sin()  # (T, dim/2) each


def rotate(x, cos, sin):
    """RoPE on the first 2 * cos.shape[-1] dimensions of each head (all of them unless rope_dimensions < head_dim);
    the rest stay unrotated. Half-split convention: the first and second halves of the rotated part form the pairs."""
    r = 2 * cos.shape[-1]
    a, b = x[..., :r].float().chunk(2, dim=-1)
    rotated = torch.cat((a * cos - b * sin, a * sin + b * cos), dim=-1).to(x.dtype)
    return rotated if r == x.shape[-1] else torch.cat((rotated, x[..., r:]), dim=-1)


def local_mask(n, window):
    """(n, W, 2W) boolean mask for local_attention: chunk c's queries against chunks c-1 and c."""
    i = torch.arange(window)[:, None]
    j = torch.arange(2 * window)[None, :]
    allowed = (j > i) & (j <= i + window)  # key at chunk offset j - W, query at offset i
    return torch.stack([allowed & (j >= window)] + [allowed] * (n - 1))  # chunk 0 has no previous chunk


def local_attention(q, k, v, window, mask=None):
    """Causal attention over the previous `window` positions (the query's own included), computed per chunk of
    `window` queries against that chunk and the one before it, so scores cost T*2W instead of T*T. `mask`: the
    (n, W, 2W) chunk mask, or a (B, 1, n, W, 2W) one from document_masks."""
    B, H, T, d = q.shape
    pad = (-T) % window
    if pad:
        q, k, v = (F.pad(t, (0, 0, 0, pad)) for t in (q, k, v))
    n = q.shape[2] // window

    def pairs(t):
        t = t.view(B, H, n, window, d)
        previous = torch.cat([torch.zeros_like(t[:, :, :1]), t[:, :, :-1]], 2)
        return torch.cat([previous, t], 3)  # (B, H, n, 2W, d)

    mask = local_mask(n, window).to(q.device) if mask is None else mask[..., :n, :, :]
    y = F.scaled_dot_product_attention(q.view(B, H, n, window, d), pairs(k), pairs(v), attn_mask=mask)
    return y.reshape(B, H, n * window, d)[:, :, :T]


def document_positions(tokens):
    """Each token's position within its document: documents start at EOS (token 0, the tokenizer's eos_id), as
    validation windows do; a window that starts inside a document counts from the window's start."""
    index = torch.arange(tokens.shape[1], device=tokens.device)
    return index - torch.cummax(torch.where(tokens == 0, index, 0), 1).values


def document_masks(position, window, chunks):
    """Masks that keep attention inside each document, from document_positions, computed once per forward pass:
    "global" (B, 1, T, T) for keys from the query's document start to the query, "local" (B, 1, n, W, 2W) for
    local_attention's chunk pairs (`chunks`: its (n, W, 2W) chunk mask), and "continues" (B, T), false at a
    document's first position (its key offset takes nothing from before it)."""
    B, T = position.shape
    index = torch.arange(T, device=position.device)
    start = index - position  # each query's document start
    masks = {"global": ((index >= start[:, :, None]) & (index <= index[:, None]))[:, None], "continues": position > 0}
    if window:
        n = -(-T // window)
        query_start = F.pad(start, (0, n * window - T)).view(B, n, window, 1)  # padded queries are discarded
        key = torch.arange(n, device=position.device)[:, None] * window + torch.arange(-window, window,
                                                                                         device=position.device)
        masks["local"] = (chunks[:n] & (key[:, None, :] >= query_start))[:, None]
    return masks


class Attention(nn.Module):
    def __init__(self, cfg, window):
        super().__init__()
        width, heads = cfg["width"], cfg["query_heads"]
        self.heads, self.window = heads, window
        global_kv = cfg.get("global_kv_heads")
        self.kv_heads = heads if window or global_kv is None else global_kv
        if type(self.kv_heads) is not int or self.kv_heads < 1 or heads % self.kv_heads:
            raise ValueError("global_kv_heads must be a positive divisor of query_heads")
        kv_width = self.kv_heads * cfg["head_dim"]
        # Partial key offset (modded-nanogpt PR 169): in global layers, each key's unrotated dimensions come from
        # the previous token, so one head can match "the token after the previous occurrence" (induction).
        self.key_offset = cfg.get("key_offset") and not window
        self.key_offset_mix = (nn.Parameter(torch.zeros(self.kv_heads))
                               if self.key_offset and cfg.get("learn_key_offset") else None)
        self.rope = cfg["rope_dimensions"]
        self.q = nn.Linear(width, width, bias=False)
        self.k = nn.Linear(width, kv_width, bias=False)
        self.v = nn.Linear(width, kv_width, bias=False)
        self.o = nn.Linear(width, width, bias=False)
        self.head_gate = nn.Linear(width, heads, bias=False) if cfg.get("attention_gate") else None
        qk = cfg.get("qk_norm")
        self.q_norm = RMSNorm(cfg["head_dim"], cfg["rmsnorm_epsilon"]) if qk else None
        self.k_norm = RMSNorm(cfg["head_dim"], cfg["rmsnorm_epsilon"]) if qk else None
        if window:  # built once for the longest window count. Shorter inputs use its first chunks.
            self.register_buffer("mask", local_mask(-(-cfg["context_tokens"] // window), window), persistent=False)

    def forward(self, x, cos, sin, masks=None):
        B, T, D = x.shape
        head_dim = D // self.heads
        # One matrix multiply for Q, K and V. They stay separate parameters with separate Muon updates.
        qkv = F.linear(x, torch.cat([self.q.weight, self.k.weight, self.v.weight]))
        q, k, v = qkv.split((D, self.k.out_features, self.v.out_features), dim=-1)
        q = q.reshape(B, T, self.heads, head_dim).transpose(1, 2)
        k = k.reshape(B, T, self.kv_heads, head_dim).transpose(1, 2)
        v = v.reshape(B, T, self.kv_heads, head_dim).transpose(1, 2)
        if self.q_norm is not None:
            q, k = self.q_norm(q), self.k_norm(k)
        q, k = rotate(q, cos, sin), rotate(k, cos, sin)
        if self.key_offset:
            current = k[..., self.rope:]
            shifted = F.pad(current[..., :-1, :], (0, 0, 1, 0))
            if masks is not None:  # a document's first key gets nothing from the previous document
                shifted = shifted * masks["continues"][:, None, :, None]
            if self.key_offset_mix is not None:
                # Mix per KV head before GQA expansion. Zero recovers the fixed offset; one uses the current key.
                mix = self.key_offset_mix.to(k.dtype)[None, :, None, None]
                shifted = torch.lerp(shifted, current, mix)
            k = torch.cat((k[..., :self.rope], shifted), dim=-1)
        if self.kv_heads != self.heads:
            # Explicit groups use the existing XPU SDPA path for both causal and document masks.
            # A future decoder should cache the compact K/V tensors before this expansion.
            group_size = self.heads // self.kv_heads
            k = k.repeat_interleave(group_size, dim=1)
            v = v.repeat_interleave(group_size, dim=1)
        if self.window:
            y = local_attention(q, k, v, self.window, self.mask if masks is None else masks["local"])
        elif masks is None:
            y = F.scaled_dot_product_attention(q, k, v, is_causal=True)
        else:  # causal within each document: an explicit mask, so not the fused causal path
            y = F.scaled_dot_product_attention(q, k, v, attn_mask=masks["global"])
        if self.head_gate is not None:  # per-head sigmoid gate on the attention output
            y = y * torch.sigmoid(self.head_gate(x)).transpose(1, 2).unsqueeze(-1).to(y.dtype)
        return self.o(y.transpose(1, 2).reshape(B, T, D))


class Block(nn.Module):
    def __init__(self, cfg, window=0):
        super().__init__()
        d, f = cfg["width"], cfg["ffn_intermediate"]
        self.attention_norm = RMSNorm(d, cfg["rmsnorm_epsilon"])
        self.attention = Attention(cfg, window)
        self.ffn_norm = RMSNorm(d, cfg["rmsnorm_epsilon"])
        self.gate = nn.Linear(d, f, bias=False)
        self.up = nn.Linear(d, f, bias=False)
        self.down = nn.Linear(f, d, bias=False)

    def forward(self, x, cos, sin, masks=None, ngram=None, ngram_weight=None):
        """`ngram`, `ngram_weight`: the per-block n-gram injection into the first ngram_dim residual dimensions,
        done here so it compiles with the block (train.make_loss compiles block by block)."""
        if ngram is not None:
            d = ngram.shape[-1]
            x = torch.cat([x[..., :d] + ngram_weight * ngram, x[..., d:]], -1)
        x = x + self.attention(self.attention_norm(x), cos, sin, masks)
        gate, up = F.linear(self.ffn_norm(x), torch.cat([self.gate.weight, self.up.weight])).chunk(2, -1)
        return x + self.down(F.silu(gate) * up)


def earlier(t, j, position=None):
    """t (B, T, ...) moved j positions later: position i holds t[:, i - j], zero (padding) where i - j falls before
    the window or, given document_positions, before the document's start, as when the document is alone."""
    shifted = F.pad(t, (0, 0) * (t.dim() - 2) + (j, 0))[:, :t.shape[1]]
    if position is None or not j:
        return shifted
    return shifted * (position >= j).view(*position.shape, *(1,) * (t.dim() - 2))


class NgramEmbedding(nn.Module):
    """Hashed n-gram embedding as in the modded-nanogpt speedrun (records 62 and 83). Each position reads a
    zero-initialized row for the bigram (and optionally trigram) ending there, times a fixed random +-1 sign
    pattern chosen by an independent hash, so n-grams sharing a row separate. By default the orders' rows are
    summed and the model adds the result to the first `ngram_dim` residual dimensions before every block with a
    learned per-block weight. With `ngram_layers` (Engram, Cheng et al. 2026), each order is read through
    `ngram_heads` independent hashes, the rows are concatenated, and NgramGate adds them at those layers."""

    ROW_MULS = {2: (36313, 27191), 3: (17351, 60961, 45259)}  # the speedrun's hash constants
    SIGN_MULS = {2: (48271, 30011), 3: (58699, 39779, 26801)}
    SIGN_ROWS = 8192

    def __init__(self, cfg):
        super().__init__()
        self.rows, self.dim = cfg["ngram_rows"], cfg["ngram_dim"]
        self.concatenate = bool(cfg.get("ngram_layers"))
        g = torch.Generator().manual_seed(1234)
        self.register_buffer("signs", torch.randint(0, 2, (self.SIGN_ROWS, self.dim), generator=g).float() * 2 - 1)
        # One channel per (order, hash head): head 0 uses the speedrun's constants, later heads random odd ones.
        self.channels = []
        for order in cfg["ngram_orders"]:
            for head in range(cfg.get("ngram_heads") or 1):
                muls = (self.ROW_MULS[order], self.SIGN_MULS[order]) if head == 0 else \
                    (torch.randint(0, 32768, (2, order), generator=g) * 2 + 1).tolist()
                self.channels.append((order, *muls))
        self.table = nn.Embedding(self.rows * len(self.channels), self.dim)
        self.weights = None if self.concatenate else nn.Parameter(torch.full((cfg["blocks"],), 0.1))

    def forward(self, tokens, position=None):
        out = []
        for channel, (order, row_muls, sign_muls) in enumerate(self.channels):
            rows = torch.zeros_like(tokens)
            signs = torch.zeros_like(tokens)
            for j, (rm, sm) in enumerate(zip(row_muls, sign_muls)):
                shifted = earlier(tokens, j, position)  # the token j positions back
                rows = rows ^ (rm * shifted)
                signs = signs ^ (sm * shifted)
            rows = rows % (self.rows - 1) + channel * self.rows
            rows[:, :order - 1] = (channel + 1) * self.rows - 1  # reserved row: too little history in the window
            if position is not None:  # or in the document
                rows = torch.where(position < order - 1, (channel + 1) * self.rows - 1, rows)
            out.append(self.table(rows) * self.signs[signs % self.SIGN_ROWS])
        return torch.cat(out, -1) if self.concatenate else sum(out)


class NgramGate(nn.Module):
    """Engram's module (Cheng et al. 2026, Eq. 3-5): from the n-gram rows e, a key and a value; a per-token gate
    alpha = sigmoid(RMSNorm(h) . RMSNorm(key) / sqrt(d)) from the hidden state h; the output alpha * value plus a
    zero-initialized causal depthwise convolution (kernel 4, dilation `dilation`) of its SiLU-activated norm,
    added to the residual stream. The convolution reads no earlier document under document_mask."""

    def __init__(self, memory, width, eps, dilation, kernel=4):
        super().__init__()
        self.key = nn.Linear(memory, width, bias=False)
        self.value = nn.Linear(memory, width, bias=False)
        self.hidden_norm = RMSNorm(width, eps)
        self.key_norm = RMSNorm(width, eps)
        self.value_norm = RMSNorm(width, eps)
        self.conv = nn.Parameter(torch.zeros(kernel, width))
        self.dilation = dilation

    def forward(self, h, e, position=None):
        k = self.key_norm(self.key(e))
        alpha = torch.sigmoid((self.hidden_norm(h) * k).sum(-1, keepdim=True) / math.sqrt(h.shape[-1]))
        v = alpha * self.value(e)
        u = self.value_norm(v)
        conv = sum(w * earlier(u, j * self.dilation, position) for j, w in enumerate(self.conv))
        return h + (F.silu(conv) + v).to(h.dtype)


class Transformer(nn.Module):
    def __init__(self, cfg, vocab_size):
        super().__init__()
        assert cfg["query_heads"] == cfg["kv_heads"] and cfg["width"] == cfg["query_heads"] * cfg["head_dim"]
        assert cfg["rope_dimensions"] <= cfg["head_dim"] and cfg["rope_dimensions"] % 2 == 0
        assert cfg["tie_embeddings"] and not cfg["linear_bias"]
        assert not cfg.get("key_offset") or cfg["rope_dimensions"] < cfg["head_dim"], \
            "key_offset (global layers only) shifts the unrotated dimensions: set rope_dimensions below head_dim"
        self.cfg = cfg
        self.embedding = nn.Embedding(vocab_size, cfg["width"])
        # local_window > 0: every global_every-th layer stays global, the others are local (Gemma-style).
        window = cfg.get("local_window", 0)
        L = cfg["blocks"]
        if cfg.get("learn_key_offset") and (not cfg.get("key_offset") or (window and L < cfg["global_every"])):
            raise ValueError("learn_key_offset requires key_offset and at least one global block")
        self.blocks = nn.ModuleList(Block(cfg, window if window and (i + 1) % cfg["global_every"] else 0)
                                    for i in range(L))
        self.final_norm = RMSNorm(cfg["width"], cfg["rmsnorm_epsilon"])
        self.ngram = NgramEmbedding(cfg) if cfg.get("ngram") else None
        # ngram_layers: the gated n-gram module before each of these blocks, instead of the per-block injection
        layers = cfg.get("ngram_layers") or []
        assert not layers or (self.ngram is not None and all(0 <= i < L for i in layers)), "ngram_layers"
        self.ngram_gates = nn.ModuleDict({str(i): NgramGate(
            self.ngram.dim * len(self.ngram.channels), cfg["width"], cfg["rmsnorm_epsilon"],
            max(cfg["ngram_orders"])) for i in layers})
        cos, sin = rotary_tables(cfg["context_tokens"], cfg["rope_dimensions"], cfg["rope_base"])
        self.register_buffer("cos", cos, persistent=False)
        self.register_buffer("sin", sin, persistent=False)
        self.reset_parameters()

    def reset_parameters(self):
        std = self.cfg["initialization_std"]
        residual = std / math.sqrt(2 * self.cfg["blocks"])
        for name, p in self.named_parameters():
            if name.endswith("norm.weight"):
                nn.init.ones_(p)
            elif name == "ngram.table.weight" and self.ngram_gates:
                # gated: random rows, as in Engram (zero rows would leave the key's RMSNorm amplifying tiny updates)
                nn.init.normal_(p, std=std)
            elif name.endswith(("head_gate.weight", "ngram.table.weight", ".conv", ".key_offset_mix")):
                nn.init.zeros_(p)
            elif name == "ngram.weights":
                continue  # set at construction
            elif name.endswith(("attention.o.weight", "down.weight")):
                nn.init.normal_(p, std=residual)
            else:
                nn.init.normal_(p, std=std)

    def head(self, x):
        return F.linear(self.final_norm(x), self.embedding.weight)

    def prelude(self, tokens):
        """The token embeddings, the document masks and positions, and the n-gram rows: everything before the
        first block (train.make_loss compiles it on its own)."""
        x = self.embedding(tokens)
        # document_mask: each document of a packed window is processed as if alone (attention, the key offset and
        # the n-grams stay inside it).
        position = document_positions(tokens) if self.cfg.get("document_mask") else None
        masks = None if position is None else document_masks(
            position, self.cfg.get("local_window", 0), next((b.attention.mask for b in self.blocks
                                                             if b.attention.window), None))
        ngram = self.ngram(tokens, position) if self.ngram is not None else None
        return x, masks, ngram, position

    def hidden(self, tokens, blocks=None, prelude=None):
        """The last block's output at every position (head() turns it into logits). `blocks` and `prelude`:
        callables run in place of self.blocks and self.prelude (train.make_loss passes compiled ones; the model
        itself stays uncompiled, so generation does not recompile at every length)."""
        assert blocks is None or len(blocks) == len(self.blocks), "one callable per block, in order"
        T = tokens.shape[1]
        cos, sin = self.cos[:T], self.sin[:T]  # positions restart at zero in every window
        x, masks, ngram, position = (prelude or self.prelude)(tokens)
        # One unbind, not one index per block: each index adds its own small backward kernels. The 0-dimensional
        # weights are inputs, not Python numbers, so every block shares one compiled graph.
        weights = self.ngram.weights.unbind(0) if ngram is not None and self.ngram.weights is not None else None
        for i, block in enumerate(blocks or self.blocks):
            if str(i) in self.ngram_gates:
                x = self.ngram_gates[str(i)](x, ngram, position)
            x = block(x, cos, sin, masks, *((ngram, weights[i]) if weights else ()))
        return x

    def forward(self, tokens, last_only=False):
        """Logits for every position, or only the last one (generation skips the rest of the output head)."""
        x = self.hidden(tokens)
        return self.head(x[:, -1:] if last_only else x)


def parameter_count(cfg, vocab_size):
    """SPEC formula for the model with every option off: V*d + L*(4*d*d + 3*d*f + 2*d) + d."""
    d, f, L = cfg["width"], cfg["ffn_intermediate"], cfg["blocks"]
    return vocab_size * d + L * (4 * d * d + 3 * d * f + 2 * d) + d