"""Decoder-only Transformer from SPEC section 4: pre-RMSNorm, RoPE, SwiGLU, tied embeddings, no biases. Options from the 600-update comparisons (see WORKING_RECORD.md), all on in config.json: qk_norm, attention_gate, local_window with global_every, and the hashed n-gram embedding (ngram, ngram_orders, ngram_rows, ngram_dim). With every option off, the model is the agreed 33.3M-parameter design.""" import math import torch import torch.nn as nn import torch.nn.functional as F OPTIONS = ("qk_norm", "attention_gate", "local_window", "global_every", "ngram", "ngram_orders", "ngram_rows", "ngram_dim", "ngram_heads", "ngram_layers", "key_offset", "document_mask", "global_kv_heads", "learn_key_offset") class RMSNorm(nn.Module): def __init__(self, width, eps): super().__init__() self.eps = eps self.weight = nn.Parameter(torch.ones(width)) def forward(self, x): # Statistics in float32 even under autocast. h = x.float() h = h * torch.rsqrt(h.pow(2).mean(-1, keepdim=True) + self.eps) return (h * self.weight).to(x.dtype) def rotary_tables(length, dim, base): inverse = 1.0 / (base ** (torch.arange(0, dim, 2, dtype=torch.float32) / dim)) angles = torch.outer(torch.arange(length, dtype=torch.float32), inverse) return angles.cos(), angles.sin() # (T, dim/2) each def rotate(x, cos, sin): """RoPE on the first 2 * cos.shape[-1] dimensions of each head (all of them unless rope_dimensions < head_dim); the rest stay unrotated. Half-split convention: the first and second halves of the rotated part form the pairs.""" r = 2 * cos.shape[-1] a, b = x[..., :r].float().chunk(2, dim=-1) rotated = torch.cat((a * cos - b * sin, a * sin + b * cos), dim=-1).to(x.dtype) return rotated if r == x.shape[-1] else torch.cat((rotated, x[..., r:]), dim=-1) def local_mask(n, window): """(n, W, 2W) boolean mask for local_attention: chunk c's queries against chunks c-1 and c.""" i = torch.arange(window)[:, None] j = torch.arange(2 * window)[None, :] allowed = (j > i) & (j <= i + window) # key at chunk offset j - W, query at offset i return torch.stack([allowed & (j >= window)] + [allowed] * (n - 1)) # chunk 0 has no previous chunk def local_attention(q, k, v, window, mask=None): """Causal attention over the previous `window` positions (the query's own included), computed per chunk of `window` queries against that chunk and the one before it, so scores cost T*2W instead of T*T. `mask`: the (n, W, 2W) chunk mask, or a (B, 1, n, W, 2W) one from document_masks.""" B, H, T, d = q.shape pad = (-T) % window if pad: q, k, v = (F.pad(t, (0, 0, 0, pad)) for t in (q, k, v)) n = q.shape[2] // window def pairs(t): t = t.view(B, H, n, window, d) previous = torch.cat([torch.zeros_like(t[:, :, :1]), t[:, :, :-1]], 2) return torch.cat([previous, t], 3) # (B, H, n, 2W, d) mask = local_mask(n, window).to(q.device) if mask is None else mask[..., :n, :, :] y = F.scaled_dot_product_attention(q.view(B, H, n, window, d), pairs(k), pairs(v), attn_mask=mask) return y.reshape(B, H, n * window, d)[:, :, :T] def document_positions(tokens): """Each token's position within its document: documents start at EOS (token 0, the tokenizer's eos_id), as validation windows do; a window that starts inside a document counts from the window's start.""" index = torch.arange(tokens.shape[1], device=tokens.device) return index - torch.cummax(torch.where(tokens == 0, index, 0), 1).values def document_masks(position, window, chunks): """Masks that keep attention inside each document, from document_positions, computed once per forward pass: "global" (B, 1, T, T) for keys from the query's document start to the query, "local" (B, 1, n, W, 2W) for local_attention's chunk pairs (`chunks`: its (n, W, 2W) chunk mask), and "continues" (B, T), false at a document's first position (its key offset takes nothing from before it).""" B, T = position.shape index = torch.arange(T, device=position.device) start = index - position # each query's document start masks = {"global": ((index >= start[:, :, None]) & (index <= index[:, None]))[:, None], "continues": position > 0} if window: n = -(-T // window) query_start = F.pad(start, (0, n * window - T)).view(B, n, window, 1) # padded queries are discarded key = torch.arange(n, device=position.device)[:, None] * window + torch.arange(-window, window, device=position.device) masks["local"] = (chunks[:n] & (key[:, None, :] >= query_start))[:, None] return masks class Attention(nn.Module): def __init__(self, cfg, window): super().__init__() width, heads = cfg["width"], cfg["query_heads"] self.heads, self.window = heads, window global_kv = cfg.get("global_kv_heads") self.kv_heads = heads if window or global_kv is None else global_kv if type(self.kv_heads) is not int or self.kv_heads < 1 or heads % self.kv_heads: raise ValueError("global_kv_heads must be a positive divisor of query_heads") kv_width = self.kv_heads * cfg["head_dim"] # Partial key offset (modded-nanogpt PR 169): in global layers, each key's unrotated dimensions come from # the previous token, so one head can match "the token after the previous occurrence" (induction). self.key_offset = cfg.get("key_offset") and not window self.key_offset_mix = (nn.Parameter(torch.zeros(self.kv_heads)) if self.key_offset and cfg.get("learn_key_offset") else None) self.rope = cfg["rope_dimensions"] self.q = nn.Linear(width, width, bias=False) self.k = nn.Linear(width, kv_width, bias=False) self.v = nn.Linear(width, kv_width, bias=False) self.o = nn.Linear(width, width, bias=False) self.head_gate = nn.Linear(width, heads, bias=False) if cfg.get("attention_gate") else None qk = cfg.get("qk_norm") self.q_norm = RMSNorm(cfg["head_dim"], cfg["rmsnorm_epsilon"]) if qk else None self.k_norm = RMSNorm(cfg["head_dim"], cfg["rmsnorm_epsilon"]) if qk else None if window: # built once for the longest window count. Shorter inputs use its first chunks. self.register_buffer("mask", local_mask(-(-cfg["context_tokens"] // window), window), persistent=False) def forward(self, x, cos, sin, masks=None): B, T, D = x.shape head_dim = D // self.heads # One matrix multiply for Q, K and V. They stay separate parameters with separate Muon updates. qkv = F.linear(x, torch.cat([self.q.weight, self.k.weight, self.v.weight])) q, k, v = qkv.split((D, self.k.out_features, self.v.out_features), dim=-1) q = q.reshape(B, T, self.heads, head_dim).transpose(1, 2) k = k.reshape(B, T, self.kv_heads, head_dim).transpose(1, 2) v = v.reshape(B, T, self.kv_heads, head_dim).transpose(1, 2) if self.q_norm is not None: q, k = self.q_norm(q), self.k_norm(k) q, k = rotate(q, cos, sin), rotate(k, cos, sin) if self.key_offset: current = k[..., self.rope:] shifted = F.pad(current[..., :-1, :], (0, 0, 1, 0)) if masks is not None: # a document's first key gets nothing from the previous document shifted = shifted * masks["continues"][:, None, :, None] if self.key_offset_mix is not None: # Mix per KV head before GQA expansion. Zero recovers the fixed offset; one uses the current key. mix = self.key_offset_mix.to(k.dtype)[None, :, None, None] shifted = torch.lerp(shifted, current, mix) k = torch.cat((k[..., :self.rope], shifted), dim=-1) if self.kv_heads != self.heads: # Explicit groups use the existing XPU SDPA path for both causal and document masks. # A future decoder should cache the compact K/V tensors before this expansion. group_size = self.heads // self.kv_heads k = k.repeat_interleave(group_size, dim=1) v = v.repeat_interleave(group_size, dim=1) if self.window: y = local_attention(q, k, v, self.window, self.mask if masks is None else masks["local"]) elif masks is None: y = F.scaled_dot_product_attention(q, k, v, is_causal=True) else: # causal within each document: an explicit mask, so not the fused causal path y = F.scaled_dot_product_attention(q, k, v, attn_mask=masks["global"]) if self.head_gate is not None: # per-head sigmoid gate on the attention output y = y * torch.sigmoid(self.head_gate(x)).transpose(1, 2).unsqueeze(-1).to(y.dtype) return self.o(y.transpose(1, 2).reshape(B, T, D)) class Block(nn.Module): def __init__(self, cfg, window=0): super().__init__() d, f = cfg["width"], cfg["ffn_intermediate"] self.attention_norm = RMSNorm(d, cfg["rmsnorm_epsilon"]) self.attention = Attention(cfg, window) self.ffn_norm = RMSNorm(d, cfg["rmsnorm_epsilon"]) self.gate = nn.Linear(d, f, bias=False) self.up = nn.Linear(d, f, bias=False) self.down = nn.Linear(f, d, bias=False) def forward(self, x, cos, sin, masks=None, ngram=None, ngram_weight=None): """`ngram`, `ngram_weight`: the per-block n-gram injection into the first ngram_dim residual dimensions, done here so it compiles with the block (train.make_loss compiles block by block).""" if ngram is not None: d = ngram.shape[-1] x = torch.cat([x[..., :d] + ngram_weight * ngram, x[..., d:]], -1) x = x + self.attention(self.attention_norm(x), cos, sin, masks) gate, up = F.linear(self.ffn_norm(x), torch.cat([self.gate.weight, self.up.weight])).chunk(2, -1) return x + self.down(F.silu(gate) * up) def earlier(t, j, position=None): """t (B, T, ...) moved j positions later: position i holds t[:, i - j], zero (padding) where i - j falls before the window or, given document_positions, before the document's start, as when the document is alone.""" shifted = F.pad(t, (0, 0) * (t.dim() - 2) + (j, 0))[:, :t.shape[1]] if position is None or not j: return shifted return shifted * (position >= j).view(*position.shape, *(1,) * (t.dim() - 2)) class NgramEmbedding(nn.Module): """Hashed n-gram embedding as in the modded-nanogpt speedrun (records 62 and 83). Each position reads a zero-initialized row for the bigram (and optionally trigram) ending there, times a fixed random +-1 sign pattern chosen by an independent hash, so n-grams sharing a row separate. By default the orders' rows are summed and the model adds the result to the first `ngram_dim` residual dimensions before every block with a learned per-block weight. With `ngram_layers` (Engram, Cheng et al. 2026), each order is read through `ngram_heads` independent hashes, the rows are concatenated, and NgramGate adds them at those layers.""" ROW_MULS = {2: (36313, 27191), 3: (17351, 60961, 45259)} # the speedrun's hash constants SIGN_MULS = {2: (48271, 30011), 3: (58699, 39779, 26801)} SIGN_ROWS = 8192 def __init__(self, cfg): super().__init__() self.rows, self.dim = cfg["ngram_rows"], cfg["ngram_dim"] self.concatenate = bool(cfg.get("ngram_layers")) g = torch.Generator().manual_seed(1234) self.register_buffer("signs", torch.randint(0, 2, (self.SIGN_ROWS, self.dim), generator=g).float() * 2 - 1) # One channel per (order, hash head): head 0 uses the speedrun's constants, later heads random odd ones. self.channels = [] for order in cfg["ngram_orders"]: for head in range(cfg.get("ngram_heads") or 1): muls = (self.ROW_MULS[order], self.SIGN_MULS[order]) if head == 0 else \ (torch.randint(0, 32768, (2, order), generator=g) * 2 + 1).tolist() self.channels.append((order, *muls)) self.table = nn.Embedding(self.rows * len(self.channels), self.dim) self.weights = None if self.concatenate else nn.Parameter(torch.full((cfg["blocks"],), 0.1)) def forward(self, tokens, position=None): out = [] for channel, (order, row_muls, sign_muls) in enumerate(self.channels): rows = torch.zeros_like(tokens) signs = torch.zeros_like(tokens) for j, (rm, sm) in enumerate(zip(row_muls, sign_muls)): shifted = earlier(tokens, j, position) # the token j positions back rows = rows ^ (rm * shifted) signs = signs ^ (sm * shifted) rows = rows % (self.rows - 1) + channel * self.rows rows[:, :order - 1] = (channel + 1) * self.rows - 1 # reserved row: too little history in the window if position is not None: # or in the document rows = torch.where(position < order - 1, (channel + 1) * self.rows - 1, rows) out.append(self.table(rows) * self.signs[signs % self.SIGN_ROWS]) return torch.cat(out, -1) if self.concatenate else sum(out) class NgramGate(nn.Module): """Engram's module (Cheng et al. 2026, Eq. 3-5): from the n-gram rows e, a key and a value; a per-token gate alpha = sigmoid(RMSNorm(h) . RMSNorm(key) / sqrt(d)) from the hidden state h; the output alpha * value plus a zero-initialized causal depthwise convolution (kernel 4, dilation `dilation`) of its SiLU-activated norm, added to the residual stream. The convolution reads no earlier document under document_mask.""" def __init__(self, memory, width, eps, dilation, kernel=4): super().__init__() self.key = nn.Linear(memory, width, bias=False) self.value = nn.Linear(memory, width, bias=False) self.hidden_norm = RMSNorm(width, eps) self.key_norm = RMSNorm(width, eps) self.value_norm = RMSNorm(width, eps) self.conv = nn.Parameter(torch.zeros(kernel, width)) self.dilation = dilation def forward(self, h, e, position=None): k = self.key_norm(self.key(e)) alpha = torch.sigmoid((self.hidden_norm(h) * k).sum(-1, keepdim=True) / math.sqrt(h.shape[-1])) v = alpha * self.value(e) u = self.value_norm(v) conv = sum(w * earlier(u, j * self.dilation, position) for j, w in enumerate(self.conv)) return h + (F.silu(conv) + v).to(h.dtype) class Transformer(nn.Module): def __init__(self, cfg, vocab_size): super().__init__() assert cfg["query_heads"] == cfg["kv_heads"] and cfg["width"] == cfg["query_heads"] * cfg["head_dim"] assert cfg["rope_dimensions"] <= cfg["head_dim"] and cfg["rope_dimensions"] % 2 == 0 assert cfg["tie_embeddings"] and not cfg["linear_bias"] assert not cfg.get("key_offset") or cfg["rope_dimensions"] < cfg["head_dim"], \ "key_offset (global layers only) shifts the unrotated dimensions: set rope_dimensions below head_dim" self.cfg = cfg self.embedding = nn.Embedding(vocab_size, cfg["width"]) # local_window > 0: every global_every-th layer stays global, the others are local (Gemma-style). window = cfg.get("local_window", 0) L = cfg["blocks"] if cfg.get("learn_key_offset") and (not cfg.get("key_offset") or (window and L < cfg["global_every"])): raise ValueError("learn_key_offset requires key_offset and at least one global block") self.blocks = nn.ModuleList(Block(cfg, window if window and (i + 1) % cfg["global_every"] else 0) for i in range(L)) self.final_norm = RMSNorm(cfg["width"], cfg["rmsnorm_epsilon"]) self.ngram = NgramEmbedding(cfg) if cfg.get("ngram") else None # ngram_layers: the gated n-gram module before each of these blocks, instead of the per-block injection layers = cfg.get("ngram_layers") or [] assert not layers or (self.ngram is not None and all(0 <= i < L for i in layers)), "ngram_layers" self.ngram_gates = nn.ModuleDict({str(i): NgramGate( self.ngram.dim * len(self.ngram.channels), cfg["width"], cfg["rmsnorm_epsilon"], max(cfg["ngram_orders"])) for i in layers}) cos, sin = rotary_tables(cfg["context_tokens"], cfg["rope_dimensions"], cfg["rope_base"]) self.register_buffer("cos", cos, persistent=False) self.register_buffer("sin", sin, persistent=False) self.reset_parameters() def reset_parameters(self): std = self.cfg["initialization_std"] residual = std / math.sqrt(2 * self.cfg["blocks"]) for name, p in self.named_parameters(): if name.endswith("norm.weight"): nn.init.ones_(p) elif name == "ngram.table.weight" and self.ngram_gates: # gated: random rows, as in Engram (zero rows would leave the key's RMSNorm amplifying tiny updates) nn.init.normal_(p, std=std) elif name.endswith(("head_gate.weight", "ngram.table.weight", ".conv", ".key_offset_mix")): nn.init.zeros_(p) elif name == "ngram.weights": continue # set at construction elif name.endswith(("attention.o.weight", "down.weight")): nn.init.normal_(p, std=residual) else: nn.init.normal_(p, std=std) def head(self, x): return F.linear(self.final_norm(x), self.embedding.weight) def prelude(self, tokens): """The token embeddings, the document masks and positions, and the n-gram rows: everything before the first block (train.make_loss compiles it on its own).""" x = self.embedding(tokens) # document_mask: each document of a packed window is processed as if alone (attention, the key offset and # the n-grams stay inside it). position = document_positions(tokens) if self.cfg.get("document_mask") else None masks = None if position is None else document_masks( position, self.cfg.get("local_window", 0), next((b.attention.mask for b in self.blocks if b.attention.window), None)) ngram = self.ngram(tokens, position) if self.ngram is not None else None return x, masks, ngram, position def hidden(self, tokens, blocks=None, prelude=None): """The last block's output at every position (head() turns it into logits). `blocks` and `prelude`: callables run in place of self.blocks and self.prelude (train.make_loss passes compiled ones; the model itself stays uncompiled, so generation does not recompile at every length).""" assert blocks is None or len(blocks) == len(self.blocks), "one callable per block, in order" T = tokens.shape[1] cos, sin = self.cos[:T], self.sin[:T] # positions restart at zero in every window x, masks, ngram, position = (prelude or self.prelude)(tokens) # One unbind, not one index per block: each index adds its own small backward kernels. The 0-dimensional # weights are inputs, not Python numbers, so every block shares one compiled graph. weights = self.ngram.weights.unbind(0) if ngram is not None and self.ngram.weights is not None else None for i, block in enumerate(blocks or self.blocks): if str(i) in self.ngram_gates: x = self.ngram_gates[str(i)](x, ngram, position) x = block(x, cos, sin, masks, *((ngram, weights[i]) if weights else ())) return x def forward(self, tokens, last_only=False): """Logits for every position, or only the last one (generation skips the rest of the output head).""" x = self.hidden(tokens) return self.head(x[:, -1:] if last_only else x) def parameter_count(cfg, vocab_size): """SPEC formula for the model with every option off: V*d + L*(4*d*d + 3*d*f + 2*d) + d.""" d, f, L = cfg["width"], cfg["ffn_intermediate"], cfg["blocks"] return vocab_size * d + L * (4 * d * d + 3 * d * f + 2 * d) + d