File size: 20,394 Bytes
b9ee6b4 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 | """Decoder-only Transformer from SPEC section 4: pre-RMSNorm, RoPE, SwiGLU, tied embeddings, no biases.
Options from the 600-update comparisons (see WORKING_RECORD.md), all on in config.json: qk_norm, attention_gate,
local_window with global_every, and the hashed n-gram embedding (ngram, ngram_orders, ngram_rows, ngram_dim).
With every option off, the model is the agreed 33.3M-parameter design."""
import math
import torch
import torch.nn as nn
import torch.nn.functional as F
OPTIONS = ("qk_norm", "attention_gate", "local_window", "global_every", "ngram", "ngram_orders", "ngram_rows",
"ngram_dim", "ngram_heads", "ngram_layers", "key_offset", "document_mask", "global_kv_heads",
"learn_key_offset")
class RMSNorm(nn.Module):
def __init__(self, width, eps):
super().__init__()
self.eps = eps
self.weight = nn.Parameter(torch.ones(width))
def forward(self, x):
# Statistics in float32 even under autocast.
h = x.float()
h = h * torch.rsqrt(h.pow(2).mean(-1, keepdim=True) + self.eps)
return (h * self.weight).to(x.dtype)
def rotary_tables(length, dim, base):
inverse = 1.0 / (base ** (torch.arange(0, dim, 2, dtype=torch.float32) / dim))
angles = torch.outer(torch.arange(length, dtype=torch.float32), inverse)
return angles.cos(), angles.sin() # (T, dim/2) each
def rotate(x, cos, sin):
"""RoPE on the first 2 * cos.shape[-1] dimensions of each head (all of them unless rope_dimensions < head_dim);
the rest stay unrotated. Half-split convention: the first and second halves of the rotated part form the pairs."""
r = 2 * cos.shape[-1]
a, b = x[..., :r].float().chunk(2, dim=-1)
rotated = torch.cat((a * cos - b * sin, a * sin + b * cos), dim=-1).to(x.dtype)
return rotated if r == x.shape[-1] else torch.cat((rotated, x[..., r:]), dim=-1)
def local_mask(n, window):
"""(n, W, 2W) boolean mask for local_attention: chunk c's queries against chunks c-1 and c."""
i = torch.arange(window)[:, None]
j = torch.arange(2 * window)[None, :]
allowed = (j > i) & (j <= i + window) # key at chunk offset j - W, query at offset i
return torch.stack([allowed & (j >= window)] + [allowed] * (n - 1)) # chunk 0 has no previous chunk
def local_attention(q, k, v, window, mask=None):
"""Causal attention over the previous `window` positions (the query's own included), computed per chunk of
`window` queries against that chunk and the one before it, so scores cost T*2W instead of T*T. `mask`: the
(n, W, 2W) chunk mask, or a (B, 1, n, W, 2W) one from document_masks."""
B, H, T, d = q.shape
pad = (-T) % window
if pad:
q, k, v = (F.pad(t, (0, 0, 0, pad)) for t in (q, k, v))
n = q.shape[2] // window
def pairs(t):
t = t.view(B, H, n, window, d)
previous = torch.cat([torch.zeros_like(t[:, :, :1]), t[:, :, :-1]], 2)
return torch.cat([previous, t], 3) # (B, H, n, 2W, d)
mask = local_mask(n, window).to(q.device) if mask is None else mask[..., :n, :, :]
y = F.scaled_dot_product_attention(q.view(B, H, n, window, d), pairs(k), pairs(v), attn_mask=mask)
return y.reshape(B, H, n * window, d)[:, :, :T]
def document_positions(tokens):
"""Each token's position within its document: documents start at EOS (token 0, the tokenizer's eos_id), as
validation windows do; a window that starts inside a document counts from the window's start."""
index = torch.arange(tokens.shape[1], device=tokens.device)
return index - torch.cummax(torch.where(tokens == 0, index, 0), 1).values
def document_masks(position, window, chunks):
"""Masks that keep attention inside each document, from document_positions, computed once per forward pass:
"global" (B, 1, T, T) for keys from the query's document start to the query, "local" (B, 1, n, W, 2W) for
local_attention's chunk pairs (`chunks`: its (n, W, 2W) chunk mask), and "continues" (B, T), false at a
document's first position (its key offset takes nothing from before it)."""
B, T = position.shape
index = torch.arange(T, device=position.device)
start = index - position # each query's document start
masks = {"global": ((index >= start[:, :, None]) & (index <= index[:, None]))[:, None], "continues": position > 0}
if window:
n = -(-T // window)
query_start = F.pad(start, (0, n * window - T)).view(B, n, window, 1) # padded queries are discarded
key = torch.arange(n, device=position.device)[:, None] * window + torch.arange(-window, window,
device=position.device)
masks["local"] = (chunks[:n] & (key[:, None, :] >= query_start))[:, None]
return masks
class Attention(nn.Module):
def __init__(self, cfg, window):
super().__init__()
width, heads = cfg["width"], cfg["query_heads"]
self.heads, self.window = heads, window
global_kv = cfg.get("global_kv_heads")
self.kv_heads = heads if window or global_kv is None else global_kv
if type(self.kv_heads) is not int or self.kv_heads < 1 or heads % self.kv_heads:
raise ValueError("global_kv_heads must be a positive divisor of query_heads")
kv_width = self.kv_heads * cfg["head_dim"]
# Partial key offset (modded-nanogpt PR 169): in global layers, each key's unrotated dimensions come from
# the previous token, so one head can match "the token after the previous occurrence" (induction).
self.key_offset = cfg.get("key_offset") and not window
self.key_offset_mix = (nn.Parameter(torch.zeros(self.kv_heads))
if self.key_offset and cfg.get("learn_key_offset") else None)
self.rope = cfg["rope_dimensions"]
self.q = nn.Linear(width, width, bias=False)
self.k = nn.Linear(width, kv_width, bias=False)
self.v = nn.Linear(width, kv_width, bias=False)
self.o = nn.Linear(width, width, bias=False)
self.head_gate = nn.Linear(width, heads, bias=False) if cfg.get("attention_gate") else None
qk = cfg.get("qk_norm")
self.q_norm = RMSNorm(cfg["head_dim"], cfg["rmsnorm_epsilon"]) if qk else None
self.k_norm = RMSNorm(cfg["head_dim"], cfg["rmsnorm_epsilon"]) if qk else None
if window: # built once for the longest window count. Shorter inputs use its first chunks.
self.register_buffer("mask", local_mask(-(-cfg["context_tokens"] // window), window), persistent=False)
def forward(self, x, cos, sin, masks=None):
B, T, D = x.shape
head_dim = D // self.heads
# One matrix multiply for Q, K and V. They stay separate parameters with separate Muon updates.
qkv = F.linear(x, torch.cat([self.q.weight, self.k.weight, self.v.weight]))
q, k, v = qkv.split((D, self.k.out_features, self.v.out_features), dim=-1)
q = q.reshape(B, T, self.heads, head_dim).transpose(1, 2)
k = k.reshape(B, T, self.kv_heads, head_dim).transpose(1, 2)
v = v.reshape(B, T, self.kv_heads, head_dim).transpose(1, 2)
if self.q_norm is not None:
q, k = self.q_norm(q), self.k_norm(k)
q, k = rotate(q, cos, sin), rotate(k, cos, sin)
if self.key_offset:
current = k[..., self.rope:]
shifted = F.pad(current[..., :-1, :], (0, 0, 1, 0))
if masks is not None: # a document's first key gets nothing from the previous document
shifted = shifted * masks["continues"][:, None, :, None]
if self.key_offset_mix is not None:
# Mix per KV head before GQA expansion. Zero recovers the fixed offset; one uses the current key.
mix = self.key_offset_mix.to(k.dtype)[None, :, None, None]
shifted = torch.lerp(shifted, current, mix)
k = torch.cat((k[..., :self.rope], shifted), dim=-1)
if self.kv_heads != self.heads:
# Explicit groups use the existing XPU SDPA path for both causal and document masks.
# A future decoder should cache the compact K/V tensors before this expansion.
group_size = self.heads // self.kv_heads
k = k.repeat_interleave(group_size, dim=1)
v = v.repeat_interleave(group_size, dim=1)
if self.window:
y = local_attention(q, k, v, self.window, self.mask if masks is None else masks["local"])
elif masks is None:
y = F.scaled_dot_product_attention(q, k, v, is_causal=True)
else: # causal within each document: an explicit mask, so not the fused causal path
y = F.scaled_dot_product_attention(q, k, v, attn_mask=masks["global"])
if self.head_gate is not None: # per-head sigmoid gate on the attention output
y = y * torch.sigmoid(self.head_gate(x)).transpose(1, 2).unsqueeze(-1).to(y.dtype)
return self.o(y.transpose(1, 2).reshape(B, T, D))
class Block(nn.Module):
def __init__(self, cfg, window=0):
super().__init__()
d, f = cfg["width"], cfg["ffn_intermediate"]
self.attention_norm = RMSNorm(d, cfg["rmsnorm_epsilon"])
self.attention = Attention(cfg, window)
self.ffn_norm = RMSNorm(d, cfg["rmsnorm_epsilon"])
self.gate = nn.Linear(d, f, bias=False)
self.up = nn.Linear(d, f, bias=False)
self.down = nn.Linear(f, d, bias=False)
def forward(self, x, cos, sin, masks=None, ngram=None, ngram_weight=None):
"""`ngram`, `ngram_weight`: the per-block n-gram injection into the first ngram_dim residual dimensions,
done here so it compiles with the block (train.make_loss compiles block by block)."""
if ngram is not None:
d = ngram.shape[-1]
x = torch.cat([x[..., :d] + ngram_weight * ngram, x[..., d:]], -1)
x = x + self.attention(self.attention_norm(x), cos, sin, masks)
gate, up = F.linear(self.ffn_norm(x), torch.cat([self.gate.weight, self.up.weight])).chunk(2, -1)
return x + self.down(F.silu(gate) * up)
def earlier(t, j, position=None):
"""t (B, T, ...) moved j positions later: position i holds t[:, i - j], zero (padding) where i - j falls before
the window or, given document_positions, before the document's start, as when the document is alone."""
shifted = F.pad(t, (0, 0) * (t.dim() - 2) + (j, 0))[:, :t.shape[1]]
if position is None or not j:
return shifted
return shifted * (position >= j).view(*position.shape, *(1,) * (t.dim() - 2))
class NgramEmbedding(nn.Module):
"""Hashed n-gram embedding as in the modded-nanogpt speedrun (records 62 and 83). Each position reads a
zero-initialized row for the bigram (and optionally trigram) ending there, times a fixed random +-1 sign
pattern chosen by an independent hash, so n-grams sharing a row separate. By default the orders' rows are
summed and the model adds the result to the first `ngram_dim` residual dimensions before every block with a
learned per-block weight. With `ngram_layers` (Engram, Cheng et al. 2026), each order is read through
`ngram_heads` independent hashes, the rows are concatenated, and NgramGate adds them at those layers."""
ROW_MULS = {2: (36313, 27191), 3: (17351, 60961, 45259)} # the speedrun's hash constants
SIGN_MULS = {2: (48271, 30011), 3: (58699, 39779, 26801)}
SIGN_ROWS = 8192
def __init__(self, cfg):
super().__init__()
self.rows, self.dim = cfg["ngram_rows"], cfg["ngram_dim"]
self.concatenate = bool(cfg.get("ngram_layers"))
g = torch.Generator().manual_seed(1234)
self.register_buffer("signs", torch.randint(0, 2, (self.SIGN_ROWS, self.dim), generator=g).float() * 2 - 1)
# One channel per (order, hash head): head 0 uses the speedrun's constants, later heads random odd ones.
self.channels = []
for order in cfg["ngram_orders"]:
for head in range(cfg.get("ngram_heads") or 1):
muls = (self.ROW_MULS[order], self.SIGN_MULS[order]) if head == 0 else \
(torch.randint(0, 32768, (2, order), generator=g) * 2 + 1).tolist()
self.channels.append((order, *muls))
self.table = nn.Embedding(self.rows * len(self.channels), self.dim)
self.weights = None if self.concatenate else nn.Parameter(torch.full((cfg["blocks"],), 0.1))
def forward(self, tokens, position=None):
out = []
for channel, (order, row_muls, sign_muls) in enumerate(self.channels):
rows = torch.zeros_like(tokens)
signs = torch.zeros_like(tokens)
for j, (rm, sm) in enumerate(zip(row_muls, sign_muls)):
shifted = earlier(tokens, j, position) # the token j positions back
rows = rows ^ (rm * shifted)
signs = signs ^ (sm * shifted)
rows = rows % (self.rows - 1) + channel * self.rows
rows[:, :order - 1] = (channel + 1) * self.rows - 1 # reserved row: too little history in the window
if position is not None: # or in the document
rows = torch.where(position < order - 1, (channel + 1) * self.rows - 1, rows)
out.append(self.table(rows) * self.signs[signs % self.SIGN_ROWS])
return torch.cat(out, -1) if self.concatenate else sum(out)
class NgramGate(nn.Module):
"""Engram's module (Cheng et al. 2026, Eq. 3-5): from the n-gram rows e, a key and a value; a per-token gate
alpha = sigmoid(RMSNorm(h) . RMSNorm(key) / sqrt(d)) from the hidden state h; the output alpha * value plus a
zero-initialized causal depthwise convolution (kernel 4, dilation `dilation`) of its SiLU-activated norm,
added to the residual stream. The convolution reads no earlier document under document_mask."""
def __init__(self, memory, width, eps, dilation, kernel=4):
super().__init__()
self.key = nn.Linear(memory, width, bias=False)
self.value = nn.Linear(memory, width, bias=False)
self.hidden_norm = RMSNorm(width, eps)
self.key_norm = RMSNorm(width, eps)
self.value_norm = RMSNorm(width, eps)
self.conv = nn.Parameter(torch.zeros(kernel, width))
self.dilation = dilation
def forward(self, h, e, position=None):
k = self.key_norm(self.key(e))
alpha = torch.sigmoid((self.hidden_norm(h) * k).sum(-1, keepdim=True) / math.sqrt(h.shape[-1]))
v = alpha * self.value(e)
u = self.value_norm(v)
conv = sum(w * earlier(u, j * self.dilation, position) for j, w in enumerate(self.conv))
return h + (F.silu(conv) + v).to(h.dtype)
class Transformer(nn.Module):
def __init__(self, cfg, vocab_size):
super().__init__()
assert cfg["query_heads"] == cfg["kv_heads"] and cfg["width"] == cfg["query_heads"] * cfg["head_dim"]
assert cfg["rope_dimensions"] <= cfg["head_dim"] and cfg["rope_dimensions"] % 2 == 0
assert cfg["tie_embeddings"] and not cfg["linear_bias"]
assert not cfg.get("key_offset") or cfg["rope_dimensions"] < cfg["head_dim"], \
"key_offset (global layers only) shifts the unrotated dimensions: set rope_dimensions below head_dim"
self.cfg = cfg
self.embedding = nn.Embedding(vocab_size, cfg["width"])
# local_window > 0: every global_every-th layer stays global, the others are local (Gemma-style).
window = cfg.get("local_window", 0)
L = cfg["blocks"]
if cfg.get("learn_key_offset") and (not cfg.get("key_offset") or (window and L < cfg["global_every"])):
raise ValueError("learn_key_offset requires key_offset and at least one global block")
self.blocks = nn.ModuleList(Block(cfg, window if window and (i + 1) % cfg["global_every"] else 0)
for i in range(L))
self.final_norm = RMSNorm(cfg["width"], cfg["rmsnorm_epsilon"])
self.ngram = NgramEmbedding(cfg) if cfg.get("ngram") else None
# ngram_layers: the gated n-gram module before each of these blocks, instead of the per-block injection
layers = cfg.get("ngram_layers") or []
assert not layers or (self.ngram is not None and all(0 <= i < L for i in layers)), "ngram_layers"
self.ngram_gates = nn.ModuleDict({str(i): NgramGate(
self.ngram.dim * len(self.ngram.channels), cfg["width"], cfg["rmsnorm_epsilon"],
max(cfg["ngram_orders"])) for i in layers})
cos, sin = rotary_tables(cfg["context_tokens"], cfg["rope_dimensions"], cfg["rope_base"])
self.register_buffer("cos", cos, persistent=False)
self.register_buffer("sin", sin, persistent=False)
self.reset_parameters()
def reset_parameters(self):
std = self.cfg["initialization_std"]
residual = std / math.sqrt(2 * self.cfg["blocks"])
for name, p in self.named_parameters():
if name.endswith("norm.weight"):
nn.init.ones_(p)
elif name == "ngram.table.weight" and self.ngram_gates:
# gated: random rows, as in Engram (zero rows would leave the key's RMSNorm amplifying tiny updates)
nn.init.normal_(p, std=std)
elif name.endswith(("head_gate.weight", "ngram.table.weight", ".conv", ".key_offset_mix")):
nn.init.zeros_(p)
elif name == "ngram.weights":
continue # set at construction
elif name.endswith(("attention.o.weight", "down.weight")):
nn.init.normal_(p, std=residual)
else:
nn.init.normal_(p, std=std)
def head(self, x):
return F.linear(self.final_norm(x), self.embedding.weight)
def prelude(self, tokens):
"""The token embeddings, the document masks and positions, and the n-gram rows: everything before the
first block (train.make_loss compiles it on its own)."""
x = self.embedding(tokens)
# document_mask: each document of a packed window is processed as if alone (attention, the key offset and
# the n-grams stay inside it).
position = document_positions(tokens) if self.cfg.get("document_mask") else None
masks = None if position is None else document_masks(
position, self.cfg.get("local_window", 0), next((b.attention.mask for b in self.blocks
if b.attention.window), None))
ngram = self.ngram(tokens, position) if self.ngram is not None else None
return x, masks, ngram, position
def hidden(self, tokens, blocks=None, prelude=None):
"""The last block's output at every position (head() turns it into logits). `blocks` and `prelude`:
callables run in place of self.blocks and self.prelude (train.make_loss passes compiled ones; the model
itself stays uncompiled, so generation does not recompile at every length)."""
assert blocks is None or len(blocks) == len(self.blocks), "one callable per block, in order"
T = tokens.shape[1]
cos, sin = self.cos[:T], self.sin[:T] # positions restart at zero in every window
x, masks, ngram, position = (prelude or self.prelude)(tokens)
# One unbind, not one index per block: each index adds its own small backward kernels. The 0-dimensional
# weights are inputs, not Python numbers, so every block shares one compiled graph.
weights = self.ngram.weights.unbind(0) if ngram is not None and self.ngram.weights is not None else None
for i, block in enumerate(blocks or self.blocks):
if str(i) in self.ngram_gates:
x = self.ngram_gates[str(i)](x, ngram, position)
x = block(x, cos, sin, masks, *((ngram, weights[i]) if weights else ()))
return x
def forward(self, tokens, last_only=False):
"""Logits for every position, or only the last one (generation skips the rest of the output head)."""
x = self.hidden(tokens)
return self.head(x[:, -1:] if last_only else x)
def parameter_count(cfg, vocab_size):
"""SPEC formula for the model with every option off: V*d + L*(4*d*d + 3*d*f + 2*d) + d."""
d, f, L = cfg["width"], cfg["ffn_intermediate"], cfg["blocks"]
return vocab_size * d + L * (4 * d * d + 3 * d * f + 2 * d) + d
|