PhiloLM / model.py
gustavecortal's picture
philosophy-lm: model card, code, tokenizer and browser files
b9ee6b4 verified
Raw History Blame Contribute Delete
20.4 kB
"""Decoder-only Transformer from SPEC section 4: pre-RMSNorm, RoPE, SwiGLU, tied embeddings, no biases.
Options from the 600-update comparisons (see WORKING_RECORD.md), all on in config.json: qk_norm, attention_gate,
local_window with global_every, and the hashed n-gram embedding (ngram, ngram_orders, ngram_rows, ngram_dim).
With every option off, the model is the agreed 33.3M-parameter design."""
import math
import torch
import torch.nn as nn
import torch.nn.functional as F
OPTIONS = ("qk_norm", "attention_gate", "local_window", "global_every", "ngram", "ngram_orders", "ngram_rows",
"ngram_dim", "ngram_heads", "ngram_layers", "key_offset", "document_mask", "global_kv_heads",
"learn_key_offset")
class RMSNorm(nn.Module):
def __init__(self, width, eps):
super().__init__()
self.eps = eps
self.weight = nn.Parameter(torch.ones(width))
def forward(self, x):
# Statistics in float32 even under autocast.
h = x.float()
h = h * torch.rsqrt(h.pow(2).mean(-1, keepdim=True) + self.eps)
return (h * self.weight).to(x.dtype)
def rotary_tables(length, dim, base):
inverse = 1.0 / (base ** (torch.arange(0, dim, 2, dtype=torch.float32) / dim))
angles = torch.outer(torch.arange(length, dtype=torch.float32), inverse)
return angles.cos(), angles.sin() # (T, dim/2) each
def rotate(x, cos, sin):
"""RoPE on the first 2 * cos.shape[-1] dimensions of each head (all of them unless rope_dimensions < head_dim);
the rest stay unrotated. Half-split convention: the first and second halves of the rotated part form the pairs."""
r = 2 * cos.shape[-1]
a, b = x[..., :r].float().chunk(2, dim=-1)
rotated = torch.cat((a * cos - b * sin, a * sin + b * cos), dim=-1).to(x.dtype)
return rotated if r == x.shape[-1] else torch.cat((rotated, x[..., r:]), dim=-1)
def local_mask(n, window):
"""(n, W, 2W) boolean mask for local_attention: chunk c's queries against chunks c-1 and c."""
i = torch.arange(window)[:, None]
j = torch.arange(2 * window)[None, :]
allowed = (j > i) & (j <= i + window) # key at chunk offset j - W, query at offset i
return torch.stack([allowed & (j >= window)] + [allowed] * (n - 1)) # chunk 0 has no previous chunk
def local_attention(q, k, v, window, mask=None):
"""Causal attention over the previous `window` positions (the query's own included), computed per chunk of
`window` queries against that chunk and the one before it, so scores cost T*2W instead of T*T. `mask`: the
(n, W, 2W) chunk mask, or a (B, 1, n, W, 2W) one from document_masks."""
B, H, T, d = q.shape
pad = (-T) % window
if pad:
q, k, v = (F.pad(t, (0, 0, 0, pad)) for t in (q, k, v))
n = q.shape[2] // window
def pairs(t):
t = t.view(B, H, n, window, d)
previous = torch.cat([torch.zeros_like(t[:, :, :1]), t[:, :, :-1]], 2)
return torch.cat([previous, t], 3) # (B, H, n, 2W, d)
mask = local_mask(n, window).to(q.device) if mask is None else mask[..., :n, :, :]
y = F.scaled_dot_product_attention(q.view(B, H, n, window, d), pairs(k), pairs(v), attn_mask=mask)
return y.reshape(B, H, n * window, d)[:, :, :T]
def document_positions(tokens):
"""Each token's position within its document: documents start at EOS (token 0, the tokenizer's eos_id), as
validation windows do; a window that starts inside a document counts from the window's start."""
index = torch.arange(tokens.shape[1], device=tokens.device)
return index - torch.cummax(torch.where(tokens == 0, index, 0), 1).values
def document_masks(position, window, chunks):
"""Masks that keep attention inside each document, from document_positions, computed once per forward pass:
"global" (B, 1, T, T) for keys from the query's document start to the query, "local" (B, 1, n, W, 2W) for
local_attention's chunk pairs (`chunks`: its (n, W, 2W) chunk mask), and "continues" (B, T), false at a
document's first position (its key offset takes nothing from before it)."""
B, T = position.shape
index = torch.arange(T, device=position.device)
start = index - position # each query's document start
masks = {"global": ((index >= start[:, :, None]) & (index <= index[:, None]))[:, None], "continues": position > 0}
if window:
n = -(-T // window)
query_start = F.pad(start, (0, n * window - T)).view(B, n, window, 1) # padded queries are discarded
key = torch.arange(n, device=position.device)[:, None] * window + torch.arange(-window, window,
device=position.device)
masks["local"] = (chunks[:n] & (key[:, None, :] >= query_start))[:, None]
return masks
class Attention(nn.Module):
def __init__(self, cfg, window):
super().__init__()
width, heads = cfg["width"], cfg["query_heads"]
self.heads, self.window = heads, window
global_kv = cfg.get("global_kv_heads")
self.kv_heads = heads if window or global_kv is None else global_kv
if type(self.kv_heads) is not int or self.kv_heads < 1 or heads % self.kv_heads:
raise ValueError("global_kv_heads must be a positive divisor of query_heads")
kv_width = self.kv_heads * cfg["head_dim"]
# Partial key offset (modded-nanogpt PR 169): in global layers, each key's unrotated dimensions come from
# the previous token, so one head can match "the token after the previous occurrence" (induction).
self.key_offset = cfg.get("key_offset") and not window
self.key_offset_mix = (nn.Parameter(torch.zeros(self.kv_heads))
if self.key_offset and cfg.get("learn_key_offset") else None)
self.rope = cfg["rope_dimensions"]
self.q = nn.Linear(width, width, bias=False)
self.k = nn.Linear(width, kv_width, bias=False)
self.v = nn.Linear(width, kv_width, bias=False)
self.o = nn.Linear(width, width, bias=False)
self.head_gate = nn.Linear(width, heads, bias=False) if cfg.get("attention_gate") else None
qk = cfg.get("qk_norm")
self.q_norm = RMSNorm(cfg["head_dim"], cfg["rmsnorm_epsilon"]) if qk else None
self.k_norm = RMSNorm(cfg["head_dim"], cfg["rmsnorm_epsilon"]) if qk else None
if window: # built once for the longest window count. Shorter inputs use its first chunks.
self.register_buffer("mask", local_mask(-(-cfg["context_tokens"] // window), window), persistent=False)
def forward(self, x, cos, sin, masks=None):
B, T, D = x.shape
head_dim = D // self.heads
# One matrix multiply for Q, K and V. They stay separate parameters with separate Muon updates.
qkv = F.linear(x, torch.cat([self.q.weight, self.k.weight, self.v.weight]))
q, k, v = qkv.split((D, self.k.out_features, self.v.out_features), dim=-1)
q = q.reshape(B, T, self.heads, head_dim).transpose(1, 2)
k = k.reshape(B, T, self.kv_heads, head_dim).transpose(1, 2)
v = v.reshape(B, T, self.kv_heads, head_dim).transpose(1, 2)
if self.q_norm is not None:
q, k = self.q_norm(q), self.k_norm(k)
q, k = rotate(q, cos, sin), rotate(k, cos, sin)
if self.key_offset:
current = k[..., self.rope:]
shifted = F.pad(current[..., :-1, :], (0, 0, 1, 0))
if masks is not None: # a document's first key gets nothing from the previous document
shifted = shifted * masks["continues"][:, None, :, None]
if self.key_offset_mix is not None:
# Mix per KV head before GQA expansion. Zero recovers the fixed offset; one uses the current key.
mix = self.key_offset_mix.to(k.dtype)[None, :, None, None]
shifted = torch.lerp(shifted, current, mix)
k = torch.cat((k[..., :self.rope], shifted), dim=-1)
if self.kv_heads != self.heads:
# Explicit groups use the existing XPU SDPA path for both causal and document masks.
# A future decoder should cache the compact K/V tensors before this expansion.
group_size = self.heads // self.kv_heads
k = k.repeat_interleave(group_size, dim=1)
v = v.repeat_interleave(group_size, dim=1)
if self.window:
y = local_attention(q, k, v, self.window, self.mask if masks is None else masks["local"])
elif masks is None:
y = F.scaled_dot_product_attention(q, k, v, is_causal=True)
else: # causal within each document: an explicit mask, so not the fused causal path
y = F.scaled_dot_product_attention(q, k, v, attn_mask=masks["global"])
if self.head_gate is not None: # per-head sigmoid gate on the attention output
y = y * torch.sigmoid(self.head_gate(x)).transpose(1, 2).unsqueeze(-1).to(y.dtype)
return self.o(y.transpose(1, 2).reshape(B, T, D))
class Block(nn.Module):
def __init__(self, cfg, window=0):
super().__init__()
d, f = cfg["width"], cfg["ffn_intermediate"]
self.attention_norm = RMSNorm(d, cfg["rmsnorm_epsilon"])
self.attention = Attention(cfg, window)
self.ffn_norm = RMSNorm(d, cfg["rmsnorm_epsilon"])
self.gate = nn.Linear(d, f, bias=False)
self.up = nn.Linear(d, f, bias=False)
self.down = nn.Linear(f, d, bias=False)
def forward(self, x, cos, sin, masks=None, ngram=None, ngram_weight=None):
"""`ngram`, `ngram_weight`: the per-block n-gram injection into the first ngram_dim residual dimensions,
done here so it compiles with the block (train.make_loss compiles block by block)."""
if ngram is not None:
d = ngram.shape[-1]
x = torch.cat([x[..., :d] + ngram_weight * ngram, x[..., d:]], -1)
x = x + self.attention(self.attention_norm(x), cos, sin, masks)
gate, up = F.linear(self.ffn_norm(x), torch.cat([self.gate.weight, self.up.weight])).chunk(2, -1)
return x + self.down(F.silu(gate) * up)
def earlier(t, j, position=None):
"""t (B, T, ...) moved j positions later: position i holds t[:, i - j], zero (padding) where i - j falls before
the window or, given document_positions, before the document's start, as when the document is alone."""
shifted = F.pad(t, (0, 0) * (t.dim() - 2) + (j, 0))[:, :t.shape[1]]
if position is None or not j:
return shifted
return shifted * (position >= j).view(*position.shape, *(1,) * (t.dim() - 2))
class NgramEmbedding(nn.Module):
"""Hashed n-gram embedding as in the modded-nanogpt speedrun (records 62 and 83). Each position reads a
zero-initialized row for the bigram (and optionally trigram) ending there, times a fixed random +-1 sign
pattern chosen by an independent hash, so n-grams sharing a row separate. By default the orders' rows are
summed and the model adds the result to the first `ngram_dim` residual dimensions before every block with a
learned per-block weight. With `ngram_layers` (Engram, Cheng et al. 2026), each order is read through
`ngram_heads` independent hashes, the rows are concatenated, and NgramGate adds them at those layers."""
ROW_MULS = {2: (36313, 27191), 3: (17351, 60961, 45259)} # the speedrun's hash constants
SIGN_MULS = {2: (48271, 30011), 3: (58699, 39779, 26801)}
SIGN_ROWS = 8192
def __init__(self, cfg):
super().__init__()
self.rows, self.dim = cfg["ngram_rows"], cfg["ngram_dim"]
self.concatenate = bool(cfg.get("ngram_layers"))
g = torch.Generator().manual_seed(1234)
self.register_buffer("signs", torch.randint(0, 2, (self.SIGN_ROWS, self.dim), generator=g).float() * 2 - 1)
# One channel per (order, hash head): head 0 uses the speedrun's constants, later heads random odd ones.
self.channels = []
for order in cfg["ngram_orders"]:
for head in range(cfg.get("ngram_heads") or 1):
muls = (self.ROW_MULS[order], self.SIGN_MULS[order]) if head == 0 else \
(torch.randint(0, 32768, (2, order), generator=g) * 2 + 1).tolist()
self.channels.append((order, *muls))
self.table = nn.Embedding(self.rows * len(self.channels), self.dim)
self.weights = None if self.concatenate else nn.Parameter(torch.full((cfg["blocks"],), 0.1))
def forward(self, tokens, position=None):
out = []
for channel, (order, row_muls, sign_muls) in enumerate(self.channels):
rows = torch.zeros_like(tokens)
signs = torch.zeros_like(tokens)
for j, (rm, sm) in enumerate(zip(row_muls, sign_muls)):
shifted = earlier(tokens, j, position) # the token j positions back
rows = rows ^ (rm * shifted)
signs = signs ^ (sm * shifted)
rows = rows % (self.rows - 1) + channel * self.rows
rows[:, :order - 1] = (channel + 1) * self.rows - 1 # reserved row: too little history in the window
if position is not None: # or in the document
rows = torch.where(position < order - 1, (channel + 1) * self.rows - 1, rows)
out.append(self.table(rows) * self.signs[signs % self.SIGN_ROWS])
return torch.cat(out, -1) if self.concatenate else sum(out)
class NgramGate(nn.Module):
"""Engram's module (Cheng et al. 2026, Eq. 3-5): from the n-gram rows e, a key and a value; a per-token gate
alpha = sigmoid(RMSNorm(h) . RMSNorm(key) / sqrt(d)) from the hidden state h; the output alpha * value plus a
zero-initialized causal depthwise convolution (kernel 4, dilation `dilation`) of its SiLU-activated norm,
added to the residual stream. The convolution reads no earlier document under document_mask."""
def __init__(self, memory, width, eps, dilation, kernel=4):
super().__init__()
self.key = nn.Linear(memory, width, bias=False)
self.value = nn.Linear(memory, width, bias=False)
self.hidden_norm = RMSNorm(width, eps)
self.key_norm = RMSNorm(width, eps)
self.value_norm = RMSNorm(width, eps)
self.conv = nn.Parameter(torch.zeros(kernel, width))
self.dilation = dilation
def forward(self, h, e, position=None):
k = self.key_norm(self.key(e))
alpha = torch.sigmoid((self.hidden_norm(h) * k).sum(-1, keepdim=True) / math.sqrt(h.shape[-1]))
v = alpha * self.value(e)
u = self.value_norm(v)
conv = sum(w * earlier(u, j * self.dilation, position) for j, w in enumerate(self.conv))
return h + (F.silu(conv) + v).to(h.dtype)
class Transformer(nn.Module):
def __init__(self, cfg, vocab_size):
super().__init__()
assert cfg["query_heads"] == cfg["kv_heads"] and cfg["width"] == cfg["query_heads"] * cfg["head_dim"]
assert cfg["rope_dimensions"] <= cfg["head_dim"] and cfg["rope_dimensions"] % 2 == 0
assert cfg["tie_embeddings"] and not cfg["linear_bias"]
assert not cfg.get("key_offset") or cfg["rope_dimensions"] < cfg["head_dim"], \
"key_offset (global layers only) shifts the unrotated dimensions: set rope_dimensions below head_dim"
self.cfg = cfg
self.embedding = nn.Embedding(vocab_size, cfg["width"])
# local_window > 0: every global_every-th layer stays global, the others are local (Gemma-style).
window = cfg.get("local_window", 0)
L = cfg["blocks"]
if cfg.get("learn_key_offset") and (not cfg.get("key_offset") or (window and L < cfg["global_every"])):
raise ValueError("learn_key_offset requires key_offset and at least one global block")
self.blocks = nn.ModuleList(Block(cfg, window if window and (i + 1) % cfg["global_every"] else 0)
for i in range(L))
self.final_norm = RMSNorm(cfg["width"], cfg["rmsnorm_epsilon"])
self.ngram = NgramEmbedding(cfg) if cfg.get("ngram") else None
# ngram_layers: the gated n-gram module before each of these blocks, instead of the per-block injection
layers = cfg.get("ngram_layers") or []
assert not layers or (self.ngram is not None and all(0 <= i < L for i in layers)), "ngram_layers"
self.ngram_gates = nn.ModuleDict({str(i): NgramGate(
self.ngram.dim * len(self.ngram.channels), cfg["width"], cfg["rmsnorm_epsilon"],
max(cfg["ngram_orders"])) for i in layers})
cos, sin = rotary_tables(cfg["context_tokens"], cfg["rope_dimensions"], cfg["rope_base"])
self.register_buffer("cos", cos, persistent=False)
self.register_buffer("sin", sin, persistent=False)
self.reset_parameters()
def reset_parameters(self):
std = self.cfg["initialization_std"]
residual = std / math.sqrt(2 * self.cfg["blocks"])
for name, p in self.named_parameters():
if name.endswith("norm.weight"):
nn.init.ones_(p)
elif name == "ngram.table.weight" and self.ngram_gates:
# gated: random rows, as in Engram (zero rows would leave the key's RMSNorm amplifying tiny updates)
nn.init.normal_(p, std=std)
elif name.endswith(("head_gate.weight", "ngram.table.weight", ".conv", ".key_offset_mix")):
nn.init.zeros_(p)
elif name == "ngram.weights":
continue # set at construction
elif name.endswith(("attention.o.weight", "down.weight")):
nn.init.normal_(p, std=residual)
else:
nn.init.normal_(p, std=std)
def head(self, x):
return F.linear(self.final_norm(x), self.embedding.weight)
def prelude(self, tokens):
"""The token embeddings, the document masks and positions, and the n-gram rows: everything before the
first block (train.make_loss compiles it on its own)."""
x = self.embedding(tokens)
# document_mask: each document of a packed window is processed as if alone (attention, the key offset and
# the n-grams stay inside it).
position = document_positions(tokens) if self.cfg.get("document_mask") else None
masks = None if position is None else document_masks(
position, self.cfg.get("local_window", 0), next((b.attention.mask for b in self.blocks
if b.attention.window), None))
ngram = self.ngram(tokens, position) if self.ngram is not None else None
return x, masks, ngram, position
def hidden(self, tokens, blocks=None, prelude=None):
"""The last block's output at every position (head() turns it into logits). `blocks` and `prelude`:
callables run in place of self.blocks and self.prelude (train.make_loss passes compiled ones; the model
itself stays uncompiled, so generation does not recompile at every length)."""
assert blocks is None or len(blocks) == len(self.blocks), "one callable per block, in order"
T = tokens.shape[1]
cos, sin = self.cos[:T], self.sin[:T] # positions restart at zero in every window
x, masks, ngram, position = (prelude or self.prelude)(tokens)
# One unbind, not one index per block: each index adds its own small backward kernels. The 0-dimensional
# weights are inputs, not Python numbers, so every block shares one compiled graph.
weights = self.ngram.weights.unbind(0) if ngram is not None and self.ngram.weights is not None else None
for i, block in enumerate(blocks or self.blocks):
if str(i) in self.ngram_gates:
x = self.ngram_gates[str(i)](x, ngram, position)
x = block(x, cos, sin, masks, *((ngram, weights[i]) if weights else ()))
return x
def forward(self, tokens, last_only=False):
"""Logits for every position, or only the last one (generation skips the rest of the output head)."""
x = self.hidden(tokens)
return self.head(x[:, -1:] if last_only else x)
def parameter_count(cfg, vocab_size):
"""SPEC formula for the model with every option off: V*d + L*(4*d*d + 3*d*f + 2*d) + d."""
d, f, L = cfg["width"], cfg["ffn_intermediate"], cfg["blocks"]
return vocab_size * d + L * (4 * d * d + 3 * d * f + 2 * d) + d