Download model.py from gustavecortal/PhiloLM: direct link, hf CLI and curl.
- Browser
- Download file 20.4 kB
-
https://huggingface.co/gustavecortal/PhiloLM/resolve/main/model.py
- Command line
-
hf download hf://gustavecortal/PhiloLM/model.py
-
curl -L -o model.py https://huggingface.co/gustavecortal/PhiloLM/resolve/main/model.py
20.4 kB
| """Decoder-only Transformer from SPEC section 4: pre-RMSNorm, RoPE, SwiGLU, tied embeddings, no biases. | |
| Options from the 600-update comparisons (see WORKING_RECORD.md), all on in config.json: qk_norm, attention_gate, | |
| local_window with global_every, and the hashed n-gram embedding (ngram, ngram_orders, ngram_rows, ngram_dim). | |
| With every option off, the model is the agreed 33.3M-parameter design.""" | |
| import math | |
| import torch | |
| import torch.nn as nn | |
| import torch.nn.functional as F | |
| OPTIONS = ("qk_norm", "attention_gate", "local_window", "global_every", "ngram", "ngram_orders", "ngram_rows", | |
| "ngram_dim", "ngram_heads", "ngram_layers", "key_offset", "document_mask", "global_kv_heads", | |
| "learn_key_offset") | |
| class RMSNorm(nn.Module): | |
| def __init__(self, width, eps): | |
| super().__init__() | |
| self.eps = eps | |
| self.weight = nn.Parameter(torch.ones(width)) | |
| def forward(self, x): | |
| # Statistics in float32 even under autocast. | |
| h = x.float() | |
| h = h * torch.rsqrt(h.pow(2).mean(-1, keepdim=True) + self.eps) | |
| return (h * self.weight).to(x.dtype) | |
| def rotary_tables(length, dim, base): | |
| inverse = 1.0 / (base ** (torch.arange(0, dim, 2, dtype=torch.float32) / dim)) | |
| angles = torch.outer(torch.arange(length, dtype=torch.float32), inverse) | |
| return angles.cos(), angles.sin() # (T, dim/2) each | |
| def rotate(x, cos, sin): | |
| """RoPE on the first 2 * cos.shape[-1] dimensions of each head (all of them unless rope_dimensions < head_dim); | |
| the rest stay unrotated. Half-split convention: the first and second halves of the rotated part form the pairs.""" | |
| r = 2 * cos.shape[-1] | |
| a, b = x[..., :r].float().chunk(2, dim=-1) | |
| rotated = torch.cat((a * cos - b * sin, a * sin + b * cos), dim=-1).to(x.dtype) | |
| return rotated if r == x.shape[-1] else torch.cat((rotated, x[..., r:]), dim=-1) | |
| def local_mask(n, window): | |
| """(n, W, 2W) boolean mask for local_attention: chunk c's queries against chunks c-1 and c.""" | |
| i = torch.arange(window)[:, None] | |
| j = torch.arange(2 * window)[None, :] | |
| allowed = (j > i) & (j <= i + window) # key at chunk offset j - W, query at offset i | |
| return torch.stack([allowed & (j >= window)] + [allowed] * (n - 1)) # chunk 0 has no previous chunk | |
| def local_attention(q, k, v, window, mask=None): | |
| """Causal attention over the previous `window` positions (the query's own included), computed per chunk of | |
| `window` queries against that chunk and the one before it, so scores cost T*2W instead of T*T. `mask`: the | |
| (n, W, 2W) chunk mask, or a (B, 1, n, W, 2W) one from document_masks.""" | |
| B, H, T, d = q.shape | |
| pad = (-T) % window | |
| if pad: | |
| q, k, v = (F.pad(t, (0, 0, 0, pad)) for t in (q, k, v)) | |
| n = q.shape[2] // window | |
| def pairs(t): | |
| t = t.view(B, H, n, window, d) | |
| previous = torch.cat([torch.zeros_like(t[:, :, :1]), t[:, :, :-1]], 2) | |
| return torch.cat([previous, t], 3) # (B, H, n, 2W, d) | |
| mask = local_mask(n, window).to(q.device) if mask is None else mask[..., :n, :, :] | |
| y = F.scaled_dot_product_attention(q.view(B, H, n, window, d), pairs(k), pairs(v), attn_mask=mask) | |
| return y.reshape(B, H, n * window, d)[:, :, :T] | |
| def document_positions(tokens): | |
| """Each token's position within its document: documents start at EOS (token 0, the tokenizer's eos_id), as | |
| validation windows do; a window that starts inside a document counts from the window's start.""" | |
| index = torch.arange(tokens.shape[1], device=tokens.device) | |
| return index - torch.cummax(torch.where(tokens == 0, index, 0), 1).values | |
| def document_masks(position, window, chunks): | |
| """Masks that keep attention inside each document, from document_positions, computed once per forward pass: | |
| "global" (B, 1, T, T) for keys from the query's document start to the query, "local" (B, 1, n, W, 2W) for | |
| local_attention's chunk pairs (`chunks`: its (n, W, 2W) chunk mask), and "continues" (B, T), false at a | |
| document's first position (its key offset takes nothing from before it).""" | |
| B, T = position.shape | |
| index = torch.arange(T, device=position.device) | |
| start = index - position # each query's document start | |
| masks = {"global": ((index >= start[:, :, None]) & (index <= index[:, None]))[:, None], "continues": position > 0} | |
| if window: | |
| n = -(-T // window) | |
| query_start = F.pad(start, (0, n * window - T)).view(B, n, window, 1) # padded queries are discarded | |
| key = torch.arange(n, device=position.device)[:, None] * window + torch.arange(-window, window, | |
| device=position.device) | |
| masks["local"] = (chunks[:n] & (key[:, None, :] >= query_start))[:, None] | |
| return masks | |
| class Attention(nn.Module): | |
| def __init__(self, cfg, window): | |
| super().__init__() | |
| width, heads = cfg["width"], cfg["query_heads"] | |
| self.heads, self.window = heads, window | |
| global_kv = cfg.get("global_kv_heads") | |
| self.kv_heads = heads if window or global_kv is None else global_kv | |
| if type(self.kv_heads) is not int or self.kv_heads < 1 or heads % self.kv_heads: | |
| raise ValueError("global_kv_heads must be a positive divisor of query_heads") | |
| kv_width = self.kv_heads * cfg["head_dim"] | |
| # Partial key offset (modded-nanogpt PR 169): in global layers, each key's unrotated dimensions come from | |
| # the previous token, so one head can match "the token after the previous occurrence" (induction). | |
| self.key_offset = cfg.get("key_offset") and not window | |
| self.key_offset_mix = (nn.Parameter(torch.zeros(self.kv_heads)) | |
| if self.key_offset and cfg.get("learn_key_offset") else None) | |
| self.rope = cfg["rope_dimensions"] | |
| self.q = nn.Linear(width, width, bias=False) | |
| self.k = nn.Linear(width, kv_width, bias=False) | |
| self.v = nn.Linear(width, kv_width, bias=False) | |
| self.o = nn.Linear(width, width, bias=False) | |
| self.head_gate = nn.Linear(width, heads, bias=False) if cfg.get("attention_gate") else None | |
| qk = cfg.get("qk_norm") | |
| self.q_norm = RMSNorm(cfg["head_dim"], cfg["rmsnorm_epsilon"]) if qk else None | |
| self.k_norm = RMSNorm(cfg["head_dim"], cfg["rmsnorm_epsilon"]) if qk else None | |
| if window: # built once for the longest window count. Shorter inputs use its first chunks. | |
| self.register_buffer("mask", local_mask(-(-cfg["context_tokens"] // window), window), persistent=False) | |
| def forward(self, x, cos, sin, masks=None): | |
| B, T, D = x.shape | |
| head_dim = D // self.heads | |
| # One matrix multiply for Q, K and V. They stay separate parameters with separate Muon updates. | |
| qkv = F.linear(x, torch.cat([self.q.weight, self.k.weight, self.v.weight])) | |
| q, k, v = qkv.split((D, self.k.out_features, self.v.out_features), dim=-1) | |
| q = q.reshape(B, T, self.heads, head_dim).transpose(1, 2) | |
| k = k.reshape(B, T, self.kv_heads, head_dim).transpose(1, 2) | |
| v = v.reshape(B, T, self.kv_heads, head_dim).transpose(1, 2) | |
| if self.q_norm is not None: | |
| q, k = self.q_norm(q), self.k_norm(k) | |
| q, k = rotate(q, cos, sin), rotate(k, cos, sin) | |
| if self.key_offset: | |
| current = k[..., self.rope:] | |
| shifted = F.pad(current[..., :-1, :], (0, 0, 1, 0)) | |
| if masks is not None: # a document's first key gets nothing from the previous document | |
| shifted = shifted * masks["continues"][:, None, :, None] | |
| if self.key_offset_mix is not None: | |
| # Mix per KV head before GQA expansion. Zero recovers the fixed offset; one uses the current key. | |
| mix = self.key_offset_mix.to(k.dtype)[None, :, None, None] | |
| shifted = torch.lerp(shifted, current, mix) | |
| k = torch.cat((k[..., :self.rope], shifted), dim=-1) | |
| if self.kv_heads != self.heads: | |
| # Explicit groups use the existing XPU SDPA path for both causal and document masks. | |
| # A future decoder should cache the compact K/V tensors before this expansion. | |
| group_size = self.heads // self.kv_heads | |
| k = k.repeat_interleave(group_size, dim=1) | |
| v = v.repeat_interleave(group_size, dim=1) | |
| if self.window: | |
| y = local_attention(q, k, v, self.window, self.mask if masks is None else masks["local"]) | |
| elif masks is None: | |
| y = F.scaled_dot_product_attention(q, k, v, is_causal=True) | |
| else: # causal within each document: an explicit mask, so not the fused causal path | |
| y = F.scaled_dot_product_attention(q, k, v, attn_mask=masks["global"]) | |
| if self.head_gate is not None: # per-head sigmoid gate on the attention output | |
| y = y * torch.sigmoid(self.head_gate(x)).transpose(1, 2).unsqueeze(-1).to(y.dtype) | |
| return self.o(y.transpose(1, 2).reshape(B, T, D)) | |
| class Block(nn.Module): | |
| def __init__(self, cfg, window=0): | |
| super().__init__() | |
| d, f = cfg["width"], cfg["ffn_intermediate"] | |
| self.attention_norm = RMSNorm(d, cfg["rmsnorm_epsilon"]) | |
| self.attention = Attention(cfg, window) | |
| self.ffn_norm = RMSNorm(d, cfg["rmsnorm_epsilon"]) | |
| self.gate = nn.Linear(d, f, bias=False) | |
| self.up = nn.Linear(d, f, bias=False) | |
| self.down = nn.Linear(f, d, bias=False) | |
| def forward(self, x, cos, sin, masks=None, ngram=None, ngram_weight=None): | |
| """`ngram`, `ngram_weight`: the per-block n-gram injection into the first ngram_dim residual dimensions, | |
| done here so it compiles with the block (train.make_loss compiles block by block).""" | |
| if ngram is not None: | |
| d = ngram.shape[-1] | |
| x = torch.cat([x[..., :d] + ngram_weight * ngram, x[..., d:]], -1) | |
| x = x + self.attention(self.attention_norm(x), cos, sin, masks) | |
| gate, up = F.linear(self.ffn_norm(x), torch.cat([self.gate.weight, self.up.weight])).chunk(2, -1) | |
| return x + self.down(F.silu(gate) * up) | |
| def earlier(t, j, position=None): | |
| """t (B, T, ...) moved j positions later: position i holds t[:, i - j], zero (padding) where i - j falls before | |
| the window or, given document_positions, before the document's start, as when the document is alone.""" | |
| shifted = F.pad(t, (0, 0) * (t.dim() - 2) + (j, 0))[:, :t.shape[1]] | |
| if position is None or not j: | |
| return shifted | |
| return shifted * (position >= j).view(*position.shape, *(1,) * (t.dim() - 2)) | |
| class NgramEmbedding(nn.Module): | |
| """Hashed n-gram embedding as in the modded-nanogpt speedrun (records 62 and 83). Each position reads a | |
| zero-initialized row for the bigram (and optionally trigram) ending there, times a fixed random +-1 sign | |
| pattern chosen by an independent hash, so n-grams sharing a row separate. By default the orders' rows are | |
| summed and the model adds the result to the first `ngram_dim` residual dimensions before every block with a | |
| learned per-block weight. With `ngram_layers` (Engram, Cheng et al. 2026), each order is read through | |
| `ngram_heads` independent hashes, the rows are concatenated, and NgramGate adds them at those layers.""" | |
| ROW_MULS = {2: (36313, 27191), 3: (17351, 60961, 45259)} # the speedrun's hash constants | |
| SIGN_MULS = {2: (48271, 30011), 3: (58699, 39779, 26801)} | |
| SIGN_ROWS = 8192 | |
| def __init__(self, cfg): | |
| super().__init__() | |
| self.rows, self.dim = cfg["ngram_rows"], cfg["ngram_dim"] | |
| self.concatenate = bool(cfg.get("ngram_layers")) | |
| g = torch.Generator().manual_seed(1234) | |
| self.register_buffer("signs", torch.randint(0, 2, (self.SIGN_ROWS, self.dim), generator=g).float() * 2 - 1) | |
| # One channel per (order, hash head): head 0 uses the speedrun's constants, later heads random odd ones. | |
| self.channels = [] | |
| for order in cfg["ngram_orders"]: | |
| for head in range(cfg.get("ngram_heads") or 1): | |
| muls = (self.ROW_MULS[order], self.SIGN_MULS[order]) if head == 0 else \ | |
| (torch.randint(0, 32768, (2, order), generator=g) * 2 + 1).tolist() | |
| self.channels.append((order, *muls)) | |
| self.table = nn.Embedding(self.rows * len(self.channels), self.dim) | |
| self.weights = None if self.concatenate else nn.Parameter(torch.full((cfg["blocks"],), 0.1)) | |
| def forward(self, tokens, position=None): | |
| out = [] | |
| for channel, (order, row_muls, sign_muls) in enumerate(self.channels): | |
| rows = torch.zeros_like(tokens) | |
| signs = torch.zeros_like(tokens) | |
| for j, (rm, sm) in enumerate(zip(row_muls, sign_muls)): | |
| shifted = earlier(tokens, j, position) # the token j positions back | |
| rows = rows ^ (rm * shifted) | |
| signs = signs ^ (sm * shifted) | |
| rows = rows % (self.rows - 1) + channel * self.rows | |
| rows[:, :order - 1] = (channel + 1) * self.rows - 1 # reserved row: too little history in the window | |
| if position is not None: # or in the document | |
| rows = torch.where(position < order - 1, (channel + 1) * self.rows - 1, rows) | |
| out.append(self.table(rows) * self.signs[signs % self.SIGN_ROWS]) | |
| return torch.cat(out, -1) if self.concatenate else sum(out) | |
| class NgramGate(nn.Module): | |
| """Engram's module (Cheng et al. 2026, Eq. 3-5): from the n-gram rows e, a key and a value; a per-token gate | |
| alpha = sigmoid(RMSNorm(h) . RMSNorm(key) / sqrt(d)) from the hidden state h; the output alpha * value plus a | |
| zero-initialized causal depthwise convolution (kernel 4, dilation `dilation`) of its SiLU-activated norm, | |
| added to the residual stream. The convolution reads no earlier document under document_mask.""" | |
| def __init__(self, memory, width, eps, dilation, kernel=4): | |
| super().__init__() | |
| self.key = nn.Linear(memory, width, bias=False) | |
| self.value = nn.Linear(memory, width, bias=False) | |
| self.hidden_norm = RMSNorm(width, eps) | |
| self.key_norm = RMSNorm(width, eps) | |
| self.value_norm = RMSNorm(width, eps) | |
| self.conv = nn.Parameter(torch.zeros(kernel, width)) | |
| self.dilation = dilation | |
| def forward(self, h, e, position=None): | |
| k = self.key_norm(self.key(e)) | |
| alpha = torch.sigmoid((self.hidden_norm(h) * k).sum(-1, keepdim=True) / math.sqrt(h.shape[-1])) | |
| v = alpha * self.value(e) | |
| u = self.value_norm(v) | |
| conv = sum(w * earlier(u, j * self.dilation, position) for j, w in enumerate(self.conv)) | |
| return h + (F.silu(conv) + v).to(h.dtype) | |
| class Transformer(nn.Module): | |
| def __init__(self, cfg, vocab_size): | |
| super().__init__() | |
| assert cfg["query_heads"] == cfg["kv_heads"] and cfg["width"] == cfg["query_heads"] * cfg["head_dim"] | |
| assert cfg["rope_dimensions"] <= cfg["head_dim"] and cfg["rope_dimensions"] % 2 == 0 | |
| assert cfg["tie_embeddings"] and not cfg["linear_bias"] | |
| assert not cfg.get("key_offset") or cfg["rope_dimensions"] < cfg["head_dim"], \ | |
| "key_offset (global layers only) shifts the unrotated dimensions: set rope_dimensions below head_dim" | |
| self.cfg = cfg | |
| self.embedding = nn.Embedding(vocab_size, cfg["width"]) | |
| # local_window > 0: every global_every-th layer stays global, the others are local (Gemma-style). | |
| window = cfg.get("local_window", 0) | |
| L = cfg["blocks"] | |
| if cfg.get("learn_key_offset") and (not cfg.get("key_offset") or (window and L < cfg["global_every"])): | |
| raise ValueError("learn_key_offset requires key_offset and at least one global block") | |
| self.blocks = nn.ModuleList(Block(cfg, window if window and (i + 1) % cfg["global_every"] else 0) | |
| for i in range(L)) | |
| self.final_norm = RMSNorm(cfg["width"], cfg["rmsnorm_epsilon"]) | |
| self.ngram = NgramEmbedding(cfg) if cfg.get("ngram") else None | |
| # ngram_layers: the gated n-gram module before each of these blocks, instead of the per-block injection | |
| layers = cfg.get("ngram_layers") or [] | |
| assert not layers or (self.ngram is not None and all(0 <= i < L for i in layers)), "ngram_layers" | |
| self.ngram_gates = nn.ModuleDict({str(i): NgramGate( | |
| self.ngram.dim * len(self.ngram.channels), cfg["width"], cfg["rmsnorm_epsilon"], | |
| max(cfg["ngram_orders"])) for i in layers}) | |
| cos, sin = rotary_tables(cfg["context_tokens"], cfg["rope_dimensions"], cfg["rope_base"]) | |
| self.register_buffer("cos", cos, persistent=False) | |
| self.register_buffer("sin", sin, persistent=False) | |
| self.reset_parameters() | |
| def reset_parameters(self): | |
| std = self.cfg["initialization_std"] | |
| residual = std / math.sqrt(2 * self.cfg["blocks"]) | |
| for name, p in self.named_parameters(): | |
| if name.endswith("norm.weight"): | |
| nn.init.ones_(p) | |
| elif name == "ngram.table.weight" and self.ngram_gates: | |
| # gated: random rows, as in Engram (zero rows would leave the key's RMSNorm amplifying tiny updates) | |
| nn.init.normal_(p, std=std) | |
| elif name.endswith(("head_gate.weight", "ngram.table.weight", ".conv", ".key_offset_mix")): | |
| nn.init.zeros_(p) | |
| elif name == "ngram.weights": | |
| continue # set at construction | |
| elif name.endswith(("attention.o.weight", "down.weight")): | |
| nn.init.normal_(p, std=residual) | |
| else: | |
| nn.init.normal_(p, std=std) | |
| def head(self, x): | |
| return F.linear(self.final_norm(x), self.embedding.weight) | |
| def prelude(self, tokens): | |
| """The token embeddings, the document masks and positions, and the n-gram rows: everything before the | |
| first block (train.make_loss compiles it on its own).""" | |
| x = self.embedding(tokens) | |
| # document_mask: each document of a packed window is processed as if alone (attention, the key offset and | |
| # the n-grams stay inside it). | |
| position = document_positions(tokens) if self.cfg.get("document_mask") else None | |
| masks = None if position is None else document_masks( | |
| position, self.cfg.get("local_window", 0), next((b.attention.mask for b in self.blocks | |
| if b.attention.window), None)) | |
| ngram = self.ngram(tokens, position) if self.ngram is not None else None | |
| return x, masks, ngram, position | |
| def hidden(self, tokens, blocks=None, prelude=None): | |
| """The last block's output at every position (head() turns it into logits). `blocks` and `prelude`: | |
| callables run in place of self.blocks and self.prelude (train.make_loss passes compiled ones; the model | |
| itself stays uncompiled, so generation does not recompile at every length).""" | |
| assert blocks is None or len(blocks) == len(self.blocks), "one callable per block, in order" | |
| T = tokens.shape[1] | |
| cos, sin = self.cos[:T], self.sin[:T] # positions restart at zero in every window | |
| x, masks, ngram, position = (prelude or self.prelude)(tokens) | |
| # One unbind, not one index per block: each index adds its own small backward kernels. The 0-dimensional | |
| # weights are inputs, not Python numbers, so every block shares one compiled graph. | |
| weights = self.ngram.weights.unbind(0) if ngram is not None and self.ngram.weights is not None else None | |
| for i, block in enumerate(blocks or self.blocks): | |
| if str(i) in self.ngram_gates: | |
| x = self.ngram_gates[str(i)](x, ngram, position) | |
| x = block(x, cos, sin, masks, *((ngram, weights[i]) if weights else ())) | |
| return x | |
| def forward(self, tokens, last_only=False): | |
| """Logits for every position, or only the last one (generation skips the rest of the output head).""" | |
| x = self.hidden(tokens) | |
| return self.head(x[:, -1:] if last_only else x) | |
| def parameter_count(cfg, vocab_size): | |
| """SPEC formula for the model with every option off: V*d + L*(4*d*d + 3*d*f + 2*d) + d.""" | |
| d, f, L = cfg["width"], cfg["ffn_intermediate"], cfg["blocks"] | |
| return vocab_size * d + L * (4 * d * d + 3 * d * f + 2 * d) + d | |