PhiloLM / decode.py
gustavecortal's picture
philosophy-lm: model card, code, tokenizer and browser files
b9ee6b4 verified
Raw History Blame Contribute Delete
5.77 kB
"""Cached decoding: next-token logits one token at a time, reusing each layer's keys and values instead of recomputing
the prefix. It gives the same logits as model(tokens)[:, -1] (to float rounding) for every prefix up to the model's
context length: global layers keep their compact KV heads (before the GQA expansion) and, with the key offset, the
previous token's unrotated key slice; local layers keep only the last `local_window` positions; the n-gram table reads
the last tokens of the current document. Batch size 1. Not used by training.
decoder = Decoder(model)
logits = decoder.prefill(prompt_ids) # (V,) for the token after the prompt
logits = decoder.step(next_id) # (V,) after appending next_id
"""
import torch
import torch.nn.functional as F
from model import rotate
EOS = 0 # documents start at EOS (model.document_positions)
class Decoder:
def __init__(self, model):
assert not model.ngram_gates, "the gated n-gram module's convolution is not cached"
self.model, self.cfg = model, model.cfg
self.masked = bool(model.cfg.get("document_mask"))
self.reset()
def reset(self):
self.tokens, self.positions = [], [] # ids, and each one's position in its document (or window)
blocks = len(self.model.blocks)
self.keys, self.values = [None] * blocks, [None] * blocks # (1, kv_heads, n, head_dim) per layer
self.base = [0] * blocks # the window position of each layer's first cached key (local layers trim)
self.previous = [None] * blocks # the previous token's unrotated key slice (key offset)
@torch.no_grad()
def prefill(self, tokens):
self.reset()
for t in tokens:
logits = self.step(t)
return logits
@torch.no_grad()
def step(self, token):
m = self.model
device = m.embedding.weight.device
p = len(self.tokens)
assert p < m.cos.shape[0], "cached decoding covers one window: the model's context length"
self.tokens.append(int(token))
self.positions.append(0 if self.masked and token == EOS else (self.positions[-1] + 1 if p else 0))
start = p - self.positions[p] # the current document's start (the window's start without document masking)
cos, sin = m.cos[p:p + 1], m.sin[p:p + 1]
x = m.embedding(torch.tensor([[token]], device=device))
inject = m.ngram is not None and m.ngram.weights is not None
if inject: # the rows of position p, from the last max-order tokens, as the full forward pass reads them
span = max(order for order, *_ in m.ngram.channels)
first = max(0, p + 1 - span) # a slice starting at 0 sees the window start as model.forward does
position = torch.tensor([self.positions[first:]], device=device) if self.masked else None
ngram = m.ngram(torch.tensor([self.tokens[first:]], device=device), position)[:, -1:]
for i, block in enumerate(m.blocks):
if inject:
d = ngram.shape[-1]
x = torch.cat([x[..., :d] + m.ngram.weights[i] * ngram, x[..., d:]], -1)
x = x + self._attention(i, block.attention, block.attention_norm(x), cos, sin, p, start)
gate, up = F.linear(block.ffn_norm(x), torch.cat([block.gate.weight, block.up.weight])).chunk(2, -1)
x = x + block.down(F.silu(gate) * up)
return m.head(x)[0, -1]
def _attention(self, i, a, x, cos, sin, p, start):
"""Attention.forward for one new position p, against the cached keys of its document (local layers: of
the last `window` positions)."""
D = x.shape[-1]
hd = D // a.heads
qkv = F.linear(x, torch.cat([a.q.weight, a.k.weight, a.v.weight]))
q, k, v = qkv.split((D, a.k.out_features, a.v.out_features), dim=-1)
q = q.reshape(1, 1, a.heads, hd).transpose(1, 2)
k = k.reshape(1, 1, a.kv_heads, hd).transpose(1, 2)
v = v.reshape(1, 1, a.kv_heads, hd).transpose(1, 2)
if a.q_norm is not None:
q, k = a.q_norm(q), a.k_norm(k)
q, k = rotate(q, cos, sin), rotate(k, cos, sin)
if a.key_offset:
current = k[..., a.rope:]
# The previous token's slice; zero at the window's first position and at a document's first position.
shifted = self.previous[i] if p > start else torch.zeros_like(current)
if a.key_offset_mix is not None:
shifted = torch.lerp(shifted, current, a.key_offset_mix.to(k.dtype)[None, :, None, None])
self.previous[i] = current
k = torch.cat((k[..., :a.rope], shifted), dim=-1)
self.keys[i] = k if self.keys[i] is None else torch.cat([self.keys[i], k], 2)
self.values[i] = v if self.values[i] is None else torch.cat([self.values[i], v], 2)
first = max(start, p - a.window + 1) if a.window else start # the first key this query sees
if a.window and first > self.base[i]: # a local layer never needs keys before its window again
self.keys[i] = self.keys[i][:, :, first - self.base[i]:]
self.values[i] = self.values[i][:, :, first - self.base[i]:]
self.base[i] = first
keys = self.keys[i][:, :, first - self.base[i]:]
values = self.values[i][:, :, first - self.base[i]:]
if a.kv_heads != a.heads:
keys = keys.repeat_interleave(a.heads // a.kv_heads, dim=1)
values = values.repeat_interleave(a.heads // a.kv_heads, dim=1)
y = F.scaled_dot_product_attention(q, keys, values)
if a.head_gate is not None:
y = y * torch.sigmoid(a.head_gate(x)).transpose(1, 2).unsqueeze(-1).to(y.dtype)
return a.o(y.transpose(1, 2).reshape(1, 1, D))