"""Cached decoding: next-token logits one token at a time, reusing each layer's keys and values instead of recomputing the prefix. It gives the same logits as model(tokens)[:, -1] (to float rounding) for every prefix up to the model's context length: global layers keep their compact KV heads (before the GQA expansion) and, with the key offset, the previous token's unrotated key slice; local layers keep only the last `local_window` positions; the n-gram table reads the last tokens of the current document. Batch size 1. Not used by training. decoder = Decoder(model) logits = decoder.prefill(prompt_ids) # (V,) for the token after the prompt logits = decoder.step(next_id) # (V,) after appending next_id """ import torch import torch.nn.functional as F from model import rotate EOS = 0 # documents start at EOS (model.document_positions) class Decoder: def __init__(self, model): assert not model.ngram_gates, "the gated n-gram module's convolution is not cached" self.model, self.cfg = model, model.cfg self.masked = bool(model.cfg.get("document_mask")) self.reset() def reset(self): self.tokens, self.positions = [], [] # ids, and each one's position in its document (or window) blocks = len(self.model.blocks) self.keys, self.values = [None] * blocks, [None] * blocks # (1, kv_heads, n, head_dim) per layer self.base = [0] * blocks # the window position of each layer's first cached key (local layers trim) self.previous = [None] * blocks # the previous token's unrotated key slice (key offset) @torch.no_grad() def prefill(self, tokens): self.reset() for t in tokens: logits = self.step(t) return logits @torch.no_grad() def step(self, token): m = self.model device = m.embedding.weight.device p = len(self.tokens) assert p < m.cos.shape[0], "cached decoding covers one window: the model's context length" self.tokens.append(int(token)) self.positions.append(0 if self.masked and token == EOS else (self.positions[-1] + 1 if p else 0)) start = p - self.positions[p] # the current document's start (the window's start without document masking) cos, sin = m.cos[p:p + 1], m.sin[p:p + 1] x = m.embedding(torch.tensor([[token]], device=device)) inject = m.ngram is not None and m.ngram.weights is not None if inject: # the rows of position p, from the last max-order tokens, as the full forward pass reads them span = max(order for order, *_ in m.ngram.channels) first = max(0, p + 1 - span) # a slice starting at 0 sees the window start as model.forward does position = torch.tensor([self.positions[first:]], device=device) if self.masked else None ngram = m.ngram(torch.tensor([self.tokens[first:]], device=device), position)[:, -1:] for i, block in enumerate(m.blocks): if inject: d = ngram.shape[-1] x = torch.cat([x[..., :d] + m.ngram.weights[i] * ngram, x[..., d:]], -1) x = x + self._attention(i, block.attention, block.attention_norm(x), cos, sin, p, start) gate, up = F.linear(block.ffn_norm(x), torch.cat([block.gate.weight, block.up.weight])).chunk(2, -1) x = x + block.down(F.silu(gate) * up) return m.head(x)[0, -1] def _attention(self, i, a, x, cos, sin, p, start): """Attention.forward for one new position p, against the cached keys of its document (local layers: of the last `window` positions).""" D = x.shape[-1] hd = D // a.heads qkv = F.linear(x, torch.cat([a.q.weight, a.k.weight, a.v.weight])) q, k, v = qkv.split((D, a.k.out_features, a.v.out_features), dim=-1) q = q.reshape(1, 1, a.heads, hd).transpose(1, 2) k = k.reshape(1, 1, a.kv_heads, hd).transpose(1, 2) v = v.reshape(1, 1, a.kv_heads, hd).transpose(1, 2) if a.q_norm is not None: q, k = a.q_norm(q), a.k_norm(k) q, k = rotate(q, cos, sin), rotate(k, cos, sin) if a.key_offset: current = k[..., a.rope:] # The previous token's slice; zero at the window's first position and at a document's first position. shifted = self.previous[i] if p > start else torch.zeros_like(current) if a.key_offset_mix is not None: shifted = torch.lerp(shifted, current, a.key_offset_mix.to(k.dtype)[None, :, None, None]) self.previous[i] = current k = torch.cat((k[..., :a.rope], shifted), dim=-1) self.keys[i] = k if self.keys[i] is None else torch.cat([self.keys[i], k], 2) self.values[i] = v if self.values[i] is None else torch.cat([self.values[i], v], 2) first = max(start, p - a.window + 1) if a.window else start # the first key this query sees if a.window and first > self.base[i]: # a local layer never needs keys before its window again self.keys[i] = self.keys[i][:, :, first - self.base[i]:] self.values[i] = self.values[i][:, :, first - self.base[i]:] self.base[i] = first keys = self.keys[i][:, :, first - self.base[i]:] values = self.values[i][:, :, first - self.base[i]:] if a.kv_heads != a.heads: keys = keys.repeat_interleave(a.heads // a.kv_heads, dim=1) values = values.repeat_interleave(a.heads // a.kv_heads, dim=1) y = F.scaled_dot_product_attention(q, keys, values) if a.head_gate is not None: y = y * torch.sigmoid(a.head_gate(x)).transpose(1, 2).unsqueeze(-1).to(y.dtype) return a.o(y.transpose(1, 2).reshape(1, 1, D))