Download decode.py from gustavecortal/PhiloLM: direct link, hf CLI and curl.
- Browser
- Download file 5.77 kB
-
https://huggingface.co/gustavecortal/PhiloLM/resolve/main/decode.py
- Command line
-
hf download hf://gustavecortal/PhiloLM/decode.py
-
curl -L -o decode.py https://huggingface.co/gustavecortal/PhiloLM/resolve/main/decode.py
5.77 kB
| """Cached decoding: next-token logits one token at a time, reusing each layer's keys and values instead of recomputing | |
| the prefix. It gives the same logits as model(tokens)[:, -1] (to float rounding) for every prefix up to the model's | |
| context length: global layers keep their compact KV heads (before the GQA expansion) and, with the key offset, the | |
| previous token's unrotated key slice; local layers keep only the last `local_window` positions; the n-gram table reads | |
| the last tokens of the current document. Batch size 1. Not used by training. | |
| decoder = Decoder(model) | |
| logits = decoder.prefill(prompt_ids) # (V,) for the token after the prompt | |
| logits = decoder.step(next_id) # (V,) after appending next_id | |
| """ | |
| import torch | |
| import torch.nn.functional as F | |
| from model import rotate | |
| EOS = 0 # documents start at EOS (model.document_positions) | |
| class Decoder: | |
| def __init__(self, model): | |
| assert not model.ngram_gates, "the gated n-gram module's convolution is not cached" | |
| self.model, self.cfg = model, model.cfg | |
| self.masked = bool(model.cfg.get("document_mask")) | |
| self.reset() | |
| def reset(self): | |
| self.tokens, self.positions = [], [] # ids, and each one's position in its document (or window) | |
| blocks = len(self.model.blocks) | |
| self.keys, self.values = [None] * blocks, [None] * blocks # (1, kv_heads, n, head_dim) per layer | |
| self.base = [0] * blocks # the window position of each layer's first cached key (local layers trim) | |
| self.previous = [None] * blocks # the previous token's unrotated key slice (key offset) | |
| def prefill(self, tokens): | |
| self.reset() | |
| for t in tokens: | |
| logits = self.step(t) | |
| return logits | |
| def step(self, token): | |
| m = self.model | |
| device = m.embedding.weight.device | |
| p = len(self.tokens) | |
| assert p < m.cos.shape[0], "cached decoding covers one window: the model's context length" | |
| self.tokens.append(int(token)) | |
| self.positions.append(0 if self.masked and token == EOS else (self.positions[-1] + 1 if p else 0)) | |
| start = p - self.positions[p] # the current document's start (the window's start without document masking) | |
| cos, sin = m.cos[p:p + 1], m.sin[p:p + 1] | |
| x = m.embedding(torch.tensor([[token]], device=device)) | |
| inject = m.ngram is not None and m.ngram.weights is not None | |
| if inject: # the rows of position p, from the last max-order tokens, as the full forward pass reads them | |
| span = max(order for order, *_ in m.ngram.channels) | |
| first = max(0, p + 1 - span) # a slice starting at 0 sees the window start as model.forward does | |
| position = torch.tensor([self.positions[first:]], device=device) if self.masked else None | |
| ngram = m.ngram(torch.tensor([self.tokens[first:]], device=device), position)[:, -1:] | |
| for i, block in enumerate(m.blocks): | |
| if inject: | |
| d = ngram.shape[-1] | |
| x = torch.cat([x[..., :d] + m.ngram.weights[i] * ngram, x[..., d:]], -1) | |
| x = x + self._attention(i, block.attention, block.attention_norm(x), cos, sin, p, start) | |
| gate, up = F.linear(block.ffn_norm(x), torch.cat([block.gate.weight, block.up.weight])).chunk(2, -1) | |
| x = x + block.down(F.silu(gate) * up) | |
| return m.head(x)[0, -1] | |
| def _attention(self, i, a, x, cos, sin, p, start): | |
| """Attention.forward for one new position p, against the cached keys of its document (local layers: of | |
| the last `window` positions).""" | |
| D = x.shape[-1] | |
| hd = D // a.heads | |
| qkv = F.linear(x, torch.cat([a.q.weight, a.k.weight, a.v.weight])) | |
| q, k, v = qkv.split((D, a.k.out_features, a.v.out_features), dim=-1) | |
| q = q.reshape(1, 1, a.heads, hd).transpose(1, 2) | |
| k = k.reshape(1, 1, a.kv_heads, hd).transpose(1, 2) | |
| v = v.reshape(1, 1, a.kv_heads, hd).transpose(1, 2) | |
| if a.q_norm is not None: | |
| q, k = a.q_norm(q), a.k_norm(k) | |
| q, k = rotate(q, cos, sin), rotate(k, cos, sin) | |
| if a.key_offset: | |
| current = k[..., a.rope:] | |
| # The previous token's slice; zero at the window's first position and at a document's first position. | |
| shifted = self.previous[i] if p > start else torch.zeros_like(current) | |
| if a.key_offset_mix is not None: | |
| shifted = torch.lerp(shifted, current, a.key_offset_mix.to(k.dtype)[None, :, None, None]) | |
| self.previous[i] = current | |
| k = torch.cat((k[..., :a.rope], shifted), dim=-1) | |
| self.keys[i] = k if self.keys[i] is None else torch.cat([self.keys[i], k], 2) | |
| self.values[i] = v if self.values[i] is None else torch.cat([self.values[i], v], 2) | |
| first = max(start, p - a.window + 1) if a.window else start # the first key this query sees | |
| if a.window and first > self.base[i]: # a local layer never needs keys before its window again | |
| self.keys[i] = self.keys[i][:, :, first - self.base[i]:] | |
| self.values[i] = self.values[i][:, :, first - self.base[i]:] | |
| self.base[i] = first | |
| keys = self.keys[i][:, :, first - self.base[i]:] | |
| values = self.values[i][:, :, first - self.base[i]:] | |
| if a.kv_heads != a.heads: | |
| keys = keys.repeat_interleave(a.heads // a.kv_heads, dim=1) | |
| values = values.repeat_interleave(a.heads // a.kv_heads, dim=1) | |
| y = F.scaled_dot_product_attention(q, keys, values) | |
| if a.head_gate is not None: | |
| y = y * torch.sigmoid(a.head_gate(x)).transpose(1, 2).unsqueeze(-1).to(y.dtype) | |
| return a.o(y.transpose(1, 2).reshape(1, 1, D)) | |