File size: 5,768 Bytes
b9ee6b4
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
"""Cached decoding: next-token logits one token at a time, reusing each layer's keys and values instead of recomputing
the prefix. It gives the same logits as model(tokens)[:, -1] (to float rounding) for every prefix up to the model's
context length: global layers keep their compact KV heads (before the GQA expansion) and, with the key offset, the
previous token's unrotated key slice; local layers keep only the last `local_window` positions; the n-gram table reads
the last tokens of the current document. Batch size 1. Not used by training.

    decoder = Decoder(model)
    logits = decoder.prefill(prompt_ids)   # (V,) for the token after the prompt
    logits = decoder.step(next_id)         # (V,) after appending next_id
"""
import torch
import torch.nn.functional as F

from model import rotate

EOS = 0  # documents start at EOS (model.document_positions)


class Decoder:
    def __init__(self, model):
        assert not model.ngram_gates, "the gated n-gram module's convolution is not cached"
        self.model, self.cfg = model, model.cfg
        self.masked = bool(model.cfg.get("document_mask"))
        self.reset()

    def reset(self):
        self.tokens, self.positions = [], []  # ids, and each one's position in its document (or window)
        blocks = len(self.model.blocks)
        self.keys, self.values = [None] * blocks, [None] * blocks  # (1, kv_heads, n, head_dim) per layer
        self.base = [0] * blocks  # the window position of each layer's first cached key (local layers trim)
        self.previous = [None] * blocks  # the previous token's unrotated key slice (key offset)

    @torch.no_grad()
    def prefill(self, tokens):
        self.reset()
        for t in tokens:
            logits = self.step(t)
        return logits

    @torch.no_grad()
    def step(self, token):
        m = self.model
        device = m.embedding.weight.device
        p = len(self.tokens)
        assert p < m.cos.shape[0], "cached decoding covers one window: the model's context length"
        self.tokens.append(int(token))
        self.positions.append(0 if self.masked and token == EOS else (self.positions[-1] + 1 if p else 0))
        start = p - self.positions[p]  # the current document's start (the window's start without document masking)
        cos, sin = m.cos[p:p + 1], m.sin[p:p + 1]
        x = m.embedding(torch.tensor([[token]], device=device))
        inject = m.ngram is not None and m.ngram.weights is not None
        if inject:  # the rows of position p, from the last max-order tokens, as the full forward pass reads them
            span = max(order for order, *_ in m.ngram.channels)
            first = max(0, p + 1 - span)  # a slice starting at 0 sees the window start as model.forward does
            position = torch.tensor([self.positions[first:]], device=device) if self.masked else None
            ngram = m.ngram(torch.tensor([self.tokens[first:]], device=device), position)[:, -1:]
        for i, block in enumerate(m.blocks):
            if inject:
                d = ngram.shape[-1]
                x = torch.cat([x[..., :d] + m.ngram.weights[i] * ngram, x[..., d:]], -1)
            x = x + self._attention(i, block.attention, block.attention_norm(x), cos, sin, p, start)
            gate, up = F.linear(block.ffn_norm(x), torch.cat([block.gate.weight, block.up.weight])).chunk(2, -1)
            x = x + block.down(F.silu(gate) * up)
        return m.head(x)[0, -1]

    def _attention(self, i, a, x, cos, sin, p, start):
        """Attention.forward for one new position p, against the cached keys of its document (local layers: of
        the last `window` positions)."""
        D = x.shape[-1]
        hd = D // a.heads
        qkv = F.linear(x, torch.cat([a.q.weight, a.k.weight, a.v.weight]))
        q, k, v = qkv.split((D, a.k.out_features, a.v.out_features), dim=-1)
        q = q.reshape(1, 1, a.heads, hd).transpose(1, 2)
        k = k.reshape(1, 1, a.kv_heads, hd).transpose(1, 2)
        v = v.reshape(1, 1, a.kv_heads, hd).transpose(1, 2)
        if a.q_norm is not None:
            q, k = a.q_norm(q), a.k_norm(k)
        q, k = rotate(q, cos, sin), rotate(k, cos, sin)
        if a.key_offset:
            current = k[..., a.rope:]
            # The previous token's slice; zero at the window's first position and at a document's first position.
            shifted = self.previous[i] if p > start else torch.zeros_like(current)
            if a.key_offset_mix is not None:
                shifted = torch.lerp(shifted, current, a.key_offset_mix.to(k.dtype)[None, :, None, None])
            self.previous[i] = current
            k = torch.cat((k[..., :a.rope], shifted), dim=-1)
        self.keys[i] = k if self.keys[i] is None else torch.cat([self.keys[i], k], 2)
        self.values[i] = v if self.values[i] is None else torch.cat([self.values[i], v], 2)
        first = max(start, p - a.window + 1) if a.window else start  # the first key this query sees
        if a.window and first > self.base[i]:  # a local layer never needs keys before its window again
            self.keys[i] = self.keys[i][:, :, first - self.base[i]:]
            self.values[i] = self.values[i][:, :, first - self.base[i]:]
            self.base[i] = first
        keys = self.keys[i][:, :, first - self.base[i]:]
        values = self.values[i][:, :, first - self.base[i]:]
        if a.kv_heads != a.heads:
            keys = keys.repeat_interleave(a.heads // a.kv_heads, dim=1)
            values = values.repeat_interleave(a.heads // a.kv_heads, dim=1)
        y = F.scaled_dot_product_attention(q, keys, values)
        if a.head_gate is not None:
            y = y * torch.sigmoid(a.head_gate(x)).transpose(1, 2).unsqueeze(-1).to(y.dtype)
        return a.o(y.transpose(1, 2).reshape(1, 1, D))